This repository has no description
0

Configure Feed

Select the types of activity you want to include in your feed.

tools.lazy-store: tokio and reqwest instead of ureq and threads

The I/O runs on a tokio runtime now:

- HTTP via reqwest (rustls on aws-lc, HTTP/2, bundled webpki roots as
before) instead of ureq. Instead of our own caching ureq::Resolver the
cache hosts are resolved once at startup and pinned with
resolve_to_addrs, which keeps getaddrinfo() (and NSS modules from the
store) out of requests once /nix/store is mounted. The retry loop
stays: reqwest-middleware/retry would not save code, as body read
errors are retried too.
- Decompression via async-compression (zstd, xz, bzip2, gzip, brotli,
concatenated members like Nix) instead of zstd/xz2/bzip2/flate2/
brotli-decompressor; the magic sniffing for narinfo/.ls stays.
- The closure walk and the .ls listings run with buffer_unordered
instead of par_map threads.
- FUSE workers stay threads; open() fetches via Handle::block_on, and
concurrent opens of a path wait on a tokio Mutex.
- The compressed size is counted with tokio-util's InspectReader instead
of our CountingReader. The NAR is still unpacked by our reader for now,
through SyncIoBridge.

+1215 -267
+1022 -78
tools/lazy-store/Cargo.lock
··· 10 10 11 11 [[package]] 12 12 name = "alloc-no-stdlib" 13 - version = "2.0.4" 13 + version = "3.0.0" 14 14 source = "registry+https://github.com/rust-lang/crates.io-index" 15 - checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" 15 + checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" 16 16 17 17 [[package]] 18 18 name = "alloc-stdlib" 19 - version = "0.2.4" 19 + version = "0.3.0" 20 20 source = "registry+https://github.com/rust-lang/crates.io-index" 21 - checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" 21 + checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" 22 22 dependencies = [ 23 23 "alloc-no-stdlib", 24 24 ] ··· 33 33 ] 34 34 35 35 [[package]] 36 + name = "async-compression" 37 + version = "0.4.51" 38 + source = "registry+https://github.com/rust-lang/crates.io-index" 39 + checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" 40 + dependencies = [ 41 + "compression-codecs", 42 + "compression-core", 43 + "pin-project-lite", 44 + "tokio", 45 + ] 46 + 47 + [[package]] 48 + name = "atomic-waker" 49 + version = "1.1.2" 50 + source = "registry+https://github.com/rust-lang/crates.io-index" 51 + checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" 52 + 53 + [[package]] 36 54 name = "autocfg" 37 55 version = "1.5.1" 38 56 source = "registry+https://github.com/rust-lang/crates.io-index" 39 57 checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" 40 58 41 59 [[package]] 42 - name = "base64" 43 - version = "0.22.1" 60 + name = "aws-lc-rs" 61 + version = "1.18.2" 44 62 source = "registry+https://github.com/rust-lang/crates.io-index" 45 - checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" 63 + checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" 64 + dependencies = [ 65 + "aws-lc-sys", 66 + "zeroize", 67 + ] 68 + 69 + [[package]] 70 + name = "aws-lc-sys" 71 + version = "0.46.0" 72 + source = "registry+https://github.com/rust-lang/crates.io-index" 73 + checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" 74 + dependencies = [ 75 + "cc", 76 + "cmake", 77 + "dunce", 78 + "fs_extra", 79 + "pkg-config", 80 + ] 81 + 82 + [[package]] 83 + name = "base64" 84 + version = "0.23.1" 85 + source = "registry+https://github.com/rust-lang/crates.io-index" 86 + checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" 46 87 47 88 [[package]] 48 89 name = "bitflags" 49 90 version = "1.3.2" 50 91 source = "registry+https://github.com/rust-lang/crates.io-index" 51 92 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 93 + 94 + [[package]] 95 + name = "bitflags" 96 + version = "2.13.2" 97 + source = "registry+https://github.com/rust-lang/crates.io-index" 98 + checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" 52 99 53 100 [[package]] 54 101 name = "block-buffer" ··· 60 107 ] 61 108 62 109 [[package]] 63 - name = "brotli-decompressor" 64 - version = "4.0.3" 110 + name = "brotli" 111 + version = "9.0.0" 65 112 source = "registry+https://github.com/rust-lang/crates.io-index" 66 - checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" 113 + checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b" 114 + dependencies = [ 115 + "alloc-no-stdlib", 116 + "alloc-stdlib", 117 + "brotli-decompressor", 118 + ] 119 + 120 + [[package]] 121 + name = "brotli-decompressor" 122 + version = "6.0.1" 123 + source = "registry+https://github.com/rust-lang/crates.io-index" 124 + checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7" 67 125 dependencies = [ 68 126 "alloc-no-stdlib", 69 127 "alloc-stdlib", 70 128 ] 71 129 72 130 [[package]] 73 - name = "bzip2" 74 - version = "0.4.4" 131 + name = "bumpalo" 132 + version = "3.20.3" 75 133 source = "registry+https://github.com/rust-lang/crates.io-index" 76 - checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" 77 - dependencies = [ 78 - "bzip2-sys", 79 - "libc", 80 - ] 134 + checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" 81 135 82 136 [[package]] 83 - name = "bzip2-sys" 84 - version = "0.1.13+1.0.8" 137 + name = "bytes" 138 + version = "1.12.1" 85 139 source = "registry+https://github.com/rust-lang/crates.io-index" 86 - checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" 140 + checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" 141 + 142 + [[package]] 143 + name = "bzip2" 144 + version = "0.6.1" 145 + source = "registry+https://github.com/rust-lang/crates.io-index" 146 + checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" 87 147 dependencies = [ 88 - "cc", 89 - "pkg-config", 148 + "libbz2-rs-sys", 90 149 ] 91 150 92 151 [[package]] ··· 117 176 checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" 118 177 119 178 [[package]] 179 + name = "cfg_aliases" 180 + version = "0.2.2" 181 + source = "registry+https://github.com/rust-lang/crates.io-index" 182 + checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" 183 + 184 + [[package]] 185 + name = "chacha20" 186 + version = "0.10.2" 187 + source = "registry+https://github.com/rust-lang/crates.io-index" 188 + checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" 189 + dependencies = [ 190 + "cfg-if", 191 + "cpufeatures 0.3.1", 192 + "rand_core", 193 + ] 194 + 195 + [[package]] 196 + name = "cmake" 197 + version = "0.1.58" 198 + source = "registry+https://github.com/rust-lang/crates.io-index" 199 + checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" 200 + dependencies = [ 201 + "cc", 202 + ] 203 + 204 + [[package]] 205 + name = "combine" 206 + version = "4.6.9" 207 + source = "registry+https://github.com/rust-lang/crates.io-index" 208 + checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" 209 + dependencies = [ 210 + "bytes", 211 + "memchr", 212 + ] 213 + 214 + [[package]] 215 + name = "compression-codecs" 216 + version = "0.4.46" 217 + source = "registry+https://github.com/rust-lang/crates.io-index" 218 + checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" 219 + dependencies = [ 220 + "brotli", 221 + "bzip2", 222 + "compression-core", 223 + "flate2", 224 + "liblzma", 225 + "memchr", 226 + "zstd", 227 + "zstd-safe", 228 + ] 229 + 230 + [[package]] 231 + name = "compression-core" 232 + version = "0.4.34" 233 + source = "registry+https://github.com/rust-lang/crates.io-index" 234 + checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" 235 + 236 + [[package]] 120 237 name = "convert_case" 121 238 version = "0.10.0" 122 239 source = "registry+https://github.com/rust-lang/crates.io-index" 123 240 checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" 124 241 dependencies = [ 125 242 "unicode-segmentation", 243 + ] 244 + 245 + [[package]] 246 + name = "core-foundation" 247 + version = "0.10.1" 248 + source = "registry+https://github.com/rust-lang/crates.io-index" 249 + checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" 250 + dependencies = [ 251 + "core-foundation-sys", 252 + "libc", 126 253 ] 127 254 128 255 [[package]] ··· 136 263 version = "0.2.17" 137 264 source = "registry+https://github.com/rust-lang/crates.io-index" 138 265 checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" 266 + dependencies = [ 267 + "libc", 268 + ] 269 + 270 + [[package]] 271 + name = "cpufeatures" 272 + version = "0.3.1" 273 + source = "registry+https://github.com/rust-lang/crates.io-index" 274 + checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" 139 275 dependencies = [ 140 276 "libc", 141 277 ] ··· 210 346 ] 211 347 212 348 [[package]] 349 + name = "dunce" 350 + version = "1.0.5" 351 + source = "registry+https://github.com/rust-lang/crates.io-index" 352 + checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" 353 + 354 + [[package]] 213 355 name = "endian-type" 214 356 version = "0.1.2" 215 357 source = "registry+https://github.com/rust-lang/crates.io-index" 216 358 checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" 359 + 360 + [[package]] 361 + name = "equivalent" 362 + version = "1.0.3" 363 + source = "registry+https://github.com/rust-lang/crates.io-index" 364 + checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" 217 365 218 366 [[package]] 219 367 name = "find-msvc-tools" ··· 233 381 ] 234 382 235 383 [[package]] 384 + name = "fnv" 385 + version = "1.0.7" 386 + source = "registry+https://github.com/rust-lang/crates.io-index" 387 + checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 388 + 389 + [[package]] 236 390 name = "form_urlencoded" 237 391 version = "1.2.2" 238 392 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 242 396 ] 243 397 244 398 [[package]] 399 + name = "fs_extra" 400 + version = "1.3.0" 401 + source = "registry+https://github.com/rust-lang/crates.io-index" 402 + checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" 403 + 404 + [[package]] 245 405 name = "fuse-backend-core" 246 406 version = "0.14.0" 247 407 source = "git+https://github.com/Kranzes/fuse-backend-rs?rev=bac50ad4cf022b3428ac483cf14fd6f71817282f#bac50ad4cf022b3428ac483cf14fd6f71817282f" 248 408 dependencies = [ 249 - "bitflags", 409 + "bitflags 1.3.2", 250 410 "libc", 251 411 "log", 252 412 "vm-memory", ··· 261 421 "fuse-backend-core", 262 422 "libc", 263 423 "log", 264 - "mio", 424 + "mio 0.8.11", 265 425 "nix", 266 426 "vm-memory", 267 427 "vmm-sys-util", ··· 284 444 version = "0.14.0" 285 445 source = "git+https://github.com/Kranzes/fuse-backend-rs?rev=bac50ad4cf022b3428ac483cf14fd6f71817282f#bac50ad4cf022b3428ac483cf14fd6f71817282f" 286 446 dependencies = [ 287 - "bitflags", 447 + "bitflags 1.3.2", 288 448 "caps", 289 449 "fuse-backend-core", 290 450 "libc", ··· 317 477 ] 318 478 319 479 [[package]] 480 + name = "futures-channel" 481 + version = "0.3.34" 482 + source = "registry+https://github.com/rust-lang/crates.io-index" 483 + checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" 484 + dependencies = [ 485 + "futures-core", 486 + ] 487 + 488 + [[package]] 489 + name = "futures-core" 490 + version = "0.3.34" 491 + source = "registry+https://github.com/rust-lang/crates.io-index" 492 + checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" 493 + 494 + [[package]] 495 + name = "futures-io" 496 + version = "0.3.34" 497 + source = "registry+https://github.com/rust-lang/crates.io-index" 498 + checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" 499 + 500 + [[package]] 501 + name = "futures-macro" 502 + version = "0.3.34" 503 + source = "registry+https://github.com/rust-lang/crates.io-index" 504 + checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" 505 + dependencies = [ 506 + "proc-macro2", 507 + "quote", 508 + "syn 3.0.9", 509 + ] 510 + 511 + [[package]] 512 + name = "futures-sink" 513 + version = "0.3.34" 514 + source = "registry+https://github.com/rust-lang/crates.io-index" 515 + checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" 516 + 517 + [[package]] 518 + name = "futures-task" 519 + version = "0.3.34" 520 + source = "registry+https://github.com/rust-lang/crates.io-index" 521 + checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" 522 + 523 + [[package]] 524 + name = "futures-util" 525 + version = "0.3.34" 526 + source = "registry+https://github.com/rust-lang/crates.io-index" 527 + checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" 528 + dependencies = [ 529 + "futures-core", 530 + "futures-io", 531 + "futures-macro", 532 + "futures-sink", 533 + "futures-task", 534 + "memchr", 535 + "pin-project-lite", 536 + "slab", 537 + ] 538 + 539 + [[package]] 320 540 name = "generic-array" 321 541 version = "0.14.7" 322 542 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 333 553 checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" 334 554 dependencies = [ 335 555 "cfg-if", 556 + "js-sys", 336 557 "libc", 337 558 "wasi", 559 + "wasm-bindgen", 338 560 ] 339 561 340 562 [[package]] ··· 344 566 checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" 345 567 dependencies = [ 346 568 "cfg-if", 569 + "js-sys", 347 570 "libc", 348 571 "r-efi", 572 + "rand_core", 573 + "wasm-bindgen", 574 + ] 575 + 576 + [[package]] 577 + name = "h2" 578 + version = "0.4.20" 579 + source = "registry+https://github.com/rust-lang/crates.io-index" 580 + checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" 581 + dependencies = [ 582 + "atomic-waker", 583 + "bytes", 584 + "fnv", 585 + "futures-core", 586 + "futures-sink", 587 + "http", 588 + "indexmap", 589 + "slab", 590 + "tokio", 591 + "tokio-util", 592 + "tracing", 349 593 ] 350 594 351 595 [[package]] ··· 366 610 "data-encoding", 367 611 "derive_more", 368 612 "serde", 613 + ] 614 + 615 + [[package]] 616 + name = "hashbrown" 617 + version = "0.17.1" 618 + source = "registry+https://github.com/rust-lang/crates.io-index" 619 + checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" 620 + 621 + [[package]] 622 + name = "http" 623 + version = "1.5.0" 624 + source = "registry+https://github.com/rust-lang/crates.io-index" 625 + checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" 626 + dependencies = [ 627 + "bytes", 628 + "itoa", 629 + ] 630 + 631 + [[package]] 632 + name = "http-body" 633 + version = "1.1.0" 634 + source = "registry+https://github.com/rust-lang/crates.io-index" 635 + checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" 636 + dependencies = [ 637 + "bytes", 638 + "http", 639 + ] 640 + 641 + [[package]] 642 + name = "http-body-util" 643 + version = "0.1.5" 644 + source = "registry+https://github.com/rust-lang/crates.io-index" 645 + checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" 646 + dependencies = [ 647 + "bytes", 648 + "futures-core", 649 + "http", 650 + "http-body", 651 + "pin-project-lite", 652 + ] 653 + 654 + [[package]] 655 + name = "httparse" 656 + version = "1.10.1" 657 + source = "registry+https://github.com/rust-lang/crates.io-index" 658 + checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" 659 + 660 + [[package]] 661 + name = "hyper" 662 + version = "1.12.0" 663 + source = "registry+https://github.com/rust-lang/crates.io-index" 664 + checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" 665 + dependencies = [ 666 + "atomic-waker", 667 + "bytes", 668 + "futures-channel", 669 + "futures-core", 670 + "h2", 671 + "http", 672 + "http-body", 673 + "httparse", 674 + "itoa", 675 + "pin-project-lite", 676 + "smallvec", 677 + "tokio", 678 + "want", 679 + ] 680 + 681 + [[package]] 682 + name = "hyper-rustls" 683 + version = "0.27.10" 684 + source = "registry+https://github.com/rust-lang/crates.io-index" 685 + checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" 686 + dependencies = [ 687 + "http", 688 + "hyper", 689 + "hyper-util", 690 + "rustls", 691 + "tokio", 692 + "tokio-rustls", 693 + "tower-service", 694 + ] 695 + 696 + [[package]] 697 + name = "hyper-util" 698 + version = "0.1.21" 699 + source = "registry+https://github.com/rust-lang/crates.io-index" 700 + checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" 701 + dependencies = [ 702 + "base64", 703 + "bytes", 704 + "futures-channel", 705 + "futures-util", 706 + "http", 707 + "http-body", 708 + "httparse", 709 + "hyper", 710 + "ipnet", 711 + "libc", 712 + "percent-encoding", 713 + "pin-project-lite", 714 + "socket2", 715 + "tokio", 716 + "tower-service", 717 + "tracing", 369 718 ] 370 719 371 720 [[package]] ··· 473 822 ] 474 823 475 824 [[package]] 825 + name = "indexmap" 826 + version = "2.14.2" 827 + source = "registry+https://github.com/rust-lang/crates.io-index" 828 + checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" 829 + dependencies = [ 830 + "equivalent", 831 + "hashbrown", 832 + ] 833 + 834 + [[package]] 835 + name = "ipnet" 836 + version = "2.12.2" 837 + source = "registry+https://github.com/rust-lang/crates.io-index" 838 + checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" 839 + 840 + [[package]] 476 841 name = "itoa" 477 842 version = "1.0.18" 478 843 source = "registry+https://github.com/rust-lang/crates.io-index" 479 844 checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" 845 + 846 + [[package]] 847 + name = "jni" 848 + version = "0.22.4" 849 + source = "registry+https://github.com/rust-lang/crates.io-index" 850 + checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" 851 + dependencies = [ 852 + "cfg-if", 853 + "combine", 854 + "jni-macros", 855 + "jni-sys", 856 + "log", 857 + "simd_cesu8", 858 + "thiserror", 859 + "walkdir", 860 + "windows-link", 861 + ] 862 + 863 + [[package]] 864 + name = "jni-macros" 865 + version = "0.22.4" 866 + source = "registry+https://github.com/rust-lang/crates.io-index" 867 + checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" 868 + dependencies = [ 869 + "proc-macro2", 870 + "quote", 871 + "rustc_version", 872 + "simd_cesu8", 873 + "syn 2.0.119", 874 + ] 875 + 876 + [[package]] 877 + name = "jni-sys" 878 + version = "0.4.1" 879 + source = "registry+https://github.com/rust-lang/crates.io-index" 880 + checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" 881 + dependencies = [ 882 + "jni-sys-macros", 883 + ] 884 + 885 + [[package]] 886 + name = "jni-sys-macros" 887 + version = "0.4.1" 888 + source = "registry+https://github.com/rust-lang/crates.io-index" 889 + checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" 890 + dependencies = [ 891 + "quote", 892 + "syn 2.0.119", 893 + ] 480 894 481 895 [[package]] 482 896 name = "jobserver" ··· 489 903 ] 490 904 491 905 [[package]] 906 + name = "js-sys" 907 + version = "0.3.106" 908 + source = "registry+https://github.com/rust-lang/crates.io-index" 909 + checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" 910 + dependencies = [ 911 + "cfg-if", 912 + "futures-util", 913 + "wasm-bindgen", 914 + ] 915 + 916 + [[package]] 492 917 name = "lazy-store" 493 918 version = "0.1.0" 494 919 dependencies = [ 495 - "brotli-decompressor", 496 - "bzip2", 497 - "flate2", 920 + "async-compression", 498 921 "fuse-backend-rs", 922 + "futures-util", 499 923 "harmonia-file-core", 500 924 "harmonia-utils-base-encoding", 501 925 "libc", 502 926 "log", 927 + "reqwest", 503 928 "serde", 504 929 "serde_json", 505 930 "sha2", 506 - "ureq", 507 - "xz2", 508 - "zstd", 931 + "tokio", 932 + "tokio-util", 933 + "webpki-root-certs", 509 934 ] 935 + 936 + [[package]] 937 + name = "libbz2-rs-sys" 938 + version = "0.2.5" 939 + source = "registry+https://github.com/rust-lang/crates.io-index" 940 + checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" 510 941 511 942 [[package]] 512 943 name = "libc" 513 944 version = "0.2.190" 514 945 source = "registry+https://github.com/rust-lang/crates.io-index" 515 946 checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" 947 + 948 + [[package]] 949 + name = "liblzma" 950 + version = "0.4.8" 951 + source = "registry+https://github.com/rust-lang/crates.io-index" 952 + checksum = "2fe0a34ca854fd4f20c07f696fc8675aec78f87d88d29f5e10257a7490a1b2e1" 953 + dependencies = [ 954 + "liblzma-sys", 955 + ] 956 + 957 + [[package]] 958 + name = "liblzma-sys" 959 + version = "0.4.9" 960 + source = "registry+https://github.com/rust-lang/crates.io-index" 961 + checksum = "7c7e3581f367a78d7b7e7ae948d023310556f0cfc13156c2e4e00e25616492b9" 962 + dependencies = [ 963 + "cc", 964 + "libc", 965 + "pkg-config", 966 + ] 516 967 517 968 [[package]] 518 969 name = "litemap" ··· 527 978 checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" 528 979 529 980 [[package]] 530 - name = "lzma-sys" 531 - version = "0.1.20" 981 + name = "lru-slab" 982 + version = "0.1.3" 532 983 source = "registry+https://github.com/rust-lang/crates.io-index" 533 - checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" 534 - dependencies = [ 535 - "cc", 536 - "libc", 537 - "pkg-config", 538 - ] 984 + checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" 539 985 540 986 [[package]] 541 987 name = "memchr" ··· 575 1021 ] 576 1022 577 1023 [[package]] 1024 + name = "mio" 1025 + version = "1.2.4" 1026 + source = "registry+https://github.com/rust-lang/crates.io-index" 1027 + checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" 1028 + dependencies = [ 1029 + "libc", 1030 + "wasi", 1031 + "windows-sys 0.61.2", 1032 + ] 1033 + 1034 + [[package]] 578 1035 name = "nibble_vec" 579 1036 version = "0.1.0" 580 1037 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 589 1046 source = "registry+https://github.com/rust-lang/crates.io-index" 590 1047 checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" 591 1048 dependencies = [ 592 - "bitflags", 1049 + "bitflags 1.3.2", 593 1050 "cfg-if", 594 1051 "libc", 595 1052 "memoffset", ··· 602 1059 checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" 603 1060 604 1061 [[package]] 1062 + name = "openssl-probe" 1063 + version = "0.2.1" 1064 + source = "registry+https://github.com/rust-lang/crates.io-index" 1065 + checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" 1066 + 1067 + [[package]] 605 1068 name = "percent-encoding" 606 1069 version = "2.3.2" 607 1070 source = "registry+https://github.com/rust-lang/crates.io-index" 608 1071 checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" 1072 + 1073 + [[package]] 1074 + name = "pin-project-lite" 1075 + version = "0.2.17" 1076 + source = "registry+https://github.com/rust-lang/crates.io-index" 1077 + checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" 609 1078 610 1079 [[package]] 611 1080 name = "pkg-config" ··· 629 1098 checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" 630 1099 dependencies = [ 631 1100 "unicode-ident", 1101 + ] 1102 + 1103 + [[package]] 1104 + name = "quinn" 1105 + version = "0.11.12" 1106 + source = "registry+https://github.com/rust-lang/crates.io-index" 1107 + checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" 1108 + dependencies = [ 1109 + "bytes", 1110 + "cfg_aliases", 1111 + "pin-project-lite", 1112 + "quinn-proto", 1113 + "quinn-udp", 1114 + "rustc-hash", 1115 + "rustls", 1116 + "socket2", 1117 + "thiserror", 1118 + "tokio", 1119 + "tracing", 1120 + "web-time", 1121 + ] 1122 + 1123 + [[package]] 1124 + name = "quinn-proto" 1125 + version = "0.11.19" 1126 + source = "registry+https://github.com/rust-lang/crates.io-index" 1127 + checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" 1128 + dependencies = [ 1129 + "aws-lc-rs", 1130 + "bytes", 1131 + "getrandom 0.4.3", 1132 + "lru-slab", 1133 + "rand", 1134 + "rand_pcg", 1135 + "ring", 1136 + "rustc-hash", 1137 + "rustls", 1138 + "rustls-pki-types", 1139 + "slab", 1140 + "thiserror", 1141 + "tinyvec", 1142 + "tracing", 1143 + "web-time", 1144 + ] 1145 + 1146 + [[package]] 1147 + name = "quinn-udp" 1148 + version = "0.5.16" 1149 + source = "registry+https://github.com/rust-lang/crates.io-index" 1150 + checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" 1151 + dependencies = [ 1152 + "cfg_aliases", 1153 + "libc", 1154 + "once_cell", 1155 + "socket2", 1156 + "tracing", 1157 + "windows-sys 0.52.0", 632 1158 ] 633 1159 634 1160 [[package]] ··· 657 1183 ] 658 1184 659 1185 [[package]] 1186 + name = "rand" 1187 + version = "0.10.3" 1188 + source = "registry+https://github.com/rust-lang/crates.io-index" 1189 + checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" 1190 + dependencies = [ 1191 + "chacha20", 1192 + "getrandom 0.4.3", 1193 + "rand_core", 1194 + ] 1195 + 1196 + [[package]] 1197 + name = "rand_core" 1198 + version = "0.10.1" 1199 + source = "registry+https://github.com/rust-lang/crates.io-index" 1200 + checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" 1201 + 1202 + [[package]] 1203 + name = "rand_pcg" 1204 + version = "0.10.2" 1205 + source = "registry+https://github.com/rust-lang/crates.io-index" 1206 + checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" 1207 + dependencies = [ 1208 + "rand_core", 1209 + ] 1210 + 1211 + [[package]] 1212 + name = "reqwest" 1213 + version = "0.13.5" 1214 + source = "registry+https://github.com/rust-lang/crates.io-index" 1215 + checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" 1216 + dependencies = [ 1217 + "base64", 1218 + "bytes", 1219 + "futures-core", 1220 + "futures-util", 1221 + "h2", 1222 + "http", 1223 + "http-body", 1224 + "http-body-util", 1225 + "hyper", 1226 + "hyper-rustls", 1227 + "hyper-util", 1228 + "js-sys", 1229 + "log", 1230 + "percent-encoding", 1231 + "pin-project-lite", 1232 + "quinn", 1233 + "rustls", 1234 + "rustls-pki-types", 1235 + "rustls-platform-verifier", 1236 + "sync_wrapper", 1237 + "tokio", 1238 + "tokio-rustls", 1239 + "tokio-util", 1240 + "tower", 1241 + "tower-http", 1242 + "tower-service", 1243 + "url", 1244 + "wasm-bindgen", 1245 + "wasm-bindgen-futures", 1246 + "wasm-streams", 1247 + "web-sys", 1248 + ] 1249 + 1250 + [[package]] 660 1251 name = "ring" 661 1252 version = "0.17.14" 662 1253 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 669 1260 "untrusted", 670 1261 "windows-sys 0.52.0", 671 1262 ] 1263 + 1264 + [[package]] 1265 + name = "rustc-hash" 1266 + version = "2.1.3" 1267 + source = "registry+https://github.com/rust-lang/crates.io-index" 1268 + checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" 672 1269 673 1270 [[package]] 674 1271 name = "rustc_version" ··· 685 1282 source = "registry+https://github.com/rust-lang/crates.io-index" 686 1283 checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" 687 1284 dependencies = [ 688 - "log", 1285 + "aws-lc-rs", 689 1286 "once_cell", 690 - "ring", 691 1287 "rustls-pki-types", 692 1288 "rustls-webpki", 693 1289 "subtle", 694 1290 "zeroize", 1291 + ] 1292 + 1293 + [[package]] 1294 + name = "rustls-native-certs" 1295 + version = "0.8.5" 1296 + source = "registry+https://github.com/rust-lang/crates.io-index" 1297 + checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" 1298 + dependencies = [ 1299 + "openssl-probe", 1300 + "rustls-pki-types", 1301 + "schannel", 1302 + "security-framework", 695 1303 ] 696 1304 697 1305 [[package]] ··· 700 1308 source = "registry+https://github.com/rust-lang/crates.io-index" 701 1309 checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" 702 1310 dependencies = [ 1311 + "web-time", 703 1312 "zeroize", 704 1313 ] 1314 + 1315 + [[package]] 1316 + name = "rustls-platform-verifier" 1317 + version = "0.7.1" 1318 + source = "registry+https://github.com/rust-lang/crates.io-index" 1319 + checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" 1320 + dependencies = [ 1321 + "core-foundation", 1322 + "core-foundation-sys", 1323 + "jni", 1324 + "log", 1325 + "once_cell", 1326 + "rustls", 1327 + "rustls-native-certs", 1328 + "rustls-platform-verifier-android", 1329 + "rustls-webpki", 1330 + "security-framework", 1331 + "security-framework-sys", 1332 + "webpki-root-certs", 1333 + "windows-sys 0.52.0", 1334 + ] 1335 + 1336 + [[package]] 1337 + name = "rustls-platform-verifier-android" 1338 + version = "0.2.0" 1339 + source = "registry+https://github.com/rust-lang/crates.io-index" 1340 + checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" 705 1341 706 1342 [[package]] 707 1343 name = "rustls-webpki" ··· 709 1345 source = "registry+https://github.com/rust-lang/crates.io-index" 710 1346 checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" 711 1347 dependencies = [ 1348 + "aws-lc-rs", 712 1349 "ring", 713 1350 "rustls-pki-types", 714 1351 "untrusted", ··· 719 1356 version = "1.0.23" 720 1357 source = "registry+https://github.com/rust-lang/crates.io-index" 721 1358 checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" 1359 + 1360 + [[package]] 1361 + name = "same-file" 1362 + version = "1.0.6" 1363 + source = "registry+https://github.com/rust-lang/crates.io-index" 1364 + checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" 1365 + dependencies = [ 1366 + "winapi-util", 1367 + ] 1368 + 1369 + [[package]] 1370 + name = "schannel" 1371 + version = "0.1.30" 1372 + source = "registry+https://github.com/rust-lang/crates.io-index" 1373 + checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" 1374 + dependencies = [ 1375 + "windows-sys 0.61.2", 1376 + ] 1377 + 1378 + [[package]] 1379 + name = "security-framework" 1380 + version = "3.7.0" 1381 + source = "registry+https://github.com/rust-lang/crates.io-index" 1382 + checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" 1383 + dependencies = [ 1384 + "bitflags 2.13.2", 1385 + "core-foundation", 1386 + "core-foundation-sys", 1387 + "libc", 1388 + "security-framework-sys", 1389 + ] 1390 + 1391 + [[package]] 1392 + name = "security-framework-sys" 1393 + version = "2.17.0" 1394 + source = "registry+https://github.com/rust-lang/crates.io-index" 1395 + checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" 1396 + dependencies = [ 1397 + "core-foundation-sys", 1398 + "libc", 1399 + ] 722 1400 723 1401 [[package]] 724 1402 name = "semver" ··· 776 1454 checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" 777 1455 dependencies = [ 778 1456 "cfg-if", 779 - "cpufeatures", 1457 + "cpufeatures 0.2.17", 780 1458 "digest", 781 1459 ] 782 1460 ··· 793 1471 checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" 794 1472 795 1473 [[package]] 1474 + name = "simd_cesu8" 1475 + version = "1.2.0" 1476 + source = "registry+https://github.com/rust-lang/crates.io-index" 1477 + checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" 1478 + dependencies = [ 1479 + "rustc_version", 1480 + "simdutf8", 1481 + ] 1482 + 1483 + [[package]] 1484 + name = "simdutf8" 1485 + version = "0.1.5" 1486 + source = "registry+https://github.com/rust-lang/crates.io-index" 1487 + checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" 1488 + 1489 + [[package]] 1490 + name = "slab" 1491 + version = "0.4.12" 1492 + source = "registry+https://github.com/rust-lang/crates.io-index" 1493 + checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" 1494 + 1495 + [[package]] 796 1496 name = "smallvec" 797 1497 version = "1.16.3" 798 1498 source = "registry+https://github.com/rust-lang/crates.io-index" 799 1499 checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" 1500 + 1501 + [[package]] 1502 + name = "socket2" 1503 + version = "0.6.5" 1504 + source = "registry+https://github.com/rust-lang/crates.io-index" 1505 + checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" 1506 + dependencies = [ 1507 + "libc", 1508 + "windows-sys 0.61.2", 1509 + ] 800 1510 801 1511 [[package]] 802 1512 name = "stable_deref_trait" ··· 830 1540 "proc-macro2", 831 1541 "quote", 832 1542 "unicode-ident", 1543 + ] 1544 + 1545 + [[package]] 1546 + name = "sync_wrapper" 1547 + version = "1.0.2" 1548 + source = "registry+https://github.com/rust-lang/crates.io-index" 1549 + checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" 1550 + dependencies = [ 1551 + "futures-core", 833 1552 ] 834 1553 835 1554 [[package]] ··· 874 1593 ] 875 1594 876 1595 [[package]] 1596 + name = "tinyvec" 1597 + version = "1.13.3" 1598 + source = "registry+https://github.com/rust-lang/crates.io-index" 1599 + checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" 1600 + 1601 + [[package]] 1602 + name = "tokio" 1603 + version = "1.53.3" 1604 + source = "registry+https://github.com/rust-lang/crates.io-index" 1605 + checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" 1606 + dependencies = [ 1607 + "bytes", 1608 + "libc", 1609 + "mio 1.2.4", 1610 + "pin-project-lite", 1611 + "socket2", 1612 + "tokio-macros", 1613 + "windows-sys 0.61.2", 1614 + ] 1615 + 1616 + [[package]] 1617 + name = "tokio-macros" 1618 + version = "2.7.2" 1619 + source = "registry+https://github.com/rust-lang/crates.io-index" 1620 + checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" 1621 + dependencies = [ 1622 + "proc-macro2", 1623 + "quote", 1624 + "syn 3.0.9", 1625 + ] 1626 + 1627 + [[package]] 1628 + name = "tokio-rustls" 1629 + version = "0.26.6" 1630 + source = "registry+https://github.com/rust-lang/crates.io-index" 1631 + checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" 1632 + dependencies = [ 1633 + "rustls", 1634 + "tokio", 1635 + ] 1636 + 1637 + [[package]] 1638 + name = "tokio-util" 1639 + version = "0.7.20" 1640 + source = "registry+https://github.com/rust-lang/crates.io-index" 1641 + checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" 1642 + dependencies = [ 1643 + "bytes", 1644 + "futures-core", 1645 + "futures-sink", 1646 + "libc", 1647 + "pin-project-lite", 1648 + "tokio", 1649 + ] 1650 + 1651 + [[package]] 1652 + name = "tower" 1653 + version = "0.5.3" 1654 + source = "registry+https://github.com/rust-lang/crates.io-index" 1655 + checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" 1656 + dependencies = [ 1657 + "futures-core", 1658 + "futures-util", 1659 + "pin-project-lite", 1660 + "sync_wrapper", 1661 + "tokio", 1662 + "tower-layer", 1663 + "tower-service", 1664 + ] 1665 + 1666 + [[package]] 1667 + name = "tower-http" 1668 + version = "0.6.11" 1669 + source = "registry+https://github.com/rust-lang/crates.io-index" 1670 + checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" 1671 + dependencies = [ 1672 + "bitflags 2.13.2", 1673 + "bytes", 1674 + "futures-util", 1675 + "http", 1676 + "http-body", 1677 + "pin-project-lite", 1678 + "tower", 1679 + "tower-layer", 1680 + "tower-service", 1681 + "url", 1682 + ] 1683 + 1684 + [[package]] 1685 + name = "tower-layer" 1686 + version = "0.3.3" 1687 + source = "registry+https://github.com/rust-lang/crates.io-index" 1688 + checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" 1689 + 1690 + [[package]] 1691 + name = "tower-service" 1692 + version = "0.3.3" 1693 + source = "registry+https://github.com/rust-lang/crates.io-index" 1694 + checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" 1695 + 1696 + [[package]] 1697 + name = "tracing" 1698 + version = "0.1.44" 1699 + source = "registry+https://github.com/rust-lang/crates.io-index" 1700 + checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" 1701 + dependencies = [ 1702 + "pin-project-lite", 1703 + "tracing-core", 1704 + ] 1705 + 1706 + [[package]] 1707 + name = "tracing-core" 1708 + version = "0.1.36" 1709 + source = "registry+https://github.com/rust-lang/crates.io-index" 1710 + checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" 1711 + dependencies = [ 1712 + "once_cell", 1713 + ] 1714 + 1715 + [[package]] 1716 + name = "try-lock" 1717 + version = "0.2.5" 1718 + source = "registry+https://github.com/rust-lang/crates.io-index" 1719 + checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" 1720 + 1721 + [[package]] 877 1722 name = "typenum" 878 1723 version = "1.20.1" 879 1724 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 902 1747 version = "0.9.0" 903 1748 source = "registry+https://github.com/rust-lang/crates.io-index" 904 1749 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" 905 - 906 - [[package]] 907 - name = "ureq" 908 - version = "2.12.1" 909 - source = "registry+https://github.com/rust-lang/crates.io-index" 910 - checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" 911 - dependencies = [ 912 - "base64", 913 - "log", 914 - "once_cell", 915 - "rustls", 916 - "rustls-pki-types", 917 - "url", 918 - "webpki-roots 0.26.11", 919 - ] 920 1750 921 1751 [[package]] 922 1752 name = "url" ··· 959 1789 source = "registry+https://github.com/rust-lang/crates.io-index" 960 1790 checksum = "506c62fdf617a5176827c2f9afbcf1be155b03a9b4bf9617a60dbc07e3a1642f" 961 1791 dependencies = [ 962 - "bitflags", 1792 + "bitflags 1.3.2", 963 1793 "libc", 1794 + ] 1795 + 1796 + [[package]] 1797 + name = "walkdir" 1798 + version = "2.5.0" 1799 + source = "registry+https://github.com/rust-lang/crates.io-index" 1800 + checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" 1801 + dependencies = [ 1802 + "same-file", 1803 + "winapi-util", 1804 + ] 1805 + 1806 + [[package]] 1807 + name = "want" 1808 + version = "0.3.2" 1809 + source = "registry+https://github.com/rust-lang/crates.io-index" 1810 + checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" 1811 + dependencies = [ 1812 + "try-lock", 964 1813 ] 965 1814 966 1815 [[package]] ··· 970 1819 checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" 971 1820 972 1821 [[package]] 973 - name = "webpki-roots" 974 - version = "0.26.11" 1822 + name = "wasm-bindgen" 1823 + version = "0.2.129" 975 1824 source = "registry+https://github.com/rust-lang/crates.io-index" 976 - checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" 1825 + checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" 977 1826 dependencies = [ 978 - "webpki-roots 1.0.9", 1827 + "cfg-if", 1828 + "once_cell", 1829 + "rustversion", 1830 + "wasm-bindgen-macro", 1831 + "wasm-bindgen-shared", 979 1832 ] 980 1833 981 1834 [[package]] 982 - name = "webpki-roots" 1835 + name = "wasm-bindgen-futures" 1836 + version = "0.4.79" 1837 + source = "registry+https://github.com/rust-lang/crates.io-index" 1838 + checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" 1839 + dependencies = [ 1840 + "js-sys", 1841 + "tokio", 1842 + "wasm-bindgen", 1843 + ] 1844 + 1845 + [[package]] 1846 + name = "wasm-bindgen-macro" 1847 + version = "0.2.129" 1848 + source = "registry+https://github.com/rust-lang/crates.io-index" 1849 + checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" 1850 + dependencies = [ 1851 + "quote", 1852 + "wasm-bindgen-macro-support", 1853 + ] 1854 + 1855 + [[package]] 1856 + name = "wasm-bindgen-macro-support" 1857 + version = "0.2.129" 1858 + source = "registry+https://github.com/rust-lang/crates.io-index" 1859 + checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" 1860 + dependencies = [ 1861 + "bumpalo", 1862 + "proc-macro2", 1863 + "quote", 1864 + "syn 3.0.9", 1865 + "wasm-bindgen-shared", 1866 + ] 1867 + 1868 + [[package]] 1869 + name = "wasm-bindgen-shared" 1870 + version = "0.2.129" 1871 + source = "registry+https://github.com/rust-lang/crates.io-index" 1872 + checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" 1873 + dependencies = [ 1874 + "unicode-ident", 1875 + ] 1876 + 1877 + [[package]] 1878 + name = "wasm-streams" 1879 + version = "0.5.0" 1880 + source = "registry+https://github.com/rust-lang/crates.io-index" 1881 + checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" 1882 + dependencies = [ 1883 + "futures-util", 1884 + "js-sys", 1885 + "wasm-bindgen", 1886 + "wasm-bindgen-futures", 1887 + "web-sys", 1888 + ] 1889 + 1890 + [[package]] 1891 + name = "web-sys" 1892 + version = "0.3.106" 1893 + source = "registry+https://github.com/rust-lang/crates.io-index" 1894 + checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" 1895 + dependencies = [ 1896 + "js-sys", 1897 + "wasm-bindgen", 1898 + ] 1899 + 1900 + [[package]] 1901 + name = "web-time" 1902 + version = "1.1.0" 1903 + source = "registry+https://github.com/rust-lang/crates.io-index" 1904 + checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" 1905 + dependencies = [ 1906 + "js-sys", 1907 + "wasm-bindgen", 1908 + ] 1909 + 1910 + [[package]] 1911 + name = "webpki-root-certs" 983 1912 version = "1.0.9" 984 1913 source = "registry+https://github.com/rust-lang/crates.io-index" 985 - checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" 1914 + checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" 986 1915 dependencies = [ 987 1916 "rustls-pki-types", 988 1917 ] ··· 1004 1933 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 1005 1934 1006 1935 [[package]] 1936 + name = "winapi-util" 1937 + version = "0.1.11" 1938 + source = "registry+https://github.com/rust-lang/crates.io-index" 1939 + checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" 1940 + dependencies = [ 1941 + "windows-sys 0.52.0", 1942 + ] 1943 + 1944 + [[package]] 1007 1945 name = "winapi-x86_64-pc-windows-gnu" 1008 1946 version = "0.4.0" 1009 1947 source = "registry+https://github.com/rust-lang/crates.io-index" 1010 1948 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 1949 + 1950 + [[package]] 1951 + name = "windows-link" 1952 + version = "0.2.1" 1953 + source = "registry+https://github.com/rust-lang/crates.io-index" 1954 + checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" 1011 1955 1012 1956 [[package]] 1013 1957 name = "windows-sys" ··· 1025 1969 checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 1026 1970 dependencies = [ 1027 1971 "windows-targets 0.52.6", 1972 + ] 1973 + 1974 + [[package]] 1975 + name = "windows-sys" 1976 + version = "0.61.2" 1977 + source = "registry+https://github.com/rust-lang/crates.io-index" 1978 + checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" 1979 + dependencies = [ 1980 + "windows-link", 1028 1981 ] 1029 1982 1030 1983 [[package]] ··· 1155 2108 checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" 1156 2109 1157 2110 [[package]] 1158 - name = "xz2" 1159 - version = "0.1.7" 1160 - source = "registry+https://github.com/rust-lang/crates.io-index" 1161 - checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" 1162 - dependencies = [ 1163 - "lzma-sys", 1164 - ] 1165 - 1166 - [[package]] 1167 2111 name = "yoke" 1168 2112 version = "0.8.3" 1169 2113 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1260 2204 1261 2205 [[package]] 1262 2206 name = "zstd" 1263 - version = "0.13.3" 2207 + version = "0.14.3" 1264 2208 source = "registry+https://github.com/rust-lang/crates.io-index" 1265 - checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" 2209 + checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" 1266 2210 dependencies = [ 1267 2211 "zstd-safe", 1268 2212 ] 1269 2213 1270 2214 [[package]] 1271 2215 name = "zstd-safe" 1272 - version = "7.3.0" 2216 + version = "8.2.0" 1273 2217 source = "registry+https://github.com/rust-lang/crates.io-index" 1274 - checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" 2218 + checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" 1275 2219 dependencies = [ 1276 2220 "zstd-sys", 1277 2221 ]
+8 -7
tools/lazy-store/Cargo.toml
··· 14 14 serde = { version = "1", features = ["derive"] } 15 15 serde_json = "1" 16 16 sha2 = "0.10" 17 - # rustls with bundled webpki roots: nothing read from /etc/ssl or the store 18 - ureq = { version = "2", default-features = false, features = ["tls"] } 19 - zstd = "0.13" 20 - xz2 = { version = "0.1", features = ["static"] } 21 - bzip2 = { version = "0.4", features = ["static"] } 22 - flate2 = "1" 23 - brotli-decompressor = "4" 17 + tokio = { version = "1", features = ["rt-multi-thread", "macros", "fs", "io-util", "net", "time", "sync"] } 18 + tokio-util = { version = "0.7", features = ["io", "io-util"] } 19 + futures-util = "0.3" 20 + # rustls (aws-lc, as harmonia-utils-hash uses it anyway) with bundled roots: 21 + # nothing read from /etc/ssl or the store 22 + reqwest = { version = "0.13", default-features = false, features = ["rustls", "http2", "stream"] } 23 + webpki-root-certs = "1" 24 + async-compression = { version = "0.4", features = ["tokio", "zstd", "xz", "bzip2", "gzip", "brotli"] } 24 25 harmonia-file-core = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 25 26 harmonia-utils-base-encoding = { git = "https://github.com/nix-community/harmonia", rev = "3edb37ae36b561c2d395980170c70bbce292bbc5" } 26 27
+13 -35
tools/lazy-store/src/closure.rs
··· 4 4 use std::collections::{BTreeMap, BTreeSet, HashMap}; 5 5 use std::io::{self, BufRead, Write}; 6 6 use std::path::Path; 7 - use std::sync::atomic::{AtomicUsize, Ordering}; 8 - use std::sync::Mutex; 9 7 8 + use futures_util::{stream, StreamExt}; 10 9 use serde::{Deserialize, Serialize}; 11 10 12 11 use crate::http::{self, Http}; ··· 50 49 51 50 /// The parsed narinfo of `name` from the first cache that has it, and its 52 51 /// text. 53 - fn narinfo( 52 + async fn narinfo( 54 53 http: &Http, 55 54 caches: &[String], 56 55 name: &str, ··· 58 57 ) -> Option<(HashMap<String, String>, Vec<u8>)> { 59 58 let h = name.split('-').next().unwrap_or(name); 60 59 for c in caches { 61 - match http.get(&format!("{c}/{h}.narinfo")) { 60 + match http.get(&format!("{c}/{h}.narinfo")).await { 62 61 Ok((body, len)) => { 63 62 stats.add("narinfo", len as u64); 64 63 let mut i = parse_narinfo(&body); ··· 76 75 None 77 76 } 78 77 79 - /// Run `f` over `items` on `threads` threads, results in order. 80 - pub fn par_map<T: Sync, R: Send>( 81 - items: &[T], 82 - threads: usize, 83 - f: impl Fn(&T) -> R + Sync, 84 - ) -> Vec<R> { 85 - let next = AtomicUsize::new(0); 86 - let out: Mutex<Vec<Option<R>>> = Mutex::new((0..items.len()).map(|_| None).collect()); 87 - std::thread::scope(|s| { 88 - for _ in 0..threads.min(items.len()) { 89 - s.spawn(|| loop { 90 - let i = next.fetch_add(1, Ordering::Relaxed); 91 - if i >= items.len() { 92 - break; 93 - } 94 - let r = f(&items[i]); 95 - out.lock().unwrap()[i] = Some(r); 96 - }); 97 - } 98 - }); 99 - out.into_inner() 100 - .unwrap() 101 - .into_iter() 102 - .map(Option::unwrap) 103 - .collect() 104 - } 105 - 106 78 /// Writes OUT/paths.json (per path: cache, NAR url, compression, hash), 107 79 /// OUT/db.dump (input for `nix-store --load-db`), OUT/narinfo (a binary 108 80 /// cache of just the narinfos) and OUT/closure.json. 109 81 /// Paths that are in none of the caches, and everything referring to them, 110 82 /// are left out: a path is registered only with its whole closure. 111 - pub fn closure(outdir: &Path) -> io::Result<()> { 83 + pub async fn closure(outdir: &Path) -> io::Result<()> { 112 84 let mut roots = BTreeSet::new(); 113 85 for l in io::stdin().lock().lines() { 114 86 let l = l?; ··· 118 90 } 119 91 } 120 92 let caches = caches(); 121 - let http = Http::new(caches.iter().map(String::as_str)); 93 + let http = Http::new(caches.iter().map(String::as_str)).await; 122 94 let stats = Stats::new(None); 123 95 let mut infos: BTreeMap<String, HashMap<String, String>> = BTreeMap::new(); 124 96 let mut texts: HashMap<String, Vec<u8>> = HashMap::new(); 125 97 let mut missing: BTreeSet<String> = BTreeSet::new(); 126 98 let mut todo: Vec<String> = roots.iter().cloned().collect(); 127 99 while !todo.is_empty() { 128 - let res = par_map(&todo, 32, |n| narinfo(&http, &caches, n, &stats)); 100 + let (http, caches, stats) = (&http, &caches, &stats); 101 + let mut res = stream::iter(std::mem::take(&mut todo)) 102 + .map(|n| async move { 103 + let i = narinfo(http, caches, &n, stats).await; 104 + (n, i) 105 + }) 106 + .buffer_unordered(32); 129 107 let mut next = BTreeSet::new(); 130 - for (name, i) in todo.iter().zip(res) { 108 + while let Some((name, i)) = res.next().await { 131 109 let Some((i, text)) = i else { 132 110 missing.insert(name.clone()); 133 111 continue;
+94 -81
tools/lazy-store/src/http.rs
··· 1 1 //! Binary cache access: GET with retries, and body decoding. 2 2 3 - use std::collections::HashMap; 4 - use std::io::{self, Read}; 5 - use std::net::{SocketAddr, ToSocketAddrs}; 6 - use std::sync::Mutex; 7 - use std::thread::sleep; 3 + use std::io; 4 + use std::pin::Pin; 8 5 use std::time::Duration; 6 + 7 + use async_compression::tokio::bufread::{ 8 + BrotliDecoder, BzDecoder, GzipDecoder, XzDecoder, ZstdDecoder, 9 + }; 10 + use futures_util::TryStreamExt; 11 + use tokio::io::{AsyncBufRead, AsyncRead, AsyncReadExt, BufReader}; 12 + use tokio::time::sleep; 13 + use tokio_util::io::StreamReader; 9 14 10 15 const TRIES: u32 = 6; 11 16 12 - /// Resolves each host once and then serves it from memory. Once the FUSE 13 - /// file system is mounted on /nix/store, getaddrinfo() could end up looking 14 - /// for NSS modules in the store, i.e. in a request to ourselves. 15 - struct Resolver(Mutex<HashMap<String, Vec<SocketAddr>>>); 16 - 17 - impl ureq::Resolver for Resolver { 18 - fn resolve(&self, netloc: &str) -> io::Result<Vec<SocketAddr>> { 19 - if let Some(a) = self.0.lock().unwrap().get(netloc) { 20 - return Ok(a.clone()); 21 - } 22 - let a: Vec<SocketAddr> = netloc.to_socket_addrs()?.collect(); 23 - self.0.lock().unwrap().insert(netloc.to_string(), a.clone()); 24 - Ok(a) 25 - } 26 - } 17 + pub type Body = Pin<Box<dyn AsyncRead + Send>>; 27 18 28 19 pub struct Http { 29 - agent: ureq::Agent, 20 + client: reqwest::Client, 30 21 } 31 22 32 23 pub enum Error { ··· 51 42 } 52 43 53 44 impl Http { 54 - /// `caches` are base URLs; their hosts are resolved right away. 55 - pub fn new<'a>(caches: impl IntoIterator<Item = &'a str>) -> Http { 56 - let mut known = HashMap::new(); 45 + /// `caches` are base URLs. Their hosts are resolved right away and then 46 + /// served from memory: once the FUSE file system is mounted on 47 + /// /nix/store, getaddrinfo() could end up looking for NSS modules in 48 + /// the store, i.e. in a request to ourselves. 49 + pub async fn new<'a>(caches: impl IntoIterator<Item = &'a str>) -> Http { 50 + // rustls with bundled roots: nothing read from /etc/ssl or the store 51 + let roots = webpki_root_certs::TLS_SERVER_ROOT_CERTS 52 + .iter() 53 + .filter_map(|c| reqwest::Certificate::from_der(c).ok()); 54 + let mut b = reqwest::Client::builder() 55 + .connect_timeout(Duration::from_secs(30)) 56 + .read_timeout(Duration::from_secs(60)) 57 + .pool_max_idle_per_host(64) 58 + .tls_certs_only(roots); 57 59 for c in caches { 58 - let Some(rest) = c 59 - .strip_prefix("https://") 60 - .or_else(|| c.strip_prefix("http://")) 61 - else { 60 + let Ok(u) = reqwest::Url::parse(c) else { 62 61 continue; 63 62 }; 64 - let host = rest.split('/').next().unwrap_or(rest); 65 - let port = if c.starts_with("https:") { 443 } else { 80 }; 66 - let netloc = if host.contains(':') { 67 - host.to_string() 68 - } else { 69 - format!("{host}:{port}") 63 + let (Some(host), Some(port)) = (u.host_str(), u.port_or_known_default()) else { 64 + continue; 70 65 }; 71 - match netloc.to_socket_addrs() { 72 - Ok(a) => { 73 - known.insert(netloc, a.collect()); 74 - } 75 - Err(e) => eprintln!("resolving {netloc}: {e}"), 76 - } 66 + match tokio::net::lookup_host((host, port)).await { 67 + Ok(a) => b = b.resolve_to_addrs(host, &a.collect::<Vec<_>>()), 68 + Err(e) => eprintln!("resolving {host}:{port}: {e}"), 69 + }; 77 70 } 78 - let agent = ureq::AgentBuilder::new() 79 - .timeout_connect(Duration::from_secs(30)) 80 - .timeout_read(Duration::from_secs(60)) 81 - .max_idle_connections_per_host(64) 82 - .resolver(Resolver(Mutex::new(known))) 83 - .build(); 84 - Http { agent } 71 + Http { 72 + client: b.build().expect("building the HTTP client"), 73 + } 85 74 } 86 75 87 76 /// GET `url`, retrying 429/5xx and transport errors with backoff (the 88 77 /// caches answer 503 under load). Returns the body reader and its 89 78 /// Content-Encoding. 90 - pub fn open(&self, url: &str) -> Result<(Box<dyn Read + Send>, Option<String>), Error> { 79 + pub async fn open(&self, url: &str) -> Result<(Body, Option<String>), Error> { 91 80 if let Some(path) = url.strip_prefix("file://") { 92 - return match std::fs::File::open(path) { 93 - Ok(f) => Ok((Box::new(f), None)), 81 + return match tokio::fs::File::open(path).await { 82 + Ok(f) => Ok((Box::pin(f), None)), 94 83 Err(e) if e.kind() == io::ErrorKind::NotFound => Err(Error::NotFound), 95 84 Err(e) => Err(Error::Other(format!("{url}: {e}"))), 96 85 }; 97 86 } 98 87 let mut n = 0; 99 88 loop { 100 - let err = match self.agent.get(url).call() { 101 - Ok(r) => { 102 - let enc = r.header("content-encoding").map(str::to_string); 103 - return Ok((Box::new(r.into_reader()), enc)); 89 + let err = match self.client.get(url).send().await { 90 + Ok(r) if r.status().is_success() => { 91 + let enc = r 92 + .headers() 93 + .get("content-encoding") 94 + .and_then(|v| v.to_str().ok()) 95 + .map(str::to_string); 96 + let body = r.bytes_stream().map_err(io::Error::other); 97 + return Ok((Box::pin(StreamReader::new(body)), enc)); 104 98 } 105 - Err(ureq::Error::Status(403 | 404, _)) => return Err(Error::NotFound), 106 - Err(ureq::Error::Status(code, _)) if code != 429 && code < 500 => { 107 - return Err(Error::Other(format!("{url}: HTTP {code}"))) 108 - } 109 - Err(ureq::Error::Status(code, _)) => format!("HTTP {code}"), 99 + Ok(r) => match r.status().as_u16() { 100 + 403 | 404 => return Err(Error::NotFound), 101 + code if code != 429 && code < 500 => { 102 + return Err(Error::Other(format!("{url}: HTTP {code}"))) 103 + } 104 + code => format!("HTTP {code}"), 105 + }, 110 106 Err(e) => e.to_string(), 111 107 }; 112 108 n += 1; ··· 114 110 return Err(Error::Other(format!("{url}: {err}"))); 115 111 } 116 112 eprintln!("retry {n}/{} {url}: {err}", TRIES - 1); 117 - sleep(Duration::from_secs((1u64 << (n - 1)).min(30))); 113 + sleep(Duration::from_secs((1u64 << (n - 1)).min(30))).await; 118 114 } 119 115 } 120 116 121 117 /// GET `url` -> decoded body. Transfer errors while reading the body are 122 118 /// retried like failed requests. 123 - pub fn get(&self, url: &str) -> Result<(Vec<u8>, usize), Error> { 119 + pub async fn get(&self, url: &str) -> Result<(Vec<u8>, usize), Error> { 124 120 let mut n = 0; 125 121 loop { 126 - let (mut r, enc) = self.open(url)?; 122 + let (mut r, enc) = self.open(url).await?; 127 123 let mut body = Vec::new(); 128 - match r.read_to_end(&mut body) { 124 + match r.read_to_end(&mut body).await { 129 125 Ok(_) => { 130 126 let len = body.len(); 131 127 let dec = decode(&body, enc.as_deref()) 128 + .await 132 129 .map_err(|e| Error::Other(format!("{url}: decoding: {e}")))?; 133 130 return Ok((dec, len)); 134 131 } ··· 138 135 return Err(Error::Other(format!("{url}: {e}"))); 139 136 } 140 137 eprintln!("retry {n}/{} {url}: {e}", TRIES - 1); 141 - sleep(Duration::from_secs((1u64 << (n - 1)).min(30))); 138 + sleep(Duration::from_secs((1u64 << (n - 1)).min(30))).await; 142 139 } 143 140 } 144 141 } ··· 146 143 } 147 144 148 145 /// Wrap `r` in a decompressor for `method` (narinfo Compression or 149 - /// Content-Encoding). 146 + /// Content-Encoding). Concatenated streams are decoded as one, like Nix. 150 147 pub fn decoder<'a>( 151 148 method: &str, 152 - r: impl Read + Send + 'a, 153 - ) -> io::Result<Box<dyn Read + Send + 'a>> { 149 + r: impl AsyncBufRead + Send + 'a, 150 + ) -> io::Result<Pin<Box<dyn AsyncRead + Send + 'a>>> { 151 + fn multi<D: AsyncRead + Send>(mut d: D, set: impl FnOnce(&mut D)) -> D { 152 + set(&mut d); 153 + d 154 + } 154 155 Ok(match method { 155 - "" | "none" | "identity" => Box::new(r), 156 - "zstd" => Box::new(zstd::stream::read::Decoder::new(r)?), 157 - "xz" => Box::new(xz2::read::XzDecoder::new_multi_decoder(r)), 158 - "bzip2" => Box::new(bzip2::read::MultiBzDecoder::new(r)), 159 - "gzip" => Box::new(flate2::read::MultiGzDecoder::new(r)), 160 - "br" => Box::new(brotli_decompressor::Decompressor::new(r, 1 << 16)), 156 + "" | "none" | "identity" => Box::pin(r), 157 + "zstd" => Box::pin(multi(ZstdDecoder::new(r), |d| d.multiple_members(true))), 158 + "xz" => Box::pin(multi(XzDecoder::new(r), |d| d.multiple_members(true))), 159 + "bzip2" => Box::pin(multi(BzDecoder::new(r), |d| d.multiple_members(true))), 160 + "gzip" => Box::pin(multi(GzipDecoder::new(r), |d| d.multiple_members(true))), 161 + "br" => Box::pin(BrotliDecoder::new(r)), 161 162 m => return Err(io::Error::other(format!("unknown compression {m:?}"))), 162 163 }) 164 + } 165 + 166 + /// `decoder` for an unbuffered body. 167 + pub fn decode_body<'a>( 168 + method: &str, 169 + r: impl AsyncRead + Send + 'a, 170 + ) -> io::Result<Pin<Box<dyn AsyncRead + Send + 'a>>> { 171 + decoder(method, BufReader::with_capacity(1 << 16, r)) 163 172 } 164 173 165 174 const MAGIC: &[(&[u8], &str)] = &[ ··· 170 179 171 180 /// Both caches store narinfo/.ls zstd-compressed (Content-Encoding), but the 172 181 /// CDN sometimes serves them without that header: sniff the magic too. 173 - pub fn decode(body: &[u8], encoding: Option<&str>) -> io::Result<Vec<u8>> { 182 + pub async fn decode(body: &[u8], encoding: Option<&str>) -> io::Result<Vec<u8>> { 174 183 let enc = match encoding { 175 184 None | Some("") | Some("identity") => { 176 185 match MAGIC.iter().find(|(m, _)| body.starts_with(m)) { ··· 181 190 Some(e) => e, 182 191 }; 183 192 let mut out = Vec::new(); 184 - decoder(enc, body)?.read_to_end(&mut out)?; 193 + decoder(enc, body)?.read_to_end(&mut out).await?; 185 194 Ok(out) 186 195 } 187 196 ··· 189 198 mod tests { 190 199 use super::*; 191 200 192 - #[test] 193 - fn decode_sniffs_zstd() { 194 - let z = zstd::encode_all(&b"{\"root\":1}"[..], 3).unwrap(); 195 - assert_eq!(decode(&z, None).unwrap(), b"{\"root\":1}"); 196 - assert_eq!(decode(&z, Some("zstd")).unwrap(), b"{\"root\":1}"); 197 - assert_eq!(decode(b"plain", None).unwrap(), b"plain"); 201 + #[tokio::test] 202 + async fn decode_sniffs_zstd() { 203 + let mut z = Vec::new(); 204 + async_compression::tokio::bufread::ZstdEncoder::new(&b"{\"root\":1}"[..]) 205 + .read_to_end(&mut z) 206 + .await 207 + .unwrap(); 208 + assert_eq!(decode(&z, None).await.unwrap(), b"{\"root\":1}"); 209 + assert_eq!(decode(&z, Some("zstd")).await.unwrap(), b"{\"root\":1}"); 210 + assert_eq!(decode(b"plain", None).await.unwrap(), b"plain"); 198 211 } 199 212 }
+63 -36
tools/lazy-store/src/lazy.rs
··· 11 11 use std::mem::ManuallyDrop; 12 12 use std::os::fd::{AsRawFd, FromRawFd}; 13 13 use std::path::{Path, PathBuf}; 14 - use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; 14 + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; 15 15 use std::sync::{Arc, Mutex, RwLock}; 16 16 use std::time::{Duration, Instant}; 17 17 ··· 20 20 Context, DirEntry, Entry, FileSystem, GetxattrReply, ListxattrReply, OpenOptions, 21 21 ZeroCopyWriter, 22 22 }; 23 + use futures_util::{stream, StreamExt}; 23 24 use harmonia_utils_base_encoding::base32; 25 + use tokio_util::io::{InspectReader, SyncIoBridge}; 24 26 25 - use crate::closure::{par_map, PathInfo}; 27 + use crate::closure::PathInfo; 26 28 use crate::http::{self, Http}; 27 - use crate::nar::{self, CountingReader, Tree}; 29 + use crate::nar::{self, Tree}; 28 30 use crate::stats::Stats; 29 31 use crate::Backing; 30 32 ··· 67 69 http: Http, 68 70 pub stats: Arc<Stats>, 69 71 /// per store path: where it is unpacked, once fetched 70 - fetched: Vec<Mutex<Option<PathBuf>>>, 72 + fetched: Vec<tokio::sync::Mutex<Option<PathBuf>>>, 73 + /// for the FUSE worker threads, to fetch on the runtime 74 + rt: tokio::runtime::Handle, 71 75 handles: RwLock<HashMap<u64, Arc<Handle>>>, 72 76 next_handle: AtomicU64, 73 77 backing: Option<Arc<Backing>>, ··· 83 87 } 84 88 85 89 impl LazyFs { 86 - pub fn new(outdir: &Path, backing: Option<Arc<Backing>>) -> io::Result<LazyFs> { 90 + pub async fn new(outdir: &Path, backing: Option<Arc<Backing>>) -> io::Result<LazyFs> { 87 91 let paths: BTreeMap<String, PathInfo> = 88 92 serde_json::from_slice(&std::fs::read(outdir.join("paths.json"))?)?; 89 93 let (names, infos): (Vec<String>, Vec<PathInfo>) = paths.into_iter().unzip(); 90 94 let cache = outdir.join("cache"); 91 95 std::fs::create_dir_all(&cache)?; 92 - let http = Http::new(infos.iter().map(|i| i.cache.as_str())); 96 + let http = Http::new(infos.iter().map(|i| i.cache.as_str())).await; 93 97 let mut fs = LazyFs { 94 - fetched: names.iter().map(|_| Mutex::new(None)).collect(), 98 + fetched: names.iter().map(|_| Default::default()).collect(), 99 + rt: tokio::runtime::Handle::current(), 95 100 names, 96 101 infos, 97 102 nodes: Vec::new(), ··· 105 110 backing_ids: Mutex::new(HashMap::new()), 106 111 backing_failed: AtomicBool::new(false), 107 112 }; 108 - fs.load_listings()?; 113 + fs.load_listings().await?; 109 114 fs.stats.save(); 110 115 Ok(fs) 111 116 } ··· 114 119 /// for paths without one) and build the inode table from them. Fails if 115 120 /// any path is left without a listing: all of them get registered as 116 121 /// valid in nix's database. 117 - fn load_listings(&mut self) -> io::Result<()> { 122 + async fn load_listings(&mut self) -> io::Result<()> { 118 123 let t = Instant::now(); 119 - let idx: Vec<usize> = (0..self.names.len()).collect(); 120 - let done = AtomicUsize::new(0); 121 - let trees = par_map(&idx, 64, |&i| { 122 - let r = self.listing(i); 123 - let n = done.fetch_add(1, Ordering::Relaxed) + 1; 124 - if n.is_multiple_of(500) { 125 - eprintln!("listings: {n}/{}", idx.len()); 124 + let total = self.names.len(); 125 + let mut trees: Vec<Option<io::Result<Tree>>> = (0..total).map(|_| None).collect(); 126 + let this = &*self; 127 + let mut done = stream::iter(0..total) 128 + .map(|i| async move { (i, this.listing(i).await) }) 129 + .buffer_unordered(64); 130 + let mut n = 0; 131 + while let Some((i, r)) = done.next().await { 132 + trees[i] = Some(r); 133 + n += 1; 134 + if n % 500 == 0 { 135 + eprintln!("listings: {n}/{total}"); 126 136 } 127 - r 128 - }); 129 - let mut root = Vec::with_capacity(trees.len()); 137 + } 138 + drop(done); 139 + let trees = trees.into_iter().map(Option::unwrap); 140 + let mut root = Vec::with_capacity(total); 130 141 self.nodes.push(Node { 131 142 kind: Kind::Dir(Vec::new()), 132 143 path: u32::MAX, 133 144 rel: "".into(), 134 145 }); 135 146 let mut failed = 0; 136 - for (i, t) in trees.into_iter().enumerate() { 147 + for (i, t) in trees.enumerate() { 137 148 match t { 138 149 Ok(t) => { 139 150 let ino = self.add(i as u32, String::new(), t); ··· 162 173 Ok(()) 163 174 } 164 175 165 - fn listing(&self, i: usize) -> io::Result<Tree> { 176 + async fn listing(&self, i: usize) -> io::Result<Tree> { 166 177 let info = &self.infos[i]; 167 178 let h = self.names[i].split('-').next().unwrap_or(""); 168 - match self.http.get(&format!("{}/{h}.ls", info.cache)) { 179 + match self.http.get(&format!("{}/{h}.ls", info.cache)).await { 169 180 Ok((body, len)) => { 170 181 self.stats.add("ls", len as u64); 171 182 Tree::from_listing(&body) 172 183 } 173 184 // no listing: fall back to the NAR 174 - Err(http::Error::NotFound) => Tree::from_path(&self.fetch(i)?), 185 + Err(http::Error::NotFound) => Tree::from_path(&self.fetch(i).await?), 175 186 Err(e) => Err(e.into()), 176 187 } 177 188 } ··· 223 234 224 235 /// Fetch store path `name` as an open() would and check that what got 225 236 /// unpacked matches its listing (`lazy-store fetch`, no FUSE needed). 226 - pub fn fetch_and_check(&self, name: &str) -> io::Result<PathBuf> { 237 + pub async fn fetch_and_check(&self, name: &str) -> io::Result<PathBuf> { 227 238 let c = CString::new(name).map_err(io::Error::other)?; 228 239 let ino = self.lookup_child(1, &c).ok_or_else(|| err(libc::ENOENT))?; 229 - let p = self.fetch(self.nodes[ino as usize - 1].path as usize)?; 240 + let p = self 241 + .fetch(self.nodes[ino as usize - 1].path as usize) 242 + .await?; 230 243 if Tree::from_path(&p)? != self.tree_of(ino) { 231 244 return Err(io::Error::other(format!( 232 245 "{name}: unpacked tree differs from its listing" ··· 310 323 311 324 /// The unpacked store path `i`, fetching it first if needed. Concurrent 312 325 /// callers for the same path wait for one fetch. 313 - fn fetch(&self, i: usize) -> io::Result<PathBuf> { 314 - let mut slot = self.fetched[i].lock().unwrap(); 326 + async fn fetch(&self, i: usize) -> io::Result<PathBuf> { 327 + let mut slot = self.fetched[i].lock().await; 315 328 if let Some(p) = &*slot { 316 329 return Ok(p.clone()); 317 330 } ··· 319 332 let dest = self.cache.join(name); 320 333 let mut last = None; 321 334 for attempt in 1..=3 { 322 - match self.fetch_once(i, &dest) { 335 + match self.fetch_once(i, &dest).await { 323 336 Ok(()) => { 324 337 *slot = Some(dest.clone()); 325 338 return Ok(dest); ··· 333 346 Err(last.unwrap()) 334 347 } 335 348 336 - fn fetch_once(&self, i: usize, dest: &Path) -> io::Result<()> { 349 + async fn fetch_once(&self, i: usize, dest: &Path) -> io::Result<()> { 337 350 let (name, info) = (&self.names[i], &self.infos[i]); 338 351 let t = Instant::now(); 339 352 let tmp = dest.with_extension("tmp"); 340 353 nar::remove(&tmp)?; 341 354 nar::remove(dest)?; 342 - let (body, _) = self.http.open(&format!("{}/{}", info.cache, info.url))?; 343 - let mut counted = CountingReader { inner: body, n: 0 }; 344 - let (digest, size) = nar::unpack(http::decoder(&info.compression, &mut counted)?, &tmp)?; 355 + let (body, _) = self 356 + .http 357 + .open(&format!("{}/{}", info.cache, info.url)) 358 + .await?; 359 + // the compressed download size 360 + let file_size = Arc::new(AtomicU64::new(0)); 361 + let counted = InspectReader::new(body, { 362 + let n = file_size.clone(); 363 + move |b: &[u8]| { 364 + n.fetch_add(b.len() as u64, Ordering::Relaxed); 365 + } 366 + }); 367 + let nar = SyncIoBridge::new(http::decode_body(&info.compression, counted)?); 368 + let unpack_to = tmp.clone(); 369 + let (digest, size) = tokio::task::spawn_blocking(move || nar::unpack(nar, &unpack_to)) 370 + .await 371 + .map_err(io::Error::other)??; 372 + let file_size = file_size.load(Ordering::Relaxed); 345 373 if !nar::hash_matches(&info.nar_hash, &digest) || size != info.nar_size { 346 374 nar::remove(&tmp)?; 347 375 return Err(io::Error::other(format!( ··· 352 380 ))); 353 381 } 354 382 std::fs::rename(&tmp, dest)?; 355 - self.stats.nar(name, counted.n, size); 383 + self.stats.nar(name, file_size, size); 356 384 eprintln!( 357 - "fetched {name}: {} bytes in {:.1} s", 358 - counted.n, 385 + "fetched {name}: {file_size} bytes in {:.1} s", 359 386 t.elapsed().as_secs_f64() 360 387 ); 361 388 Ok(()) ··· 451 478 if acc != libc::O_RDONLY || flags as i32 & libc::O_TRUNC != 0 { 452 479 return Err(err(libc::EROFS)); 453 480 } 454 - let root = self.fetch(n.path as usize).map_err(|e| { 481 + let root = self.rt.block_on(self.fetch(n.path as usize)).map_err(|e| { 455 482 eprintln!("open {}/{}: {e}", self.names[n.path as usize], n.rel); 456 483 err(libc::EIO) 457 484 })?;
+12 -10
tools/lazy-store/src/main.rs
··· 127 127 } 128 128 129 129 fn mount( 130 + rt: &tokio::runtime::Runtime, 130 131 outdir: &Path, 131 132 upper: &Path, 132 133 mnt: &Path, ··· 135 136 ) -> io::Result<()> { 136 137 raise_nofile(); 137 138 let backing = use_passthrough.then(|| Arc::new(Backing::default())); 138 - let lower = lazy::LazyFs::new(outdir, backing.clone())?; 139 + let lower = rt.block_on(lazy::LazyFs::new(outdir, backing.clone()))?; 139 140 140 141 let upper_fs = { 141 142 let cfg = passthrough::Config { ··· 196 197 fn main() { 197 198 let _ = log::set_logger(&Logger).map(|_| log::set_max_level(log::LevelFilter::Warn)); 198 199 let args: Vec<String> = std::env::args().skip(1).collect(); 200 + let rt = tokio::runtime::Runtime::new().expect("starting the tokio runtime"); 199 201 let r = match args.first().map(String::as_str) { 200 - Some("closure") if args.len() == 2 => closure::closure(Path::new(&args[1])), 201 - Some("fetch") if args.len() >= 3 => { 202 - lazy::LazyFs::new(Path::new(&args[1]), None).and_then(|fs| { 203 - for name in &args[2..] { 204 - println!("{}", fs.fetch_and_check(name)?.display()); 205 - } 206 - Ok(()) 207 - }) 208 - } 202 + Some("closure") if args.len() == 2 => rt.block_on(closure::closure(Path::new(&args[1]))), 203 + Some("fetch") if args.len() >= 3 => rt.block_on(async { 204 + let fs = lazy::LazyFs::new(Path::new(&args[1]), None).await?; 205 + for name in &args[2..] { 206 + println!("{}", fs.fetch_and_check(name).await?.display()); 207 + } 208 + Ok(()) 209 + }), 209 210 Some("mount") if args.len() >= 4 => { 210 211 let (mut workers, mut pt) = (64, true); 211 212 let mut rest = args[4..].iter(); ··· 222 223 } 223 224 } 224 225 mount( 226 + &rt, 225 227 &PathBuf::from(&args[1]), 226 228 Path::new(&args[2]), 227 229 Path::new(&args[3]),
+3 -20
tools/lazy-store/src/nar.rs
··· 42 42 } 43 43 fn tree(FileTree(t): FileTree<Size>) -> Tree { 44 44 match t { 45 - FileSystemObject::Directory(d) => Tree::Dir( 46 - d.entries 47 - .into_iter() 48 - .map(|(k, c)| (k, tree(*c))) 49 - .collect(), 50 - ), 45 + FileSystemObject::Directory(d) => { 46 + Tree::Dir(d.entries.into_iter().map(|(k, c)| (k, tree(*c))).collect()) 47 + } 51 48 FileSystemObject::Regular(r) => Tree::File { 52 49 size: r.contents.size, 53 50 exec: r.executable, ··· 95 92 fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { 96 93 let n = self.inner.read(buf)?; 97 94 self.hasher.update(&buf[..n]); 98 - self.n += n as u64; 99 - Ok(n) 100 - } 101 - } 102 - 103 - /// Counts the bytes read from `inner` (the compressed download size). 104 - pub struct CountingReader<R> { 105 - pub inner: R, 106 - pub n: u64, 107 - } 108 - 109 - impl<R: Read> Read for CountingReader<R> { 110 - fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { 111 - let n = self.inner.read(buf)?; 112 95 self.n += n as u64; 113 96 Ok(n) 114 97 }