This repository has no description
0

Configure Feed

Select the types of activity you want to include in your feed.

tools.lazy-store: init

FUSE file system for CI's machine builds, ported from the experiment on
ci-lazy-fuse (.github/exp/lazy-store). It mounts on /nix/store the real
store dir (all writes go there) merged with store paths of
cache.nixos.org and cache.lassul.us, whose metadata comes from their
.ls listings and whose NAR is fetched, checked and unpacked on the first
open() of a file in them; reads go through kernel FUSE passthrough. In
the experiment, rebuilding neoprism's system-units, etc, system-path and
toplevel downloaded 62 MiB instead of the 3,455 MiB of an eager
substitution.

Changes from the experiment:
- closure takes the store paths to make valid on stdin (what
`nix build --dry-run` would fetch) instead of walking derivations, and
leaves out paths whose closure is not entirely in the caches, so that
nix never sees a valid path with a missing reference; nix substitutes
those as usual.
- closure also writes the fetched narinfos as a file:// binary cache:
`nix-store --load-db` registers no signatures, and niks3's
--ignore-upstream-key and hestia's upstream-cache-filter go by the
signatures in the local database, so without them both would upload
(and thereby download) every cache.nixos.org path in an output's
closure. `nix store copy-sigs` adds them from there.
- mount fails if a path has no listing instead of leaving it out, as
every path gets registered.

Static, so that once it serves /nix/store it loads nothing from there.
A flake package, so CI builds it once and pushes it to cache.lassul.us,
and the jobs substitute it. passthru.tests.fetch covers fetch, closure
and the registration with signatures against a file:// cache.

+3634
+1189
tools/lazy-store/Cargo.lock
··· 1 + # This file is automatically @generated by Cargo. 2 + # It is not intended for manual editing. 3 + version = 4 4 + 5 + [[package]] 6 + name = "adler2" 7 + version = "2.0.1" 8 + source = "registry+https://github.com/rust-lang/crates.io-index" 9 + checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" 10 + 11 + [[package]] 12 + name = "alloc-no-stdlib" 13 + version = "2.0.4" 14 + source = "registry+https://github.com/rust-lang/crates.io-index" 15 + checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" 16 + 17 + [[package]] 18 + name = "alloc-stdlib" 19 + version = "0.2.4" 20 + source = "registry+https://github.com/rust-lang/crates.io-index" 21 + checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" 22 + dependencies = [ 23 + "alloc-no-stdlib", 24 + ] 25 + 26 + [[package]] 27 + name = "arc-swap" 28 + version = "1.9.2" 29 + source = "registry+https://github.com/rust-lang/crates.io-index" 30 + checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" 31 + dependencies = [ 32 + "rustversion", 33 + ] 34 + 35 + [[package]] 36 + name = "autocfg" 37 + version = "1.5.1" 38 + source = "registry+https://github.com/rust-lang/crates.io-index" 39 + checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" 40 + 41 + [[package]] 42 + name = "base64" 43 + version = "0.22.1" 44 + source = "registry+https://github.com/rust-lang/crates.io-index" 45 + checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" 46 + 47 + [[package]] 48 + name = "bitflags" 49 + version = "1.3.2" 50 + source = "registry+https://github.com/rust-lang/crates.io-index" 51 + checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 52 + 53 + [[package]] 54 + name = "block-buffer" 55 + version = "0.10.4" 56 + source = "registry+https://github.com/rust-lang/crates.io-index" 57 + checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" 58 + dependencies = [ 59 + "generic-array", 60 + ] 61 + 62 + [[package]] 63 + name = "brotli-decompressor" 64 + version = "4.0.3" 65 + source = "registry+https://github.com/rust-lang/crates.io-index" 66 + checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" 67 + dependencies = [ 68 + "alloc-no-stdlib", 69 + "alloc-stdlib", 70 + ] 71 + 72 + [[package]] 73 + name = "bzip2" 74 + version = "0.4.4" 75 + source = "registry+https://github.com/rust-lang/crates.io-index" 76 + checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" 77 + dependencies = [ 78 + "bzip2-sys", 79 + "libc", 80 + ] 81 + 82 + [[package]] 83 + name = "bzip2-sys" 84 + version = "0.1.13+1.0.8" 85 + source = "registry+https://github.com/rust-lang/crates.io-index" 86 + checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" 87 + dependencies = [ 88 + "cc", 89 + "pkg-config", 90 + ] 91 + 92 + [[package]] 93 + name = "caps" 94 + version = "0.5.6" 95 + source = "registry+https://github.com/rust-lang/crates.io-index" 96 + checksum = "fd1ddba47aba30b6a889298ad0109c3b8dcb0e8fc993b459daa7067d46f865e0" 97 + dependencies = [ 98 + "libc", 99 + ] 100 + 101 + [[package]] 102 + name = "cc" 103 + version = "1.8.0" 104 + source = "registry+https://github.com/rust-lang/crates.io-index" 105 + checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" 106 + dependencies = [ 107 + "find-msvc-tools", 108 + "jobserver", 109 + "libc", 110 + "shlex", 111 + ] 112 + 113 + [[package]] 114 + name = "cfg-if" 115 + version = "1.0.5" 116 + source = "registry+https://github.com/rust-lang/crates.io-index" 117 + checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" 118 + 119 + [[package]] 120 + name = "core-foundation-sys" 121 + version = "0.8.7" 122 + source = "registry+https://github.com/rust-lang/crates.io-index" 123 + checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" 124 + 125 + [[package]] 126 + name = "cpufeatures" 127 + version = "0.2.17" 128 + source = "registry+https://github.com/rust-lang/crates.io-index" 129 + checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" 130 + dependencies = [ 131 + "libc", 132 + ] 133 + 134 + [[package]] 135 + name = "crc32fast" 136 + version = "1.5.2" 137 + source = "registry+https://github.com/rust-lang/crates.io-index" 138 + checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" 139 + dependencies = [ 140 + "cfg-if", 141 + ] 142 + 143 + [[package]] 144 + name = "crypto-common" 145 + version = "0.1.7" 146 + source = "registry+https://github.com/rust-lang/crates.io-index" 147 + checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" 148 + dependencies = [ 149 + "generic-array", 150 + "typenum", 151 + ] 152 + 153 + [[package]] 154 + name = "digest" 155 + version = "0.10.7" 156 + source = "registry+https://github.com/rust-lang/crates.io-index" 157 + checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" 158 + dependencies = [ 159 + "block-buffer", 160 + "crypto-common", 161 + ] 162 + 163 + [[package]] 164 + name = "displaydoc" 165 + version = "0.2.7" 166 + source = "registry+https://github.com/rust-lang/crates.io-index" 167 + checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" 168 + dependencies = [ 169 + "proc-macro2", 170 + "quote", 171 + "syn", 172 + ] 173 + 174 + [[package]] 175 + name = "endian-type" 176 + version = "0.1.2" 177 + source = "registry+https://github.com/rust-lang/crates.io-index" 178 + checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" 179 + 180 + [[package]] 181 + name = "find-msvc-tools" 182 + version = "0.1.14" 183 + source = "registry+https://github.com/rust-lang/crates.io-index" 184 + checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" 185 + 186 + [[package]] 187 + name = "flate2" 188 + version = "1.1.10" 189 + source = "registry+https://github.com/rust-lang/crates.io-index" 190 + checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" 191 + dependencies = [ 192 + "crc32fast", 193 + "miniz_oxide", 194 + "zlib-rs", 195 + ] 196 + 197 + [[package]] 198 + name = "form_urlencoded" 199 + version = "1.2.2" 200 + source = "registry+https://github.com/rust-lang/crates.io-index" 201 + checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" 202 + dependencies = [ 203 + "percent-encoding", 204 + ] 205 + 206 + [[package]] 207 + name = "fuse-backend-core" 208 + version = "0.14.0" 209 + source = "git+https://github.com/Kranzes/fuse-backend-rs?rev=bac50ad4cf022b3428ac483cf14fd6f71817282f#bac50ad4cf022b3428ac483cf14fd6f71817282f" 210 + dependencies = [ 211 + "bitflags", 212 + "libc", 213 + "log", 214 + "vm-memory", 215 + ] 216 + 217 + [[package]] 218 + name = "fuse-backend-fusedev" 219 + version = "0.14.0" 220 + source = "git+https://github.com/Kranzes/fuse-backend-rs?rev=bac50ad4cf022b3428ac483cf14fd6f71817282f#bac50ad4cf022b3428ac483cf14fd6f71817282f" 221 + dependencies = [ 222 + "core-foundation-sys", 223 + "fuse-backend-core", 224 + "libc", 225 + "log", 226 + "mio", 227 + "nix", 228 + "vm-memory", 229 + "vmm-sys-util", 230 + ] 231 + 232 + [[package]] 233 + name = "fuse-backend-overlayfs" 234 + version = "0.14.0" 235 + source = "git+https://github.com/Kranzes/fuse-backend-rs?rev=bac50ad4cf022b3428ac483cf14fd6f71817282f#bac50ad4cf022b3428ac483cf14fd6f71817282f" 236 + dependencies = [ 237 + "fuse-backend-core", 238 + "libc", 239 + "log", 240 + "radix_trie", 241 + "vmm-sys-util", 242 + ] 243 + 244 + [[package]] 245 + name = "fuse-backend-passthrough" 246 + version = "0.14.0" 247 + source = "git+https://github.com/Kranzes/fuse-backend-rs?rev=bac50ad4cf022b3428ac483cf14fd6f71817282f#bac50ad4cf022b3428ac483cf14fd6f71817282f" 248 + dependencies = [ 249 + "bitflags", 250 + "caps", 251 + "fuse-backend-core", 252 + "libc", 253 + "log", 254 + "vm-memory", 255 + "vmm-sys-util", 256 + ] 257 + 258 + [[package]] 259 + name = "fuse-backend-rs" 260 + version = "0.14.0" 261 + source = "git+https://github.com/Kranzes/fuse-backend-rs?rev=bac50ad4cf022b3428ac483cf14fd6f71817282f#bac50ad4cf022b3428ac483cf14fd6f71817282f" 262 + dependencies = [ 263 + "fuse-backend-core", 264 + "fuse-backend-fusedev", 265 + "fuse-backend-overlayfs", 266 + "fuse-backend-passthrough", 267 + "fuse-backend-vfs", 268 + ] 269 + 270 + [[package]] 271 + name = "fuse-backend-vfs" 272 + version = "0.14.0" 273 + source = "git+https://github.com/Kranzes/fuse-backend-rs?rev=bac50ad4cf022b3428ac483cf14fd6f71817282f#bac50ad4cf022b3428ac483cf14fd6f71817282f" 274 + dependencies = [ 275 + "arc-swap", 276 + "fuse-backend-core", 277 + "libc", 278 + "log", 279 + ] 280 + 281 + [[package]] 282 + name = "generic-array" 283 + version = "0.14.7" 284 + source = "registry+https://github.com/rust-lang/crates.io-index" 285 + checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" 286 + dependencies = [ 287 + "typenum", 288 + "version_check", 289 + ] 290 + 291 + [[package]] 292 + name = "getrandom" 293 + version = "0.2.17" 294 + source = "registry+https://github.com/rust-lang/crates.io-index" 295 + checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" 296 + dependencies = [ 297 + "cfg-if", 298 + "libc", 299 + "wasi", 300 + ] 301 + 302 + [[package]] 303 + name = "getrandom" 304 + version = "0.4.3" 305 + source = "registry+https://github.com/rust-lang/crates.io-index" 306 + checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" 307 + dependencies = [ 308 + "cfg-if", 309 + "libc", 310 + "r-efi", 311 + ] 312 + 313 + [[package]] 314 + name = "icu_collections" 315 + version = "2.3.0" 316 + source = "registry+https://github.com/rust-lang/crates.io-index" 317 + checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" 318 + dependencies = [ 319 + "displaydoc", 320 + "potential_utf", 321 + "utf8_iter", 322 + "yoke", 323 + "zerofrom", 324 + "zerovec", 325 + ] 326 + 327 + [[package]] 328 + name = "icu_locale_core" 329 + version = "2.3.0" 330 + source = "registry+https://github.com/rust-lang/crates.io-index" 331 + checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" 332 + dependencies = [ 333 + "displaydoc", 334 + "litemap", 335 + "tinystr", 336 + "writeable", 337 + "zerovec", 338 + ] 339 + 340 + [[package]] 341 + name = "icu_normalizer" 342 + version = "2.3.0" 343 + source = "registry+https://github.com/rust-lang/crates.io-index" 344 + checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" 345 + dependencies = [ 346 + "icu_collections", 347 + "icu_normalizer_data", 348 + "icu_properties", 349 + "icu_provider", 350 + "smallvec", 351 + "zerovec", 352 + ] 353 + 354 + [[package]] 355 + name = "icu_normalizer_data" 356 + version = "2.3.0" 357 + source = "registry+https://github.com/rust-lang/crates.io-index" 358 + checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" 359 + 360 + [[package]] 361 + name = "icu_properties" 362 + version = "2.3.0" 363 + source = "registry+https://github.com/rust-lang/crates.io-index" 364 + checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" 365 + dependencies = [ 366 + "displaydoc", 367 + "icu_collections", 368 + "icu_locale_core", 369 + "icu_properties_data", 370 + "icu_provider", 371 + "zerotrie", 372 + "zerovec", 373 + ] 374 + 375 + [[package]] 376 + name = "icu_properties_data" 377 + version = "2.3.0" 378 + source = "registry+https://github.com/rust-lang/crates.io-index" 379 + checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" 380 + 381 + [[package]] 382 + name = "icu_provider" 383 + version = "2.3.1" 384 + source = "registry+https://github.com/rust-lang/crates.io-index" 385 + checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" 386 + dependencies = [ 387 + "displaydoc", 388 + "icu_locale_core", 389 + "writeable", 390 + "yoke", 391 + "zerofrom", 392 + "zerotrie", 393 + "zerovec", 394 + ] 395 + 396 + [[package]] 397 + name = "idna" 398 + version = "1.1.0" 399 + source = "registry+https://github.com/rust-lang/crates.io-index" 400 + checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" 401 + dependencies = [ 402 + "idna_adapter", 403 + "smallvec", 404 + "utf8_iter", 405 + ] 406 + 407 + [[package]] 408 + name = "idna_adapter" 409 + version = "1.2.2" 410 + source = "registry+https://github.com/rust-lang/crates.io-index" 411 + checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" 412 + dependencies = [ 413 + "icu_normalizer", 414 + "icu_properties", 415 + ] 416 + 417 + [[package]] 418 + name = "itoa" 419 + version = "1.0.18" 420 + source = "registry+https://github.com/rust-lang/crates.io-index" 421 + checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" 422 + 423 + [[package]] 424 + name = "jobserver" 425 + version = "0.1.35" 426 + source = "registry+https://github.com/rust-lang/crates.io-index" 427 + checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" 428 + dependencies = [ 429 + "getrandom 0.4.3", 430 + "libc", 431 + ] 432 + 433 + [[package]] 434 + name = "lazy-store" 435 + version = "0.1.0" 436 + dependencies = [ 437 + "brotli-decompressor", 438 + "bzip2", 439 + "flate2", 440 + "fuse-backend-rs", 441 + "libc", 442 + "log", 443 + "serde", 444 + "serde_json", 445 + "sha2", 446 + "ureq", 447 + "xz2", 448 + "zstd", 449 + ] 450 + 451 + [[package]] 452 + name = "libc" 453 + version = "0.2.190" 454 + source = "registry+https://github.com/rust-lang/crates.io-index" 455 + checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" 456 + 457 + [[package]] 458 + name = "litemap" 459 + version = "0.8.3" 460 + source = "registry+https://github.com/rust-lang/crates.io-index" 461 + checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" 462 + 463 + [[package]] 464 + name = "log" 465 + version = "0.4.34" 466 + source = "registry+https://github.com/rust-lang/crates.io-index" 467 + checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" 468 + 469 + [[package]] 470 + name = "lzma-sys" 471 + version = "0.1.20" 472 + source = "registry+https://github.com/rust-lang/crates.io-index" 473 + checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" 474 + dependencies = [ 475 + "cc", 476 + "libc", 477 + "pkg-config", 478 + ] 479 + 480 + [[package]] 481 + name = "memchr" 482 + version = "2.8.3" 483 + source = "registry+https://github.com/rust-lang/crates.io-index" 484 + checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" 485 + 486 + [[package]] 487 + name = "memoffset" 488 + version = "0.6.5" 489 + source = "registry+https://github.com/rust-lang/crates.io-index" 490 + checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" 491 + dependencies = [ 492 + "autocfg", 493 + ] 494 + 495 + [[package]] 496 + name = "miniz_oxide" 497 + version = "0.9.1" 498 + source = "registry+https://github.com/rust-lang/crates.io-index" 499 + checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" 500 + dependencies = [ 501 + "adler2", 502 + "simd-adler32", 503 + ] 504 + 505 + [[package]] 506 + name = "mio" 507 + version = "0.8.11" 508 + source = "registry+https://github.com/rust-lang/crates.io-index" 509 + checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" 510 + dependencies = [ 511 + "libc", 512 + "log", 513 + "wasi", 514 + "windows-sys 0.48.0", 515 + ] 516 + 517 + [[package]] 518 + name = "nibble_vec" 519 + version = "0.1.0" 520 + source = "registry+https://github.com/rust-lang/crates.io-index" 521 + checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" 522 + dependencies = [ 523 + "smallvec", 524 + ] 525 + 526 + [[package]] 527 + name = "nix" 528 + version = "0.24.3" 529 + source = "registry+https://github.com/rust-lang/crates.io-index" 530 + checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" 531 + dependencies = [ 532 + "bitflags", 533 + "cfg-if", 534 + "libc", 535 + "memoffset", 536 + ] 537 + 538 + [[package]] 539 + name = "once_cell" 540 + version = "1.21.4" 541 + source = "registry+https://github.com/rust-lang/crates.io-index" 542 + checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" 543 + 544 + [[package]] 545 + name = "percent-encoding" 546 + version = "2.3.2" 547 + source = "registry+https://github.com/rust-lang/crates.io-index" 548 + checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" 549 + 550 + [[package]] 551 + name = "pkg-config" 552 + version = "0.3.34" 553 + source = "registry+https://github.com/rust-lang/crates.io-index" 554 + checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" 555 + 556 + [[package]] 557 + name = "potential_utf" 558 + version = "0.1.6" 559 + source = "registry+https://github.com/rust-lang/crates.io-index" 560 + checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" 561 + dependencies = [ 562 + "zerovec", 563 + ] 564 + 565 + [[package]] 566 + name = "proc-macro2" 567 + version = "1.0.107" 568 + source = "registry+https://github.com/rust-lang/crates.io-index" 569 + checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" 570 + dependencies = [ 571 + "unicode-ident", 572 + ] 573 + 574 + [[package]] 575 + name = "quote" 576 + version = "1.0.47" 577 + source = "registry+https://github.com/rust-lang/crates.io-index" 578 + checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" 579 + dependencies = [ 580 + "proc-macro2", 581 + ] 582 + 583 + [[package]] 584 + name = "r-efi" 585 + version = "6.0.0" 586 + source = "registry+https://github.com/rust-lang/crates.io-index" 587 + checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" 588 + 589 + [[package]] 590 + name = "radix_trie" 591 + version = "0.2.1" 592 + source = "registry+https://github.com/rust-lang/crates.io-index" 593 + checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" 594 + dependencies = [ 595 + "endian-type", 596 + "nibble_vec", 597 + ] 598 + 599 + [[package]] 600 + name = "ring" 601 + version = "0.17.14" 602 + source = "registry+https://github.com/rust-lang/crates.io-index" 603 + checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" 604 + dependencies = [ 605 + "cc", 606 + "cfg-if", 607 + "getrandom 0.2.17", 608 + "libc", 609 + "untrusted", 610 + "windows-sys 0.52.0", 611 + ] 612 + 613 + [[package]] 614 + name = "rustls" 615 + version = "0.23.46" 616 + source = "registry+https://github.com/rust-lang/crates.io-index" 617 + checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" 618 + dependencies = [ 619 + "log", 620 + "once_cell", 621 + "ring", 622 + "rustls-pki-types", 623 + "rustls-webpki", 624 + "subtle", 625 + "zeroize", 626 + ] 627 + 628 + [[package]] 629 + name = "rustls-pki-types" 630 + version = "1.15.1" 631 + source = "registry+https://github.com/rust-lang/crates.io-index" 632 + checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" 633 + dependencies = [ 634 + "zeroize", 635 + ] 636 + 637 + [[package]] 638 + name = "rustls-webpki" 639 + version = "0.103.15" 640 + source = "registry+https://github.com/rust-lang/crates.io-index" 641 + checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" 642 + dependencies = [ 643 + "ring", 644 + "rustls-pki-types", 645 + "untrusted", 646 + ] 647 + 648 + [[package]] 649 + name = "rustversion" 650 + version = "1.0.23" 651 + source = "registry+https://github.com/rust-lang/crates.io-index" 652 + checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" 653 + 654 + [[package]] 655 + name = "serde" 656 + version = "1.0.229" 657 + source = "registry+https://github.com/rust-lang/crates.io-index" 658 + checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" 659 + dependencies = [ 660 + "serde_core", 661 + "serde_derive", 662 + ] 663 + 664 + [[package]] 665 + name = "serde_core" 666 + version = "1.0.229" 667 + source = "registry+https://github.com/rust-lang/crates.io-index" 668 + checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" 669 + dependencies = [ 670 + "serde_derive", 671 + ] 672 + 673 + [[package]] 674 + name = "serde_derive" 675 + version = "1.0.229" 676 + source = "registry+https://github.com/rust-lang/crates.io-index" 677 + checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" 678 + dependencies = [ 679 + "proc-macro2", 680 + "quote", 681 + "syn", 682 + ] 683 + 684 + [[package]] 685 + name = "serde_json" 686 + version = "1.0.154" 687 + source = "registry+https://github.com/rust-lang/crates.io-index" 688 + checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" 689 + dependencies = [ 690 + "itoa", 691 + "memchr", 692 + "serde", 693 + "serde_core", 694 + "zmij", 695 + ] 696 + 697 + [[package]] 698 + name = "sha2" 699 + version = "0.10.9" 700 + source = "registry+https://github.com/rust-lang/crates.io-index" 701 + checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" 702 + dependencies = [ 703 + "cfg-if", 704 + "cpufeatures", 705 + "digest", 706 + ] 707 + 708 + [[package]] 709 + name = "shlex" 710 + version = "2.0.1" 711 + source = "registry+https://github.com/rust-lang/crates.io-index" 712 + checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" 713 + 714 + [[package]] 715 + name = "simd-adler32" 716 + version = "0.3.10" 717 + source = "registry+https://github.com/rust-lang/crates.io-index" 718 + checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" 719 + 720 + [[package]] 721 + name = "smallvec" 722 + version = "1.16.3" 723 + source = "registry+https://github.com/rust-lang/crates.io-index" 724 + checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" 725 + 726 + [[package]] 727 + name = "stable_deref_trait" 728 + version = "1.2.1" 729 + source = "registry+https://github.com/rust-lang/crates.io-index" 730 + checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" 731 + 732 + [[package]] 733 + name = "subtle" 734 + version = "2.6.1" 735 + source = "registry+https://github.com/rust-lang/crates.io-index" 736 + checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" 737 + 738 + [[package]] 739 + name = "syn" 740 + version = "3.0.9" 741 + source = "registry+https://github.com/rust-lang/crates.io-index" 742 + checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" 743 + dependencies = [ 744 + "proc-macro2", 745 + "quote", 746 + "unicode-ident", 747 + ] 748 + 749 + [[package]] 750 + name = "synstructure" 751 + version = "0.14.0" 752 + source = "registry+https://github.com/rust-lang/crates.io-index" 753 + checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" 754 + dependencies = [ 755 + "proc-macro2", 756 + "quote", 757 + "syn", 758 + ] 759 + 760 + [[package]] 761 + name = "thiserror" 762 + version = "2.0.21" 763 + source = "registry+https://github.com/rust-lang/crates.io-index" 764 + checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" 765 + dependencies = [ 766 + "thiserror-impl", 767 + ] 768 + 769 + [[package]] 770 + name = "thiserror-impl" 771 + version = "2.0.21" 772 + source = "registry+https://github.com/rust-lang/crates.io-index" 773 + checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" 774 + dependencies = [ 775 + "proc-macro2", 776 + "quote", 777 + "syn", 778 + ] 779 + 780 + [[package]] 781 + name = "tinystr" 782 + version = "0.8.4" 783 + source = "registry+https://github.com/rust-lang/crates.io-index" 784 + checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" 785 + dependencies = [ 786 + "displaydoc", 787 + "zerovec", 788 + ] 789 + 790 + [[package]] 791 + name = "typenum" 792 + version = "1.20.1" 793 + source = "registry+https://github.com/rust-lang/crates.io-index" 794 + checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" 795 + 796 + [[package]] 797 + name = "unicode-ident" 798 + version = "1.0.27" 799 + source = "registry+https://github.com/rust-lang/crates.io-index" 800 + checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" 801 + 802 + [[package]] 803 + name = "untrusted" 804 + version = "0.9.0" 805 + source = "registry+https://github.com/rust-lang/crates.io-index" 806 + checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" 807 + 808 + [[package]] 809 + name = "ureq" 810 + version = "2.12.1" 811 + source = "registry+https://github.com/rust-lang/crates.io-index" 812 + checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" 813 + dependencies = [ 814 + "base64", 815 + "log", 816 + "once_cell", 817 + "rustls", 818 + "rustls-pki-types", 819 + "url", 820 + "webpki-roots 0.26.11", 821 + ] 822 + 823 + [[package]] 824 + name = "url" 825 + version = "2.5.8" 826 + source = "registry+https://github.com/rust-lang/crates.io-index" 827 + checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" 828 + dependencies = [ 829 + "form_urlencoded", 830 + "idna", 831 + "percent-encoding", 832 + "serde", 833 + ] 834 + 835 + [[package]] 836 + name = "utf8_iter" 837 + version = "1.0.4" 838 + source = "registry+https://github.com/rust-lang/crates.io-index" 839 + checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" 840 + 841 + [[package]] 842 + name = "version_check" 843 + version = "0.9.5" 844 + source = "registry+https://github.com/rust-lang/crates.io-index" 845 + checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" 846 + 847 + [[package]] 848 + name = "vm-memory" 849 + version = "0.17.1" 850 + source = "registry+https://github.com/rust-lang/crates.io-index" 851 + checksum = "f39348a049689cabd3377cdd9182bf526ec76a6f823b79903896452e9d7a7380" 852 + dependencies = [ 853 + "libc", 854 + "thiserror", 855 + "winapi", 856 + ] 857 + 858 + [[package]] 859 + name = "vmm-sys-util" 860 + version = "0.15.0" 861 + source = "registry+https://github.com/rust-lang/crates.io-index" 862 + checksum = "506c62fdf617a5176827c2f9afbcf1be155b03a9b4bf9617a60dbc07e3a1642f" 863 + dependencies = [ 864 + "bitflags", 865 + "libc", 866 + ] 867 + 868 + [[package]] 869 + name = "wasi" 870 + version = "0.11.1+wasi-snapshot-preview1" 871 + source = "registry+https://github.com/rust-lang/crates.io-index" 872 + checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" 873 + 874 + [[package]] 875 + name = "webpki-roots" 876 + version = "0.26.11" 877 + source = "registry+https://github.com/rust-lang/crates.io-index" 878 + checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" 879 + dependencies = [ 880 + "webpki-roots 1.0.9", 881 + ] 882 + 883 + [[package]] 884 + name = "webpki-roots" 885 + version = "1.0.9" 886 + source = "registry+https://github.com/rust-lang/crates.io-index" 887 + checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" 888 + dependencies = [ 889 + "rustls-pki-types", 890 + ] 891 + 892 + [[package]] 893 + name = "winapi" 894 + version = "0.3.9" 895 + source = "registry+https://github.com/rust-lang/crates.io-index" 896 + checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 897 + dependencies = [ 898 + "winapi-i686-pc-windows-gnu", 899 + "winapi-x86_64-pc-windows-gnu", 900 + ] 901 + 902 + [[package]] 903 + name = "winapi-i686-pc-windows-gnu" 904 + version = "0.4.0" 905 + source = "registry+https://github.com/rust-lang/crates.io-index" 906 + checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 907 + 908 + [[package]] 909 + name = "winapi-x86_64-pc-windows-gnu" 910 + version = "0.4.0" 911 + source = "registry+https://github.com/rust-lang/crates.io-index" 912 + checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 913 + 914 + [[package]] 915 + name = "windows-sys" 916 + version = "0.48.0" 917 + source = "registry+https://github.com/rust-lang/crates.io-index" 918 + checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" 919 + dependencies = [ 920 + "windows-targets 0.48.5", 921 + ] 922 + 923 + [[package]] 924 + name = "windows-sys" 925 + version = "0.52.0" 926 + source = "registry+https://github.com/rust-lang/crates.io-index" 927 + checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" 928 + dependencies = [ 929 + "windows-targets 0.52.6", 930 + ] 931 + 932 + [[package]] 933 + name = "windows-targets" 934 + version = "0.48.5" 935 + source = "registry+https://github.com/rust-lang/crates.io-index" 936 + checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" 937 + dependencies = [ 938 + "windows_aarch64_gnullvm 0.48.5", 939 + "windows_aarch64_msvc 0.48.5", 940 + "windows_i686_gnu 0.48.5", 941 + "windows_i686_msvc 0.48.5", 942 + "windows_x86_64_gnu 0.48.5", 943 + "windows_x86_64_gnullvm 0.48.5", 944 + "windows_x86_64_msvc 0.48.5", 945 + ] 946 + 947 + [[package]] 948 + name = "windows-targets" 949 + version = "0.52.6" 950 + source = "registry+https://github.com/rust-lang/crates.io-index" 951 + checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" 952 + dependencies = [ 953 + "windows_aarch64_gnullvm 0.52.6", 954 + "windows_aarch64_msvc 0.52.6", 955 + "windows_i686_gnu 0.52.6", 956 + "windows_i686_gnullvm", 957 + "windows_i686_msvc 0.52.6", 958 + "windows_x86_64_gnu 0.52.6", 959 + "windows_x86_64_gnullvm 0.52.6", 960 + "windows_x86_64_msvc 0.52.6", 961 + ] 962 + 963 + [[package]] 964 + name = "windows_aarch64_gnullvm" 965 + version = "0.48.5" 966 + source = "registry+https://github.com/rust-lang/crates.io-index" 967 + checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" 968 + 969 + [[package]] 970 + name = "windows_aarch64_gnullvm" 971 + version = "0.52.6" 972 + source = "registry+https://github.com/rust-lang/crates.io-index" 973 + checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" 974 + 975 + [[package]] 976 + name = "windows_aarch64_msvc" 977 + version = "0.48.5" 978 + source = "registry+https://github.com/rust-lang/crates.io-index" 979 + checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" 980 + 981 + [[package]] 982 + name = "windows_aarch64_msvc" 983 + version = "0.52.6" 984 + source = "registry+https://github.com/rust-lang/crates.io-index" 985 + checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" 986 + 987 + [[package]] 988 + name = "windows_i686_gnu" 989 + version = "0.48.5" 990 + source = "registry+https://github.com/rust-lang/crates.io-index" 991 + checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" 992 + 993 + [[package]] 994 + name = "windows_i686_gnu" 995 + version = "0.52.6" 996 + source = "registry+https://github.com/rust-lang/crates.io-index" 997 + checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" 998 + 999 + [[package]] 1000 + name = "windows_i686_gnullvm" 1001 + version = "0.52.6" 1002 + source = "registry+https://github.com/rust-lang/crates.io-index" 1003 + checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" 1004 + 1005 + [[package]] 1006 + name = "windows_i686_msvc" 1007 + version = "0.48.5" 1008 + source = "registry+https://github.com/rust-lang/crates.io-index" 1009 + checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" 1010 + 1011 + [[package]] 1012 + name = "windows_i686_msvc" 1013 + version = "0.52.6" 1014 + source = "registry+https://github.com/rust-lang/crates.io-index" 1015 + checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" 1016 + 1017 + [[package]] 1018 + name = "windows_x86_64_gnu" 1019 + version = "0.48.5" 1020 + source = "registry+https://github.com/rust-lang/crates.io-index" 1021 + checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" 1022 + 1023 + [[package]] 1024 + name = "windows_x86_64_gnu" 1025 + version = "0.52.6" 1026 + source = "registry+https://github.com/rust-lang/crates.io-index" 1027 + checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" 1028 + 1029 + [[package]] 1030 + name = "windows_x86_64_gnullvm" 1031 + version = "0.48.5" 1032 + source = "registry+https://github.com/rust-lang/crates.io-index" 1033 + checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" 1034 + 1035 + [[package]] 1036 + name = "windows_x86_64_gnullvm" 1037 + version = "0.52.6" 1038 + source = "registry+https://github.com/rust-lang/crates.io-index" 1039 + checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" 1040 + 1041 + [[package]] 1042 + name = "windows_x86_64_msvc" 1043 + version = "0.48.5" 1044 + source = "registry+https://github.com/rust-lang/crates.io-index" 1045 + checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" 1046 + 1047 + [[package]] 1048 + name = "windows_x86_64_msvc" 1049 + version = "0.52.6" 1050 + source = "registry+https://github.com/rust-lang/crates.io-index" 1051 + checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" 1052 + 1053 + [[package]] 1054 + name = "writeable" 1055 + version = "0.6.4" 1056 + source = "registry+https://github.com/rust-lang/crates.io-index" 1057 + checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" 1058 + 1059 + [[package]] 1060 + name = "xz2" 1061 + version = "0.1.7" 1062 + source = "registry+https://github.com/rust-lang/crates.io-index" 1063 + checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" 1064 + dependencies = [ 1065 + "lzma-sys", 1066 + ] 1067 + 1068 + [[package]] 1069 + name = "yoke" 1070 + version = "0.8.3" 1071 + source = "registry+https://github.com/rust-lang/crates.io-index" 1072 + checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" 1073 + dependencies = [ 1074 + "stable_deref_trait", 1075 + "yoke-derive", 1076 + "zerofrom", 1077 + ] 1078 + 1079 + [[package]] 1080 + name = "yoke-derive" 1081 + version = "0.8.4" 1082 + source = "registry+https://github.com/rust-lang/crates.io-index" 1083 + checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" 1084 + dependencies = [ 1085 + "proc-macro2", 1086 + "quote", 1087 + "syn", 1088 + "synstructure", 1089 + ] 1090 + 1091 + [[package]] 1092 + name = "zerofrom" 1093 + version = "0.1.8" 1094 + source = "registry+https://github.com/rust-lang/crates.io-index" 1095 + checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" 1096 + dependencies = [ 1097 + "zerofrom-derive", 1098 + ] 1099 + 1100 + [[package]] 1101 + name = "zerofrom-derive" 1102 + version = "0.1.8" 1103 + source = "registry+https://github.com/rust-lang/crates.io-index" 1104 + checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" 1105 + dependencies = [ 1106 + "proc-macro2", 1107 + "quote", 1108 + "syn", 1109 + "synstructure", 1110 + ] 1111 + 1112 + [[package]] 1113 + name = "zeroize" 1114 + version = "1.9.1" 1115 + source = "registry+https://github.com/rust-lang/crates.io-index" 1116 + checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" 1117 + 1118 + [[package]] 1119 + name = "zerotrie" 1120 + version = "0.2.5" 1121 + source = "registry+https://github.com/rust-lang/crates.io-index" 1122 + checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" 1123 + dependencies = [ 1124 + "displaydoc", 1125 + "yoke", 1126 + "zerofrom", 1127 + ] 1128 + 1129 + [[package]] 1130 + name = "zerovec" 1131 + version = "0.11.8" 1132 + source = "registry+https://github.com/rust-lang/crates.io-index" 1133 + checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" 1134 + dependencies = [ 1135 + "yoke", 1136 + "zerofrom", 1137 + "zerovec-derive", 1138 + ] 1139 + 1140 + [[package]] 1141 + name = "zerovec-derive" 1142 + version = "0.11.6" 1143 + source = "registry+https://github.com/rust-lang/crates.io-index" 1144 + checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" 1145 + dependencies = [ 1146 + "proc-macro2", 1147 + "quote", 1148 + "syn", 1149 + ] 1150 + 1151 + [[package]] 1152 + name = "zlib-rs" 1153 + version = "0.6.8" 1154 + source = "registry+https://github.com/rust-lang/crates.io-index" 1155 + checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" 1156 + 1157 + [[package]] 1158 + name = "zmij" 1159 + version = "1.0.23" 1160 + source = "registry+https://github.com/rust-lang/crates.io-index" 1161 + checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" 1162 + 1163 + [[package]] 1164 + name = "zstd" 1165 + version = "0.13.3" 1166 + source = "registry+https://github.com/rust-lang/crates.io-index" 1167 + checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" 1168 + dependencies = [ 1169 + "zstd-safe", 1170 + ] 1171 + 1172 + [[package]] 1173 + name = "zstd-safe" 1174 + version = "7.3.0" 1175 + source = "registry+https://github.com/rust-lang/crates.io-index" 1176 + checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" 1177 + dependencies = [ 1178 + "zstd-sys", 1179 + ] 1180 + 1181 + [[package]] 1182 + name = "zstd-sys" 1183 + version = "2.1.2+zstd.1.5.7" 1184 + source = "registry+https://github.com/rust-lang/crates.io-index" 1185 + checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" 1186 + dependencies = [ 1187 + "cc", 1188 + "pkg-config", 1189 + ]
+26
tools/lazy-store/Cargo.toml
··· 1 + [package] 2 + name = "lazy-store" 3 + version = "0.1.0" 4 + edition = "2021" 5 + description = "Lazy nix store FUSE file system for CI's machine builds" 6 + publish = false 7 + 8 + [dependencies] 9 + # Kranzes' combined branch: kernel FUSE passthrough, async/io_uring and 10 + # overlayfs fixes on top of cloud-hypervisor/fuse-backend-rs. 11 + fuse-backend-rs = { git = "https://github.com/Kranzes/fuse-backend-rs", rev = "bac50ad4cf022b3428ac483cf14fd6f71817282f", features = ["fusedev"] } 12 + libc = "0.2" 13 + log = "0.4" 14 + serde = { version = "1", features = ["derive"] } 15 + serde_json = "1" 16 + sha2 = "0.10" 17 + # rustls with bundled webpki roots: nothing read from /etc/ssl or the store 18 + ureq = { version = "2", default-features = false, features = ["tls"] } 19 + zstd = "0.13" 20 + xz2 = { version = "0.1", features = ["static"] } 21 + bzip2 = { version = "0.4", features = ["static"] } 22 + flate2 = "1" 23 + brotli-decompressor = "4" 24 + 25 + [profile.release] 26 + debug = 1
+13
tools/lazy-store/flake-module.nix
··· 1 + { ... }: 2 + { 3 + perSystem = 4 + { lib, pkgs, ... }: 5 + { 6 + # FUSE file system that serves /nix/store paths from the binary caches 7 + # without downloading them up front, for CI's machine builds 8 + # (.github/lazy-store.sh). 9 + packages = lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { 10 + lazy-store = pkgs.callPackage ./package.nix { }; 11 + }; 12 + }; 13 + }
+131
tools/lazy-store/package.nix
··· 1 + { 2 + lib, 3 + rustPlatform, 4 + glibc, 5 + runCommand, 6 + nix, 7 + zstd, 8 + xz, 9 + jq, 10 + }: 11 + rustPlatform.buildRustPackage (finalAttrs: { 12 + pname = "lazy-store"; 13 + version = "0.1.0"; 14 + src = lib.fileset.toSource { 15 + root = ./.; 16 + fileset = lib.fileset.unions [ 17 + ./Cargo.toml 18 + ./Cargo.lock 19 + ./src 20 + ]; 21 + }; 22 + cargoLock = { 23 + lockFile = ./Cargo.lock; 24 + outputHashes."fuse-backend-rs-0.14.0" = "sha256-pSnYq8IhfpxtnTS7bnE8GHAVP+yZ+hItChTEmGUP7xM="; 25 + }; 26 + # Static, so that once it serves /nix/store it loads nothing from there. 27 + # (Not glibc.static in buildInputs: build scripts would link against it.) 28 + env.RUSTFLAGS = "-C target-feature=+crt-static -L native=${glibc.static}/lib"; 29 + 30 + doInstallCheck = true; 31 + installCheckPhase = '' 32 + if $READELF -l $out/bin/lazy-store | grep -q INTERP; then 33 + echo "lazy-store is not static"; exit 1 34 + fi 35 + ($out/bin/lazy-store || true) 2>&1 | grep -q usage 36 + ''; 37 + 38 + # closure and the lower layer against a file:// cache, without FUSE: 39 + # nix build .#lazy-store.tests.fetch 40 + passthru.tests.fetch = 41 + runCommand "lazy-store-fetch-test" 42 + { 43 + nativeBuildInputs = [ 44 + finalAttrs.finalPackage 45 + nix 46 + zstd 47 + xz 48 + jq 49 + ]; 50 + } 51 + '' 52 + export HOME=$TMPDIR NIX_STATE_DIR=$TMPDIR/state NIX_LOG_DIR=$TMPDIR/log 53 + mkdir -p src/a/sub cache/nar out closure 54 + echo hello > src/a/file 55 + printf '#!/bin/sh\n' > src/a/sub/run 56 + chmod +x src/a/sub/run 57 + ln -s file src/a/link 58 + echo single > src/b 59 + cp -a src/a src/c 60 + 61 + A=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-a 62 + B=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-b 63 + C=cccccccccccccccccccccccccccccccc-c 64 + D=dddddddddddddddddddddddddddddddd-d 65 + E=eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-e 66 + ls='{"version":1,"root":{"type":"directory","entries":{ 67 + "file":{"type":"regular","size":6}, 68 + "link":{"type":"symlink","target":"file"}, 69 + "sub":{"type":"directory","entries":{"run":{"type":"regular","size":10,"executable":true}}}}}}' 70 + # zstd .ls served without Content-Encoding (sniffed); b has none 71 + echo "$ls" | zstd -q > cache/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.ls 72 + echo "$ls" | zstd -q > cache/cccccccccccccccccccccccccccccccc.ls 73 + 74 + nix-store --dump src/a | zstd -q > cache/nar/a.nar.zst 75 + nix-store --dump src/b | xz > cache/nar/b.nar.xz 76 + nix-store --dump src/c | zstd -q > cache/nar/c.nar.zst 77 + info() { # name file compression hash size 78 + jq -n --arg c "file://$PWD/cache" --arg u "$2" --arg z "$3" --arg h "$4" --argjson s "$5" \ 79 + '{($ARGS.positional[0]): {cache: $c, url: $u, compression: $z, narHash: $h, narSize: $s, fileSize: 0}}' \ 80 + --args "$1" 81 + } 82 + size() { nix-store --dump "$1" | wc -c; } 83 + { 84 + info $A nar/a.nar.zst zstd "sha256:$(nix-hash --type sha256 --base32 src/a)" "$(size src/a)" 85 + info $B nar/b.nar.xz xz "sha256:$(nix-hash --type sha256 src/b)" "$(size src/b)" 86 + # wrong hash: must not be unpacked 87 + info $C nar/c.nar.zst zstd "sha256:$(nix-hash --type sha256 --base32 src/b)" "$(size src/c)" 88 + } | jq -s add > out/paths.json 89 + 90 + lazy-store fetch out $A $B 91 + diff -r src/a out/cache/$A 92 + diff src/b out/cache/$B 93 + test "$(stat -c %a out/cache/$A/sub/run)" = 555 94 + test "$(stat -c %a out/cache/$A/file)" = 444 95 + test "$(readlink out/cache/$A/link)" = file 96 + jq -e '.ls_count == 2 and .nar_count == 2' out/stats.json 97 + if lazy-store fetch out $C; then 98 + echo "NAR with a wrong hash was accepted"; exit 1 99 + fi 100 + test ! -e out/cache/$C 101 + 102 + # a refers to b; d refers to e, which no cache has: d is left out 103 + narinfo() { # name url references 104 + printf 'StorePath: /nix/store/%s\nURL: %s\nCompression: zstd\nNarHash: sha256:%s\nNarSize: 8\nReferences: %s\nSig: test-1:%s\n' \ 105 + "$1" "$2" "$(nix-hash --type sha256 --base32 src/b)" "$3" "$(printf %64s | base64 -w0)" \ 106 + > "cache/''${1%%-*}.narinfo" 107 + } 108 + narinfo $A nar/a.nar.zst "$A $B" 109 + narinfo $B nar/b.nar.xz "" 110 + narinfo $D nar/d.nar.zst "$E" 111 + printf '/nix/store/%s\n' $A $D | LAZY_STORE_CACHES="file://$PWD/cache" lazy-store closure closure 112 + jq -e '.paths == 2 and .missing == ["'$E'"] and .incomplete == 1' closure/closure.json 113 + jq -e 'keys == ["'$A'", "'$B'"]' closure/paths.json 114 + if grep -q $D closure/db.dump; then 115 + echo "path with an incomplete closure was registered"; exit 1 116 + fi 117 + 118 + # registered as .github/lazy-store.sh does, with the signatures 119 + export NIX_CONFIG="experimental-features = nix-command" 120 + root="local?root=$PWD/root" 121 + nix-store --store "$root" --load-db < closure/db.dump 122 + printf '/nix/store/%s\n' $A $B | 123 + nix store copy-sigs --store "$root" -s "file://$PWD/closure/narinfo" --stdin 124 + nix path-info --store "$root" --sigs /nix/store/$A | grep -q "[[:space:]]test-1:$(printf %64s | base64 -w0)$" 125 + test "$(nix-store --store "$root" -q --references /nix/store/$A | wc -l)" = 2 126 + touch $out 127 + ''; 128 + 129 + meta.mainProgram = "lazy-store"; 130 + meta.platforms = lib.platforms.linux; 131 + })
+236
tools/lazy-store/src/closure.rs
··· 1 + //! `closure OUT`: the closure of the store paths on stdin, from the caches' 2 + //! narinfos only. 3 + 4 + use std::collections::{BTreeMap, BTreeSet, HashMap}; 5 + use std::io::{self, BufRead, Write}; 6 + use std::path::Path; 7 + use std::sync::atomic::{AtomicUsize, Ordering}; 8 + use std::sync::Mutex; 9 + 10 + use serde::{Deserialize, Serialize}; 11 + 12 + use crate::http::{self, Http}; 13 + use crate::stats::Stats; 14 + use crate::STORE; 15 + 16 + /// Per store path, what the lazy layer needs to fetch it (OUT/paths.json). 17 + #[derive(Serialize, Deserialize, Clone, Debug)] 18 + #[serde(rename_all = "camelCase")] 19 + pub struct PathInfo { 20 + pub cache: String, 21 + pub url: String, 22 + pub compression: String, 23 + pub nar_hash: String, 24 + pub nar_size: u64, 25 + pub file_size: u64, 26 + } 27 + 28 + pub fn caches() -> Vec<String> { 29 + std::env::var("LAZY_STORE_CACHES") 30 + .unwrap_or_else(|_| "https://cache.nixos.org https://cache.lassul.us".into()) 31 + .split_whitespace() 32 + .map(str::to_string) 33 + .collect() 34 + } 35 + 36 + fn parse_narinfo(text: &[u8]) -> HashMap<String, String> { 37 + String::from_utf8_lossy(text) 38 + .lines() 39 + .filter_map(|l| l.split_once(": ")) 40 + .map(|(k, v)| (k.to_string(), v.to_string())) 41 + .collect() 42 + } 43 + 44 + fn references(i: &HashMap<String, String>) -> impl Iterator<Item = &str> { 45 + i.get("References") 46 + .map(String::as_str) 47 + .unwrap_or("") 48 + .split_whitespace() 49 + } 50 + 51 + /// The parsed narinfo of `name` from the first cache that has it, and its 52 + /// text. 53 + fn narinfo( 54 + http: &Http, 55 + caches: &[String], 56 + name: &str, 57 + stats: &Stats, 58 + ) -> Option<(HashMap<String, String>, Vec<u8>)> { 59 + let h = name.split('-').next().unwrap_or(name); 60 + for c in caches { 61 + match http.get(&format!("{c}/{h}.narinfo")) { 62 + Ok((body, len)) => { 63 + stats.add("narinfo", len as u64); 64 + let mut i = parse_narinfo(&body); 65 + i.insert("cache".into(), c.clone()); 66 + return Some((i, body)); 67 + } 68 + Err(http::Error::NotFound) => {} 69 + // still failing after the retries: try the next cache 70 + Err(e) => { 71 + eprintln!("giving up on {c}/{h}.narinfo: {e}"); 72 + stats.add("narinfo_error", 0); 73 + } 74 + } 75 + } 76 + None 77 + } 78 + 79 + /// Run `f` over `items` on `threads` threads, results in order. 80 + pub fn par_map<T: Sync, R: Send>( 81 + items: &[T], 82 + threads: usize, 83 + f: impl Fn(&T) -> R + Sync, 84 + ) -> Vec<R> { 85 + let next = AtomicUsize::new(0); 86 + let out: Mutex<Vec<Option<R>>> = Mutex::new((0..items.len()).map(|_| None).collect()); 87 + std::thread::scope(|s| { 88 + for _ in 0..threads.min(items.len()) { 89 + s.spawn(|| loop { 90 + let i = next.fetch_add(1, Ordering::Relaxed); 91 + if i >= items.len() { 92 + break; 93 + } 94 + let r = f(&items[i]); 95 + out.lock().unwrap()[i] = Some(r); 96 + }); 97 + } 98 + }); 99 + out.into_inner() 100 + .unwrap() 101 + .into_iter() 102 + .map(Option::unwrap) 103 + .collect() 104 + } 105 + 106 + /// Writes OUT/paths.json (per path: cache, NAR url, compression, hash), 107 + /// OUT/db.dump (input for `nix-store --load-db`), OUT/narinfo (a binary 108 + /// cache of just the narinfos) and OUT/closure.json. 109 + /// Paths that are in none of the caches, and everything referring to them, 110 + /// are left out: a path is registered only with its whole closure. 111 + pub fn closure(outdir: &Path) -> io::Result<()> { 112 + let mut roots = BTreeSet::new(); 113 + for l in io::stdin().lock().lines() { 114 + let l = l?; 115 + let l = l.trim(); 116 + if !l.is_empty() { 117 + roots.insert(l.strip_prefix(STORE).unwrap_or(l).to_string()); 118 + } 119 + } 120 + let caches = caches(); 121 + let http = Http::new(caches.iter().map(String::as_str)); 122 + let stats = Stats::new(None); 123 + let mut infos: BTreeMap<String, HashMap<String, String>> = BTreeMap::new(); 124 + let mut texts: HashMap<String, Vec<u8>> = HashMap::new(); 125 + let mut missing: BTreeSet<String> = BTreeSet::new(); 126 + let mut todo: Vec<String> = roots.iter().cloned().collect(); 127 + while !todo.is_empty() { 128 + let res = par_map(&todo, 32, |n| narinfo(&http, &caches, n, &stats)); 129 + let mut next = BTreeSet::new(); 130 + for (name, i) in todo.iter().zip(res) { 131 + let Some((i, text)) = i else { 132 + missing.insert(name.clone()); 133 + continue; 134 + }; 135 + next.extend(references(&i).map(str::to_string)); 136 + infos.insert(name.clone(), i); 137 + texts.insert(name.clone(), text); 138 + } 139 + todo = next 140 + .into_iter() 141 + .filter(|r| !infos.contains_key(r) && !missing.contains(r)) 142 + .collect(); 143 + } 144 + 145 + let mut referrers: HashMap<String, Vec<String>> = HashMap::new(); 146 + for (n, i) in &infos { 147 + for r in references(i).filter(|r| *r != n.as_str()) { 148 + referrers.entry(r.to_string()).or_default().push(n.clone()); 149 + } 150 + } 151 + let mut incomplete = BTreeSet::new(); 152 + let mut todo: Vec<&String> = missing.iter().collect(); 153 + while let Some(m) = todo.pop() { 154 + for r in referrers.get(m).into_iter().flatten() { 155 + if incomplete.insert(r.clone()) { 156 + todo.push(r); 157 + } 158 + } 159 + } 160 + infos.retain(|n, _| !incomplete.contains(n)); 161 + 162 + let mut paths = BTreeMap::new(); 163 + for (n, i) in &infos { 164 + let get = |k: &str| { 165 + i.get(k) 166 + .cloned() 167 + .ok_or_else(|| io::Error::other(format!("{n}: narinfo without {k}"))) 168 + }; 169 + paths.insert( 170 + n.clone(), 171 + PathInfo { 172 + cache: get("cache")?, 173 + url: get("URL")?, 174 + compression: i 175 + .get("Compression") 176 + .cloned() 177 + .unwrap_or_else(|| "none".into()), 178 + nar_hash: get("NarHash")?, 179 + nar_size: get("NarSize")?.parse().map_err(io::Error::other)?, 180 + file_size: i.get("FileSize").and_then(|s| s.parse().ok()).unwrap_or(0), 181 + }, 182 + ); 183 + } 184 + std::fs::write(outdir.join("paths.json"), serde_json::to_vec(&paths)?)?; 185 + 186 + let mut db = io::BufWriter::new(std::fs::File::create(outdir.join("db.dump"))?); 187 + for (n, i) in &infos { 188 + let refs: Vec<&str> = references(i).collect(); 189 + let d = i.get("Deriver").map(String::as_str).unwrap_or(""); 190 + let deriver = if d.ends_with(".drv") { 191 + format!("{STORE}{d}") 192 + } else { 193 + String::new() 194 + }; 195 + writeln!( 196 + db, 197 + "{STORE}{n}\n{}\n{}\n{deriver}\n{}", 198 + i["NarHash"], 199 + i["NarSize"], 200 + refs.len() 201 + )?; 202 + for r in refs { 203 + writeln!(db, "{STORE}{r}")?; 204 + } 205 + } 206 + db.flush()?; 207 + 208 + // the narinfos as a binary cache, for their signatures: `nix-store 209 + // --load-db` registers none, `nix store copy-sigs` adds them 210 + let dir = outdir.join("narinfo"); 211 + std::fs::create_dir_all(&dir)?; 212 + std::fs::write(dir.join("nix-cache-info"), "StoreDir: /nix/store\n")?; 213 + for n in infos.keys() { 214 + let h = n.split('-').next().unwrap_or(n); 215 + std::fs::write(dir.join(format!("{h}.narinfo")), &texts[n])?; 216 + } 217 + 218 + let mut by_cache: BTreeMap<String, u64> = BTreeMap::new(); 219 + for p in paths.values() { 220 + *by_cache.entry(p.cache.clone()).or_default() += 1; 221 + } 222 + let mut summary = serde_json::json!({ 223 + "roots": roots.len(), 224 + "paths": infos.len(), 225 + "missing": missing, 226 + "incomplete": incomplete.len(), 227 + "by_cache": by_cache, 228 + "nar_size": paths.values().map(|p| p.nar_size).sum::<u64>(), 229 + "file_size": paths.values().map(|p| p.file_size).sum::<u64>(), 230 + }); 231 + summary.as_object_mut().unwrap().extend(stats.counters()); 232 + let s = serde_json::to_string_pretty(&summary)?; 233 + std::fs::write(outdir.join("closure.json"), &s)?; 234 + println!("{s}"); 235 + Ok(()) 236 + }
+199
tools/lazy-store/src/http.rs
··· 1 + //! Binary cache access: GET with retries, and body decoding. 2 + 3 + use std::collections::HashMap; 4 + use std::io::{self, Read}; 5 + use std::net::{SocketAddr, ToSocketAddrs}; 6 + use std::sync::Mutex; 7 + use std::thread::sleep; 8 + use std::time::Duration; 9 + 10 + const TRIES: u32 = 6; 11 + 12 + /// Resolves each host once and then serves it from memory. Once the FUSE 13 + /// file system is mounted on /nix/store, getaddrinfo() could end up looking 14 + /// for NSS modules in the store, i.e. in a request to ourselves. 15 + struct Resolver(Mutex<HashMap<String, Vec<SocketAddr>>>); 16 + 17 + impl ureq::Resolver for Resolver { 18 + fn resolve(&self, netloc: &str) -> io::Result<Vec<SocketAddr>> { 19 + if let Some(a) = self.0.lock().unwrap().get(netloc) { 20 + return Ok(a.clone()); 21 + } 22 + let a: Vec<SocketAddr> = netloc.to_socket_addrs()?.collect(); 23 + self.0.lock().unwrap().insert(netloc.to_string(), a.clone()); 24 + Ok(a) 25 + } 26 + } 27 + 28 + pub struct Http { 29 + agent: ureq::Agent, 30 + } 31 + 32 + pub enum Error { 33 + /// 403/404, or a missing file for file:// caches 34 + NotFound, 35 + Other(String), 36 + } 37 + 38 + impl std::fmt::Display for Error { 39 + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 40 + match self { 41 + Error::NotFound => write!(f, "not found"), 42 + Error::Other(e) => write!(f, "{e}"), 43 + } 44 + } 45 + } 46 + 47 + impl From<Error> for io::Error { 48 + fn from(e: Error) -> io::Error { 49 + io::Error::other(e.to_string()) 50 + } 51 + } 52 + 53 + impl Http { 54 + /// `caches` are base URLs; their hosts are resolved right away. 55 + pub fn new<'a>(caches: impl IntoIterator<Item = &'a str>) -> Http { 56 + let mut known = HashMap::new(); 57 + for c in caches { 58 + let Some(rest) = c 59 + .strip_prefix("https://") 60 + .or_else(|| c.strip_prefix("http://")) 61 + else { 62 + continue; 63 + }; 64 + let host = rest.split('/').next().unwrap_or(rest); 65 + let port = if c.starts_with("https:") { 443 } else { 80 }; 66 + let netloc = if host.contains(':') { 67 + host.to_string() 68 + } else { 69 + format!("{host}:{port}") 70 + }; 71 + match netloc.to_socket_addrs() { 72 + Ok(a) => { 73 + known.insert(netloc, a.collect()); 74 + } 75 + Err(e) => eprintln!("resolving {netloc}: {e}"), 76 + } 77 + } 78 + let agent = ureq::AgentBuilder::new() 79 + .timeout_connect(Duration::from_secs(30)) 80 + .timeout_read(Duration::from_secs(60)) 81 + .max_idle_connections_per_host(64) 82 + .resolver(Resolver(Mutex::new(known))) 83 + .build(); 84 + Http { agent } 85 + } 86 + 87 + /// GET `url`, retrying 429/5xx and transport errors with backoff (the 88 + /// caches answer 503 under load). Returns the body reader and its 89 + /// Content-Encoding. 90 + pub fn open(&self, url: &str) -> Result<(Box<dyn Read + Send>, Option<String>), Error> { 91 + if let Some(path) = url.strip_prefix("file://") { 92 + return match std::fs::File::open(path) { 93 + Ok(f) => Ok((Box::new(f), None)), 94 + Err(e) if e.kind() == io::ErrorKind::NotFound => Err(Error::NotFound), 95 + Err(e) => Err(Error::Other(format!("{url}: {e}"))), 96 + }; 97 + } 98 + let mut n = 0; 99 + loop { 100 + let err = match self.agent.get(url).call() { 101 + Ok(r) => { 102 + let enc = r.header("content-encoding").map(str::to_string); 103 + return Ok((Box::new(r.into_reader()), enc)); 104 + } 105 + Err(ureq::Error::Status(403 | 404, _)) => return Err(Error::NotFound), 106 + Err(ureq::Error::Status(code, _)) if code != 429 && code < 500 => { 107 + return Err(Error::Other(format!("{url}: HTTP {code}"))) 108 + } 109 + Err(ureq::Error::Status(code, _)) => format!("HTTP {code}"), 110 + Err(e) => e.to_string(), 111 + }; 112 + n += 1; 113 + if n == TRIES { 114 + return Err(Error::Other(format!("{url}: {err}"))); 115 + } 116 + eprintln!("retry {n}/{} {url}: {err}", TRIES - 1); 117 + sleep(Duration::from_secs((1u64 << (n - 1)).min(30))); 118 + } 119 + } 120 + 121 + /// GET `url` -> decoded body. Transfer errors while reading the body are 122 + /// retried like failed requests. 123 + pub fn get(&self, url: &str) -> Result<(Vec<u8>, usize), Error> { 124 + let mut n = 0; 125 + loop { 126 + let (mut r, enc) = self.open(url)?; 127 + let mut body = Vec::new(); 128 + match r.read_to_end(&mut body) { 129 + Ok(_) => { 130 + let len = body.len(); 131 + let dec = decode(&body, enc.as_deref()) 132 + .map_err(|e| Error::Other(format!("{url}: decoding: {e}")))?; 133 + return Ok((dec, len)); 134 + } 135 + Err(e) => { 136 + n += 1; 137 + if n == TRIES { 138 + return Err(Error::Other(format!("{url}: {e}"))); 139 + } 140 + eprintln!("retry {n}/{} {url}: {e}", TRIES - 1); 141 + sleep(Duration::from_secs((1u64 << (n - 1)).min(30))); 142 + } 143 + } 144 + } 145 + } 146 + } 147 + 148 + /// Wrap `r` in a decompressor for `method` (narinfo Compression or 149 + /// Content-Encoding). 150 + pub fn decoder<'a>( 151 + method: &str, 152 + r: impl Read + Send + 'a, 153 + ) -> io::Result<Box<dyn Read + Send + 'a>> { 154 + Ok(match method { 155 + "" | "none" | "identity" => Box::new(r), 156 + "zstd" => Box::new(zstd::stream::read::Decoder::new(r)?), 157 + "xz" => Box::new(xz2::read::XzDecoder::new_multi_decoder(r)), 158 + "bzip2" => Box::new(bzip2::read::MultiBzDecoder::new(r)), 159 + "gzip" => Box::new(flate2::read::MultiGzDecoder::new(r)), 160 + "br" => Box::new(brotli_decompressor::Decompressor::new(r, 1 << 16)), 161 + m => return Err(io::Error::other(format!("unknown compression {m:?}"))), 162 + }) 163 + } 164 + 165 + const MAGIC: &[(&[u8], &str)] = &[ 166 + (b"\x28\xb5\x2f\xfd", "zstd"), 167 + (b"\xfd7zXZ\x00", "xz"), 168 + (b"\x1f\x8b", "gzip"), 169 + ]; 170 + 171 + /// Both caches store narinfo/.ls zstd-compressed (Content-Encoding), but the 172 + /// CDN sometimes serves them without that header: sniff the magic too. 173 + pub fn decode(body: &[u8], encoding: Option<&str>) -> io::Result<Vec<u8>> { 174 + let enc = match encoding { 175 + None | Some("") | Some("identity") => { 176 + match MAGIC.iter().find(|(m, _)| body.starts_with(m)) { 177 + Some((_, t)) => *t, 178 + None => return Ok(body.to_vec()), 179 + } 180 + } 181 + Some(e) => e, 182 + }; 183 + let mut out = Vec::new(); 184 + decoder(enc, body)?.read_to_end(&mut out)?; 185 + Ok(out) 186 + } 187 + 188 + #[cfg(test)] 189 + mod tests { 190 + use super::*; 191 + 192 + #[test] 193 + fn decode_sniffs_zstd() { 194 + let z = zstd::encode_all(&b"{\"root\":1}"[..], 3).unwrap(); 195 + assert_eq!(decode(&z, None).unwrap(), b"{\"root\":1}"); 196 + assert_eq!(decode(&z, Some("zstd")).unwrap(), b"{\"root\":1}"); 197 + assert_eq!(decode(b"plain", None).unwrap(), b"plain"); 198 + } 199 + }
+611
tools/lazy-store/src/lazy.rs
··· 1 + //! The lower layer: all store paths of OUT/paths.json, read-only. Metadata 2 + //! comes from the caches' `.ls` listings, all fetched up front and kept in 3 + //! memory; the first open() of a file in a store path fetches, verifies 4 + //! (NarHash/NarSize) and unpacks that path's NAR. Opens of a path that is 5 + //! being fetched wait for that fetch. 6 + 7 + use std::collections::{BTreeMap, HashMap}; 8 + use std::ffi::{CStr, CString}; 9 + use std::fs::File; 10 + use std::io; 11 + use std::mem::ManuallyDrop; 12 + use std::os::fd::{AsRawFd, FromRawFd}; 13 + use std::path::{Path, PathBuf}; 14 + use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; 15 + use std::sync::{Arc, Mutex, RwLock}; 16 + use std::time::{Duration, Instant}; 17 + 18 + use fuse_backend_rs::abi::fuse_abi::stat64; 19 + use fuse_backend_rs::api::filesystem::{ 20 + Context, DirEntry, Entry, FileSystem, GetxattrReply, ListxattrReply, OpenOptions, 21 + ZeroCopyWriter, 22 + }; 23 + 24 + use crate::closure::{par_map, PathInfo}; 25 + use crate::http::{self, Http}; 26 + use crate::nar::{self, CountingReader, Tree}; 27 + use crate::stats::Stats; 28 + use crate::Backing; 29 + 30 + /// The store paths never change: let the kernel cache everything. 31 + pub const TTL: Duration = Duration::from_secs(86400); 32 + 33 + /// Set on the inode numbers (st_ino) of this layer, so they don't collide 34 + /// with the upper layer's. 35 + pub const INO_TAG: u64 = 1 << 62; 36 + 37 + enum Kind { 38 + /// children sorted by name 39 + Dir(Vec<(Box<str>, u32)>), 40 + File { 41 + size: u64, 42 + exec: bool, 43 + }, 44 + Symlink(Box<[u8]>), 45 + } 46 + 47 + struct Node { 48 + kind: Kind, 49 + /// index into `LazyFs::names` 50 + path: u32, 51 + /// path below the store path (empty for its root) 52 + rel: Box<str>, 53 + } 54 + 55 + struct Handle { 56 + file: File, 57 + ino: u64, 58 + } 59 + 60 + pub struct LazyFs { 61 + names: Vec<String>, 62 + infos: Vec<PathInfo>, 63 + /// inode `i` is `nodes[i - 1]`; inode 1 is the store dir 64 + nodes: Vec<Node>, 65 + cache: PathBuf, 66 + http: Http, 67 + pub stats: Arc<Stats>, 68 + /// per store path: where it is unpacked, once fetched 69 + fetched: Vec<Mutex<Option<PathBuf>>>, 70 + handles: RwLock<HashMap<u64, Arc<Handle>>>, 71 + next_handle: AtomicU64, 72 + backing: Option<Arc<Backing>>, 73 + passthrough: AtomicBool, 74 + /// backing id and open count per inode: the kernel wants all 75 + /// passthrough opens of an inode to use the same backing file 76 + backing_ids: Mutex<HashMap<u64, (u32, usize)>>, 77 + backing_failed: AtomicBool, 78 + } 79 + 80 + fn err(e: i32) -> io::Error { 81 + io::Error::from_raw_os_error(e) 82 + } 83 + 84 + impl LazyFs { 85 + pub fn new(outdir: &Path, backing: Option<Arc<Backing>>) -> io::Result<LazyFs> { 86 + let paths: BTreeMap<String, PathInfo> = 87 + serde_json::from_slice(&std::fs::read(outdir.join("paths.json"))?)?; 88 + let (names, infos): (Vec<String>, Vec<PathInfo>) = paths.into_iter().unzip(); 89 + let cache = outdir.join("cache"); 90 + std::fs::create_dir_all(&cache)?; 91 + let http = Http::new(infos.iter().map(|i| i.cache.as_str())); 92 + let mut fs = LazyFs { 93 + fetched: names.iter().map(|_| Mutex::new(None)).collect(), 94 + names, 95 + infos, 96 + nodes: Vec::new(), 97 + cache, 98 + http, 99 + stats: Arc::new(Stats::new(Some(outdir.join("stats.json")))), 100 + handles: RwLock::new(HashMap::new()), 101 + next_handle: AtomicU64::new(1), 102 + backing, 103 + passthrough: AtomicBool::new(false), 104 + backing_ids: Mutex::new(HashMap::new()), 105 + backing_failed: AtomicBool::new(false), 106 + }; 107 + fs.load_listings()?; 108 + fs.stats.save(); 109 + Ok(fs) 110 + } 111 + 112 + /// Fetch the `.ls` listing of every store path (falling back to the NAR 113 + /// for paths without one) and build the inode table from them. Fails if 114 + /// any path is left without a listing: all of them get registered as 115 + /// valid in nix's database. 116 + fn load_listings(&mut self) -> io::Result<()> { 117 + let t = Instant::now(); 118 + let idx: Vec<usize> = (0..self.names.len()).collect(); 119 + let done = AtomicUsize::new(0); 120 + let trees = par_map(&idx, 64, |&i| { 121 + let r = self.listing(i); 122 + let n = done.fetch_add(1, Ordering::Relaxed) + 1; 123 + if n.is_multiple_of(500) { 124 + eprintln!("listings: {n}/{}", idx.len()); 125 + } 126 + r 127 + }); 128 + let mut root = Vec::with_capacity(trees.len()); 129 + self.nodes.push(Node { 130 + kind: Kind::Dir(Vec::new()), 131 + path: u32::MAX, 132 + rel: "".into(), 133 + }); 134 + let mut failed = 0; 135 + for (i, t) in trees.into_iter().enumerate() { 136 + match t { 137 + Ok(t) => { 138 + let ino = self.add(i as u32, String::new(), t); 139 + root.push((self.names[i].clone().into_boxed_str(), ino)); 140 + } 141 + Err(e) => { 142 + eprintln!("no listing for {}: {e}", self.names[i]); 143 + failed += 1; 144 + } 145 + } 146 + } 147 + if failed > 0 { 148 + return Err(io::Error::other(format!( 149 + "{failed} of {} store paths without a listing", 150 + self.names.len() 151 + ))); 152 + } 153 + root.sort(); 154 + self.nodes[0].kind = Kind::Dir(root); 155 + eprintln!( 156 + "listings: {} store paths, {} inodes in {:.1} s", 157 + self.names.len(), 158 + self.nodes.len(), 159 + t.elapsed().as_secs_f64(), 160 + ); 161 + Ok(()) 162 + } 163 + 164 + fn listing(&self, i: usize) -> io::Result<Tree> { 165 + let info = &self.infos[i]; 166 + let h = self.names[i].split('-').next().unwrap_or(""); 167 + match self.http.get(&format!("{}/{h}.ls", info.cache)) { 168 + Ok((body, len)) => { 169 + self.stats.add("ls", len as u64); 170 + Tree::from_listing(&body) 171 + } 172 + // no listing: fall back to the NAR 173 + Err(http::Error::NotFound) => Tree::from_path(&self.fetch(i)?), 174 + Err(e) => Err(e.into()), 175 + } 176 + } 177 + 178 + /// Add the nodes of `t` (at `rel` in store path `path`), return its inode. 179 + fn add(&mut self, path: u32, rel: String, t: Tree) -> u32 { 180 + let ino = self.nodes.len() as u32 + 1; 181 + let (kind, entries) = match t { 182 + Tree::File { size, exec } => (Kind::File { size, exec }, None), 183 + Tree::Symlink(s) => (Kind::Symlink(s.into_bytes().into()), None), 184 + Tree::Dir(e) => (Kind::Dir(Vec::new()), Some(e)), 185 + }; 186 + self.nodes.push(Node { 187 + kind, 188 + path, 189 + rel: rel.clone().into(), 190 + }); 191 + if let Some(entries) = entries { 192 + let mut children = Vec::with_capacity(entries.len()); 193 + for (name, c) in entries { 194 + let r = if rel.is_empty() { 195 + name.clone() 196 + } else { 197 + format!("{rel}/{name}") 198 + }; 199 + children.push((name.into_boxed_str(), self.add(path, r, c))); 200 + } 201 + // BTreeMap order is sorted already 202 + self.nodes[ino as usize - 1].kind = Kind::Dir(children); 203 + } 204 + ino 205 + } 206 + 207 + /// The listing of `ino` as built from the `.ls` (for `fetch`). 208 + fn tree_of(&self, ino: u64) -> Tree { 209 + match &self.nodes[ino as usize - 1].kind { 210 + Kind::Dir(c) => Tree::Dir( 211 + c.iter() 212 + .map(|(n, i)| (n.to_string(), self.tree_of(*i as u64))) 213 + .collect(), 214 + ), 215 + Kind::File { size, exec } => Tree::File { 216 + size: *size, 217 + exec: *exec, 218 + }, 219 + Kind::Symlink(t) => Tree::Symlink(String::from_utf8_lossy(t).into_owned()), 220 + } 221 + } 222 + 223 + /// Fetch store path `name` as an open() would and check that what got 224 + /// unpacked matches its listing (`lazy-store fetch`, no FUSE needed). 225 + pub fn fetch_and_check(&self, name: &str) -> io::Result<PathBuf> { 226 + let c = CString::new(name).map_err(io::Error::other)?; 227 + let ino = self.lookup_child(1, &c).ok_or_else(|| err(libc::ENOENT))?; 228 + let p = self.fetch(self.nodes[ino as usize - 1].path as usize)?; 229 + if Tree::from_path(&p)? != self.tree_of(ino) { 230 + return Err(io::Error::other(format!( 231 + "{name}: unpacked tree differs from its listing" 232 + ))); 233 + } 234 + Ok(p) 235 + } 236 + 237 + pub fn set_passthrough(&self, on: bool) { 238 + self.passthrough 239 + .store(on && self.backing.is_some(), Ordering::Relaxed); 240 + } 241 + 242 + fn node(&self, ino: u64) -> io::Result<&Node> { 243 + (ino as usize) 244 + .checked_sub(1) 245 + .and_then(|i| self.nodes.get(i)) 246 + .ok_or_else(|| err(libc::ENOENT)) 247 + } 248 + 249 + pub fn has_root_entry(&self, name: &CStr) -> bool { 250 + self.lookup_child(1, name).is_some() 251 + } 252 + 253 + pub fn root_entries(&self) -> impl Iterator<Item = (&str, u64, u32)> { 254 + let Kind::Dir(c) = &self.nodes[0].kind else { 255 + unreachable!() 256 + }; 257 + c.iter() 258 + .map(|(n, i)| (&**n, *i as u64, self.dtype(*i as u64))) 259 + } 260 + 261 + fn lookup_child(&self, parent: u64, name: &CStr) -> Option<u64> { 262 + let Kind::Dir(c) = &self.node(parent).ok()?.kind else { 263 + return None; 264 + }; 265 + let name = name.to_str().ok()?; 266 + c.binary_search_by(|(n, _)| (**n).cmp(name)) 267 + .ok() 268 + .map(|i| c[i].1 as u64) 269 + } 270 + 271 + fn dtype(&self, ino: u64) -> u32 { 272 + match self.nodes[ino as usize - 1].kind { 273 + Kind::Dir(_) => libc::DT_DIR as u32, 274 + Kind::File { .. } => libc::DT_REG as u32, 275 + Kind::Symlink(_) => libc::DT_LNK as u32, 276 + } 277 + } 278 + 279 + fn stat(&self, ino: u64) -> io::Result<stat64> { 280 + let n = self.node(ino)?; 281 + // Safe: stat64 is plain old data. 282 + let mut st: stat64 = unsafe { std::mem::zeroed() }; 283 + st.st_ino = ino | INO_TAG; 284 + st.st_blksize = 4096; 285 + st.st_atime = 1; 286 + st.st_mtime = 1; 287 + st.st_ctime = 1; 288 + match &n.kind { 289 + Kind::Dir(_) => { 290 + st.st_mode = libc::S_IFDIR | 0o555; 291 + // 1: "subdirectory count unknown", which keeps find(1) 292 + // from skipping entries based on it 293 + st.st_nlink = 1; 294 + } 295 + Kind::File { size, exec } => { 296 + st.st_mode = libc::S_IFREG | if *exec { 0o555 } else { 0o444 }; 297 + st.st_nlink = 1; 298 + st.st_size = *size as i64; 299 + st.st_blocks = size.div_ceil(512) as i64; 300 + } 301 + Kind::Symlink(t) => { 302 + st.st_mode = libc::S_IFLNK | 0o777; 303 + st.st_nlink = 1; 304 + st.st_size = t.len() as i64; 305 + } 306 + } 307 + Ok(st) 308 + } 309 + 310 + /// The unpacked store path `i`, fetching it first if needed. Concurrent 311 + /// callers for the same path wait for one fetch. 312 + fn fetch(&self, i: usize) -> io::Result<PathBuf> { 313 + let mut slot = self.fetched[i].lock().unwrap(); 314 + if let Some(p) = &*slot { 315 + return Ok(p.clone()); 316 + } 317 + let name = &self.names[i]; 318 + let dest = self.cache.join(name); 319 + let mut last = None; 320 + for attempt in 1..=3 { 321 + match self.fetch_once(i, &dest) { 322 + Ok(()) => { 323 + *slot = Some(dest.clone()); 324 + return Ok(dest); 325 + } 326 + Err(e) => { 327 + eprintln!("fetching {name} (attempt {attempt}/3): {e}"); 328 + last = Some(e); 329 + } 330 + } 331 + } 332 + Err(last.unwrap()) 333 + } 334 + 335 + fn fetch_once(&self, i: usize, dest: &Path) -> io::Result<()> { 336 + let (name, info) = (&self.names[i], &self.infos[i]); 337 + let t = Instant::now(); 338 + let tmp = dest.with_extension("tmp"); 339 + nar::remove(&tmp)?; 340 + nar::remove(dest)?; 341 + let (body, _) = self.http.open(&format!("{}/{}", info.cache, info.url))?; 342 + let mut counted = CountingReader { inner: body, n: 0 }; 343 + let (digest, size) = nar::unpack(http::decoder(&info.compression, &mut counted)?, &tmp)?; 344 + if !nar::hash_matches(&info.nar_hash, &digest) || size != info.nar_size { 345 + nar::remove(&tmp)?; 346 + return Err(io::Error::other(format!( 347 + "{name}: NAR hash mismatch, sha256:{} ({size} bytes) != {} ({} bytes)", 348 + nar::nix32(&digest), 349 + info.nar_hash, 350 + info.nar_size 351 + ))); 352 + } 353 + std::fs::rename(&tmp, dest)?; 354 + self.stats.nar(name, counted.n, size); 355 + eprintln!( 356 + "fetched {name}: {} bytes in {:.1} s", 357 + counted.n, 358 + t.elapsed().as_secs_f64() 359 + ); 360 + Ok(()) 361 + } 362 + 363 + fn entry(&self, ino: u64) -> io::Result<Entry> { 364 + Ok(Entry { 365 + inode: ino, 366 + generation: 0, 367 + attr: self.stat(ino)?, 368 + attr_flags: 0, 369 + attr_timeout: TTL, 370 + entry_timeout: TTL, 371 + }) 372 + } 373 + 374 + fn backing_id(&self, ino: u64, file: &File) -> Option<u32> { 375 + if !self.passthrough.load(Ordering::Relaxed) { 376 + return None; 377 + } 378 + let backing = self.backing.as_ref()?; 379 + let mut ids = self.backing_ids.lock().unwrap(); 380 + if let Some((id, n)) = ids.get_mut(&ino) { 381 + *n += 1; 382 + return Some(*id); 383 + } 384 + match backing.open_fd(file.as_raw_fd()) { 385 + Ok(id) => { 386 + ids.insert(ino, (id, 1)); 387 + Some(id) 388 + } 389 + Err(e) => { 390 + if !self.backing_failed.swap(true, Ordering::Relaxed) { 391 + eprintln!( 392 + "lower: FUSE passthrough backing open failed, reading via the daemon: {e}" 393 + ); 394 + } 395 + None 396 + } 397 + } 398 + } 399 + 400 + fn release_backing(&self, ino: u64) { 401 + let mut ids = self.backing_ids.lock().unwrap(); 402 + if let Some((id, n)) = ids.get_mut(&ino) { 403 + *n -= 1; 404 + if *n == 0 { 405 + let id = *id; 406 + ids.remove(&ino); 407 + drop(ids); 408 + if let Some(b) = &self.backing { 409 + b.close_id(id); 410 + } 411 + } 412 + } 413 + } 414 + } 415 + 416 + impl FileSystem for LazyFs { 417 + type Inode = u64; 418 + type Handle = u64; 419 + 420 + fn lookup(&self, _ctx: &Context, parent: u64, name: &CStr) -> io::Result<Entry> { 421 + match self.lookup_child(parent, name) { 422 + Some(ino) => self.entry(ino), 423 + None => Err(err(libc::ENOENT)), 424 + } 425 + } 426 + 427 + fn getattr(&self, _ctx: &Context, ino: u64, _h: Option<u64>) -> io::Result<(stat64, Duration)> { 428 + Ok((self.stat(ino)?, TTL)) 429 + } 430 + 431 + fn readlink(&self, _ctx: &Context, ino: u64) -> io::Result<Vec<u8>> { 432 + match &self.node(ino)?.kind { 433 + Kind::Symlink(t) => Ok(t.to_vec()), 434 + _ => Err(err(libc::EINVAL)), 435 + } 436 + } 437 + 438 + fn open( 439 + &self, 440 + _ctx: &Context, 441 + ino: u64, 442 + flags: u32, 443 + _fuse_flags: u32, 444 + ) -> io::Result<(Option<u64>, OpenOptions, Option<u32>)> { 445 + let n = self.node(ino)?; 446 + if !matches!(n.kind, Kind::File { .. }) { 447 + return Err(err(libc::EISDIR)); 448 + } 449 + let acc = flags as i32 & libc::O_ACCMODE; 450 + if acc != libc::O_RDONLY || flags as i32 & libc::O_TRUNC != 0 { 451 + return Err(err(libc::EROFS)); 452 + } 453 + let root = self.fetch(n.path as usize).map_err(|e| { 454 + eprintln!("open {}/{}: {e}", self.names[n.path as usize], n.rel); 455 + err(libc::EIO) 456 + })?; 457 + let file = File::open(if n.rel.is_empty() { 458 + root 459 + } else { 460 + root.join(&*n.rel) 461 + })?; 462 + let backing = self.backing_id(ino, &file); 463 + let h = self.next_handle.fetch_add(1, Ordering::Relaxed); 464 + self.handles 465 + .write() 466 + .unwrap() 467 + .insert(h, Arc::new(Handle { file, ino })); 468 + Ok(match backing { 469 + Some(id) => (Some(h), OpenOptions::PASSTHROUGH, Some(id)), 470 + None => (Some(h), OpenOptions::KEEP_CACHE, None), 471 + }) 472 + } 473 + 474 + fn read( 475 + &self, 476 + _ctx: &Context, 477 + _ino: u64, 478 + handle: u64, 479 + w: &mut dyn ZeroCopyWriter, 480 + size: u32, 481 + offset: u64, 482 + _lock_owner: Option<u64>, 483 + _flags: u32, 484 + ) -> io::Result<usize> { 485 + let h = self 486 + .handles 487 + .read() 488 + .unwrap() 489 + .get(&handle) 490 + .cloned() 491 + .ok_or_else(|| err(libc::EBADF))?; 492 + // Safe: borrows the fd of `h.file`, which outlives `f`, without closing it. 493 + let mut f = ManuallyDrop::new(unsafe { File::from_raw_fd(h.file.as_raw_fd()) }); 494 + w.write_from(&mut *f, size as usize, offset) 495 + } 496 + 497 + fn lseek( 498 + &self, 499 + _ctx: &Context, 500 + _ino: u64, 501 + handle: u64, 502 + offset: u64, 503 + whence: u32, 504 + ) -> io::Result<u64> { 505 + let h = self 506 + .handles 507 + .read() 508 + .unwrap() 509 + .get(&handle) 510 + .cloned() 511 + .ok_or_else(|| err(libc::EBADF))?; 512 + // Safe: plain syscall on an fd we own. 513 + let r = unsafe { libc::lseek(h.file.as_raw_fd(), offset as i64, whence as i32) }; 514 + if r < 0 { 515 + return Err(io::Error::last_os_error()); 516 + } 517 + Ok(r as u64) 518 + } 519 + 520 + fn flush(&self, _ctx: &Context, _ino: u64, _handle: u64, _lock_owner: u64) -> io::Result<()> { 521 + Ok(()) 522 + } 523 + 524 + fn fsync(&self, _ctx: &Context, _ino: u64, _datasync: bool, _handle: u64) -> io::Result<()> { 525 + Ok(()) 526 + } 527 + 528 + fn release( 529 + &self, 530 + _ctx: &Context, 531 + _ino: u64, 532 + _flags: u32, 533 + handle: u64, 534 + _flush: bool, 535 + _flock_release: bool, 536 + _lock_owner: Option<u64>, 537 + ) -> io::Result<()> { 538 + if let Some(h) = self.handles.write().unwrap().remove(&handle) { 539 + self.release_backing(h.ino); 540 + } 541 + Ok(()) 542 + } 543 + 544 + fn opendir( 545 + &self, 546 + _ctx: &Context, 547 + ino: u64, 548 + _flags: u32, 549 + ) -> io::Result<(Option<u64>, OpenOptions)> { 550 + match self.node(ino)?.kind { 551 + Kind::Dir(_) => Ok((None, OpenOptions::KEEP_CACHE | OpenOptions::CACHE_DIR)), 552 + _ => Err(err(libc::ENOTDIR)), 553 + } 554 + } 555 + 556 + fn readdir( 557 + &self, 558 + _ctx: &Context, 559 + ino: u64, 560 + _handle: u64, 561 + _size: u32, 562 + offset: u64, 563 + add_entry: &mut dyn FnMut(DirEntry) -> io::Result<usize>, 564 + ) -> io::Result<()> { 565 + let Kind::Dir(c) = &self.node(ino)?.kind else { 566 + return Err(err(libc::ENOTDIR)); 567 + }; 568 + for (i, (name, child)) in c.iter().enumerate().skip(offset as usize) { 569 + let child = *child as u64; 570 + let n = add_entry(DirEntry { 571 + ino: child | INO_TAG, 572 + offset: i as u64 + 1, 573 + type_: self.dtype(child), 574 + name: name.as_bytes(), 575 + })?; 576 + if n == 0 { 577 + break; 578 + } 579 + } 580 + Ok(()) 581 + } 582 + 583 + fn releasedir(&self, _ctx: &Context, _ino: u64, _flags: u32, _handle: u64) -> io::Result<()> { 584 + Ok(()) 585 + } 586 + 587 + fn getxattr( 588 + &self, 589 + _ctx: &Context, 590 + _ino: u64, 591 + _name: &CStr, 592 + _size: u32, 593 + ) -> io::Result<GetxattrReply> { 594 + Err(err(libc::ENODATA)) 595 + } 596 + 597 + fn listxattr(&self, _ctx: &Context, _ino: u64, size: u32) -> io::Result<ListxattrReply> { 598 + Ok(if size == 0 { 599 + ListxattrReply::Count(0) 600 + } else { 601 + ListxattrReply::Names(Vec::new()) 602 + }) 603 + } 604 + 605 + fn access(&self, _ctx: &Context, _ino: u64, mask: u32) -> io::Result<()> { 606 + if mask as i32 & libc::W_OK != 0 { 607 + return Err(err(libc::EROFS)); 608 + } 609 + Ok(()) 610 + } 611 + }
+238
tools/lazy-store/src/main.rs
··· 1 + //! Lazy nix store for CI's machine builds (.github/lazy-store.sh). 2 + //! 3 + //! closure OUT collect the closure of the store paths on stdin 4 + //! (one per line) from the caches by fetching only 5 + //! narinfos. Writes OUT/paths.json (per path: 6 + //! cache, NAR url, compression, hash), OUT/db.dump 7 + //! (input for `nix-store --load-db`), OUT/narinfo 8 + //! (their narinfos as a binary cache, for `nix 9 + //! store copy-sigs`) and OUT/closure.json. 10 + //! fetch OUT NAME... fetch store paths of OUT/paths.json like the 11 + //! first open() would, and check them against 12 + //! their listing (testing without FUSE) 13 + //! mount OUT UPPER MNT mount the store dir UPPER, merged with the paths 14 + //! of OUT/paths.json, at MNT (see storefs.rs). 15 + //! Stats go to OUT/stats.json. 16 + //! [--workers N] FUSE worker threads (default 64) 17 + //! [--no-passthrough] don't use kernel FUSE passthrough 18 + 19 + mod closure; 20 + mod http; 21 + mod lazy; 22 + mod nar; 23 + mod stats; 24 + mod storefs; 25 + 26 + use std::fs::File; 27 + use std::io; 28 + use std::os::fd::{AsRawFd, BorrowedFd, RawFd}; 29 + use std::path::{Path, PathBuf}; 30 + use std::sync::{Arc, OnceLock}; 31 + 32 + use fuse_backend_rs::api::server::Server; 33 + use fuse_backend_rs::passthrough::{self, BackingFiles, PassthroughFs}; 34 + use fuse_backend_rs::transport::{ 35 + fuse_backing_close, fuse_backing_open, FuseServing, FuseSession, SyncFuseServing, 36 + SyncServingConfig, 37 + }; 38 + 39 + pub const STORE: &str = "/nix/store/"; 40 + 41 + /// Registers backing files for kernel FUSE passthrough on the session's 42 + /// /dev/fuse fd, which only exists once mounted. 43 + #[derive(Default)] 44 + pub struct Backing(OnceLock<File>); 45 + 46 + impl Backing { 47 + pub fn open_fd(&self, fd: RawFd) -> io::Result<u32> { 48 + let f = self 49 + .0 50 + .get() 51 + .ok_or_else(|| io::Error::from_raw_os_error(libc::ENODEV))?; 52 + fuse_backing_open(f, fd) 53 + } 54 + 55 + pub fn close_id(&self, id: u32) { 56 + if let Some(f) = self.0.get() { 57 + if let Err(e) = fuse_backing_close(f, id) { 58 + eprintln!("closing backing id {id}: {e}"); 59 + } 60 + } 61 + } 62 + } 63 + 64 + struct SharedBacking(Arc<Backing>); 65 + 66 + impl BackingFiles for SharedBacking { 67 + fn open(&self, fd: BorrowedFd<'_>) -> io::Result<u32> { 68 + self.0.open_fd(fd.as_raw_fd()) 69 + } 70 + 71 + fn close(&self, id: u32) -> io::Result<()> { 72 + let f = self 73 + .0 74 + .0 75 + .get() 76 + .ok_or_else(|| io::Error::from_raw_os_error(libc::ENODEV))?; 77 + fuse_backing_close(f, id) 78 + } 79 + } 80 + 81 + /// fuse-backend-rs logs through `log`: warnings and errors to stderr. 82 + struct Logger; 83 + 84 + impl log::Log for Logger { 85 + fn enabled(&self, m: &log::Metadata) -> bool { 86 + m.level() <= log::Level::Warn 87 + } 88 + 89 + fn log(&self, r: &log::Record) { 90 + if self.enabled(r.metadata()) { 91 + eprintln!("{} {}: {}", r.level(), r.target(), r.args()); 92 + } 93 + } 94 + 95 + fn flush(&self) {} 96 + } 97 + 98 + /// PassthroughFs keeps an O_PATH fd per inode the kernel looked up and 99 + /// hasn't forgotten, so the default soft limit of 1024 runs out within the 100 + /// first build. Raise the limit as far as the kernel allows (we run as 101 + /// root, so the hard limit too, up to fs.nr_open). 102 + fn raise_nofile() { 103 + let nr_open = std::fs::read_to_string("/proc/sys/fs/nr_open") 104 + .ok() 105 + .and_then(|s| s.trim().parse::<libc::rlim_t>().ok()) 106 + .unwrap_or(1 << 20); 107 + let mut rl = libc::rlimit { 108 + rlim_cur: 0, 109 + rlim_max: 0, 110 + }; 111 + unsafe { 112 + libc::getrlimit(libc::RLIMIT_NOFILE, &mut rl); 113 + let want = libc::rlimit { 114 + rlim_cur: nr_open, 115 + rlim_max: nr_open.max(rl.rlim_max), 116 + }; 117 + if libc::setrlimit(libc::RLIMIT_NOFILE, &want) != 0 { 118 + let soft = libc::rlimit { 119 + rlim_cur: rl.rlim_max, 120 + rlim_max: rl.rlim_max, 121 + }; 122 + libc::setrlimit(libc::RLIMIT_NOFILE, &soft); 123 + } 124 + libc::getrlimit(libc::RLIMIT_NOFILE, &mut rl); 125 + } 126 + eprintln!("RLIMIT_NOFILE: {} (hard {})", rl.rlim_cur, rl.rlim_max); 127 + } 128 + 129 + fn mount( 130 + outdir: &Path, 131 + upper: &Path, 132 + mnt: &Path, 133 + workers: usize, 134 + use_passthrough: bool, 135 + ) -> io::Result<()> { 136 + raise_nofile(); 137 + let backing = use_passthrough.then(|| Arc::new(Backing::default())); 138 + let lower = lazy::LazyFs::new(outdir, backing.clone())?; 139 + 140 + let upper_fs = { 141 + let cfg = passthrough::Config { 142 + root_dir: upper 143 + .to_str() 144 + .ok_or_else(|| io::Error::other("non-UTF-8 upper dir"))? 145 + .into(), 146 + do_import: true, 147 + xattr: false, 148 + ..Default::default() 149 + }; 150 + let mut fs = PassthroughFs::<()>::new(cfg)?; 151 + if let Some(b) = &backing { 152 + fs.set_backing_files(Arc::new(SharedBacking(b.clone()))); 153 + } 154 + fs 155 + }; 156 + let stats = lower.stats.clone(); 157 + let fs = storefs::StoreFs::new(upper_fs, upper.to_path_buf(), lower); 158 + 159 + let mut se = FuseSession::new(mnt, "lazystore", "", false).map_err(io::Error::other)?; 160 + se.mount().map_err(io::Error::other)?; 161 + if let Some(b) = &backing { 162 + let f = se 163 + .get_fuse_file() 164 + .ok_or_else(|| io::Error::other("no fuse fd"))? 165 + .try_clone()?; 166 + let _ = b.0.set(f); 167 + } 168 + let server = Arc::new(Server::new(fs)); 169 + let serving = SyncFuseServing::new( 170 + se, 171 + server, 172 + SyncServingConfig { 173 + workers, 174 + blocking: true, 175 + }, 176 + ) 177 + .map_err(io::Error::other)?; 178 + eprintln!( 179 + "mounted {} on {} ({workers} workers)", 180 + upper.display(), 181 + mnt.display() 182 + ); 183 + serving.wait(); 184 + stats.save(); 185 + eprintln!("FUSE main loop exited"); 186 + Ok(()) 187 + } 188 + 189 + fn usage() -> ! { 190 + eprintln!( 191 + "usage: lazy-store closure OUT <PATHS | fetch OUT NAME... | mount OUT UPPER MNT [--workers N] [--no-passthrough]" 192 + ); 193 + std::process::exit(2) 194 + } 195 + 196 + fn main() { 197 + let _ = log::set_logger(&Logger).map(|_| log::set_max_level(log::LevelFilter::Warn)); 198 + let args: Vec<String> = std::env::args().skip(1).collect(); 199 + let r = match args.first().map(String::as_str) { 200 + Some("closure") if args.len() == 2 => closure::closure(Path::new(&args[1])), 201 + Some("fetch") if args.len() >= 3 => { 202 + lazy::LazyFs::new(Path::new(&args[1]), None).and_then(|fs| { 203 + for name in &args[2..] { 204 + println!("{}", fs.fetch_and_check(name)?.display()); 205 + } 206 + Ok(()) 207 + }) 208 + } 209 + Some("mount") if args.len() >= 4 => { 210 + let (mut workers, mut pt) = (64, true); 211 + let mut rest = args[4..].iter(); 212 + while let Some(a) = rest.next() { 213 + match a.as_str() { 214 + "--workers" => { 215 + workers = rest 216 + .next() 217 + .and_then(|n| n.parse().ok()) 218 + .unwrap_or_else(|| usage()) 219 + } 220 + "--no-passthrough" => pt = false, 221 + _ => usage(), 222 + } 223 + } 224 + mount( 225 + &PathBuf::from(&args[1]), 226 + Path::new(&args[2]), 227 + Path::new(&args[3]), 228 + workers, 229 + pt, 230 + ) 231 + } 232 + _ => usage(), 233 + }; 234 + if let Err(e) = r { 235 + eprintln!("error: {e}"); 236 + std::process::exit(1); 237 + } 238 + }
+371
tools/lazy-store/src/nar.rs
··· 1 + //! NAR unpacking and store path listings. 2 + 3 + use std::collections::BTreeMap; 4 + use std::fs; 5 + use std::io::{self, Read, Write}; 6 + use std::os::unix::ffi::OsStrExt; 7 + use std::os::unix::fs::{symlink, MetadataExt, PermissionsExt}; 8 + use std::path::Path; 9 + 10 + use sha2::{Digest, Sha256}; 11 + 12 + const NIX32: &[u8] = b"0123456789abcdfghijklmnpqrsvwxyz"; 13 + 14 + pub fn nix32(b: &[u8]) -> String { 15 + let n = (b.len() * 8 - 1) / 5 + 1; 16 + (0..n) 17 + .rev() 18 + .map(|i| { 19 + let bit = i * 5; 20 + let (j, k) = (bit / 8, bit % 8); 21 + let mut c = (b[j] as u32) >> k; 22 + if j + 1 < b.len() { 23 + c |= (b[j + 1] as u32) << (8 - k); 24 + } 25 + NIX32[(c & 0x1f) as usize] as char 26 + }) 27 + .collect() 28 + } 29 + 30 + /// Does `digest` match a narinfo NarHash (`sha256:<nix32>` or base16)? 31 + pub fn hash_matches(expected: &str, digest: &[u8]) -> bool { 32 + let Some(h) = expected.strip_prefix("sha256:") else { 33 + return false; 34 + }; 35 + let hex: String = digest.iter().map(|b| format!("{b:02x}")).collect(); 36 + h == nix32(digest) || h == hex 37 + } 38 + 39 + /// The metadata of a store path, as in the caches' `.ls` listings. 40 + #[derive(Debug, Clone, PartialEq)] 41 + pub enum Tree { 42 + Dir(BTreeMap<String, Tree>), 43 + File { size: u64, exec: bool }, 44 + Symlink(String), 45 + } 46 + 47 + impl Tree { 48 + /// Parse a `.ls` listing (`{"version": 1, "root": {...}}`). 49 + pub fn from_listing(body: &[u8]) -> io::Result<Tree> { 50 + let v: serde_json::Value = serde_json::from_slice(body).map_err(io::Error::other)?; 51 + Tree::from_json(&v["root"]) 52 + } 53 + 54 + fn from_json(v: &serde_json::Value) -> io::Result<Tree> { 55 + let bad = || io::Error::other(format!("bad listing node {v}")); 56 + Ok(match v["type"].as_str().ok_or_else(bad)? { 57 + "directory" => { 58 + let mut m = BTreeMap::new(); 59 + if let Some(e) = v["entries"].as_object() { 60 + for (k, c) in e { 61 + m.insert(k.clone(), Tree::from_json(c)?); 62 + } 63 + } 64 + Tree::Dir(m) 65 + } 66 + "regular" => Tree::File { 67 + size: v["size"].as_u64().unwrap_or(0), 68 + exec: v["executable"].as_bool().unwrap_or(false), 69 + }, 70 + "symlink" => Tree::Symlink(v["target"].as_str().ok_or_else(bad)?.to_string()), 71 + _ => return Err(bad()), 72 + }) 73 + } 74 + 75 + /// Listing of an unpacked store path (for paths without `.ls`). 76 + pub fn from_path(p: &Path) -> io::Result<Tree> { 77 + let m = fs::symlink_metadata(p)?; 78 + let t = m.file_type(); 79 + Ok(if t.is_dir() { 80 + let mut e = BTreeMap::new(); 81 + for d in fs::read_dir(p)? { 82 + let d = d?; 83 + let name = d.file_name().into_string().map_err(|n| { 84 + io::Error::other(format!("non-UTF-8 name {n:?} in {}", p.display())) 85 + })?; 86 + e.insert(name, Tree::from_path(&d.path())?); 87 + } 88 + Tree::Dir(e) 89 + } else if t.is_symlink() { 90 + Tree::Symlink(fs::read_link(p)?.to_string_lossy().into_owned()) 91 + } else { 92 + Tree::File { 93 + size: m.size(), 94 + exec: m.mode() & 0o100 != 0, 95 + } 96 + }) 97 + } 98 + } 99 + 100 + /// Reads through to `inner`, hashing and counting what passes. 101 + pub struct HashingReader<R> { 102 + inner: R, 103 + pub hasher: Sha256, 104 + pub n: u64, 105 + } 106 + 107 + impl<R: Read> Read for HashingReader<R> { 108 + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { 109 + let n = self.inner.read(buf)?; 110 + self.hasher.update(&buf[..n]); 111 + self.n += n as u64; 112 + Ok(n) 113 + } 114 + } 115 + 116 + /// Counts the bytes read from `inner` (the compressed download size). 117 + pub struct CountingReader<R> { 118 + pub inner: R, 119 + pub n: u64, 120 + } 121 + 122 + impl<R: Read> Read for CountingReader<R> { 123 + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { 124 + let n = self.inner.read(buf)?; 125 + self.n += n as u64; 126 + Ok(n) 127 + } 128 + } 129 + 130 + fn bad(msg: impl Into<String>) -> io::Error { 131 + io::Error::new(io::ErrorKind::InvalidData, format!("NAR: {}", msg.into())) 132 + } 133 + 134 + struct Nar<R> { 135 + r: HashingReader<R>, 136 + } 137 + 138 + impl<R: Read> Nar<R> { 139 + fn int(&mut self) -> io::Result<u64> { 140 + let mut b = [0u8; 8]; 141 + self.r.read_exact(&mut b)?; 142 + Ok(u64::from_le_bytes(b)) 143 + } 144 + 145 + fn pad(&mut self, n: u64) -> io::Result<()> { 146 + let p = ((8 - n % 8) % 8) as usize; 147 + let mut b = [0u8; 8]; 148 + self.r.read_exact(&mut b[..p])?; 149 + if b[..p].iter().any(|&x| x != 0) { 150 + return Err(bad("non-zero padding")); 151 + } 152 + Ok(()) 153 + } 154 + 155 + fn str(&mut self, limit: u64) -> io::Result<Vec<u8>> { 156 + let n = self.int()?; 157 + if n > limit { 158 + return Err(bad("string too long")); 159 + } 160 + let mut s = vec![0u8; n as usize]; 161 + self.r.read_exact(&mut s)?; 162 + self.pad(n)?; 163 + Ok(s) 164 + } 165 + 166 + fn expect(&mut self, s: &[u8]) -> io::Result<()> { 167 + let got = self.str(4096)?; 168 + if got != s { 169 + return Err(bad(format!( 170 + "expected {:?}, got {:?}", 171 + String::from_utf8_lossy(s), 172 + String::from_utf8_lossy(&got) 173 + ))); 174 + } 175 + Ok(()) 176 + } 177 + 178 + fn node(&mut self, dest: &Path) -> io::Result<()> { 179 + self.expect(b"(")?; 180 + self.expect(b"type")?; 181 + match &self.str(4096)?[..] { 182 + b"regular" => { 183 + let mut tag = self.str(4096)?; 184 + let mut mode = 0o444; 185 + if tag == b"executable" { 186 + self.expect(b"")?; 187 + tag = self.str(4096)?; 188 + mode = 0o555; 189 + } 190 + if tag != b"contents" { 191 + return Err(bad("expected contents")); 192 + } 193 + let n = self.int()?; 194 + let mut out = io::BufWriter::with_capacity(1 << 20, fs::File::create(dest)?); 195 + let copied = io::copy(&mut (&mut self.r).take(n), &mut out)?; 196 + if copied != n { 197 + return Err(bad("short file contents")); 198 + } 199 + out.flush()?; 200 + self.pad(n)?; 201 + fs::set_permissions(dest, fs::Permissions::from_mode(mode))?; 202 + self.expect(b")")?; 203 + } 204 + b"symlink" => { 205 + self.expect(b"target")?; 206 + let t = self.str(4096)?; 207 + symlink(std::ffi::OsStr::from_bytes(&t), dest)?; 208 + self.expect(b")")?; 209 + } 210 + b"directory" => { 211 + fs::create_dir(dest)?; 212 + loop { 213 + let tag = self.str(4096)?; 214 + if tag == b")" { 215 + break; 216 + } 217 + if tag != b"entry" { 218 + return Err(bad("expected entry")); 219 + } 220 + self.expect(b"(")?; 221 + self.expect(b"name")?; 222 + let name = self.str(255)?; 223 + if name.is_empty() || name == b"." || name == b".." || name.contains(&b'/') { 224 + return Err(bad("bad entry name")); 225 + } 226 + self.expect(b"node")?; 227 + let name = String::from_utf8(name).map_err(|_| bad("non-UTF-8 name"))?; 228 + self.node(&dest.join(name))?; 229 + self.expect(b")")?; 230 + } 231 + fs::set_permissions(dest, fs::Permissions::from_mode(0o555))?; 232 + } 233 + t => { 234 + return Err(bad(format!( 235 + "unknown type {:?}", 236 + String::from_utf8_lossy(t) 237 + ))) 238 + } 239 + } 240 + Ok(()) 241 + } 242 + } 243 + 244 + /// Unpack the NAR read from `r` to `dest` (which must not exist). Returns 245 + /// the NAR's sha256 and size; checking them is up to the caller. Fails if 246 + /// anything follows the NAR. 247 + pub fn unpack(r: impl Read, dest: &Path) -> io::Result<(Vec<u8>, u64)> { 248 + let mut nar = Nar { 249 + r: HashingReader { 250 + inner: r, 251 + hasher: Sha256::new(), 252 + n: 0, 253 + }, 254 + }; 255 + nar.expect(b"nix-archive-1")?; 256 + nar.node(dest)?; 257 + let mut rest = [0u8; 1]; 258 + if nar.r.inner.read(&mut rest)? != 0 { 259 + return Err(bad("trailing data")); 260 + } 261 + let n = nar.r.n; 262 + Ok((nar.r.hasher.finalize().to_vec(), n)) 263 + } 264 + 265 + /// Remove a (partially) unpacked tree, including read-only directories. 266 + pub fn remove(p: &Path) -> io::Result<()> { 267 + let m = match fs::symlink_metadata(p) { 268 + Ok(m) => m, 269 + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), 270 + Err(e) => return Err(e), 271 + }; 272 + if m.is_dir() { 273 + fs::set_permissions(p, fs::Permissions::from_mode(0o755))?; 274 + for d in fs::read_dir(p)? { 275 + remove(&d?.path())?; 276 + } 277 + fs::remove_dir(p) 278 + } else { 279 + fs::remove_file(p) 280 + } 281 + } 282 + 283 + #[cfg(test)] 284 + mod tests { 285 + use super::*; 286 + 287 + fn s(out: &mut Vec<u8>, b: &[u8]) { 288 + out.extend_from_slice(&(b.len() as u64).to_le_bytes()); 289 + out.extend_from_slice(b); 290 + out.resize(out.len() + (8 - b.len() % 8) % 8, 0); 291 + } 292 + 293 + fn nar() -> Vec<u8> { 294 + let mut o = Vec::new(); 295 + for t in [&b"nix-archive-1"[..], b"(", b"type", b"directory"] { 296 + s(&mut o, t); 297 + } 298 + for t in [&b"entry"[..], b"(", b"name", b"bin", b"node", b"("] { 299 + s(&mut o, t); 300 + } 301 + for t in [ 302 + &b"type"[..], 303 + b"regular", 304 + b"executable", 305 + b"", 306 + b"contents", 307 + b"#!/bin/sh\n", 308 + ] { 309 + s(&mut o, t); 310 + } 311 + for t in [&b")"[..], b")"] { 312 + s(&mut o, t); 313 + } 314 + for t in [&b"entry"[..], b"(", b"name", b"link", b"node", b"("] { 315 + s(&mut o, t); 316 + } 317 + for t in [ 318 + &b"type"[..], 319 + b"symlink", 320 + b"target", 321 + b"bin", 322 + b")", 323 + b")", 324 + b")", 325 + ] { 326 + s(&mut o, t); 327 + } 328 + o 329 + } 330 + 331 + #[test] 332 + fn nix32_matches_nix() { 333 + // nix hash convert --to nix32 sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU= 334 + let d = Sha256::digest(b""); 335 + assert_eq!( 336 + nix32(&d), 337 + "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" 338 + ); 339 + } 340 + 341 + #[test] 342 + fn unpack_and_list() { 343 + let dir = std::env::temp_dir().join(format!("lazy-store-test-{}", std::process::id())); 344 + let _ = remove(&dir); 345 + let data = nar(); 346 + let (h, n) = unpack(&data[..], &dir).unwrap(); 347 + assert_eq!(n, data.len() as u64); 348 + assert_eq!(h, Sha256::digest(&data).to_vec()); 349 + let t = Tree::from_path(&dir).unwrap(); 350 + let mut e = BTreeMap::new(); 351 + e.insert( 352 + "bin".into(), 353 + Tree::File { 354 + size: 10, 355 + exec: true, 356 + }, 357 + ); 358 + e.insert("link".into(), Tree::Symlink("bin".into())); 359 + assert_eq!(t, Tree::Dir(e)); 360 + let ls = br#"{"version":1,"root":{"type":"directory","entries":{ 361 + "bin":{"type":"regular","size":10,"executable":true,"narOffset":96}, 362 + "link":{"type":"symlink","target":"bin"}}}}"#; 363 + assert_eq!(Tree::from_listing(ls).unwrap(), t); 364 + remove(&dir).unwrap(); 365 + 366 + let mut trailing = data.clone(); 367 + trailing.push(0); 368 + assert!(unpack(&trailing[..], &dir).is_err()); 369 + remove(&dir).unwrap(); 370 + } 371 + }
+62
tools/lazy-store/src/stats.rs
··· 1 + //! Counters written to OUT/stats.json (and merged into closure.json). 2 + 3 + use std::collections::BTreeMap; 4 + use std::path::PathBuf; 5 + use std::sync::Mutex; 6 + 7 + #[derive(Default)] 8 + struct Inner { 9 + c: BTreeMap<String, u64>, 10 + nars: Vec<(String, u64, u64)>, 11 + } 12 + 13 + #[derive(Default)] 14 + pub struct Stats { 15 + path: Option<PathBuf>, 16 + inner: Mutex<Inner>, 17 + } 18 + 19 + impl Stats { 20 + pub fn new(path: Option<PathBuf>) -> Stats { 21 + Stats { 22 + path, 23 + ..Default::default() 24 + } 25 + } 26 + 27 + pub fn add(&self, kind: &str, nbytes: u64) { 28 + let mut i = self.inner.lock().unwrap(); 29 + *i.c.entry(format!("{kind}_count")).or_default() += 1; 30 + *i.c.entry(format!("{kind}_bytes")).or_default() += nbytes; 31 + } 32 + 33 + pub fn nar(&self, name: &str, file_bytes: u64, nar_bytes: u64) { 34 + { 35 + let mut i = self.inner.lock().unwrap(); 36 + i.nars.push((name.to_string(), file_bytes, nar_bytes)); 37 + *i.c.entry("nar_count".into()).or_default() += 1; 38 + *i.c.entry("nar_bytes".into()).or_default() += file_bytes; 39 + *i.c.entry("nar_unpacked_bytes".into()).or_default() += nar_bytes; 40 + } 41 + self.save(); 42 + } 43 + 44 + pub fn counters(&self) -> serde_json::Map<String, serde_json::Value> { 45 + let i = self.inner.lock().unwrap(); 46 + i.c.iter().map(|(k, v)| (k.clone(), (*v).into())).collect() 47 + } 48 + 49 + pub fn save(&self) { 50 + let Some(path) = &self.path else { return }; 51 + let i = self.inner.lock().unwrap(); 52 + let mut j: serde_json::Map<_, _> = 53 + i.c.iter().map(|(k, v)| (k.clone(), (*v).into())).collect(); 54 + j.insert("nars".into(), serde_json::to_value(&i.nars).unwrap()); 55 + let tmp = path.with_extension("json.tmp"); 56 + let r = std::fs::write(&tmp, serde_json::to_vec(&j).unwrap()) 57 + .and_then(|_| std::fs::rename(&tmp, path)); 58 + if let Err(e) = r { 59 + eprintln!("writing {}: {e}", path.display()); 60 + } 61 + } 62 + }
+558
tools/lazy-store/src/storefs.rs
··· 1 + //! The file system mounted on /nix/store: the runner's real store (upper, 2 + //! PassthroughFs, all writes go there) merged with the lazy store paths 3 + //! (lower, LazyFs). 4 + //! 5 + //! Unlike a general overlay, the two only meet in the root directory: every 6 + //! entry of the store dir, i.e. every store path, is entirely in one layer 7 + //! (the upper one if it is in both). So nothing ever needs copying up, 8 + //! below the root each request goes straight to the layer of its inode, and 9 + //! both layers can hand their open files to the kernel (FUSE passthrough). 10 + //! Lower inodes are the LazyFs ones with LOWER set; PassthroughFs inode 11 + //! numbers stay below 2^56. 12 + 13 + use std::collections::{HashMap, HashSet}; 14 + use std::ffi::CStr; 15 + use std::io; 16 + use std::os::unix::fs::{DirEntryExt, FileTypeExt}; 17 + use std::path::PathBuf; 18 + use std::sync::atomic::{AtomicU64, Ordering}; 19 + use std::sync::{Arc, Mutex}; 20 + use std::time::Duration; 21 + 22 + use fuse_backend_rs::abi::fuse_abi::{stat64, statvfs64, CreateIn, FsOptions, SetattrValid}; 23 + use fuse_backend_rs::api::filesystem::{ 24 + Context, DirEntry, Entry, FileSystem, GetxattrReply, ListxattrReply, OpenOptions, 25 + ZeroCopyReader, ZeroCopyWriter, ROOT_ID, 26 + }; 27 + use fuse_backend_rs::passthrough::PassthroughFs; 28 + 29 + use crate::lazy::{LazyFs, INO_TAG}; 30 + 31 + const LOWER: u64 = INO_TAG; 32 + 33 + fn err(e: i32) -> io::Error { 34 + io::Error::from_raw_os_error(e) 35 + } 36 + 37 + fn is_lower(ino: u64) -> bool { 38 + ino & LOWER != 0 39 + } 40 + 41 + fn errno(e: &io::Error) -> i32 { 42 + e.raw_os_error().unwrap_or(0) 43 + } 44 + 45 + struct RootEnt { 46 + name: Vec<u8>, 47 + ino: u64, 48 + type_: u32, 49 + } 50 + 51 + pub struct StoreFs { 52 + upper: PassthroughFs<()>, 53 + upper_dir: PathBuf, 54 + pub lower: LazyFs, 55 + /// open handles of the root directory: a snapshot of the merged listing 56 + root_dirs: Mutex<HashMap<u64, Arc<Vec<RootEnt>>>>, 57 + next_root_handle: AtomicU64, 58 + } 59 + 60 + macro_rules! by_layer { 61 + ($self:ident, $ino:expr, |$fs:ident, $i:ident| $e:expr) => { 62 + if is_lower($ino) { 63 + let $fs = &$self.lower; 64 + let $i = $ino & !LOWER; 65 + $e 66 + } else { 67 + let $fs = &$self.upper; 68 + let $i = $ino; 69 + $e 70 + } 71 + }; 72 + } 73 + 74 + /// Writes to the lower layer. 75 + macro_rules! upper_only { 76 + ($self:ident, $ino:expr, |$fs:ident| $e:expr) => { 77 + if is_lower($ino) { 78 + Err(err(libc::EROFS)) 79 + } else { 80 + let $fs = &$self.upper; 81 + $e 82 + } 83 + }; 84 + } 85 + 86 + fn tag(mut e: Entry) -> Entry { 87 + if e.inode != 0 { 88 + e.inode |= LOWER; 89 + } 90 + e 91 + } 92 + 93 + impl StoreFs { 94 + pub fn new(upper: PassthroughFs<()>, upper_dir: PathBuf, lower: LazyFs) -> StoreFs { 95 + StoreFs { 96 + upper, 97 + upper_dir, 98 + lower, 99 + root_dirs: Mutex::new(HashMap::new()), 100 + next_root_handle: AtomicU64::new(1), 101 + } 102 + } 103 + 104 + /// Creating `name` in `parent`: fail if it would hide a lower store path. 105 + fn check_new(&self, parent: u64, name: &CStr) -> io::Result<()> { 106 + if is_lower(parent) { 107 + return Err(err(libc::EROFS)); 108 + } 109 + if parent == ROOT_ID && self.lower.has_root_entry(name) { 110 + return Err(err(libc::EEXIST)); 111 + } 112 + Ok(()) 113 + } 114 + 115 + /// Removing `name` from `parent` in the upper layer failed with `e`. 116 + fn rm_err(&self, parent: u64, name: &CStr, e: io::Error) -> io::Error { 117 + if parent == ROOT_ID && errno(&e) == libc::ENOENT && self.lower.has_root_entry(name) { 118 + err(libc::EROFS) 119 + } else { 120 + e 121 + } 122 + } 123 + 124 + fn root_listing(&self) -> io::Result<Vec<RootEnt>> { 125 + let mut out = Vec::new(); 126 + let mut seen = HashSet::new(); 127 + for d in std::fs::read_dir(&self.upper_dir)? { 128 + let d = d?; 129 + let t = d.file_type()?; 130 + let type_ = if t.is_dir() { 131 + libc::DT_DIR 132 + } else if t.is_symlink() { 133 + libc::DT_LNK 134 + } else if t.is_file() { 135 + libc::DT_REG 136 + } else if t.is_char_device() { 137 + libc::DT_CHR 138 + } else { 139 + libc::DT_UNKNOWN 140 + }; 141 + let name = d.file_name().into_encoded_bytes(); 142 + seen.insert(name.clone()); 143 + out.push(RootEnt { 144 + name, 145 + ino: d.ino(), 146 + type_: type_ as u32, 147 + }); 148 + } 149 + for (name, ino, type_) in self.lower.root_entries() { 150 + if !seen.contains(name.as_bytes()) { 151 + out.push(RootEnt { 152 + name: name.as_bytes().to_vec(), 153 + ino: ino | LOWER, 154 + type_, 155 + }); 156 + } 157 + } 158 + Ok(out) 159 + } 160 + } 161 + 162 + impl FileSystem for StoreFs { 163 + type Inode = u64; 164 + type Handle = u64; 165 + 166 + fn init(&self, capable: FsOptions) -> io::Result<FsOptions> { 167 + let mut opts = self.upper.init(capable)?; 168 + let pt = opts.contains(FsOptions::PASSTHROUGH); 169 + self.lower.set_passthrough(pt); 170 + eprintln!( 171 + "init: kernel FUSE passthrough {}", 172 + if pt { 173 + "on" 174 + } else if capable.contains(FsOptions::PASSTHROUGH) { 175 + "offered by the kernel but off" 176 + } else { 177 + "not offered by the kernel" 178 + } 179 + ); 180 + // the merged root has no readdirplus 181 + opts.remove( 182 + FsOptions::DO_READDIRPLUS | FsOptions::READDIRPLUS_AUTO | FsOptions::PERFILE_DAX, 183 + ); 184 + Ok(opts) 185 + } 186 + 187 + fn destroy(&self) { 188 + eprintln!("FUSE connection destroyed"); 189 + } 190 + 191 + fn lookup(&self, ctx: &Context, parent: u64, name: &CStr) -> io::Result<Entry> { 192 + if parent == ROOT_ID { 193 + let up = self.upper.lookup(ctx, ROOT_ID, name); 194 + match &up { 195 + Ok(e) if e.inode != 0 => return up, 196 + Ok(_) => {} 197 + Err(e) if errno(e) == libc::ENOENT => {} 198 + Err(_) => return up, 199 + } 200 + return match self.lower.lookup(ctx, 1, name) { 201 + Ok(e) => Ok(tag(e)), 202 + Err(_) => up, 203 + }; 204 + } 205 + if is_lower(parent) { 206 + return self.lower.lookup(ctx, parent & !LOWER, name).map(tag); 207 + } 208 + self.upper.lookup(ctx, parent, name) 209 + } 210 + 211 + fn forget(&self, ctx: &Context, ino: u64, count: u64) { 212 + if !is_lower(ino) { 213 + self.upper.forget(ctx, ino, count) 214 + } 215 + } 216 + 217 + fn batch_forget(&self, ctx: &Context, requests: Vec<(u64, u64)>) { 218 + let up: Vec<_> = requests 219 + .into_iter() 220 + .filter(|(i, _)| !is_lower(*i)) 221 + .collect(); 222 + self.upper.batch_forget(ctx, up) 223 + } 224 + 225 + fn getattr(&self, ctx: &Context, ino: u64, h: Option<u64>) -> io::Result<(stat64, Duration)> { 226 + by_layer!(self, ino, |fs, i| fs.getattr(ctx, i, h)) 227 + } 228 + 229 + fn setattr( 230 + &self, 231 + ctx: &Context, 232 + ino: u64, 233 + attr: stat64, 234 + h: Option<u64>, 235 + valid: SetattrValid, 236 + ) -> io::Result<(stat64, Duration)> { 237 + upper_only!(self, ino, |fs| fs.setattr(ctx, ino, attr, h, valid)) 238 + } 239 + 240 + fn readlink(&self, ctx: &Context, ino: u64) -> io::Result<Vec<u8>> { 241 + by_layer!(self, ino, |fs, i| fs.readlink(ctx, i)) 242 + } 243 + 244 + fn symlink( 245 + &self, 246 + ctx: &Context, 247 + linkname: &CStr, 248 + parent: u64, 249 + name: &CStr, 250 + ) -> io::Result<Entry> { 251 + self.check_new(parent, name)?; 252 + self.upper.symlink(ctx, linkname, parent, name) 253 + } 254 + 255 + fn mknod( 256 + &self, 257 + ctx: &Context, 258 + parent: u64, 259 + name: &CStr, 260 + mode: u32, 261 + rdev: u32, 262 + umask: u32, 263 + ) -> io::Result<Entry> { 264 + self.check_new(parent, name)?; 265 + self.upper.mknod(ctx, parent, name, mode, rdev, umask) 266 + } 267 + 268 + fn mkdir( 269 + &self, 270 + ctx: &Context, 271 + parent: u64, 272 + name: &CStr, 273 + mode: u32, 274 + umask: u32, 275 + ) -> io::Result<Entry> { 276 + self.check_new(parent, name)?; 277 + self.upper.mkdir(ctx, parent, name, mode, umask) 278 + } 279 + 280 + fn unlink(&self, ctx: &Context, parent: u64, name: &CStr) -> io::Result<()> { 281 + upper_only!(self, parent, |fs| fs 282 + .unlink(ctx, parent, name) 283 + .map_err(|e| self.rm_err(parent, name, e))) 284 + } 285 + 286 + fn rmdir(&self, ctx: &Context, parent: u64, name: &CStr) -> io::Result<()> { 287 + upper_only!(self, parent, |fs| fs 288 + .rmdir(ctx, parent, name) 289 + .map_err(|e| self.rm_err(parent, name, e))) 290 + } 291 + 292 + fn rename( 293 + &self, 294 + ctx: &Context, 295 + olddir: u64, 296 + oldname: &CStr, 297 + newdir: u64, 298 + newname: &CStr, 299 + flags: u32, 300 + ) -> io::Result<()> { 301 + if is_lower(olddir) || is_lower(newdir) { 302 + return Err(err(libc::EXDEV)); 303 + } 304 + // Renaming onto a lower store path hides it, as an overlay would. 305 + self.upper 306 + .rename(ctx, olddir, oldname, newdir, newname, flags) 307 + .map_err(|e| match self.rm_err(olddir, oldname, e) { 308 + e if errno(&e) == libc::EROFS => err(libc::EXDEV), 309 + e => e, 310 + }) 311 + } 312 + 313 + fn link(&self, ctx: &Context, ino: u64, newparent: u64, newname: &CStr) -> io::Result<Entry> { 314 + if is_lower(ino) { 315 + return Err(err(libc::EXDEV)); 316 + } 317 + self.check_new(newparent, newname)?; 318 + self.upper.link(ctx, ino, newparent, newname) 319 + } 320 + 321 + fn open( 322 + &self, 323 + ctx: &Context, 324 + ino: u64, 325 + flags: u32, 326 + fuse_flags: u32, 327 + ) -> io::Result<(Option<u64>, OpenOptions, Option<u32>)> { 328 + by_layer!(self, ino, |fs, i| fs.open(ctx, i, flags, fuse_flags)) 329 + } 330 + 331 + fn create( 332 + &self, 333 + ctx: &Context, 334 + parent: u64, 335 + name: &CStr, 336 + args: CreateIn, 337 + ) -> io::Result<(Entry, Option<u64>, OpenOptions, Option<u32>)> { 338 + self.check_new(parent, name)?; 339 + self.upper.create(ctx, parent, name, args) 340 + } 341 + 342 + fn read( 343 + &self, 344 + ctx: &Context, 345 + ino: u64, 346 + handle: u64, 347 + w: &mut dyn ZeroCopyWriter, 348 + size: u32, 349 + offset: u64, 350 + lock_owner: Option<u64>, 351 + flags: u32, 352 + ) -> io::Result<usize> { 353 + by_layer!(self, ino, |fs, i| fs 354 + .read(ctx, i, handle, w, size, offset, lock_owner, flags)) 355 + } 356 + 357 + fn write( 358 + &self, 359 + ctx: &Context, 360 + ino: u64, 361 + handle: u64, 362 + r: &mut dyn ZeroCopyReader, 363 + size: u32, 364 + offset: u64, 365 + lock_owner: Option<u64>, 366 + delayed_write: bool, 367 + flags: u32, 368 + fuse_flags: u32, 369 + ) -> io::Result<usize> { 370 + upper_only!(self, ino, |fs| fs.write( 371 + ctx, 372 + ino, 373 + handle, 374 + r, 375 + size, 376 + offset, 377 + lock_owner, 378 + delayed_write, 379 + flags, 380 + fuse_flags 381 + )) 382 + } 383 + 384 + fn flush(&self, ctx: &Context, ino: u64, handle: u64, lock_owner: u64) -> io::Result<()> { 385 + by_layer!(self, ino, |fs, i| fs.flush(ctx, i, handle, lock_owner)) 386 + } 387 + 388 + fn fsync(&self, ctx: &Context, ino: u64, datasync: bool, handle: u64) -> io::Result<()> { 389 + by_layer!(self, ino, |fs, i| fs.fsync(ctx, i, datasync, handle)) 390 + } 391 + 392 + fn fallocate( 393 + &self, 394 + ctx: &Context, 395 + ino: u64, 396 + handle: u64, 397 + mode: u32, 398 + offset: u64, 399 + length: u64, 400 + ) -> io::Result<()> { 401 + upper_only!(self, ino, |fs| fs 402 + .fallocate(ctx, ino, handle, mode, offset, length)) 403 + } 404 + 405 + fn release( 406 + &self, 407 + ctx: &Context, 408 + ino: u64, 409 + flags: u32, 410 + handle: u64, 411 + flush: bool, 412 + flock_release: bool, 413 + lock_owner: Option<u64>, 414 + ) -> io::Result<()> { 415 + by_layer!(self, ino, |fs, i| fs.release( 416 + ctx, 417 + i, 418 + flags, 419 + handle, 420 + flush, 421 + flock_release, 422 + lock_owner 423 + )) 424 + } 425 + 426 + fn statfs(&self, ctx: &Context, _ino: u64) -> io::Result<statvfs64> { 427 + self.upper.statfs(ctx, ROOT_ID) 428 + } 429 + 430 + fn setxattr( 431 + &self, 432 + ctx: &Context, 433 + ino: u64, 434 + name: &CStr, 435 + value: &[u8], 436 + flags: u32, 437 + ) -> io::Result<()> { 438 + upper_only!(self, ino, |fs| fs.setxattr(ctx, ino, name, value, flags)) 439 + } 440 + 441 + fn getxattr( 442 + &self, 443 + ctx: &Context, 444 + ino: u64, 445 + name: &CStr, 446 + size: u32, 447 + ) -> io::Result<GetxattrReply> { 448 + by_layer!(self, ino, |fs, i| fs.getxattr(ctx, i, name, size)) 449 + } 450 + 451 + fn listxattr(&self, ctx: &Context, ino: u64, size: u32) -> io::Result<ListxattrReply> { 452 + by_layer!(self, ino, |fs, i| fs.listxattr(ctx, i, size)) 453 + } 454 + 455 + fn removexattr(&self, ctx: &Context, ino: u64, name: &CStr) -> io::Result<()> { 456 + upper_only!(self, ino, |fs| fs.removexattr(ctx, ino, name)) 457 + } 458 + 459 + fn opendir( 460 + &self, 461 + ctx: &Context, 462 + ino: u64, 463 + flags: u32, 464 + ) -> io::Result<(Option<u64>, OpenOptions)> { 465 + if ino == ROOT_ID { 466 + let h = self.next_root_handle.fetch_add(1, Ordering::Relaxed); 467 + let l = Arc::new(self.root_listing()?); 468 + self.root_dirs.lock().unwrap().insert(h, l); 469 + return Ok((Some(h), OpenOptions::empty())); 470 + } 471 + by_layer!(self, ino, |fs, i| fs.opendir(ctx, i, flags)) 472 + } 473 + 474 + fn readdir( 475 + &self, 476 + ctx: &Context, 477 + ino: u64, 478 + handle: u64, 479 + size: u32, 480 + offset: u64, 481 + add_entry: &mut dyn FnMut(DirEntry) -> io::Result<usize>, 482 + ) -> io::Result<()> { 483 + if ino == ROOT_ID { 484 + let l = self 485 + .root_dirs 486 + .lock() 487 + .unwrap() 488 + .get(&handle) 489 + .cloned() 490 + .ok_or_else(|| err(libc::EBADF))?; 491 + for (i, e) in l.iter().enumerate().skip(offset as usize) { 492 + let n = add_entry(DirEntry { 493 + ino: e.ino, 494 + offset: i as u64 + 1, 495 + type_: e.type_, 496 + name: &e.name, 497 + })?; 498 + if n == 0 { 499 + break; 500 + } 501 + } 502 + return Ok(()); 503 + } 504 + by_layer!(self, ino, |fs, i| fs 505 + .readdir(ctx, i, handle, size, offset, add_entry)) 506 + } 507 + 508 + fn fsyncdir(&self, ctx: &Context, ino: u64, datasync: bool, handle: u64) -> io::Result<()> { 509 + if ino == ROOT_ID { 510 + return std::fs::File::open(&self.upper_dir)?.sync_all(); 511 + } 512 + by_layer!(self, ino, |fs, i| fs.fsyncdir(ctx, i, datasync, handle)) 513 + } 514 + 515 + fn releasedir(&self, ctx: &Context, ino: u64, flags: u32, handle: u64) -> io::Result<()> { 516 + if ino == ROOT_ID { 517 + self.root_dirs.lock().unwrap().remove(&handle); 518 + return Ok(()); 519 + } 520 + by_layer!(self, ino, |fs, i| fs.releasedir(ctx, i, flags, handle)) 521 + } 522 + 523 + fn access(&self, ctx: &Context, ino: u64, mask: u32) -> io::Result<()> { 524 + by_layer!(self, ino, |fs, i| fs.access(ctx, i, mask)) 525 + } 526 + 527 + fn lseek( 528 + &self, 529 + ctx: &Context, 530 + ino: u64, 531 + handle: u64, 532 + offset: u64, 533 + whence: u32, 534 + ) -> io::Result<u64> { 535 + by_layer!(self, ino, |fs, i| fs.lseek(ctx, i, handle, offset, whence)) 536 + } 537 + 538 + fn copy_file_range( 539 + &self, 540 + ctx: &Context, 541 + inode_in: u64, 542 + fh_in: u64, 543 + offset_in: u64, 544 + inode_out: u64, 545 + fh_out: u64, 546 + offset_out: u64, 547 + len: u64, 548 + flags: u64, 549 + ) -> io::Result<u32> { 550 + if is_lower(inode_in) || is_lower(inode_out) { 551 + // makes the caller fall back to read/write 552 + return Err(err(libc::EXDEV)); 553 + } 554 + self.upper.copy_file_range( 555 + ctx, inode_in, fh_in, offset_in, inode_out, fh_out, offset_out, len, flags, 556 + ) 557 + } 558 + }