configs.nix-farm: add nix-grpc-store farm and niks3 cache on neoprism · hermes.pds.lassul.us/superconfig@6e0f7f5

+48

flake.lock

··· 551 551 "type": "github" 552 552 } 553 553 }, 554 + "niks3": { 555 + "inputs": { 556 + "nixpkgs": [ 557 + "nixpkgs" 558 + ], 559 + "treefmt-nix": [ 560 + "treefmt-nix" 561 + ] 562 + }, 563 + "locked": { 564 + "lastModified": 1791698001, 565 + "narHash": "sha256-w2WpjabRT4UeEFKN8r0gm5JHQPZFI4mB3y/fW9t/ve0=", 566 + "owner": "Mic92", 567 + "repo": "niks3", 568 + "rev": "2d5a6b14e0df5c8fecc27571b79c40563c98ec32", 569 + "type": "github" 570 + }, 571 + "original": { 572 + "owner": "Mic92", 573 + "repo": "niks3", 574 + "type": "github" 575 + } 576 + }, 554 577 "nix-darwin": { 555 578 "inputs": { 556 579 "nixpkgs": [ ··· 589 612 "original": { 590 613 "owner": "LnL7", 591 614 "repo": "nix-darwin", 615 + "type": "github" 616 + } 617 + }, 618 + "nix-grpc-store": { 619 + "inputs": { 620 + "niks3": [ 621 + "niks3" 622 + ], 623 + "nixpkgs": [ 624 + "nixpkgs" 625 + ] 626 + }, 627 + "locked": { 628 + "lastModified": 1791642748, 629 + "narHash": "sha256-mHzxNM2g/U2cENihXpbnpAsY+xziZ440LzfIWFe49mo=", 630 + "owner": "Mic92", 631 + "repo": "nix-grpc-store", 632 + "rev": "37d20b8ee7881cd63bb75d3a303efacbd3e83be4", 633 + "type": "github" 634 + }, 635 + "original": { 636 + "owner": "Mic92", 637 + "repo": "nix-grpc-store", 592 638 "type": "github" 593 639 } 594 640 }, ··· 920 966 "le_menu": "le_menu", 921 967 "llm-agents": "llm-agents", 922 968 "nether": "nether", 969 + "niks3": "niks3", 923 970 "nix-darwin": "nix-darwin_2", 971 + "nix-grpc-store": "nix-grpc-store", 924 972 "nix-index-database": "nix-index-database", 925 973 "nix-rosetta-builder": "nix-rosetta-builder", 926 974 "nixos-facter": "nixos-facter",

+10

flake.nix

View file

··· 113 113 sigexec.url = "git+https://git.geninf.io/Lassulus/sigexec"; 114 114 sigexec.inputs.nixpkgs.follows = "nixpkgs"; 115 115 116 + # build farm (configs/nix-farm): remote builds over grpc:// ... 117 + nix-grpc-store.url = "github:Mic92/nix-grpc-store"; 118 + nix-grpc-store.inputs.nixpkgs.follows = "nixpkgs"; 119 + nix-grpc-store.inputs.niks3.follows = "niks3"; 120 + 121 + # ... and the S3-backed binary cache the outputs land in 122 + niks3.url = "github:Mic92/niks3"; 123 + niks3.inputs.nixpkgs.follows = "nixpkgs"; 124 + niks3.inputs.treefmt-nix.follows = "treefmt-nix"; 125 + 116 126 }; 117 127 118 128 outputs =

+12

configs/default.nix

View file

··· 136 136 }; 137 137 } 138 138 ) 139 + ( 140 + # binary cache of the neoprism build farm (configs/nix-farm). The key 141 + # only exists once `clan vars generate neoprism` has run; until then 142 + # the cache is left out. 143 + let 144 + pub = self + "/vars/per-machine/neoprism/nix-cache-signing/key.pub/value"; 145 + in 146 + lib.mkIf (builtins.pathExists pub) { 147 + nix.settings.extra-substituters = [ "https://cache.lassul.us" ]; 148 + nix.settings.extra-trusted-public-keys = [ (lib.removeSuffix "\n" (builtins.readFile pub)) ]; 149 + } 150 + ) 139 151 { 140 152 #for sshuttle 141 153 environment.systemPackages = [

+92

configs/nix-farm/builder.nix

View file

··· 1 + { 2 + config, 3 + lib, 4 + pkgs, 5 + self, 6 + ... 7 + }: 8 + let 9 + vars = config.clan.core.vars.generators; 10 + cachePub = self + "/vars/per-machine/neoprism/nix-cache-signing/key.pub/value"; 11 + credDir = "/run/credentials/nix-grpc-daemon.service"; 12 + in 13 + { 14 + imports = [ 15 + self.inputs.nix-grpc-store.nixosModules.server 16 + ./ca.nix 17 + ]; 18 + 19 + # Server cert for the daemon, signed by the shared farm CA, plus a copy of 20 + # the CA cert to verify clients against. Secret files for the same reason 21 + # as in ca.nix. Root-owned: the daemon gets them via LoadCredential below. 22 + clan.core.vars.generators.nix-farm-server = { 23 + dependencies = [ "nix-farm-ca" ]; 24 + files."server.key" = { }; 25 + files."server.crt" = { }; 26 + files."ca.crt" = { }; 27 + runtimeInputs = [ pkgs.openssl ]; 28 + script = '' 29 + openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ 30 + -keyout "$out/server.key" -out server.csr -subj "/CN=neoprism" 31 + printf '%s\n' \ 32 + "basicConstraints=critical,CA:FALSE" \ 33 + "keyUsage=critical,digitalSignature" \ 34 + "extendedKeyUsage=serverAuth" \ 35 + "subjectAltName=DNS:neoprism.r,DNS:neoprism.lassul.us" > server.ext 36 + openssl x509 -req -in server.csr \ 37 + -CA "$in/nix-farm-ca/ca.crt" -CAkey "$in/nix-farm-ca/ca.key" \ 38 + -CAcreateserial -days 3650 -extfile server.ext -out "$out/server.crt" 39 + cp "$in/nix-farm-ca/ca.crt" "$out/ca.crt" 40 + ''; 41 + }; 42 + 43 + # Standalone node: schedules onto itself, publishes outputs to niks3. 44 + services.nix-grpc-daemon = { 45 + enable = true; 46 + roles = [ 47 + "builder" 48 + "scheduler" 49 + ]; 50 + listen = "[::]:50051"; 51 + # gRPC's cert file watcher treats mtime 0 as a stat failure and never 52 + # loads the key pair, and clan unpacks /run/secrets with mtime 0. systemd 53 + # credentials are fresh copies with a real mtime. 54 + tls = { 55 + certFile = "${credDir}/server.crt"; 56 + keyFile = "${credDir}/server.key"; 57 + clientCaFile = "${credDir}/ca.crt"; 58 + }; 59 + # `nix build --store grpc://… --eval-store auto` needs write (README). 60 + accessRules = [ 61 + { 62 + cn = "lass"; 63 + role = "write"; 64 + } 65 + ]; 66 + niks3 = { 67 + package = self.inputs.niks3.packages.${pkgs.stdenv.hostPlatform.system}.niks3; 68 + # same box, skip nginx/ACME 69 + url = "http://${config.services.niks3.httpAddr}"; 70 + tokenFile = vars.niks3.files."api-token-grpc".path; 71 + cacheUrl = config.services.niks3.cacheUrl; 72 + # empty until `clan vars generate neoprism` wrote the key 73 + publicKeys = lib.optional (builtins.pathExists cachePub) ( 74 + lib.removeSuffix "\n" (builtins.readFile cachePub) 75 + ); 76 + }; 77 + }; 78 + 79 + systemd.services.nix-grpc-daemon.serviceConfig = { 80 + LoadCredential = map (f: "${f}:${vars.nix-farm-server.files.${f}.path}") [ 81 + "server.crt" 82 + "server.key" 83 + "ca.crt" 84 + ]; 85 + # Upstream's 2G default assumes the unit only proxies, but `niks3 push` 86 + # runs in this cgroup too and got OOM-killed on large NARs (kernel 87 + # modules), taking the whole daemon down with it. 88 + MemoryMax = "8G"; 89 + }; 90 + 91 + networking.firewall.interfaces.retiolum.allowedTCPPorts = [ 50051 ]; 92 + }

+35

configs/nix-farm/ca.nix

View file

··· 1 + { pkgs, ... }: 2 + # Private CA of the build farm plus the client cert lass uses towards it. 3 + # Shared, so every machine importing this sees the same CA. Clan requires a 4 + # shared generator to be defined identically on all machines, so nothing is 5 + # deployed from here: per-machine generators (nix-farm-server in builder.nix, 6 + # nix-farm-client in client.nix) depend on it and copy out the files they 7 + # need with their own owner/mode. The certs are not secret, but as secret 8 + # files their paths are known before `clan vars generate` ran, so the configs 9 + # still evaluate until then. 10 + { 11 + clan.core.vars.generators.nix-farm-ca = { 12 + share = true; 13 + files."ca.key".deploy = false; 14 + files."ca.crt".deploy = false; 15 + files."client.key".deploy = false; 16 + files."client.crt".deploy = false; 17 + runtimeInputs = [ pkgs.openssl ]; 18 + script = '' 19 + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ 20 + -keyout "$out/ca.key" -out "$out/ca.crt" -days 3650 \ 21 + -subj "/CN=nix-farm-ca" \ 22 + -addext "basicConstraints=critical,CA:TRUE" \ 23 + -addext "keyUsage=critical,keyCertSign,cRLSign" 24 + 25 + openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ 26 + -keyout "$out/client.key" -out client.csr -subj "/CN=lass" 27 + printf '%s\n' \ 28 + "basicConstraints=critical,CA:FALSE" \ 29 + "keyUsage=critical,digitalSignature" \ 30 + "extendedKeyUsage=clientAuth" > client.ext 31 + openssl x509 -req -in client.csr -CA "$out/ca.crt" -CAkey "$out/ca.key" \ 32 + -CAcreateserial -days 3650 -extfile client.ext -out "$out/client.crt" 33 + ''; 34 + }; 35 + }

+116

configs/nix-farm/cache.nix

View file

··· 1 + { 2 + config, 3 + pkgs, 4 + self, 5 + ... 6 + }: 7 + let 8 + # TODO: create this bucket (public) + an application key for it in B2 9 + bucket = "lassulus-nix-cache"; 10 + region = "eu-central-003"; 11 + endpoint = "s3.${region}.backblazeb2.com"; 12 + 13 + vars = config.clan.core.vars.generators; 14 + in 15 + { 16 + imports = [ self.inputs.niks3.nixosModules.niks3 ]; 17 + 18 + # B2 credentials (prompted), the niks3 API token (generated; one copy per 19 + # service user that reads it). 20 + clan.core.vars.generators.niks3 = { 21 + prompts.b2_key_id = { 22 + description = "Backblaze B2 keyID with access to the ${bucket} bucket"; 23 + type = "line"; 24 + }; 25 + prompts.b2_application_key = { 26 + description = "Backblaze B2 applicationKey for that keyID"; 27 + type = "hidden"; 28 + }; 29 + files."s3-access-key".owner = "niks3"; 30 + files."s3-secret-key".owner = "niks3"; 31 + files."api-token".owner = "niks3"; 32 + files."api-token-grpc".owner = "nix-grpc-daemon"; 33 + runtimeInputs = [ 34 + pkgs.coreutils 35 + pkgs.openssl 36 + ]; 37 + script = '' 38 + tr -d '\n' < "$prompts/b2_key_id" > "$out/s3-access-key" 39 + tr -d '\n' < "$prompts/b2_application_key" > "$out/s3-secret-key" 40 + openssl rand -hex 32 | tr -d '\n' > "$out/api-token" 41 + cp "$out/api-token" "$out/api-token-grpc" 42 + ''; 43 + }; 44 + 45 + # Public half is read by configs/default.nix from the vars dir. 46 + clan.core.vars.generators.nix-cache-signing = { 47 + files."key.sec".owner = "niks3"; 48 + files."key.pub".secret = false; 49 + runtimeInputs = [ pkgs.nix ]; 50 + script = '' 51 + nix --extra-experimental-features nix-command key generate-secret \ 52 + --key-name cache.lassul.us-1 > "$out/key.sec" 53 + nix --extra-experimental-features nix-command key convert-secret-to-public \ 54 + < "$out/key.sec" > "$out/key.pub" 55 + ''; 56 + }; 57 + 58 + services.niks3 = { 59 + enable = true; 60 + s3 = { 61 + inherit endpoint bucket region; 62 + accessKeyFile = vars.niks3.files."s3-access-key".path; 63 + secretKeyFile = vars.niks3.files."s3-secret-key".path; 64 + }; 65 + apiTokenFile = vars.niks3.files."api-token".path; 66 + signKeyFiles = [ vars.nix-cache-signing.files."key.sec".path ]; 67 + cacheUrl = "https://cache.lassul.us"; 68 + nginx = { 69 + enable = true; 70 + domain = "niks3.lassul.us"; 71 + }; 72 + }; 73 + 74 + # Reads straight from the public bucket. niks3 uploads the metadata 75 + # zstd-compressed with Content-Encoding: zstd; set it here so nix decodes 76 + # it whether or not the bucket hands the header back. Same rules as 77 + # clan-infra's Fastly config (modules/terranix/cache-new.nix): narinfo, .ls, 78 + # realisations/ and log/. NARs (nar/*.nar.zst) are plain objects, nix 79 + # decodes them via the narinfo's Compression field. 80 + # `resolve`: looked up at runtime via resolved's stub, so an unresolvable 81 + # B2 at startup doesn't keep nginx (and every other vhost) from starting. 82 + services.nginx.appendHttpConfig = '' 83 + upstream cache_lassul_us_b2 { 84 + zone cache_lassul_us_b2 64k; 85 + resolver 127.0.0.53 ipv6=off; 86 + server ${endpoint}:443 resolve; 87 + } 88 + map $uri $cache_lassul_us_encoding { 89 + default ""; 90 + ~\.(narinfo|ls)$ zstd; 91 + ~^/realisations/ zstd; 92 + ~^/log/ zstd; 93 + } 94 + ''; 95 + services.nginx.virtualHosts."cache.lassul.us" = { 96 + enableACME = true; 97 + forceSSL = true; 98 + locations."/".extraConfig = '' 99 + proxy_pass https://cache_lassul_us_b2/${bucket}/; 100 + proxy_set_header Host ${endpoint}; 101 + proxy_ssl_server_name on; 102 + proxy_ssl_name ${endpoint}; 103 + # cache keys never have a query string; don't forward one to S3 104 + set $args ""; 105 + # S3 answers 403 for keys missing from a bucket it won't list 106 + proxy_intercept_errors on; 107 + error_page 403 =404 @notfound; 108 + 109 + gzip off; 110 + proxy_hide_header Content-Encoding; 111 + # only sent on 2xx/3xx, and not at all when the map yields "" 112 + add_header Content-Encoding $cache_lassul_us_encoding; 113 + ''; 114 + locations."@notfound".return = "404"; 115 + }; 116 + }

+77

configs/nix-farm/client.nix

View file

··· 1 + { 2 + config, 3 + lib, 4 + options, 5 + self, 6 + ... 7 + }: 8 + # Client of the neoprism build farm (configs/nix-farm), for NixOS and 9 + # nix-darwin. The plugin finds client.crt/client.key in 10 + # /var/lib/nix-grpc-store by itself; the CA has to be passed explicitly: 11 + # nix build --store 'grpc://neoprism.r:50051?ca-cert=/var/lib/nix-grpc-store/ca.crt' \ 12 + # --eval-store auto .#nixosConfigurations.<m>.config.system.build.toplevel 13 + let 14 + files = [ 15 + "ca.crt" 16 + "client.crt" 17 + "client.key" 18 + ]; 19 + client = config.clan.core.vars.generators.nix-farm-client.files; 20 + in 21 + { 22 + imports = [ 23 + # darwinModules.default is only the daemon; the grpc:// plugin is in the 24 + # client module, which only sets nix.settings.plugin-files and works on 25 + # nix-darwin as is. 26 + "${self.inputs.nix-grpc-store}/nixos/client.nix" 27 + ./ca.nix 28 + ]; 29 + 30 + config = lib.mkMerge [ 31 + { programs.nix-grpc-store.enable = true; } 32 + 33 + # Per-machine copy of the shared CA's client files (see ca.nix). 34 + { 35 + clan.core.vars.generators.nix-farm-client = { 36 + dependencies = [ "nix-farm-ca" ]; 37 + files = lib.genAttrs files (_: { }); 38 + script = lib.concatMapStrings (f: '' 39 + cp "$in/nix-farm-ca/${f}" "$out/${f}" 40 + '') files; 41 + }; 42 + } 43 + 44 + # Branch on options, not pkgs: the latter recurses through the attr names. 45 + (lib.optionalAttrs (options ? system.primaryUser) { 46 + # clan's /run/secrets installer is Linux-only, so these come in as 47 + # activation secrets (root-owned, below secretLocation) and are copied 48 + # to where the plugin looks by default, readable by the user running nix. 49 + clan.core.vars.generators.nix-farm-client.files = lib.genAttrs files (_: { 50 + deploy = true; 51 + neededFor = "activation"; 52 + }); 53 + # Skipped while the secrets have not been uploaded yet. 54 + system.activationScripts.postActivation.text = '' 55 + install -d -m 0755 /var/lib/nix-grpc-store 56 + '' 57 + + lib.concatMapStrings (f: '' 58 + if [ -e ${client.${f}.path} ]; then 59 + install -m 0400 -o ${config.system.primaryUser} ${client.${f}.path} /var/lib/nix-grpc-store/${f} 60 + fi 61 + '') files; 62 + }) 63 + 64 + (lib.optionalAttrs (options ? systemd.tmpfiles) { 65 + # Regular /run/secrets, readable by wheel (the users running nix) and 66 + # root (nix-daemon's build hook), linked to where the plugin looks. 67 + clan.core.vars.generators.nix-farm-client.files = lib.genAttrs files (_: { 68 + deploy = true; 69 + group = "wheel"; 70 + mode = "0440"; 71 + }); 72 + systemd.tmpfiles.rules = map ( 73 + f: "L+ /var/lib/nix-grpc-store/${f} - - - - ${client.${f}.path}" 74 + ) files; 75 + }) 76 + ]; 77 + }

+1

machines/barnacle/configuration.nix

View file

··· 10 10 self.inputs.nix-index-database.darwinModules.nix-index 11 11 self.inputs.nix-rosetta-builder.darwinModules.default 12 12 ./retiolum.nix 13 + ../../configs/nix-farm/client.nix 13 14 ]; 14 15 clan.core.networking.targetHost = "root@localhost"; 15 16 programs.nix-index-database.comma.enable = true;

+5

machines/neoprism/config.nix

View file

··· 136 136 137 137 # nixpkgs PR triage dashboard (review.lassul.us) 138 138 ../../configs/nixpkgs-triage.nix 139 + 140 + # build farm (grpc://neoprism.r:50051) + binary cache (cache.lassul.us, 141 + # niks3.lassul.us) 142 + ../../configs/nix-farm/cache.nix 143 + ../../configs/nix-farm/builder.nix 139 144 ]; 140 145 141 146 # lassul.us shouldn't be the default vhost here (nginx.nix already sets one)

Configure Feed

Configure Feed