lass.pds.lassul.us / superconfig
This repository has no description
0
Fork 1

Configure Feed

Select the types of activity you want to include in your feed.

Merge branch 'Lassulus/build-farm'

author
lassulus date (Oct 12, 2026, 8:53 AM CEST) commit be6c7d60 parent 85941bf6
+766 -1
+48
flake.lock
··· 551 551 "type": "github" 552 552 } 553 553 }, 554 + "niks3": { 555 + "inputs": { 556 + "nixpkgs": [ 557 + "nixpkgs" 558 + ], 559 + "treefmt-nix": [ 560 + "treefmt-nix" 561 + ] 562 + }, 563 + "locked": { 564 + "lastModified": 1791782818, 565 + "narHash": "sha256-a9ps1IFkL2ynoUnKb1t/9VvfMYFMd4oLxq7JlqqotK4=", 566 + "owner": "lassulus", 567 + "repo": "niks3", 568 + "rev": "96bb25d227672be9a1fc46da43baf0d0cc2bebd0", 569 + "type": "github" 570 + }, 571 + "original": { 572 + "owner": "lassulus", 573 + "repo": "niks3", 574 + "type": "github" 575 + } 576 + }, 554 577 "nix-darwin": { 555 578 "inputs": { 556 579 "nixpkgs": [ ··· 589 612 "original": { 590 613 "owner": "LnL7", 591 614 "repo": "nix-darwin", 615 + "type": "github" 616 + } 617 + }, 618 + "nix-grpc-store": { 619 + "inputs": { 620 + "niks3": [ 621 + "niks3" 622 + ], 623 + "nixpkgs": [ 624 + "nixpkgs" 625 + ] 626 + }, 627 + "locked": { 628 + "lastModified": 1791642748, 629 + "narHash": "sha256-mHzxNM2g/U2cENihXpbnpAsY+xziZ440LzfIWFe49mo=", 630 + "owner": "Mic92", 631 + "repo": "nix-grpc-store", 632 + "rev": "37d20b8ee7881cd63bb75d3a303efacbd3e83be4", 633 + "type": "github" 634 + }, 635 + "original": { 636 + "owner": "Mic92", 637 + "repo": "nix-grpc-store", 592 638 "type": "github" 593 639 } 594 640 }, ··· 920 966 "le_menu": "le_menu", 921 967 "llm-agents": "llm-agents", 922 968 "nether": "nether", 969 + "niks3": "niks3", 923 970 "nix-darwin": "nix-darwin_2", 971 + "nix-grpc-store": "nix-grpc-store", 924 972 "nix-index-database": "nix-index-database", 925 973 "nix-rosetta-builder": "nix-rosetta-builder", 926 974 "nixos-facter": "nixos-facter",
+13
flake.nix
··· 113 113 sigexec.url = "git+https://git.geninf.io/Lassulus/sigexec"; 114 114 sigexec.inputs.nixpkgs.follows = "nixpkgs"; 115 115 116 + # build farm (configs/nix-farm): remote builds over grpc:// ... 117 + nix-grpc-store.url = "github:Mic92/nix-grpc-store"; 118 + nix-grpc-store.inputs.nixpkgs.follows = "nixpkgs"; 119 + nix-grpc-store.inputs.niks3.follows = "niks3"; 120 + 121 + # ... and the S3-backed binary cache the outputs land in. Fork for 122 + # `push --ignore-upstream-key` (not upstream yet). It rebases onto 123 + # Mic92/niks3 daily (its .github/workflows/rebase-upstream.yml), so 124 + # `nix flake update niks3` follows upstream. 125 + niks3.url = "github:lassulus/niks3"; 126 + niks3.inputs.nixpkgs.follows = "nixpkgs"; 127 + niks3.inputs.treefmt-nix.follows = "treefmt-nix"; 128 + 116 129 }; 117 130 118 131 outputs =
+12
configs/default.nix
··· 136 136 }; 137 137 } 138 138 ) 139 + ( 140 + # binary cache of the neoprism build farm (configs/nix-farm). The key 141 + # only exists once `clan vars generate neoprism` has run; until then 142 + # the cache is left out. 143 + let 144 + pub = self + "/vars/per-machine/neoprism/nix-cache-signing/key.pub/value"; 145 + in 146 + lib.mkIf (builtins.pathExists pub) { 147 + nix.settings.extra-substituters = [ "https://cache.lassul.us" ]; 148 + nix.settings.extra-trusted-public-keys = [ (lib.removeSuffix "\n" (builtins.readFile pub)) ]; 149 + } 150 + ) 139 151 { 140 152 #for sshuttle 141 153 environment.systemPackages = [
+78
.github/workflows/farm.yml
··· 1 + name: farm 2 + 3 + # On-demand nix-grpc-store farm (tools/ci-farm): one farm job (scheduler, 4 + # balancer, client, builder) and a pool of worker jobs that build the 5 + # requested machines together, each derivation once. Outputs land in 6 + # cache.lassul.us via niks3. 7 + # 8 + # Needs the FARM_KEY secret: the farm's TLS private key, matching 9 + # tools/ci-farm/farm.crt (`pass show ci-farm/key`). 10 + 11 + on: 12 + workflow_dispatch: 13 + inputs: 14 + machines: 15 + description: NixOS machines to build, space separated 16 + default: ignavia neoprism green 17 + push: 18 + paths: 19 + - .github/workflows/farm.yml 20 + - tools/ci-farm/** 21 + 22 + permissions: 23 + contents: read 24 + id-token: write # niks3 push 25 + actions: read # workers watch the farm job 26 + 27 + # The farm's iroh endpoint id is fixed, so only one run at a time. 28 + concurrency: 29 + group: farm 30 + 31 + env: 32 + # must match the worker matrix below 33 + FARM_WORKERS: 2 34 + MACHINES: ${{ inputs.machines || 'ignavia neoprism green' }} 35 + 36 + jobs: 37 + farm: 38 + name: farm 39 + runs-on: ubuntu-latest 40 + steps: 41 + - uses: actions/checkout@v4 42 + - uses: cachix/install-nix-action@v31 43 + with: 44 + extra_nix_config: | 45 + trusted-users = root runner 46 + extra-substituters = https://cache.lassul.us 47 + extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0= 48 + # Builders need each other's outputs as soon as they are pushed; 49 + # a remembered miss (default: 1 h) fails the build instead. 50 + narinfo-cache-negative-ttl = 0 51 + - run: nix build -L .#ci-farm -o ci-farm 52 + - name: build ${{ env.MACHINES }} 53 + env: 54 + FARM_KEY: ${{ secrets.FARM_KEY }} 55 + # MACHINES unquoted: one argument per machine 56 + run: ./ci-farm/bin/ci-farm farm $MACHINES 57 + 58 + worker: 59 + name: worker ${{ matrix.index }} 60 + runs-on: ubuntu-latest 61 + strategy: 62 + matrix: 63 + index: [1, 2] # FARM_WORKERS 64 + steps: 65 + - uses: actions/checkout@v4 66 + - uses: cachix/install-nix-action@v31 67 + with: 68 + extra_nix_config: | 69 + trusted-users = root runner 70 + extra-substituters = https://cache.lassul.us 71 + extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0= 72 + # see the farm job 73 + narinfo-cache-negative-ttl = 0 74 + - run: nix build -L .#ci-farm -o ci-farm 75 + - env: 76 + FARM_KEY: ${{ secrets.FARM_KEY }} 77 + GH_TOKEN: ${{ github.token }} 78 + run: ./ci-farm/bin/ci-farm worker ${{ matrix.index }}
+97
configs/nix-farm/builder.nix
··· 1 + { 2 + config, 3 + lib, 4 + pkgs, 5 + self, 6 + ... 7 + }: 8 + let 9 + vars = config.clan.core.vars.generators; 10 + cachePub = self + "/vars/per-machine/neoprism/nix-cache-signing/key.pub/value"; 11 + credDir = "/run/credentials/nix-grpc-daemon.service"; 12 + in 13 + { 14 + imports = [ 15 + self.inputs.nix-grpc-store.nixosModules.server 16 + ./ca.nix 17 + ]; 18 + 19 + # Server cert for the daemon, signed by the shared farm CA, plus a copy of 20 + # the CA cert to verify clients against. Secret files for the same reason 21 + # as in ca.nix. Root-owned: the daemon gets them via LoadCredential below. 22 + clan.core.vars.generators.nix-farm-server = { 23 + dependencies = [ "nix-farm-ca" ]; 24 + files."server.key" = { }; 25 + files."server.crt" = { }; 26 + files."ca.crt" = { }; 27 + runtimeInputs = [ pkgs.openssl ]; 28 + script = '' 29 + openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ 30 + -keyout "$out/server.key" -out server.csr -subj "/CN=neoprism" 31 + printf '%s\n' \ 32 + "basicConstraints=critical,CA:FALSE" \ 33 + "keyUsage=critical,digitalSignature" \ 34 + "extendedKeyUsage=serverAuth" \ 35 + "subjectAltName=DNS:neoprism.r,DNS:neoprism.lassul.us" > server.ext 36 + openssl x509 -req -in server.csr \ 37 + -CA "$in/nix-farm-ca/ca.crt" -CAkey "$in/nix-farm-ca/ca.key" \ 38 + -CAcreateserial -days 3650 -extfile server.ext -out "$out/server.crt" 39 + cp "$in/nix-farm-ca/ca.crt" "$out/ca.crt" 40 + ''; 41 + }; 42 + 43 + # Standalone node: schedules onto itself, publishes outputs to niks3. 44 + services.nix-grpc-daemon = { 45 + enable = true; 46 + roles = [ 47 + "builder" 48 + "scheduler" 49 + ]; 50 + listen = "[::]:50051"; 51 + # gRPC's cert file watcher treats mtime 0 as a stat failure and never 52 + # loads the key pair, and clan unpacks /run/secrets with mtime 0. systemd 53 + # credentials are fresh copies with a real mtime. 54 + tls = { 55 + certFile = "${credDir}/server.crt"; 56 + keyFile = "${credDir}/server.key"; 57 + clientCaFile = "${credDir}/ca.crt"; 58 + }; 59 + # `nix build --store grpc://… --eval-store auto` needs write (README). 60 + accessRules = [ 61 + { 62 + cn = "lass"; 63 + role = "write"; 64 + } 65 + ]; 66 + niks3 = { 67 + package = self.inputs.niks3.packages.${pkgs.stdenv.hostPlatform.system}.niks3; 68 + # same box, skip nginx/ACME 69 + url = "http://${config.services.niks3.httpAddr}"; 70 + tokenFile = vars.niks3.files."api-token-grpc".path; 71 + cacheUrl = config.services.niks3.cacheUrl; 72 + # empty until `clan vars generate neoprism` wrote the key 73 + publicKeys = lib.optional (builtins.pathExists cachePub) ( 74 + lib.removeSuffix "\n" (builtins.readFile cachePub) 75 + ); 76 + # cache.nixos.org keeps serving what it signed; only push our own paths 77 + pushFlags = [ 78 + "--ignore-upstream-key" 79 + "cache.nixos.org-1" 80 + ]; 81 + }; 82 + }; 83 + 84 + systemd.services.nix-grpc-daemon.serviceConfig = { 85 + LoadCredential = map (f: "${f}:${vars.nix-farm-server.files.${f}.path}") [ 86 + "server.crt" 87 + "server.key" 88 + "ca.crt" 89 + ]; 90 + # Upstream's 2G default assumes the unit only proxies, but `niks3 push` 91 + # runs in this cgroup too and got OOM-killed on large NARs (kernel 92 + # modules), taking the whole daemon down with it. 93 + MemoryMax = "8G"; 94 + }; 95 + 96 + networking.firewall.interfaces.retiolum.allowedTCPPorts = [ 50051 ]; 97 + }
+35
configs/nix-farm/ca.nix
··· 1 + { pkgs, ... }: 2 + # Private CA of the build farm plus the client cert lass uses towards it. 3 + # Shared, so every machine importing this sees the same CA. Clan requires a 4 + # shared generator to be defined identically on all machines, so nothing is 5 + # deployed from here: per-machine generators (nix-farm-server in builder.nix, 6 + # nix-farm-client in client.nix) depend on it and copy out the files they 7 + # need with their own owner/mode. The certs are not secret, but as secret 8 + # files their paths are known before `clan vars generate` ran, so the configs 9 + # still evaluate until then. 10 + { 11 + clan.core.vars.generators.nix-farm-ca = { 12 + share = true; 13 + files."ca.key".deploy = false; 14 + files."ca.crt".deploy = false; 15 + files."client.key".deploy = false; 16 + files."client.crt".deploy = false; 17 + runtimeInputs = [ pkgs.openssl ]; 18 + script = '' 19 + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ 20 + -keyout "$out/ca.key" -out "$out/ca.crt" -days 3650 \ 21 + -subj "/CN=nix-farm-ca" \ 22 + -addext "basicConstraints=critical,CA:TRUE" \ 23 + -addext "keyUsage=critical,keyCertSign,cRLSign" 24 + 25 + openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ 26 + -keyout "$out/client.key" -out client.csr -subj "/CN=lass" 27 + printf '%s\n' \ 28 + "basicConstraints=critical,CA:FALSE" \ 29 + "keyUsage=critical,digitalSignature" \ 30 + "extendedKeyUsage=clientAuth" > client.ext 31 + openssl x509 -req -in client.csr -CA "$out/ca.crt" -CAkey "$out/ca.key" \ 32 + -CAcreateserial -days 3650 -extfile client.ext -out "$out/client.crt" 33 + ''; 34 + }; 35 + }
+123
configs/nix-farm/cache.nix
··· 1 + { 2 + config, 3 + pkgs, 4 + self, 5 + ... 6 + }: 7 + let 8 + # TODO: create this bucket (public) + an application key for it in B2 9 + bucket = "lassulus-nix-cache"; 10 + region = "eu-central-003"; 11 + endpoint = "s3.${region}.backblazeb2.com"; 12 + 13 + vars = config.clan.core.vars.generators; 14 + in 15 + { 16 + imports = [ self.inputs.niks3.nixosModules.niks3 ]; 17 + 18 + # B2 credentials (prompted), the niks3 API token (generated; one copy per 19 + # service user that reads it). 20 + clan.core.vars.generators.niks3 = { 21 + prompts.b2_key_id = { 22 + description = "Backblaze B2 keyID with access to the ${bucket} bucket"; 23 + type = "line"; 24 + }; 25 + prompts.b2_application_key = { 26 + description = "Backblaze B2 applicationKey for that keyID"; 27 + type = "hidden"; 28 + }; 29 + files."s3-access-key".owner = "niks3"; 30 + files."s3-secret-key".owner = "niks3"; 31 + files."api-token".owner = "niks3"; 32 + files."api-token-grpc".owner = "nix-grpc-daemon"; 33 + runtimeInputs = [ 34 + pkgs.coreutils 35 + pkgs.openssl 36 + ]; 37 + script = '' 38 + tr -d '\n' < "$prompts/b2_key_id" > "$out/s3-access-key" 39 + tr -d '\n' < "$prompts/b2_application_key" > "$out/s3-secret-key" 40 + openssl rand -hex 32 | tr -d '\n' > "$out/api-token" 41 + cp "$out/api-token" "$out/api-token-grpc" 42 + ''; 43 + }; 44 + 45 + # Public half is read by configs/default.nix from the vars dir. 46 + clan.core.vars.generators.nix-cache-signing = { 47 + files."key.sec".owner = "niks3"; 48 + files."key.pub".secret = false; 49 + runtimeInputs = [ pkgs.nix ]; 50 + script = '' 51 + nix --extra-experimental-features nix-command key generate-secret \ 52 + --key-name cache.lassul.us-1 > "$out/key.sec" 53 + nix --extra-experimental-features nix-command key convert-secret-to-public \ 54 + < "$out/key.sec" > "$out/key.pub" 55 + ''; 56 + }; 57 + 58 + services.niks3 = { 59 + enable = true; 60 + s3 = { 61 + inherit endpoint bucket region; 62 + accessKeyFile = vars.niks3.files."s3-access-key".path; 63 + secretKeyFile = vars.niks3.files."s3-secret-key".path; 64 + }; 65 + apiTokenFile = vars.niks3.files."api-token".path; 66 + signKeyFiles = [ vars.nix-cache-signing.files."key.sec".path ]; 67 + cacheUrl = "https://cache.lassul.us"; 68 + # The CI build farm (tools/ci-farm) pushes with the job's GitHub OIDC 69 + # token. Fork PRs don't get one. 70 + oidc.providers.github = { 71 + issuer = "https://token.actions.githubusercontent.com"; 72 + audience = "https://niks3.lassul.us"; 73 + boundClaims.repository = [ "lassulus/superconfig" ]; 74 + }; 75 + nginx = { 76 + enable = true; 77 + domain = "niks3.lassul.us"; 78 + }; 79 + }; 80 + 81 + # Reads straight from the public bucket. niks3 uploads the metadata 82 + # zstd-compressed with Content-Encoding: zstd; set it here so nix decodes 83 + # it whether or not the bucket hands the header back. Same rules as 84 + # clan-infra's Fastly config (modules/terranix/cache-new.nix): narinfo, .ls, 85 + # realisations/ and log/. NARs (nar/*.nar.zst) are plain objects, nix 86 + # decodes them via the narinfo's Compression field. 87 + # `resolve`: looked up at runtime via resolved's stub, so an unresolvable 88 + # B2 at startup doesn't keep nginx (and every other vhost) from starting. 89 + services.nginx.appendHttpConfig = '' 90 + upstream cache_lassul_us_b2 { 91 + zone cache_lassul_us_b2 64k; 92 + resolver 127.0.0.53 ipv6=off; 93 + server ${endpoint}:443 resolve; 94 + } 95 + map $uri $cache_lassul_us_encoding { 96 + default ""; 97 + ~\.(narinfo|ls)$ zstd; 98 + ~^/realisations/ zstd; 99 + ~^/log/ zstd; 100 + } 101 + ''; 102 + services.nginx.virtualHosts."cache.lassul.us" = { 103 + enableACME = true; 104 + forceSSL = true; 105 + locations."/".extraConfig = '' 106 + proxy_pass https://cache_lassul_us_b2/${bucket}/; 107 + proxy_set_header Host ${endpoint}; 108 + proxy_ssl_server_name on; 109 + proxy_ssl_name ${endpoint}; 110 + # cache keys never have a query string; don't forward one to S3 111 + set $args ""; 112 + # S3 answers 403 for keys missing from a bucket it won't list 113 + proxy_intercept_errors on; 114 + error_page 403 =404 @notfound; 115 + 116 + gzip off; 117 + proxy_hide_header Content-Encoding; 118 + # only sent on 2xx/3xx, and not at all when the map yields "" 119 + add_header Content-Encoding $cache_lassul_us_encoding; 120 + ''; 121 + locations."@notfound".return = "404"; 122 + }; 123 + }
+77
configs/nix-farm/client.nix
··· 1 + { 2 + config, 3 + lib, 4 + options, 5 + self, 6 + ... 7 + }: 8 + # Client of the neoprism build farm (configs/nix-farm), for NixOS and 9 + # nix-darwin. The plugin finds client.crt/client.key in 10 + # /var/lib/nix-grpc-store by itself; the CA has to be passed explicitly: 11 + # nix build --store 'grpc://neoprism.r:50051?ca-cert=/var/lib/nix-grpc-store/ca.crt' \ 12 + # --eval-store auto .#nixosConfigurations.<m>.config.system.build.toplevel 13 + let 14 + files = [ 15 + "ca.crt" 16 + "client.crt" 17 + "client.key" 18 + ]; 19 + client = config.clan.core.vars.generators.nix-farm-client.files; 20 + in 21 + { 22 + imports = [ 23 + # darwinModules.default is only the daemon; the grpc:// plugin is in the 24 + # client module, which only sets nix.settings.plugin-files and works on 25 + # nix-darwin as is. 26 + "${self.inputs.nix-grpc-store}/nixos/client.nix" 27 + ./ca.nix 28 + ]; 29 + 30 + config = lib.mkMerge [ 31 + { programs.nix-grpc-store.enable = true; } 32 + 33 + # Per-machine copy of the shared CA's client files (see ca.nix). 34 + { 35 + clan.core.vars.generators.nix-farm-client = { 36 + dependencies = [ "nix-farm-ca" ]; 37 + files = lib.genAttrs files (_: { }); 38 + script = lib.concatMapStrings (f: '' 39 + cp "$in/nix-farm-ca/${f}" "$out/${f}" 40 + '') files; 41 + }; 42 + } 43 + 44 + # Branch on options, not pkgs: the latter recurses through the attr names. 45 + (lib.optionalAttrs (options ? system.primaryUser) { 46 + # clan's /run/secrets installer is Linux-only, so these come in as 47 + # activation secrets (root-owned, below secretLocation) and are copied 48 + # to where the plugin looks by default, readable by the user running nix. 49 + clan.core.vars.generators.nix-farm-client.files = lib.genAttrs files (_: { 50 + deploy = true; 51 + neededFor = "activation"; 52 + }); 53 + # Skipped while the secrets have not been uploaded yet. 54 + system.activationScripts.postActivation.text = '' 55 + install -d -m 0755 /var/lib/nix-grpc-store 56 + '' 57 + + lib.concatMapStrings (f: '' 58 + if [ -e ${client.${f}.path} ]; then 59 + install -m 0400 -o ${config.system.primaryUser} ${client.${f}.path} /var/lib/nix-grpc-store/${f} 60 + fi 61 + '') files; 62 + }) 63 + 64 + (lib.optionalAttrs (options ? systemd.tmpfiles) { 65 + # Regular /run/secrets, readable by wheel (the users running nix) and 66 + # root (nix-daemon's build hook), linked to where the plugin looks. 67 + clan.core.vars.generators.nix-farm-client.files = lib.genAttrs files (_: { 68 + deploy = true; 69 + group = "wheel"; 70 + mode = "0440"; 71 + }); 72 + systemd.tmpfiles.rules = map ( 73 + f: "L+ /var/lib/nix-grpc-store/${f} - - - - ${client.${f}.path}" 74 + ) files; 75 + }) 76 + ]; 77 + }
+1
machines/barnacle/configuration.nix
··· 10 10 self.inputs.nix-index-database.darwinModules.nix-index 11 11 self.inputs.nix-rosetta-builder.darwinModules.default 12 12 ./retiolum.nix 13 + ../../configs/nix-farm/client.nix 13 14 ]; 14 15 clan.core.networking.targetHost = "root@localhost"; 15 16 programs.nix-index-database.comma.enable = true;
+3 -1
machines/green/config.nix
··· 14 14 15 15 ../../configs/pass.nix 16 16 17 - ../../configs/et-server.nix 17 + # disabled: eternal-terminal 7.0.0 fails to build in the current nixpkgs 18 + # (cmake_pch.hxx.gch error) 19 + # ../../configs/et-server.nix 18 20 19 21 ../../configs/atuin-server.nix 20 22 ../../configs/autoupdate.nix
+5
machines/neoprism/config.nix
··· 136 136 137 137 # nixpkgs PR triage dashboard (review.lassul.us) 138 138 ../../configs/nixpkgs-triage.nix 139 + 140 + # build farm (grpc://neoprism.r:50051) + binary cache (cache.lassul.us, 141 + # niks3.lassul.us) 142 + ../../configs/nix-farm/cache.nix 143 + ../../configs/nix-farm/builder.nix 139 144 ]; 140 145 141 146 # lassul.us shouldn't be the default vhost here (nginx.nix already sets one)
+192
tools/ci-farm/ci-farm.sh
··· 1 + # shellcheck shell=bash 2 + # (writeShellApplication adds the shebang and `set -euo pipefail`.) 3 + 4 + # On-demand nix-grpc-store farm across GitHub Actions runners. 5 + # 6 + # Runners can't reach each other, so every hop goes through an iroh tunnel 7 + # (dumbpipe) that ends on 127.0.0.1 on both sides: 8 + # 9 + # farm job worker job i 10 + # client ─► envoy :50051 daemon :5100 (builder) 11 + # ├─► daemon :50052 ◄─ iroh ─ :5101 scheduler session 12 + # └─► :511i ─────────── iroh ─► :5100 builds 13 + # daemon :50052 (scheduler + builder) 14 + # 15 + # Every daemon and envoy present the same TLS identity (FARM_KEY with 16 + # farm.crt, SAN 127.0.0.1) and accept only it, as `trusted`. The iroh 17 + # endpoint keys are derived from FARM_KEY, so it is the only secret. Outputs 18 + # go to niks3 with the job's GitHub OIDC token. 19 + 20 + state=/tmp/ci-farm 21 + niks3_url=https://niks3.lassul.us 22 + 23 + mkdir -p "$state" 24 + chmod 700 "$state" 25 + printf '%s\n' "${FARM_KEY:?FARM_KEY must hold the farm private key}" >"$state/farm.key" 26 + chmod 600 "$state/farm.key" 27 + unset FARM_KEY 28 + # Where envoy's config (flake-module.nix) expects it, next to the key. 29 + install -m 644 "$FARM_CRT" "$state/farm.crt" 30 + FARM_CRT=$state/farm.crt 31 + 32 + seed=$(sha256sum <"$state/farm.key" | cut -d' ' -f1) 33 + derive() { printf '%s:%s' "$seed" "$1" | sha256sum | cut -d' ' -f1; } 34 + ticket() { IROH_SECRET=$(derive "iroh-$1") dumbpipe generate-ticket; } 35 + 36 + trap 'kill $(jobs -p) 2>/dev/null || true' EXIT 37 + 38 + # The GitHub OIDC token of the job, refreshed in the background (niks3 re-reads 39 + # the file, the daemon notices the new inode). Outside Actions, point 40 + # NIKS3_TOKEN_FILE at a niks3 API token instead. 41 + niks3_token() { 42 + curl -fsS -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \ 43 + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" | 44 + jq -er .value >"$state/niks3-token.new" 45 + mv "$state/niks3-token.new" "$state/niks3-token" 46 + } 47 + if [ -n "${NIKS3_TOKEN_FILE:-}" ]; then 48 + ln -sf "$NIKS3_TOKEN_FILE" "$state/niks3-token" 49 + else 50 + niks3_token 51 + (while sleep 120; do niks3_token || echo "ci-farm: refreshing the niks3 token failed" >&2; done) & 52 + fi 53 + 54 + daemon=( 55 + nix-grpc-daemon 56 + --proxy-socket /nix/var/nix/daemon-socket/socket 57 + --tls-cert "$FARM_CRT" 58 + --tls-key "$state/farm.key" 59 + --client-ca "$FARM_CRT" 60 + --allow farm=trusted 61 + --niks3 "$niks3_url" 62 + --niks3-token-file "$state/niks3-token" 63 + # cache.nixos.org keeps serving what it signed; only push our own paths 64 + --niks3-push "niks3 push --ignore-upstream-key cache.nixos.org-1" 65 + --min-free 2G 66 + ) 67 + 68 + wait_http() { 69 + for _ in $(seq 60); do 70 + curl -fs -o /dev/null "$1" && return 71 + sleep 1 72 + done 73 + echo "ci-farm: $1 did not come up" >&2 74 + exit 1 75 + } 76 + 77 + # Workers appear in envoy's clusters once their session with the scheduler is 78 + # up. Their tunnels resolve the farm through iroh discovery, which takes a 79 + # while after the farm comes online, so give them time before handing out 80 + # work. Builds start anyway after the timeout, with whoever has joined. 81 + wait_workers() { 82 + local want=$1 have=0 83 + for _ in $(seq 60); do 84 + have=$(curl -fs http://127.0.0.1:9901/clusters | 85 + grep -cE '::127\.0\.0\.1:511[0-9]+::health_flags::healthy$' || true) 86 + [ "$have" -ge "$want" ] && break 87 + sleep 5 88 + done 89 + echo "ci-farm: $have of $want workers joined" 90 + } 91 + 92 + # ci-farm farm ARG...: ARG is a NixOS machine of this flake (its toplevel is 93 + # built) or a /nix/store/….drv. 94 + farm() { 95 + local workers=${FARM_WORKERS:?FARM_WORKERS: number of worker jobs} 96 + IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 & 97 + for i in $(seq "$workers"); do 98 + IROH_SECRET=$(derive "iroh-farm-dial-$i") \ 99 + dumbpipe connect-tcp --addr "127.0.0.1:$((5110 + i))" "$(ticket "worker-$i")" & 100 + done 101 + "${daemon[@]}" --role builder,scheduler --listen 127.0.0.1:50052 \ 102 + --advertise 127.0.0.1:50052 --worker-name farm & 103 + envoy -c "$ENVOY_CONFIG" --log-level warn & 104 + wait_http http://127.0.0.1:9901/ready 105 + wait_workers "$workers" 106 + 107 + local store="grpc://127.0.0.1:50051?insecure=1&system=x86_64-linux" 108 + declare -A builds 109 + for arg in "$@"; do 110 + if [[ $arg == /nix/store/*.drv ]]; then 111 + drv=$arg 112 + else 113 + # One eval process per machine: all hosts at once don't fit in 16 GB. 114 + drv=$(nix eval --raw ".#nixosConfigurations.$arg.config.system.build.toplevel.drvPath") 115 + fi 116 + echo "ci-farm: $arg: $drv" 117 + "$NIX_WITH_PLUGIN" build --no-link --print-out-paths --store "$store" --eval-store auto "$drv^*" 2>&1 | 118 + sed -u "s|^|[$arg] |" & 119 + builds[$arg]=$! 120 + done 121 + 122 + local failed=() 123 + for arg in "${!builds[@]}"; do 124 + wait "${builds[$arg]}" || failed+=("$arg") 125 + done 126 + if [ ${#failed[@]} -gt 0 ]; then 127 + echo "ci-farm: failed: ${failed[*]}" >&2 128 + exit 1 129 + fi 130 + } 131 + 132 + # Outside Actions there is no farm job to watch: run until killed. 133 + farm_done() { 134 + [ -n "${GITHUB_RUN_ID:-}" ] && 135 + [ "$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs" \ 136 + --jq '.jobs[] | select(.name == "farm") | .status')" = completed ] 137 + } 138 + 139 + # A dumbpipe dialer started before the farm is discoverable never reaches it, 140 + # not even once the farm is online. Probe with short-lived dialers until the 141 + # farm daemon answers a gRPC health check, then start the long-lived one. 142 + wait_farm() { 143 + for _ in $(seq 180); do 144 + IROH_SECRET=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n') \ 145 + dumbpipe connect-tcp --addr 127.0.0.1:5102 "$1" 2>/dev/null & 146 + local probe=$! 147 + sleep 3 148 + # Empty grpc.health.v1.HealthCheckRequest: 0 = uncompressed, 0 bytes. 149 + if printf '\0\0\0\0\0' | curl -fs -o /dev/null --max-time 10 --http2 \ 150 + -H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \ 151 + --cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \ 152 + https://127.0.0.1:5102/grpc.health.v1.Health/Check; then 153 + kill "$probe" 154 + return 155 + fi 156 + kill "$probe" 157 + sleep 5 158 + done 159 + echo "ci-farm: the farm did not come up" >&2 160 + exit 1 161 + } 162 + 163 + worker() { 164 + local i=${1:?worker index} farm 165 + farm=$(ticket farm) 166 + IROH_SECRET=$(derive "iroh-worker-$i") dumbpipe listen-tcp --host 127.0.0.1:5100 & 167 + wait_farm "$farm" 168 + echo "ci-farm: the farm is up" 169 + IROH_SECRET=$(derive "iroh-worker-dial-$i") \ 170 + dumbpipe connect-tcp --addr 127.0.0.1:5101 "$farm" & 171 + "${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \ 172 + --advertise "127.0.0.1:$((5110 + i))" --worker-name "worker-$i" & 173 + local pid=$! 174 + until farm_done; do 175 + kill -0 "$pid" || exit 1 176 + sleep 15 177 + done 178 + # Drains: running builds finish and publish first. 179 + kill -TERM "$pid" 180 + wait "$pid" || true 181 + } 182 + 183 + cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm worker INDEX} 184 + shift 185 + case $cmd in 186 + farm) farm "$@" ;; 187 + worker) worker "$@" ;; 188 + *) 189 + echo "ci-farm: unknown command $cmd" >&2 190 + exit 1 191 + ;; 192 + esac
+12
tools/ci-farm/farm.crt
··· 1 + -----BEGIN CERTIFICATE----- 2 + MIIBtTCCAVugAwIBAgIUf55GJik0D9Ps1ijr4MNBQhxPH30wCgYIKoZIzj0EAwIw 3 + DzENMAsGA1UEAwwEZmFybTAeFw0yNjEwMTExNzA4MTNaFw0zNjEwMDgxNzA4MTNa 4 + MA8xDTALBgNVBAMMBGZhcm0wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAStbhFn 5 + CJvDygTYQUGc3Ck18WSNS2A5DhGvL0bT3BHpOpNTUSeZW+RWW3QemIHFkQjEfhYS 6 + oTKieGTlWgtQIJepo4GUMIGRMB0GA1UdDgQWBBQ/TD12Xjs5LPh2YLGeL8CDhKut 7 + LzAfBgNVHSMEGDAWgBQ/TD12Xjs5LPh2YLGeL8CDhKutLzAPBgNVHREECDAGhwR/ 8 + AAABMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQWMBQG 9 + CCsGAQUFBwMBBggrBgEFBQcDAjAKBggqhkjOPQQDAgNIADBFAiAvWiuqtBuhoDO1 10 + 6mlRKmqkwStUqhPDSTaQvaKOTAUVBgIhAMGDqzX7JCaZhEwdXwg9HfYGDQFqzseY 11 + 2o0he+UhHyTX 12 + -----END CERTIFICATE-----
+69
tools/ci-farm/flake-module.nix
··· 1 + { inputs, ... }: 2 + { 3 + perSystem = 4 + { 5 + lib, 6 + pkgs, 7 + system, 8 + ... 9 + }: 10 + let 11 + grpc = inputs.nix-grpc-store.packages.${system}; 12 + 13 + # Envoy as configured by nix-grpc-store's NixOS module, rendered for a 14 + # plain Linux runner. Plaintext towards the local client, the farm 15 + # identity towards the nodes (they verify it and grant it `trusted`, so 16 + # it can read the builder list from the scheduler). 17 + envoyConfig = 18 + (pkgs.formats.json { }).generate "ci-farm-envoy.json" 19 + (inputs.nixpkgs.lib.nixosSystem { 20 + modules = [ 21 + inputs.nix-grpc-store.nixosModules.lb 22 + { 23 + nixpkgs.pkgs = pkgs; 24 + services.nix-grpc-farm-lb = { 25 + enable = true; 26 + listen = "127.0.0.1:50051"; 27 + scheduler = "127.0.0.1:50052"; 28 + systems = [ system ]; 29 + # Written by ci-farm.sh. Envoy watches the directories of 30 + # these files, so they must exist on the runner. 31 + tls.upstream = { 32 + certFile = "/tmp/ci-farm/farm.crt"; 33 + keyFile = "/tmp/ci-farm/farm.key"; 34 + caFile = "/tmp/ci-farm/farm.crt"; 35 + }; 36 + }; 37 + } 38 + ]; 39 + }).config.services.envoy.settings; 40 + in 41 + { 42 + # ci-farm farm MACHINE... scheduler + builder + balancer, builds the 43 + # toplevels of MACHINE... through the farm 44 + # ci-farm worker INDEX builder, until the farm job has finished 45 + # 46 + # Runs in .github/workflows/farm.yml. See ci-farm.sh for the layout. 47 + packages = lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux { 48 + ci-farm = pkgs.writeShellApplication { 49 + name = "ci-farm"; 50 + runtimeInputs = [ 51 + grpc.default 52 + inputs.niks3.packages.${system}.niks3 53 + pkgs.coreutils 54 + pkgs.curl 55 + pkgs.dumbpipe 56 + pkgs.envoy-bin 57 + pkgs.gh 58 + pkgs.jq 59 + ]; 60 + runtimeEnv = { 61 + FARM_CRT = ./farm.crt; 62 + ENVOY_CONFIG = envoyConfig; 63 + NIX_WITH_PLUGIN = lib.getExe grpc.nix-with-plugin; 64 + }; 65 + text = builtins.readFile ./ci-farm.sh; 66 + }; 67 + }; 68 + }; 69 + }
+1
vars/per-machine/neoprism/nix-cache-signing/key.pub/value
··· 1 + cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=