This repository has no description
tools.ci-farm: add on-demand nix-grpc-store farm on GitHub Actions
A farm job (scheduler, builder, envoy, client) and a pool of worker jobs
build the requested machines together, each derivation once, and push
the outputs to niks3 with the job's GitHub OIDC token.
Runners can't reach each other, so every hop is a dumbpipe (iroh)
tunnel ending on 127.0.0.1. All daemons and envoy share one TLS
identity (FARM_KEY secret, farm.crt committed, SAN 127.0.0.1) and only
trust it. The iroh endpoint keys are derived from FARM_KEY, so it is the
only secret. Workers probe the farm with fresh dialers before starting
their long-lived one: a dumbpipe dialer started before the farm is
discoverable never reaches it.
configs.nix-farm: niks3 accepts GitHub OIDC tokens of this repository.
author
lassulus
date
(Oct 11, 2026, 7:48 PM CEST)
commit
422ce325
422ce3251eec1e508ca9d4ec193198efe18770e8
parent
fb5c4fa7
fb5c4fa7b476f4fcb6e55258c8905d27b62c4c69
+351
+73
.github/workflows/farm.yml
+73
.github/workflows/farm.yml
···
1
+
name: farm
2
+
3
+
# On-demand nix-grpc-store farm (tools/ci-farm): one farm job (scheduler,
4
+
# balancer, client, builder) and a pool of worker jobs that build the
5
+
# requested machines together, each derivation once. Outputs land in
6
+
# cache.lassul.us via niks3.
7
+
#
8
+
# Needs the FARM_KEY secret: the farm's TLS private key, matching
9
+
# tools/ci-farm/farm.crt (`pass show ci-farm/key`).
10
+
11
+
on:
12
+
workflow_dispatch:
13
+
inputs:
14
+
machines:
15
+
description: NixOS machines to build, space separated
16
+
default: ignavia neoprism green
17
+
push:
18
+
paths:
19
+
- .github/workflows/farm.yml
20
+
- tools/ci-farm/**
21
+
22
+
permissions:
23
+
contents: read
24
+
id-token: write # niks3 push
25
+
actions: read # workers watch the farm job
26
+
27
+
# The farm's iroh endpoint id is fixed, so only one run at a time.
28
+
concurrency:
29
+
group: farm
30
+
31
+
env:
32
+
# must match the worker matrix below
33
+
FARM_WORKERS: 2
34
+
MACHINES: ${{ inputs.machines || 'ignavia neoprism green' }}
35
+
36
+
jobs:
37
+
farm:
38
+
name: farm
39
+
runs-on: ubuntu-latest
40
+
steps:
41
+
- uses: actions/checkout@v4
42
+
- uses: cachix/install-nix-action@v31
43
+
with:
44
+
extra_nix_config: |
45
+
trusted-users = root runner
46
+
extra-substituters = https://cache.lassul.us
47
+
extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
48
+
- run: nix build -L .#ci-farm -o ci-farm
49
+
- name: build ${{ env.MACHINES }}
50
+
env:
51
+
FARM_KEY: ${{ secrets.FARM_KEY }}
52
+
# MACHINES unquoted: one argument per machine
53
+
run: ./ci-farm/bin/ci-farm farm $MACHINES
54
+
55
+
worker:
56
+
name: worker ${{ matrix.index }}
57
+
runs-on: ubuntu-latest
58
+
strategy:
59
+
matrix:
60
+
index: [1, 2] # FARM_WORKERS
61
+
steps:
62
+
- uses: actions/checkout@v4
63
+
- uses: cachix/install-nix-action@v31
64
+
with:
65
+
extra_nix_config: |
66
+
trusted-users = root runner
67
+
extra-substituters = https://cache.lassul.us
68
+
extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
69
+
- run: nix build -L .#ci-farm -o ci-farm
70
+
- env:
71
+
FARM_KEY: ${{ secrets.FARM_KEY }}
72
+
GH_TOKEN: ${{ github.token }}
73
+
run: ./ci-farm/bin/ci-farm worker ${{ matrix.index }}
+7
configs/nix-farm/cache.nix
+7
configs/nix-farm/cache.nix
···
65
65
apiTokenFile = vars.niks3.files."api-token".path;
66
66
signKeyFiles = [ vars.nix-cache-signing.files."key.sec".path ];
67
67
cacheUrl = "https://cache.lassul.us";
68
+
# The CI build farm (tools/ci-farm) pushes with the job's GitHub OIDC
69
+
# token. Fork PRs don't get one.
70
+
oidc.providers.github = {
71
+
issuer = "https://token.actions.githubusercontent.com";
72
+
audience = "https://niks3.lassul.us";
73
+
boundClaims.repository = [ "lassulus/superconfig" ];
74
+
};
68
75
nginx = {
69
76
enable = true;
70
77
domain = "niks3.lassul.us";
+190
tools/ci-farm/ci-farm.sh
+190
tools/ci-farm/ci-farm.sh
···
1
+
# shellcheck shell=bash
2
+
# (writeShellApplication adds the shebang and `set -euo pipefail`.)
3
+
4
+
# On-demand nix-grpc-store farm across GitHub Actions runners.
5
+
#
6
+
# Runners can't reach each other, so every hop goes through an iroh tunnel
7
+
# (dumbpipe) that ends on 127.0.0.1 on both sides:
8
+
#
9
+
# farm job worker job i
10
+
# client ─► envoy :50051 daemon :5100 (builder)
11
+
# ├─► daemon :50052 ◄─ iroh ─ :5101 scheduler session
12
+
# └─► :511i ─────────── iroh ─► :5100 builds
13
+
# daemon :50052 (scheduler + builder)
14
+
#
15
+
# Every daemon and envoy present the same TLS identity (FARM_KEY with
16
+
# farm.crt, SAN 127.0.0.1) and accept only it, as `trusted`. The iroh
17
+
# endpoint keys are derived from FARM_KEY, so it is the only secret. Outputs
18
+
# go to niks3 with the job's GitHub OIDC token.
19
+
20
+
state=/tmp/ci-farm
21
+
niks3_url=https://niks3.lassul.us
22
+
23
+
mkdir -p "$state"
24
+
chmod 700 "$state"
25
+
printf '%s\n' "${FARM_KEY:?FARM_KEY must hold the farm private key}" >"$state/farm.key"
26
+
chmod 600 "$state/farm.key"
27
+
unset FARM_KEY
28
+
# Where envoy's config (flake-module.nix) expects it, next to the key.
29
+
install -m 644 "$FARM_CRT" "$state/farm.crt"
30
+
FARM_CRT=$state/farm.crt
31
+
32
+
seed=$(sha256sum <"$state/farm.key" | cut -d' ' -f1)
33
+
derive() { printf '%s:%s' "$seed" "$1" | sha256sum | cut -d' ' -f1; }
34
+
ticket() { IROH_SECRET=$(derive "iroh-$1") dumbpipe generate-ticket; }
35
+
36
+
trap 'kill $(jobs -p) 2>/dev/null || true' EXIT
37
+
38
+
# The GitHub OIDC token of the job, refreshed in the background (niks3 re-reads
39
+
# the file, the daemon notices the new inode). Outside Actions, point
40
+
# NIKS3_TOKEN_FILE at a niks3 API token instead.
41
+
niks3_token() {
42
+
curl -fsS -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \
43
+
"$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" |
44
+
jq -er .value >"$state/niks3-token.new"
45
+
mv "$state/niks3-token.new" "$state/niks3-token"
46
+
}
47
+
if [ -n "${NIKS3_TOKEN_FILE:-}" ]; then
48
+
ln -sf "$NIKS3_TOKEN_FILE" "$state/niks3-token"
49
+
else
50
+
niks3_token
51
+
(while sleep 120; do niks3_token || echo "ci-farm: refreshing the niks3 token failed" >&2; done) &
52
+
fi
53
+
54
+
daemon=(
55
+
nix-grpc-daemon
56
+
--proxy-socket /nix/var/nix/daemon-socket/socket
57
+
--tls-cert "$FARM_CRT"
58
+
--tls-key "$state/farm.key"
59
+
--client-ca "$FARM_CRT"
60
+
--allow farm=trusted
61
+
--niks3 "$niks3_url"
62
+
--niks3-token-file "$state/niks3-token"
63
+
--min-free 2G
64
+
)
65
+
66
+
wait_http() {
67
+
for _ in $(seq 60); do
68
+
curl -fs -o /dev/null "$1" && return
69
+
sleep 1
70
+
done
71
+
echo "ci-farm: $1 did not come up" >&2
72
+
exit 1
73
+
}
74
+
75
+
# Workers appear in envoy's clusters once their session with the scheduler is
76
+
# up. Their tunnels resolve the farm through iroh discovery, which takes a
77
+
# while after the farm comes online, so give them time before handing out
78
+
# work. Builds start anyway after the timeout, with whoever has joined.
79
+
wait_workers() {
80
+
local want=$1 have=0
81
+
for _ in $(seq 60); do
82
+
have=$(curl -fs http://127.0.0.1:9901/clusters |
83
+
grep -cE '::127\.0\.0\.1:511[0-9]+::health_flags::healthy$' || true)
84
+
[ "$have" -ge "$want" ] && break
85
+
sleep 5
86
+
done
87
+
echo "ci-farm: $have of $want workers joined"
88
+
}
89
+
90
+
# ci-farm farm ARG...: ARG is a NixOS machine of this flake (its toplevel is
91
+
# built) or a /nix/store/….drv.
92
+
farm() {
93
+
local workers=${FARM_WORKERS:?FARM_WORKERS: number of worker jobs}
94
+
IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 &
95
+
for i in $(seq "$workers"); do
96
+
IROH_SECRET=$(derive "iroh-farm-dial-$i") \
97
+
dumbpipe connect-tcp --addr "127.0.0.1:$((5110 + i))" "$(ticket "worker-$i")" &
98
+
done
99
+
"${daemon[@]}" --role builder,scheduler --listen 127.0.0.1:50052 \
100
+
--advertise 127.0.0.1:50052 --worker-name farm &
101
+
envoy -c "$ENVOY_CONFIG" --log-level warn &
102
+
wait_http http://127.0.0.1:9901/ready
103
+
wait_workers "$workers"
104
+
105
+
local store="grpc://127.0.0.1:50051?insecure=1&system=x86_64-linux"
106
+
declare -A builds
107
+
for arg in "$@"; do
108
+
if [[ $arg == /nix/store/*.drv ]]; then
109
+
drv=$arg
110
+
else
111
+
# One eval process per machine: all hosts at once don't fit in 16 GB.
112
+
drv=$(nix eval --raw ".#nixosConfigurations.$arg.config.system.build.toplevel.drvPath")
113
+
fi
114
+
echo "ci-farm: $arg: $drv"
115
+
"$NIX_WITH_PLUGIN" build --no-link --print-out-paths --store "$store" --eval-store auto "$drv^*" 2>&1 |
116
+
sed -u "s|^|[$arg] |" &
117
+
builds[$arg]=$!
118
+
done
119
+
120
+
local failed=()
121
+
for arg in "${!builds[@]}"; do
122
+
wait "${builds[$arg]}" || failed+=("$arg")
123
+
done
124
+
if [ ${#failed[@]} -gt 0 ]; then
125
+
echo "ci-farm: failed: ${failed[*]}" >&2
126
+
exit 1
127
+
fi
128
+
}
129
+
130
+
# Outside Actions there is no farm job to watch: run until killed.
131
+
farm_done() {
132
+
[ -n "${GITHUB_RUN_ID:-}" ] &&
133
+
[ "$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs" \
134
+
--jq '.jobs[] | select(.name == "farm") | .status')" = completed ]
135
+
}
136
+
137
+
# A dumbpipe dialer started before the farm is discoverable never reaches it,
138
+
# not even once the farm is online. Probe with short-lived dialers until the
139
+
# farm daemon answers a gRPC health check, then start the long-lived one.
140
+
wait_farm() {
141
+
for _ in $(seq 180); do
142
+
IROH_SECRET=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n') \
143
+
dumbpipe connect-tcp --addr 127.0.0.1:5102 "$1" 2>/dev/null &
144
+
local probe=$!
145
+
sleep 3
146
+
# Empty grpc.health.v1.HealthCheckRequest: 0 = uncompressed, 0 bytes.
147
+
if printf '\0\0\0\0\0' | curl -fs -o /dev/null --max-time 10 --http2 \
148
+
-H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
149
+
--cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
150
+
https://127.0.0.1:5102/grpc.health.v1.Health/Check; then
151
+
kill "$probe"
152
+
return
153
+
fi
154
+
kill "$probe"
155
+
sleep 5
156
+
done
157
+
echo "ci-farm: the farm did not come up" >&2
158
+
exit 1
159
+
}
160
+
161
+
worker() {
162
+
local i=${1:?worker index} farm
163
+
farm=$(ticket farm)
164
+
IROH_SECRET=$(derive "iroh-worker-$i") dumbpipe listen-tcp --host 127.0.0.1:5100 &
165
+
wait_farm "$farm"
166
+
echo "ci-farm: the farm is up"
167
+
IROH_SECRET=$(derive "iroh-worker-dial-$i") \
168
+
dumbpipe connect-tcp --addr 127.0.0.1:5101 "$farm" &
169
+
"${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \
170
+
--advertise "127.0.0.1:$((5110 + i))" --worker-name "worker-$i" &
171
+
local pid=$!
172
+
until farm_done; do
173
+
kill -0 "$pid" || exit 1
174
+
sleep 15
175
+
done
176
+
# Drains: running builds finish and publish first.
177
+
kill -TERM "$pid"
178
+
wait "$pid" || true
179
+
}
180
+
181
+
cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm worker INDEX}
182
+
shift
183
+
case $cmd in
184
+
farm) farm "$@" ;;
185
+
worker) worker "$@" ;;
186
+
*)
187
+
echo "ci-farm: unknown command $cmd" >&2
188
+
exit 1
189
+
;;
190
+
esac
+12
tools/ci-farm/farm.crt
+12
tools/ci-farm/farm.crt
···
1
+
-----BEGIN CERTIFICATE-----
2
+
MIIBtTCCAVugAwIBAgIUf55GJik0D9Ps1ijr4MNBQhxPH30wCgYIKoZIzj0EAwIw
3
+
DzENMAsGA1UEAwwEZmFybTAeFw0yNjEwMTExNzA4MTNaFw0zNjEwMDgxNzA4MTNa
4
+
MA8xDTALBgNVBAMMBGZhcm0wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAStbhFn
5
+
CJvDygTYQUGc3Ck18WSNS2A5DhGvL0bT3BHpOpNTUSeZW+RWW3QemIHFkQjEfhYS
6
+
oTKieGTlWgtQIJepo4GUMIGRMB0GA1UdDgQWBBQ/TD12Xjs5LPh2YLGeL8CDhKut
7
+
LzAfBgNVHSMEGDAWgBQ/TD12Xjs5LPh2YLGeL8CDhKutLzAPBgNVHREECDAGhwR/
8
+
AAABMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQWMBQG
9
+
CCsGAQUFBwMBBggrBgEFBQcDAjAKBggqhkjOPQQDAgNIADBFAiAvWiuqtBuhoDO1
10
+
6mlRKmqkwStUqhPDSTaQvaKOTAUVBgIhAMGDqzX7JCaZhEwdXwg9HfYGDQFqzseY
11
+
2o0he+UhHyTX
12
+
-----END CERTIFICATE-----
+69
tools/ci-farm/flake-module.nix
+69
tools/ci-farm/flake-module.nix
···
1
+
{ inputs, ... }:
2
+
{
3
+
perSystem =
4
+
{
5
+
lib,
6
+
pkgs,
7
+
system,
8
+
...
9
+
}:
10
+
let
11
+
grpc = inputs.nix-grpc-store.packages.${system};
12
+
13
+
# Envoy as configured by nix-grpc-store's NixOS module, rendered for a
14
+
# plain Linux runner. Plaintext towards the local client, the farm
15
+
# identity towards the nodes (they verify it and grant it `trusted`, so
16
+
# it can read the builder list from the scheduler).
17
+
envoyConfig =
18
+
(pkgs.formats.json { }).generate "ci-farm-envoy.json"
19
+
(inputs.nixpkgs.lib.nixosSystem {
20
+
modules = [
21
+
inputs.nix-grpc-store.nixosModules.lb
22
+
{
23
+
nixpkgs.pkgs = pkgs;
24
+
services.nix-grpc-farm-lb = {
25
+
enable = true;
26
+
listen = "127.0.0.1:50051";
27
+
scheduler = "127.0.0.1:50052";
28
+
systems = [ system ];
29
+
# Written by ci-farm.sh. Envoy watches the directories of
30
+
# these files, so they must exist on the runner.
31
+
tls.upstream = {
32
+
certFile = "/tmp/ci-farm/farm.crt";
33
+
keyFile = "/tmp/ci-farm/farm.key";
34
+
caFile = "/tmp/ci-farm/farm.crt";
35
+
};
36
+
};
37
+
}
38
+
];
39
+
}).config.services.envoy.settings;
40
+
in
41
+
{
42
+
# ci-farm farm MACHINE... scheduler + builder + balancer, builds the
43
+
# toplevels of MACHINE... through the farm
44
+
# ci-farm worker INDEX builder, until the farm job has finished
45
+
#
46
+
# Runs in .github/workflows/farm.yml. See ci-farm.sh for the layout.
47
+
packages = lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {
48
+
ci-farm = pkgs.writeShellApplication {
49
+
name = "ci-farm";
50
+
runtimeInputs = [
51
+
grpc.default
52
+
inputs.niks3.packages.${system}.niks3
53
+
pkgs.coreutils
54
+
pkgs.curl
55
+
pkgs.dumbpipe
56
+
pkgs.envoy-bin
57
+
pkgs.gh
58
+
pkgs.jq
59
+
];
60
+
runtimeEnv = {
61
+
FARM_CRT = ./farm.crt;
62
+
ENVOY_CONFIG = envoyConfig;
63
+
NIX_WITH_PLUGIN = lib.getExe grpc.nix-with-plugin;
64
+
};
65
+
text = builtins.readFile ./ci-farm.sh;
66
+
};
67
+
};
68
+
};
69
+
}