This repository has no description
11 kB
281 lines
1# shellcheck shell=bash
2# (writeShellApplication adds the shebang and `set -euo pipefail`.)
3
4# On-demand nix-grpc-store farm across GitHub Actions runners.
5#
6# Runners can't reach each other, so every hop goes through an iroh tunnel
7# (dumbpipe) that ends on 127.0.0.1 on both sides:
8#
9# farm job machine job i
10# envoy :50051 ◄─────────────── iroh ─ :5103 ◄─ client (machine i)
11# ├─► daemon :50052 ◄──────── iroh ─ :5101 scheduler session
12# └─► :511i ───────────────── iroh ─► :5100 daemon (builder)
13# daemon :50052 (scheduler)
14#
15# Every daemon and envoy present the same TLS identity (FARM_KEY with
16# farm.crt, SAN 127.0.0.1) and accept only it, as `trusted`. The iroh
17# endpoint keys are derived from FARM_KEY, so it is the only secret. Outputs
18# go to niks3 with the job's GitHub OIDC token, before a build counts as
19# done, so a machine job can leave once its own machine is built.
20
21state=/tmp/ci-farm
22niks3_url=https://niks3.lassul.us
23
24mkdir -p "$state"
25chmod 700 "$state"
26printf '%s\n' "${FARM_KEY:?FARM_KEY must hold the farm private key}" >"$state/farm.key"
27chmod 600 "$state/farm.key"
28unset FARM_KEY
29# Where envoy's config (flake-module.nix) expects it, next to the key.
30install -m 644 "$FARM_CRT" "$state/farm.crt"
31FARM_CRT=$state/farm.crt
32
33seed=$(sha256sum <"$state/farm.key" | cut -d' ' -f1)
34derive() { printf '%s:%s' "$seed" "$1" | sha256sum | cut -d' ' -f1; }
35ticket() { IROH_SECRET=$(derive "iroh-$1") dumbpipe generate-ticket; }
36
37trap 'kill $(jobs -p) 2>/dev/null || true' EXIT
38
39# The GitHub OIDC token of the job, refreshed in the background (niks3 re-reads
40# the file, the daemon notices the new inode). Outside Actions, point
41# NIKS3_TOKEN_FILE at a niks3 API token instead. A failed fetch keeps the
42# current token: `set -e` doesn't apply in `niks3_token || ...`, so every step
43# returns explicitly.
44niks3_token() {
45 local token
46 token=$(curl -fsS --retry 4 --retry-all-errors \
47 -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \
48 "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" | jq -er .value) || return 1
49 [ -n "$token" ] || return 1
50 printf '%s\n' "$token" >"$state/niks3-token.new" &&
51 mv "$state/niks3-token.new" "$state/niks3-token"
52}
53# Refreshes every 2 minutes, every 20 seconds after a failure. The token is
54# valid for 5 minutes: once it is 4 minutes old a builder (machine writes
55# builder.pid) could not publish its outputs anymore, so drain it, and the
56# scheduler hands its builds to the others.
57refresh_niks3_token() {
58 local fetched=$SECONDS wait=120
59 while sleep "$wait"; do
60 if niks3_token; then
61 fetched=$SECONDS wait=120
62 continue
63 fi
64 wait=20
65 echo "ci-farm: refreshing the niks3 token failed, it is $((SECONDS - fetched))s old" >&2
66 if [ $((SECONDS - fetched)) -ge 240 ] && [ -s "$state/builder.pid" ]; then
67 echo "ci-farm: no fresh niks3 token, draining the builder" >&2
68 kill -TERM "$(<"$state/builder.pid")" 2>/dev/null || true
69 rm -f "$state/builder.pid"
70 fi
71 done
72}
73if [ -n "${NIKS3_TOKEN_FILE:-}" ]; then
74 ln -sf "$NIKS3_TOKEN_FILE" "$state/niks3-token"
75else
76 niks3_token
77 refresh_niks3_token &
78fi
79
80daemon=(
81 nix-grpc-daemon
82 --proxy-socket /nix/var/nix/daemon-socket/socket
83 --tls-cert "$FARM_CRT"
84 --tls-key "$state/farm.key"
85 --client-ca "$FARM_CRT"
86 --allow farm=trusted
87 --niks3 "$niks3_url"
88 --niks3-token-file "$state/niks3-token"
89 # cache.nixos.org keeps serving what it signed; only push our own paths
90 --niks3-push "niks3 push --ignore-upstream-key cache.nixos.org-1"
91 --min-free 2G
92)
93
94wait_http() {
95 for _ in $(seq 60); do
96 curl -fs -o /dev/null "$1" && return
97 sleep 1
98 done
99 echo "ci-farm: $1 did not come up" >&2
100 exit 1
101}
102
103# The jobs NAME... of this run have all finished. In a workflow called from
104# another (ci.yml) the API names them "<caller job> / NAME". Outside Actions
105# there are no jobs to watch: never.
106jobs_done() {
107 local jobs name
108 [ -n "${GITHUB_RUN_ID:-}" ] || return 1
109 jobs=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs?per_page=100" \
110 --jq '.jobs[] | "\(.status) \(.name | sub("^.* / "; ""))"') || return 1
111 for name in "$@"; do
112 grep -qxF "completed $name" <<<"$jobs" || return 1
113 done
114}
115
116# ci-farm farm MACHINE...: scheduler and balancer for the machine jobs
117# (builder i builds the i-th MACHINE), until they have all finished.
118farm() {
119 local i port clusters
120 "${daemon[@]}" --role scheduler --listen 127.0.0.1:50052 2>"$state/daemon.log" &
121 local pid=$!
122 tail -f "$state/daemon.log" >&2 &
123 envoy -c "$ENVOY_CONFIG" --log-level warn &
124 wait_http http://127.0.0.1:9901/ready
125 # A daemon without the builder role schedules only once niks3 has made it
126 # the leader.
127 for _ in $(seq 60); do
128 grep -q event=scheduler_take_over "$state/daemon.log" && break
129 sleep 2
130 done
131 if ! grep -q event=scheduler_take_over "$state/daemon.log"; then
132 echo "ci-farm: the scheduler did not get the lead from niks3" >&2
133 exit 1
134 fi
135 IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 &
136 IROH_SECRET=$(derive iroh-lb) dumbpipe listen-tcp --host 127.0.0.1:50051 &
137 echo "ci-farm: the farm is up"
138
139 # Builder i shows up in envoy's clusters once its scheduler session is up,
140 # i.e. its own iroh endpoint is online. Only then dial it (a dialer started
141 # before the endpoint is discoverable never reaches it), and dial again
142 # when the tunnel dies or envoy's health checks fail through it for a
143 # minute.
144 declare -A dialers=() since=()
145 until jobs_done "$@"; do
146 kill -0 "$pid" || exit 1
147 clusters=$(curl -fs http://127.0.0.1:9901/clusters) || clusters=
148 for i in $(seq $#); do
149 port=$((5110 + i))
150 grep -qF "::127.0.0.1:$port::health_flags::" <<<"$clusters" || continue
151 if [ -n "${dialers[$i]:-}" ] && kill -0 "${dialers[$i]}" 2>/dev/null; then
152 if grep -qxE ".*::127\.0\.0\.1:$port::health_flags::healthy" <<<"$clusters"; then
153 since[$i]=$SECONDS
154 continue
155 fi
156 [ $((SECONDS - since[$i])) -lt 60 ] && continue
157 kill "${dialers[$i]}" 2>/dev/null || true
158 fi
159 IROH_SECRET=$(derive "iroh-farm-dial-$i") \
160 dumbpipe connect-tcp --addr "127.0.0.1:$port" "$(ticket "worker-$i")" &
161 dialers[$i]=$!
162 since[$i]=$SECONDS
163 done
164 sleep 15
165 done
166}
167
168# grpc.health.v1 Check on 127.0.0.1:PORT over the farm's TLS, prints the
169# response headers and trailers. An empty HealthCheckRequest: 0 = uncompressed,
170# 0 bytes.
171health() {
172 printf '\0\0\0\0\0' | curl -s -o /dev/null -D - --max-time 10 --http2 \
173 -H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
174 --cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
175 "https://127.0.0.1:$1/grpc.health.v1.Health/Check"
176}
177
178# A dumbpipe dialer started before the farm is discoverable never reaches it,
179# not even once the farm is online. Probe PORT with short-lived dialers to
180# TICKET until it answers an HTTP/2 request over the farm's TLS, then leave
181# the long-lived dialer to the caller. Gives up once the farm job has
182# finished (checked every 2 minutes, the API is rate limited).
183wait_farm() {
184 local port=$1 ticket=$2 probe n=0
185 while true; do
186 IROH_SECRET=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n') \
187 dumbpipe connect-tcp --addr "127.0.0.1:$port" "$ticket" 2>/dev/null &
188 probe=$!
189 sleep 3
190 if health "$port" >/dev/null; then
191 kill "$probe" 2>/dev/null || true
192 return
193 fi
194 kill "$probe" 2>/dev/null || true
195 sleep 5
196 n=$((n + 1))
197 if [ $((n % 15)) -eq 0 ] && jobs_done coordinator; then
198 echo "ci-farm: the farm did not come up" >&2
199 exit 1
200 fi
201 done
202}
203
204# Envoy (via PORT) has a builder to send requests to only once one has joined
205# the scheduler and the farm has dialed it and health checked it through the
206# tunnel; until then it answers itself, with grpc-status 14 ("no healthy
207# upstream", or "remote connection failure" for a builder not dialed yet).
208# A client gives up on a store it has never reached, so wait until a health
209# check through envoy reaches a builder (grpc-status 0). Gives up when the
210# builder PID dies or the farm job has finished, like wait_farm.
211wait_builder() {
212 local port=$1 pid=$2 n=0
213 until health "$port" | grep -qE '^grpc-status: 0\s*$'; do
214 kill -0 "$pid" 2>/dev/null || exit 1
215 sleep 5
216 n=$((n + 1))
217 if [ $((n % 24)) -eq 0 ] && jobs_done coordinator; then
218 echo "ci-farm: the farm has no builder" >&2
219 exit 1
220 fi
221 done
222}
223
224# ci-farm machine INDEX NAME: builder INDEX of the farm (NAME is the farm's
225# INDEX-th MACHINE) while it builds the toplevel of NAME through the farm.
226# Leaves once that is done: draining finishes its running builds, whose
227# outputs are in niks3 then, and the scheduler hands the rest to the others.
228machine() {
229 local i=${1:?machine index} name=${2:?machine name} farm lb eval pid status=0
230 farm=$(ticket farm)
231 lb=$(ticket lb)
232 nix eval --raw ".#nixosConfigurations.$name.config.system.build.toplevel.drvPath" >"$state/drv" &
233 eval=$!
234 IROH_SECRET=$(derive "iroh-worker-$i") dumbpipe listen-tcp --host 127.0.0.1:5100 &
235 wait_farm 5102 "$farm"
236 wait_farm 5104 "$lb"
237 echo "ci-farm: the farm is up"
238 IROH_SECRET=$(derive "iroh-worker-dial-$i") \
239 dumbpipe connect-tcp --addr 127.0.0.1:5101 "$farm" &
240 IROH_SECRET=$(derive "iroh-worker-lb-$i") \
241 dumbpipe connect-tcp --addr 127.0.0.1:5103 "$lb" &
242 "${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \
243 --advertise "127.0.0.1:$((5110 + i))" --worker-name "$name" &
244 pid=$!
245 echo "$pid" >"$state/builder.pid"
246
247 if wait "$eval"; then
248 echo "ci-farm: $name: $(<"$state/drv")"
249 wait_builder 5103 "$pid"
250 "$NIX_WITH_PLUGIN" build --no-link --print-out-paths --eval-store auto \
251 --store "grpc://127.0.0.1:5103?system=x86_64-linux&ca-cert=$FARM_CRT&client-cert=$FARM_CRT&client-key=$state/farm.key" \
252 "$(<"$state/drv")^*" || status=$?
253 else
254 status=$?
255 fi
256 [ "$status" -eq 0 ] || echo "ci-farm: $name failed" >&2
257
258 # Drains: running builds finish and publish first.
259 kill -TERM "$pid" 2>/dev/null || true
260 wait "$pid" || true
261 return "$status"
262}
263
264# ci-farm push PATH...: pushes the closures of PATH... to niks3, e.g. the
265# farm's own closure, which no other cache has.
266push() {
267 niks3 push --server-url "$niks3_url" --auth-token-path "$state/niks3-token" \
268 --ignore-upstream-key cache.nixos.org-1 "$@"
269}
270
271cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm machine INDEX NAME | ci-farm push PATH...}
272shift
273case $cmd in
274farm) farm "$@" ;;
275machine) machine "$@" ;;
276push) push "$@" ;;
277*)
278 echo "ci-farm: unknown command $cmd" >&2
279 exit 1
280 ;;
281esac