This repository has no description
9.9 kB
255 lines
1# shellcheck shell=bash
2# (writeShellApplication adds the shebang and `set -euo pipefail`.)
3
4# On-demand nix-grpc-store farm across GitHub Actions runners.
5#
6# Runners can't reach each other, so every hop goes through an iroh tunnel
7# (dumbpipe) that ends on 127.0.0.1 on both sides:
8#
9# farm job machine job i
10# envoy :50051 ◄─────────────── iroh ─ :5103 ◄─ client (machine i)
11# ├─► daemon :50052 ◄──────── iroh ─ :5101 scheduler session
12# └─► :511i ───────────────── iroh ─► :5100 daemon (builder)
13# daemon :50052 (scheduler)
14#
15# Every daemon and envoy present the same TLS identity (FARM_KEY with
16# farm.crt, SAN 127.0.0.1) and accept only it, as `trusted`. The iroh
17# endpoint keys are derived from FARM_KEY, so it is the only secret. Outputs
18# go to niks3 with the job's GitHub OIDC token, before a build counts as
19# done, so a machine job can leave once its own machine is built.
20
21state=/tmp/ci-farm
22niks3_url=https://niks3.lassul.us
23
24mkdir -p "$state"
25chmod 700 "$state"
26printf '%s\n' "${FARM_KEY:?FARM_KEY must hold the farm private key}" >"$state/farm.key"
27chmod 600 "$state/farm.key"
28unset FARM_KEY
29# Where envoy's config (flake-module.nix) expects it, next to the key.
30install -m 644 "$FARM_CRT" "$state/farm.crt"
31FARM_CRT=$state/farm.crt
32
33seed=$(sha256sum <"$state/farm.key" | cut -d' ' -f1)
34derive() { printf '%s:%s' "$seed" "$1" | sha256sum | cut -d' ' -f1; }
35ticket() { IROH_SECRET=$(derive "iroh-$1") dumbpipe generate-ticket; }
36
37trap 'kill $(jobs -p) 2>/dev/null || true' EXIT
38
39# The GitHub OIDC token of the job, refreshed in the background (niks3 re-reads
40# the file, the daemon notices the new inode). Outside Actions, point
41# NIKS3_TOKEN_FILE at a niks3 API token instead.
42niks3_token() {
43 curl -fsS -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \
44 "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" |
45 jq -er .value >"$state/niks3-token.new"
46 mv "$state/niks3-token.new" "$state/niks3-token"
47}
48if [ -n "${NIKS3_TOKEN_FILE:-}" ]; then
49 ln -sf "$NIKS3_TOKEN_FILE" "$state/niks3-token"
50else
51 niks3_token
52 (while sleep 120; do niks3_token || echo "ci-farm: refreshing the niks3 token failed" >&2; done) &
53fi
54
55daemon=(
56 nix-grpc-daemon
57 --proxy-socket /nix/var/nix/daemon-socket/socket
58 --tls-cert "$FARM_CRT"
59 --tls-key "$state/farm.key"
60 --client-ca "$FARM_CRT"
61 --allow farm=trusted
62 --niks3 "$niks3_url"
63 --niks3-token-file "$state/niks3-token"
64 # cache.nixos.org keeps serving what it signed; only push our own paths
65 --niks3-push "niks3 push --ignore-upstream-key cache.nixos.org-1"
66 --min-free 2G
67)
68
69wait_http() {
70 for _ in $(seq 60); do
71 curl -fs -o /dev/null "$1" && return
72 sleep 1
73 done
74 echo "ci-farm: $1 did not come up" >&2
75 exit 1
76}
77
78# The jobs NAME... of this run have all finished. In a workflow called from
79# another (ci.yml) the API names them "<caller job> / NAME". Outside Actions
80# there are no jobs to watch: never.
81jobs_done() {
82 local jobs name
83 [ -n "${GITHUB_RUN_ID:-}" ] || return 1
84 jobs=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs?per_page=100" \
85 --jq '.jobs[] | "\(.status) \(.name | sub("^.* / "; ""))"') || return 1
86 for name in "$@"; do
87 grep -qxF "completed $name" <<<"$jobs" || return 1
88 done
89}
90
91# ci-farm farm MACHINE...: scheduler and balancer for the machine jobs
92# (builder i builds the i-th MACHINE), until they have all finished.
93farm() {
94 local i port clusters
95 "${daemon[@]}" --role scheduler --listen 127.0.0.1:50052 2>"$state/daemon.log" &
96 local pid=$!
97 tail -f "$state/daemon.log" >&2 &
98 envoy -c "$ENVOY_CONFIG" --log-level warn &
99 wait_http http://127.0.0.1:9901/ready
100 # A daemon without the builder role schedules only once niks3 has made it
101 # the leader.
102 for _ in $(seq 60); do
103 grep -q event=scheduler_take_over "$state/daemon.log" && break
104 sleep 2
105 done
106 if ! grep -q event=scheduler_take_over "$state/daemon.log"; then
107 echo "ci-farm: the scheduler did not get the lead from niks3" >&2
108 exit 1
109 fi
110 IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 &
111 IROH_SECRET=$(derive iroh-lb) dumbpipe listen-tcp --host 127.0.0.1:50051 &
112 echo "ci-farm: the farm is up"
113
114 # Builder i shows up in envoy's clusters once its scheduler session is up,
115 # i.e. its own iroh endpoint is online. Only then dial it (a dialer started
116 # before the endpoint is discoverable never reaches it), and dial again
117 # when the tunnel dies or envoy's health checks fail through it for a
118 # minute.
119 declare -A dialers=() since=()
120 until jobs_done "$@"; do
121 kill -0 "$pid" || exit 1
122 clusters=$(curl -fs http://127.0.0.1:9901/clusters) || clusters=
123 for i in $(seq $#); do
124 port=$((5110 + i))
125 grep -qF "::127.0.0.1:$port::health_flags::" <<<"$clusters" || continue
126 if [ -n "${dialers[$i]:-}" ] && kill -0 "${dialers[$i]}" 2>/dev/null; then
127 if grep -qxE ".*::127\.0\.0\.1:$port::health_flags::healthy" <<<"$clusters"; then
128 since[$i]=$SECONDS
129 continue
130 fi
131 [ $((SECONDS - since[$i])) -lt 60 ] && continue
132 kill "${dialers[$i]}" 2>/dev/null || true
133 fi
134 IROH_SECRET=$(derive "iroh-farm-dial-$i") \
135 dumbpipe connect-tcp --addr "127.0.0.1:$port" "$(ticket "worker-$i")" &
136 dialers[$i]=$!
137 since[$i]=$SECONDS
138 done
139 sleep 15
140 done
141}
142
143# grpc.health.v1 Check on 127.0.0.1:PORT over the farm's TLS, prints the
144# response headers and trailers. An empty HealthCheckRequest: 0 = uncompressed,
145# 0 bytes.
146health() {
147 printf '\0\0\0\0\0' | curl -s -o /dev/null -D - --max-time 10 --http2 \
148 -H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
149 --cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
150 "https://127.0.0.1:$1/grpc.health.v1.Health/Check"
151}
152
153# A dumbpipe dialer started before the farm is discoverable never reaches it,
154# not even once the farm is online. Probe PORT with short-lived dialers to
155# TICKET until it answers an HTTP/2 request over the farm's TLS, then leave
156# the long-lived dialer to the caller. Gives up once the farm job has
157# finished (checked every 2 minutes, the API is rate limited).
158wait_farm() {
159 local port=$1 ticket=$2 probe n=0
160 while true; do
161 IROH_SECRET=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n') \
162 dumbpipe connect-tcp --addr "127.0.0.1:$port" "$ticket" 2>/dev/null &
163 probe=$!
164 sleep 3
165 if health "$port" >/dev/null; then
166 kill "$probe" 2>/dev/null || true
167 return
168 fi
169 kill "$probe" 2>/dev/null || true
170 sleep 5
171 n=$((n + 1))
172 if [ $((n % 15)) -eq 0 ] && jobs_done coordinator; then
173 echo "ci-farm: the farm did not come up" >&2
174 exit 1
175 fi
176 done
177}
178
179# Envoy (via PORT) has a builder to send requests to only once one has joined
180# the scheduler and the farm has dialed it and health checked it through the
181# tunnel; until then it answers itself, with grpc-status 14 ("no healthy
182# upstream", or "remote connection failure" for a builder not dialed yet).
183# A client gives up on a store it has never reached, so wait until a health
184# check through envoy reaches a builder (grpc-status 0). Gives up when the
185# builder PID dies or the farm job has finished, like wait_farm.
186wait_builder() {
187 local port=$1 pid=$2 n=0
188 until health "$port" | grep -qE '^grpc-status: 0\s*$'; do
189 kill -0 "$pid" 2>/dev/null || exit 1
190 sleep 5
191 n=$((n + 1))
192 if [ $((n % 24)) -eq 0 ] && jobs_done coordinator; then
193 echo "ci-farm: the farm has no builder" >&2
194 exit 1
195 fi
196 done
197}
198
199# ci-farm machine INDEX NAME: builder INDEX of the farm (NAME is the farm's
200# INDEX-th MACHINE) while it builds the toplevel of NAME through the farm.
201# Leaves once that is done: draining finishes its running builds, whose
202# outputs are in niks3 then, and the scheduler hands the rest to the others.
203machine() {
204 local i=${1:?machine index} name=${2:?machine name} farm lb eval pid status=0
205 farm=$(ticket farm)
206 lb=$(ticket lb)
207 nix eval --raw ".#nixosConfigurations.$name.config.system.build.toplevel.drvPath" >"$state/drv" &
208 eval=$!
209 IROH_SECRET=$(derive "iroh-worker-$i") dumbpipe listen-tcp --host 127.0.0.1:5100 &
210 wait_farm 5102 "$farm"
211 wait_farm 5104 "$lb"
212 echo "ci-farm: the farm is up"
213 IROH_SECRET=$(derive "iroh-worker-dial-$i") \
214 dumbpipe connect-tcp --addr 127.0.0.1:5101 "$farm" &
215 IROH_SECRET=$(derive "iroh-worker-lb-$i") \
216 dumbpipe connect-tcp --addr 127.0.0.1:5103 "$lb" &
217 "${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \
218 --advertise "127.0.0.1:$((5110 + i))" --worker-name "$name" &
219 pid=$!
220
221 if wait "$eval"; then
222 echo "ci-farm: $name: $(<"$state/drv")"
223 wait_builder 5103 "$pid"
224 "$NIX_WITH_PLUGIN" build --no-link --print-out-paths --eval-store auto \
225 --store "grpc://127.0.0.1:5103?system=x86_64-linux&ca-cert=$FARM_CRT&client-cert=$FARM_CRT&client-key=$state/farm.key" \
226 "$(<"$state/drv")^*" || status=$?
227 else
228 status=$?
229 fi
230 [ "$status" -eq 0 ] || echo "ci-farm: $name failed" >&2
231
232 # Drains: running builds finish and publish first.
233 kill -TERM "$pid" 2>/dev/null || true
234 wait "$pid" || true
235 return "$status"
236}
237
238# ci-farm push PATH...: pushes the closures of PATH... to niks3, e.g. the
239# farm's own closure, which no other cache has.
240push() {
241 niks3 push --server-url "$niks3_url" --auth-token-path "$state/niks3-token" \
242 --ignore-upstream-key cache.nixos.org-1 "$@"
243}
244
245cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm machine INDEX NAME | ci-farm push PATH...}
246shift
247case $cmd in
248farm) farm "$@" ;;
249machine) machine "$@" ;;
250push) push "$@" ;;
251*)
252 echo "ci-farm: unknown command $cmd" >&2
253 exit 1
254 ;;
255esac