This repository has no description
8.9 kB
228 lines
1# shellcheck shell=bash
2# (writeShellApplication adds the shebang and `set -euo pipefail`.)
3
4# On-demand nix-grpc-store farm across GitHub Actions runners.
5#
6# Runners can't reach each other, so every hop goes through an iroh tunnel
7# (dumbpipe) that ends on 127.0.0.1 on both sides:
8#
9# farm job machine job i
10# envoy :50051 ◄─────────────── iroh ─ :5103 ◄─ client (machine i)
11# ├─► daemon :50052 ◄──────── iroh ─ :5101 scheduler session
12# └─► :511i ───────────────── iroh ─► :5100 daemon (builder)
13# daemon :50052 (scheduler)
14#
15# Every daemon and envoy present the same TLS identity (FARM_KEY with
16# farm.crt, SAN 127.0.0.1) and accept only it, as `trusted`. The iroh
17# endpoint keys are derived from FARM_KEY, so it is the only secret. Outputs
18# go to niks3 with the job's GitHub OIDC token, before a build counts as
19# done, so a machine job can leave once its own machine is built.
20
21state=/tmp/ci-farm
22niks3_url=https://niks3.lassul.us
23
24mkdir -p "$state"
25chmod 700 "$state"
26printf '%s\n' "${FARM_KEY:?FARM_KEY must hold the farm private key}" >"$state/farm.key"
27chmod 600 "$state/farm.key"
28unset FARM_KEY
29# Where envoy's config (flake-module.nix) expects it, next to the key.
30install -m 644 "$FARM_CRT" "$state/farm.crt"
31FARM_CRT=$state/farm.crt
32
33seed=$(sha256sum <"$state/farm.key" | cut -d' ' -f1)
34derive() { printf '%s:%s' "$seed" "$1" | sha256sum | cut -d' ' -f1; }
35ticket() { IROH_SECRET=$(derive "iroh-$1") dumbpipe generate-ticket; }
36
37trap 'kill $(jobs -p) 2>/dev/null || true' EXIT
38
39# The GitHub OIDC token of the job, refreshed in the background (niks3 re-reads
40# the file, the daemon notices the new inode). Outside Actions, point
41# NIKS3_TOKEN_FILE at a niks3 API token instead.
42niks3_token() {
43 curl -fsS -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \
44 "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" |
45 jq -er .value >"$state/niks3-token.new"
46 mv "$state/niks3-token.new" "$state/niks3-token"
47}
48if [ -n "${NIKS3_TOKEN_FILE:-}" ]; then
49 ln -sf "$NIKS3_TOKEN_FILE" "$state/niks3-token"
50else
51 niks3_token
52 (while sleep 120; do niks3_token || echo "ci-farm: refreshing the niks3 token failed" >&2; done) &
53fi
54
55daemon=(
56 nix-grpc-daemon
57 --proxy-socket /nix/var/nix/daemon-socket/socket
58 --tls-cert "$FARM_CRT"
59 --tls-key "$state/farm.key"
60 --client-ca "$FARM_CRT"
61 --allow farm=trusted
62 --niks3 "$niks3_url"
63 --niks3-token-file "$state/niks3-token"
64 # cache.nixos.org keeps serving what it signed; only push our own paths
65 --niks3-push "niks3 push --ignore-upstream-key cache.nixos.org-1"
66 --min-free 2G
67)
68
69wait_http() {
70 for _ in $(seq 60); do
71 curl -fs -o /dev/null "$1" && return
72 sleep 1
73 done
74 echo "ci-farm: $1 did not come up" >&2
75 exit 1
76}
77
78# The jobs NAME... of this run have all finished. In a workflow called from
79# another (ci.yml) the API names them "<caller job> / NAME". Outside Actions
80# there are no jobs to watch: never.
81jobs_done() {
82 local jobs name
83 [ -n "${GITHUB_RUN_ID:-}" ] || return 1
84 jobs=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs?per_page=100" \
85 --jq '.jobs[] | "\(.status) \(.name | sub("^.* / "; ""))"') || return 1
86 for name in "$@"; do
87 grep -qxF "completed $name" <<<"$jobs" || return 1
88 done
89}
90
91# ci-farm farm MACHINE...: scheduler and balancer for the machine jobs
92# (builder i builds the i-th MACHINE), until they have all finished.
93farm() {
94 local i port clusters
95 "${daemon[@]}" --role scheduler --listen 127.0.0.1:50052 2>"$state/daemon.log" &
96 local pid=$!
97 tail -f "$state/daemon.log" >&2 &
98 envoy -c "$ENVOY_CONFIG" --log-level warn &
99 wait_http http://127.0.0.1:9901/ready
100 # A daemon without the builder role schedules only once niks3 has made it
101 # the leader.
102 for _ in $(seq 60); do
103 grep -q event=scheduler_take_over "$state/daemon.log" && break
104 sleep 2
105 done
106 if ! grep -q event=scheduler_take_over "$state/daemon.log"; then
107 echo "ci-farm: the scheduler did not get the lead from niks3" >&2
108 exit 1
109 fi
110 IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 &
111 IROH_SECRET=$(derive iroh-lb) dumbpipe listen-tcp --host 127.0.0.1:50051 &
112 echo "ci-farm: the farm is up"
113
114 # Builder i shows up in envoy's clusters once its scheduler session is up,
115 # i.e. its own iroh endpoint is online. Only then dial it (a dialer started
116 # before the endpoint is discoverable never reaches it), and dial again
117 # when the tunnel dies or envoy's health checks fail through it for a
118 # minute.
119 declare -A dialers=() since=()
120 until jobs_done "$@"; do
121 kill -0 "$pid" || exit 1
122 clusters=$(curl -fs http://127.0.0.1:9901/clusters) || clusters=
123 for i in $(seq $#); do
124 port=$((5110 + i))
125 grep -qF "::127.0.0.1:$port::health_flags::" <<<"$clusters" || continue
126 if [ -n "${dialers[$i]:-}" ] && kill -0 "${dialers[$i]}" 2>/dev/null; then
127 if grep -qxE ".*::127\.0\.0\.1:$port::health_flags::healthy" <<<"$clusters"; then
128 since[$i]=$SECONDS
129 continue
130 fi
131 [ $((SECONDS - since[$i])) -lt 60 ] && continue
132 kill "${dialers[$i]}" 2>/dev/null || true
133 fi
134 IROH_SECRET=$(derive "iroh-farm-dial-$i") \
135 dumbpipe connect-tcp --addr "127.0.0.1:$port" "$(ticket "worker-$i")" &
136 dialers[$i]=$!
137 since[$i]=$SECONDS
138 done
139 sleep 15
140 done
141}
142
143# A dumbpipe dialer started before the farm is discoverable never reaches it,
144# not even once the farm is online. Probe PORT with short-lived dialers to
145# TICKET until it answers an HTTP/2 request over the farm's TLS, then leave
146# the long-lived dialer to the caller. Gives up once the farm job has
147# finished (checked every 2 minutes, the API is rate limited).
148wait_farm() {
149 local port=$1 ticket=$2 probe n=0
150 while true; do
151 IROH_SECRET=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n') \
152 dumbpipe connect-tcp --addr "127.0.0.1:$port" "$ticket" 2>/dev/null &
153 probe=$!
154 sleep 3
155 # Empty grpc.health.v1.HealthCheckRequest: 0 = uncompressed, 0 bytes.
156 if printf '\0\0\0\0\0' | curl -s -o /dev/null --max-time 10 --http2 \
157 -H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
158 --cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
159 "https://127.0.0.1:$port/grpc.health.v1.Health/Check"; then
160 kill "$probe" 2>/dev/null || true
161 return
162 fi
163 kill "$probe" 2>/dev/null || true
164 sleep 5
165 n=$((n + 1))
166 if [ $((n % 15)) -eq 0 ] && jobs_done coordinator; then
167 echo "ci-farm: the farm did not come up" >&2
168 exit 1
169 fi
170 done
171}
172
173# ci-farm machine INDEX NAME: builder INDEX of the farm (NAME is the farm's
174# INDEX-th MACHINE) while it builds the toplevel of NAME through the farm.
175# Leaves once that is done: draining finishes its running builds, whose
176# outputs are in niks3 then, and the scheduler hands the rest to the others.
177machine() {
178 local i=${1:?machine index} name=${2:?machine name} farm lb eval pid status=0
179 farm=$(ticket farm)
180 lb=$(ticket lb)
181 nix eval --raw ".#nixosConfigurations.$name.config.system.build.toplevel.drvPath" >"$state/drv" &
182 eval=$!
183 IROH_SECRET=$(derive "iroh-worker-$i") dumbpipe listen-tcp --host 127.0.0.1:5100 &
184 wait_farm 5102 "$farm"
185 wait_farm 5104 "$lb"
186 echo "ci-farm: the farm is up"
187 IROH_SECRET=$(derive "iroh-worker-dial-$i") \
188 dumbpipe connect-tcp --addr 127.0.0.1:5101 "$farm" &
189 IROH_SECRET=$(derive "iroh-worker-lb-$i") \
190 dumbpipe connect-tcp --addr 127.0.0.1:5103 "$lb" &
191 "${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \
192 --advertise "127.0.0.1:$((5110 + i))" --worker-name "$name" &
193 pid=$!
194
195 if wait "$eval"; then
196 echo "ci-farm: $name: $(<"$state/drv")"
197 "$NIX_WITH_PLUGIN" build --no-link --print-out-paths --eval-store auto \
198 --store "grpc://127.0.0.1:5103?system=x86_64-linux&ca-cert=$FARM_CRT&client-cert=$FARM_CRT&client-key=$state/farm.key" \
199 "$(<"$state/drv")^*" || status=$?
200 else
201 status=$?
202 fi
203 [ "$status" -eq 0 ] || echo "ci-farm: $name failed" >&2
204
205 # Drains: running builds finish and publish first.
206 kill -TERM "$pid" 2>/dev/null || true
207 wait "$pid" || true
208 return "$status"
209}
210
211# ci-farm push PATH...: pushes the closures of PATH... to niks3, e.g. the
212# farm's own closure, which no other cache has.
213push() {
214 niks3 push --server-url "$niks3_url" --auth-token-path "$state/niks3-token" \
215 --ignore-upstream-key cache.nixos.org-1 "$@"
216}
217
218cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm machine INDEX NAME | ci-farm push PATH...}
219shift
220case $cmd in
221farm) farm "$@" ;;
222machine) machine "$@" ;;
223push) push "$@" ;;
224*)
225 echo "ci-farm: unknown command $cmd" >&2
226 exit 1
227 ;;
228esac