forked from
lass.pds.lassul.us/superconfig
This repository has no description
7.3 kB
210 lines
1# shellcheck shell=bash
2# (writeShellApplication adds the shebang and `set -euo pipefail`.)
3
4# On-demand nix-grpc-store farm across GitHub Actions runners.
5#
6# Runners can't reach each other, so every hop goes through an iroh tunnel
7# (dumbpipe) that ends on 127.0.0.1 on both sides:
8#
9# farm job worker job i
10# client ─► envoy :50051 daemon :5100 (builder)
11# ├─► daemon :50052 ◄─ iroh ─ :5101 scheduler session
12# └─► :511i ─────────── iroh ─► :5100 builds
13# daemon :50052 (scheduler + builder)
14#
15# Every daemon and envoy present the same TLS identity (FARM_KEY with
16# farm.crt, SAN 127.0.0.1) and accept only it, as `trusted`. The iroh
17# endpoint keys are derived from FARM_KEY, so it is the only secret. Outputs
18# go to niks3 with the job's GitHub OIDC token.
19
20state=/tmp/ci-farm
21niks3_url=https://niks3.lassul.us
22
23mkdir -p "$state"
24chmod 700 "$state"
25printf '%s\n' "${FARM_KEY:?FARM_KEY must hold the farm private key}" >"$state/farm.key"
26chmod 600 "$state/farm.key"
27unset FARM_KEY
28# Where envoy's config (flake-module.nix) expects it, next to the key.
29install -m 644 "$FARM_CRT" "$state/farm.crt"
30FARM_CRT=$state/farm.crt
31
32seed=$(sha256sum <"$state/farm.key" | cut -d' ' -f1)
33derive() { printf '%s:%s' "$seed" "$1" | sha256sum | cut -d' ' -f1; }
34ticket() { IROH_SECRET=$(derive "iroh-$1") dumbpipe generate-ticket; }
35
36trap 'kill $(jobs -p) 2>/dev/null || true' EXIT
37
38# The GitHub OIDC token of the job, refreshed in the background (niks3 re-reads
39# the file, the daemon notices the new inode). Outside Actions, point
40# NIKS3_TOKEN_FILE at a niks3 API token instead.
41niks3_token() {
42 curl -fsS -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \
43 "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" |
44 jq -er .value >"$state/niks3-token.new"
45 mv "$state/niks3-token.new" "$state/niks3-token"
46}
47if [ -n "${NIKS3_TOKEN_FILE:-}" ]; then
48 ln -sf "$NIKS3_TOKEN_FILE" "$state/niks3-token"
49else
50 niks3_token
51 (while sleep 120; do niks3_token || echo "ci-farm: refreshing the niks3 token failed" >&2; done) &
52fi
53
54daemon=(
55 nix-grpc-daemon
56 --proxy-socket /nix/var/nix/daemon-socket/socket
57 --tls-cert "$FARM_CRT"
58 --tls-key "$state/farm.key"
59 --client-ca "$FARM_CRT"
60 --allow farm=trusted
61 --niks3 "$niks3_url"
62 --niks3-token-file "$state/niks3-token"
63 # cache.nixos.org keeps serving what it signed; only push our own paths
64 --niks3-push "niks3 push --ignore-upstream-key cache.nixos.org-1"
65 --min-free 2G
66)
67
68wait_http() {
69 for _ in $(seq 60); do
70 curl -fs -o /dev/null "$1" && return
71 sleep 1
72 done
73 echo "ci-farm: $1 did not come up" >&2
74 exit 1
75}
76
77# Workers appear in envoy's clusters once their session with the scheduler is
78# up. Their tunnels resolve the farm through iroh discovery, which takes a
79# while after the farm comes online, so give them time before handing out
80# work. Builds start anyway after the timeout, with whoever has joined.
81wait_workers() {
82 local want=$1 have=0
83 for _ in $(seq 60); do
84 have=$(curl -fs http://127.0.0.1:9901/clusters |
85 grep -cE '::127\.0\.0\.1:511[0-9]+::health_flags::healthy$' || true)
86 [ "$have" -ge "$want" ] && break
87 sleep 5
88 done
89 echo "ci-farm: $have of $want workers joined"
90}
91
92# ci-farm farm ARG...: ARG is a NixOS machine of this flake (its toplevel is
93# built) or a /nix/store/….drv.
94farm() {
95 local workers=${FARM_WORKERS:?FARM_WORKERS: number of worker jobs}
96 IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 &
97 for i in $(seq "$workers"); do
98 IROH_SECRET=$(derive "iroh-farm-dial-$i") \
99 dumbpipe connect-tcp --addr "127.0.0.1:$((5110 + i))" "$(ticket "worker-$i")" &
100 done
101 "${daemon[@]}" --role builder,scheduler --listen 127.0.0.1:50052 \
102 --advertise 127.0.0.1:50052 --worker-name farm &
103 envoy -c "$ENVOY_CONFIG" --log-level warn &
104 wait_http http://127.0.0.1:9901/ready
105 wait_workers "$workers"
106
107 local store="grpc://127.0.0.1:50051?insecure=1&system=x86_64-linux"
108 declare -A builds=() evals=()
109 local failed=()
110 build() {
111 echo "ci-farm: $1: $2"
112 "$NIX_WITH_PLUGIN" build --no-link --print-out-paths --store "$store" --eval-store auto "$2^*" 2>&1 |
113 sed -u "s|^|[$1] |" &
114 builds[$1]=$!
115 }
116 # Waits for one eval, then starts its build.
117 eval_done() {
118 local pid status=0 arg
119 wait -n -p pid "${!evals[@]}" || status=$?
120 arg=${evals[$pid]}
121 unset "evals[$pid]"
122 if [ "$status" -eq 0 ]; then
123 build "$arg" "$(<"$state/$arg.drv")"
124 else
125 failed+=("$arg")
126 fi
127 }
128 for arg in "$@"; do
129 if [[ $arg == /nix/store/*.drv ]]; then
130 build "$arg" "$arg"
131 continue
132 fi
133 # One eval process per machine, at most 3 at a time: all hosts at once
134 # don't fit in 16 GB.
135 while [ ${#evals[@]} -ge 3 ]; do eval_done; done
136 nix eval --raw ".#nixosConfigurations.$arg.config.system.build.toplevel.drvPath" >"$state/$arg.drv" &
137 evals[$!]=$arg
138 done
139 while [ ${#evals[@]} -gt 0 ]; do eval_done; done
140
141 for arg in "${!builds[@]}"; do
142 wait "${builds[$arg]}" || failed+=("$arg")
143 done
144 if [ ${#failed[@]} -gt 0 ]; then
145 echo "ci-farm: failed: ${failed[*]}" >&2
146 exit 1
147 fi
148}
149
150# Outside Actions there is no farm job to watch: run until killed.
151farm_done() {
152 [ -n "${GITHUB_RUN_ID:-}" ] &&
153 [ "$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs" \
154 --jq '.jobs[] | select(.name == "farm") | .status')" = completed ]
155}
156
157# A dumbpipe dialer started before the farm is discoverable never reaches it,
158# not even once the farm is online. Probe with short-lived dialers until the
159# farm daemon answers a gRPC health check, then start the long-lived one.
160wait_farm() {
161 for _ in $(seq 180); do
162 IROH_SECRET=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n') \
163 dumbpipe connect-tcp --addr 127.0.0.1:5102 "$1" 2>/dev/null &
164 local probe=$!
165 sleep 3
166 # Empty grpc.health.v1.HealthCheckRequest: 0 = uncompressed, 0 bytes.
167 if printf '\0\0\0\0\0' | curl -fs -o /dev/null --max-time 10 --http2 \
168 -H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
169 --cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
170 https://127.0.0.1:5102/grpc.health.v1.Health/Check; then
171 kill "$probe"
172 return
173 fi
174 kill "$probe"
175 sleep 5
176 done
177 echo "ci-farm: the farm did not come up" >&2
178 exit 1
179}
180
181worker() {
182 local i=${1:?worker index} farm
183 farm=$(ticket farm)
184 IROH_SECRET=$(derive "iroh-worker-$i") dumbpipe listen-tcp --host 127.0.0.1:5100 &
185 wait_farm "$farm"
186 echo "ci-farm: the farm is up"
187 IROH_SECRET=$(derive "iroh-worker-dial-$i") \
188 dumbpipe connect-tcp --addr 127.0.0.1:5101 "$farm" &
189 "${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \
190 --advertise "127.0.0.1:$((5110 + i))" --worker-name "worker-$i" &
191 local pid=$!
192 until farm_done; do
193 kill -0 "$pid" || exit 1
194 sleep 15
195 done
196 # Drains: running builds finish and publish first.
197 kill -TERM "$pid"
198 wait "$pid" || true
199}
200
201cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm worker INDEX}
202shift
203case $cmd in
204farm) farm "$@" ;;
205worker) worker "$@" ;;
206*)
207 echo "ci-farm: unknown command $cmd" >&2
208 exit 1
209 ;;
210esac