This repository has no description
6.7 kB
190 lines
1# shellcheck shell=bash
2# (writeShellApplication adds the shebang and `set -euo pipefail`.)
3
4# On-demand nix-grpc-store farm across GitHub Actions runners.
5#
6# Runners can't reach each other, so every hop goes through an iroh tunnel
7# (dumbpipe) that ends on 127.0.0.1 on both sides:
8#
9# farm job worker job i
10# client ─► envoy :50051 daemon :5100 (builder)
11# ├─► daemon :50052 ◄─ iroh ─ :5101 scheduler session
12# └─► :511i ─────────── iroh ─► :5100 builds
13# daemon :50052 (scheduler + builder)
14#
15# Every daemon and envoy present the same TLS identity (FARM_KEY with
16# farm.crt, SAN 127.0.0.1) and accept only it, as `trusted`. The iroh
17# endpoint keys are derived from FARM_KEY, so it is the only secret. Outputs
18# go to niks3 with the job's GitHub OIDC token.
19
20state=/tmp/ci-farm
21niks3_url=https://niks3.lassul.us
22
23mkdir -p "$state"
24chmod 700 "$state"
25printf '%s\n' "${FARM_KEY:?FARM_KEY must hold the farm private key}" >"$state/farm.key"
26chmod 600 "$state/farm.key"
27unset FARM_KEY
28# Where envoy's config (flake-module.nix) expects it, next to the key.
29install -m 644 "$FARM_CRT" "$state/farm.crt"
30FARM_CRT=$state/farm.crt
31
32seed=$(sha256sum <"$state/farm.key" | cut -d' ' -f1)
33derive() { printf '%s:%s' "$seed" "$1" | sha256sum | cut -d' ' -f1; }
34ticket() { IROH_SECRET=$(derive "iroh-$1") dumbpipe generate-ticket; }
35
36trap 'kill $(jobs -p) 2>/dev/null || true' EXIT
37
38# The GitHub OIDC token of the job, refreshed in the background (niks3 re-reads
39# the file, the daemon notices the new inode). Outside Actions, point
40# NIKS3_TOKEN_FILE at a niks3 API token instead.
41niks3_token() {
42 curl -fsS -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \
43 "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" |
44 jq -er .value >"$state/niks3-token.new"
45 mv "$state/niks3-token.new" "$state/niks3-token"
46}
47if [ -n "${NIKS3_TOKEN_FILE:-}" ]; then
48 ln -sf "$NIKS3_TOKEN_FILE" "$state/niks3-token"
49else
50 niks3_token
51 (while sleep 120; do niks3_token || echo "ci-farm: refreshing the niks3 token failed" >&2; done) &
52fi
53
54daemon=(
55 nix-grpc-daemon
56 --proxy-socket /nix/var/nix/daemon-socket/socket
57 --tls-cert "$FARM_CRT"
58 --tls-key "$state/farm.key"
59 --client-ca "$FARM_CRT"
60 --allow farm=trusted
61 --niks3 "$niks3_url"
62 --niks3-token-file "$state/niks3-token"
63 --min-free 2G
64)
65
66wait_http() {
67 for _ in $(seq 60); do
68 curl -fs -o /dev/null "$1" && return
69 sleep 1
70 done
71 echo "ci-farm: $1 did not come up" >&2
72 exit 1
73}
74
75# Workers appear in envoy's clusters once their session with the scheduler is
76# up. Their tunnels resolve the farm through iroh discovery, which takes a
77# while after the farm comes online, so give them time before handing out
78# work. Builds start anyway after the timeout, with whoever has joined.
79wait_workers() {
80 local want=$1 have=0
81 for _ in $(seq 60); do
82 have=$(curl -fs http://127.0.0.1:9901/clusters |
83 grep -cE '::127\.0\.0\.1:511[0-9]+::health_flags::healthy$' || true)
84 [ "$have" -ge "$want" ] && break
85 sleep 5
86 done
87 echo "ci-farm: $have of $want workers joined"
88}
89
90# ci-farm farm ARG...: ARG is a NixOS machine of this flake (its toplevel is
91# built) or a /nix/store/….drv.
92farm() {
93 local workers=${FARM_WORKERS:?FARM_WORKERS: number of worker jobs}
94 IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 &
95 for i in $(seq "$workers"); do
96 IROH_SECRET=$(derive "iroh-farm-dial-$i") \
97 dumbpipe connect-tcp --addr "127.0.0.1:$((5110 + i))" "$(ticket "worker-$i")" &
98 done
99 "${daemon[@]}" --role builder,scheduler --listen 127.0.0.1:50052 \
100 --advertise 127.0.0.1:50052 --worker-name farm &
101 envoy -c "$ENVOY_CONFIG" --log-level warn &
102 wait_http http://127.0.0.1:9901/ready
103 wait_workers "$workers"
104
105 local store="grpc://127.0.0.1:50051?insecure=1&system=x86_64-linux"
106 declare -A builds
107 for arg in "$@"; do
108 if [[ $arg == /nix/store/*.drv ]]; then
109 drv=$arg
110 else
111 # One eval process per machine: all hosts at once don't fit in 16 GB.
112 drv=$(nix eval --raw ".#nixosConfigurations.$arg.config.system.build.toplevel.drvPath")
113 fi
114 echo "ci-farm: $arg: $drv"
115 "$NIX_WITH_PLUGIN" build --no-link --print-out-paths --store "$store" --eval-store auto "$drv^*" 2>&1 |
116 sed -u "s|^|[$arg] |" &
117 builds[$arg]=$!
118 done
119
120 local failed=()
121 for arg in "${!builds[@]}"; do
122 wait "${builds[$arg]}" || failed+=("$arg")
123 done
124 if [ ${#failed[@]} -gt 0 ]; then
125 echo "ci-farm: failed: ${failed[*]}" >&2
126 exit 1
127 fi
128}
129
130# Outside Actions there is no farm job to watch: run until killed.
131farm_done() {
132 [ -n "${GITHUB_RUN_ID:-}" ] &&
133 [ "$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs" \
134 --jq '.jobs[] | select(.name == "farm") | .status')" = completed ]
135}
136
137# A dumbpipe dialer started before the farm is discoverable never reaches it,
138# not even once the farm is online. Probe with short-lived dialers until the
139# farm daemon answers a gRPC health check, then start the long-lived one.
140wait_farm() {
141 for _ in $(seq 180); do
142 IROH_SECRET=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n') \
143 dumbpipe connect-tcp --addr 127.0.0.1:5102 "$1" 2>/dev/null &
144 local probe=$!
145 sleep 3
146 # Empty grpc.health.v1.HealthCheckRequest: 0 = uncompressed, 0 bytes.
147 if printf '\0\0\0\0\0' | curl -fs -o /dev/null --max-time 10 --http2 \
148 -H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
149 --cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
150 https://127.0.0.1:5102/grpc.health.v1.Health/Check; then
151 kill "$probe"
152 return
153 fi
154 kill "$probe"
155 sleep 5
156 done
157 echo "ci-farm: the farm did not come up" >&2
158 exit 1
159}
160
161worker() {
162 local i=${1:?worker index} farm
163 farm=$(ticket farm)
164 IROH_SECRET=$(derive "iroh-worker-$i") dumbpipe listen-tcp --host 127.0.0.1:5100 &
165 wait_farm "$farm"
166 echo "ci-farm: the farm is up"
167 IROH_SECRET=$(derive "iroh-worker-dial-$i") \
168 dumbpipe connect-tcp --addr 127.0.0.1:5101 "$farm" &
169 "${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \
170 --advertise "127.0.0.1:$((5110 + i))" --worker-name "worker-$i" &
171 local pid=$!
172 until farm_done; do
173 kill -0 "$pid" || exit 1
174 sleep 15
175 done
176 # Drains: running builds finish and publish first.
177 kill -TERM "$pid"
178 wait "$pid" || true
179}
180
181cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm worker INDEX}
182shift
183case $cmd in
184farm) farm "$@" ;;
185worker) worker "$@" ;;
186*)
187 echo "ci-farm: unknown command $cmd" >&2
188 exit 1
189 ;;
190esac