This repository has no description
6.8 kB
192 lines
1# shellcheck shell=bash
2# (writeShellApplication adds the shebang and `set -euo pipefail`.)
3
4# On-demand nix-grpc-store farm across GitHub Actions runners.
5#
6# Runners can't reach each other, so every hop goes through an iroh tunnel
7# (dumbpipe) that ends on 127.0.0.1 on both sides:
8#
9# farm job worker job i
10# client ─► envoy :50051 daemon :5100 (builder)
11# ├─► daemon :50052 ◄─ iroh ─ :5101 scheduler session
12# └─► :511i ─────────── iroh ─► :5100 builds
13# daemon :50052 (scheduler + builder)
14#
15# Every daemon and envoy present the same TLS identity (FARM_KEY with
16# farm.crt, SAN 127.0.0.1) and accept only it, as `trusted`. The iroh
17# endpoint keys are derived from FARM_KEY, so it is the only secret. Outputs
18# go to niks3 with the job's GitHub OIDC token.
19
20state=/tmp/ci-farm
21niks3_url=https://niks3.lassul.us
22
23mkdir -p "$state"
24chmod 700 "$state"
25printf '%s\n' "${FARM_KEY:?FARM_KEY must hold the farm private key}" >"$state/farm.key"
26chmod 600 "$state/farm.key"
27unset FARM_KEY
28# Where envoy's config (flake-module.nix) expects it, next to the key.
29install -m 644 "$FARM_CRT" "$state/farm.crt"
30FARM_CRT=$state/farm.crt
31
32seed=$(sha256sum <"$state/farm.key" | cut -d' ' -f1)
33derive() { printf '%s:%s' "$seed" "$1" | sha256sum | cut -d' ' -f1; }
34ticket() { IROH_SECRET=$(derive "iroh-$1") dumbpipe generate-ticket; }
35
36trap 'kill $(jobs -p) 2>/dev/null || true' EXIT
37
38# The GitHub OIDC token of the job, refreshed in the background (niks3 re-reads
39# the file, the daemon notices the new inode). Outside Actions, point
40# NIKS3_TOKEN_FILE at a niks3 API token instead.
41niks3_token() {
42 curl -fsS -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \
43 "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" |
44 jq -er .value >"$state/niks3-token.new"
45 mv "$state/niks3-token.new" "$state/niks3-token"
46}
47if [ -n "${NIKS3_TOKEN_FILE:-}" ]; then
48 ln -sf "$NIKS3_TOKEN_FILE" "$state/niks3-token"
49else
50 niks3_token
51 (while sleep 120; do niks3_token || echo "ci-farm: refreshing the niks3 token failed" >&2; done) &
52fi
53
54daemon=(
55 nix-grpc-daemon
56 --proxy-socket /nix/var/nix/daemon-socket/socket
57 --tls-cert "$FARM_CRT"
58 --tls-key "$state/farm.key"
59 --client-ca "$FARM_CRT"
60 --allow farm=trusted
61 --niks3 "$niks3_url"
62 --niks3-token-file "$state/niks3-token"
63 # cache.nixos.org keeps serving what it signed; only push our own paths
64 --niks3-push "niks3 push --ignore-upstream-key cache.nixos.org-1"
65 --min-free 2G
66)
67
68wait_http() {
69 for _ in $(seq 60); do
70 curl -fs -o /dev/null "$1" && return
71 sleep 1
72 done
73 echo "ci-farm: $1 did not come up" >&2
74 exit 1
75}
76
77# Workers appear in envoy's clusters once their session with the scheduler is
78# up. Their tunnels resolve the farm through iroh discovery, which takes a
79# while after the farm comes online, so give them time before handing out
80# work. Builds start anyway after the timeout, with whoever has joined.
81wait_workers() {
82 local want=$1 have=0
83 for _ in $(seq 60); do
84 have=$(curl -fs http://127.0.0.1:9901/clusters |
85 grep -cE '::127\.0\.0\.1:511[0-9]+::health_flags::healthy$' || true)
86 [ "$have" -ge "$want" ] && break
87 sleep 5
88 done
89 echo "ci-farm: $have of $want workers joined"
90}
91
92# ci-farm farm ARG...: ARG is a NixOS machine of this flake (its toplevel is
93# built) or a /nix/store/….drv.
94farm() {
95 local workers=${FARM_WORKERS:?FARM_WORKERS: number of worker jobs}
96 IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 &
97 for i in $(seq "$workers"); do
98 IROH_SECRET=$(derive "iroh-farm-dial-$i") \
99 dumbpipe connect-tcp --addr "127.0.0.1:$((5110 + i))" "$(ticket "worker-$i")" &
100 done
101 "${daemon[@]}" --role builder,scheduler --listen 127.0.0.1:50052 \
102 --advertise 127.0.0.1:50052 --worker-name farm &
103 envoy -c "$ENVOY_CONFIG" --log-level warn &
104 wait_http http://127.0.0.1:9901/ready
105 wait_workers "$workers"
106
107 local store="grpc://127.0.0.1:50051?insecure=1&system=x86_64-linux"
108 declare -A builds
109 for arg in "$@"; do
110 if [[ $arg == /nix/store/*.drv ]]; then
111 drv=$arg
112 else
113 # One eval process per machine: all hosts at once don't fit in 16 GB.
114 drv=$(nix eval --raw ".#nixosConfigurations.$arg.config.system.build.toplevel.drvPath")
115 fi
116 echo "ci-farm: $arg: $drv"
117 "$NIX_WITH_PLUGIN" build --no-link --print-out-paths --store "$store" --eval-store auto "$drv^*" 2>&1 |
118 sed -u "s|^|[$arg] |" &
119 builds[$arg]=$!
120 done
121
122 local failed=()
123 for arg in "${!builds[@]}"; do
124 wait "${builds[$arg]}" || failed+=("$arg")
125 done
126 if [ ${#failed[@]} -gt 0 ]; then
127 echo "ci-farm: failed: ${failed[*]}" >&2
128 exit 1
129 fi
130}
131
132# Outside Actions there is no farm job to watch: run until killed.
133farm_done() {
134 [ -n "${GITHUB_RUN_ID:-}" ] &&
135 [ "$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs" \
136 --jq '.jobs[] | select(.name == "farm") | .status')" = completed ]
137}
138
139# A dumbpipe dialer started before the farm is discoverable never reaches it,
140# not even once the farm is online. Probe with short-lived dialers until the
141# farm daemon answers a gRPC health check, then start the long-lived one.
142wait_farm() {
143 for _ in $(seq 180); do
144 IROH_SECRET=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n') \
145 dumbpipe connect-tcp --addr 127.0.0.1:5102 "$1" 2>/dev/null &
146 local probe=$!
147 sleep 3
148 # Empty grpc.health.v1.HealthCheckRequest: 0 = uncompressed, 0 bytes.
149 if printf '\0\0\0\0\0' | curl -fs -o /dev/null --max-time 10 --http2 \
150 -H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
151 --cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
152 https://127.0.0.1:5102/grpc.health.v1.Health/Check; then
153 kill "$probe"
154 return
155 fi
156 kill "$probe"
157 sleep 5
158 done
159 echo "ci-farm: the farm did not come up" >&2
160 exit 1
161}
162
163worker() {
164 local i=${1:?worker index} farm
165 farm=$(ticket farm)
166 IROH_SECRET=$(derive "iroh-worker-$i") dumbpipe listen-tcp --host 127.0.0.1:5100 &
167 wait_farm "$farm"
168 echo "ci-farm: the farm is up"
169 IROH_SECRET=$(derive "iroh-worker-dial-$i") \
170 dumbpipe connect-tcp --addr 127.0.0.1:5101 "$farm" &
171 "${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \
172 --advertise "127.0.0.1:$((5110 + i))" --worker-name "worker-$i" &
173 local pid=$!
174 until farm_done; do
175 kill -0 "$pid" || exit 1
176 sleep 15
177 done
178 # Drains: running builds finish and publish first.
179 kill -TERM "$pid"
180 wait "$pid" || true
181}
182
183cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm worker INDEX}
184shift
185case $cmd in
186farm) farm "$@" ;;
187worker) worker "$@" ;;
188*)
189 echo "ci-farm: unknown command $cmd" >&2
190 exit 1
191 ;;
192esac