This repository has no description
tools.ci-farm: wait for a healthy builder behind the balancer before building
The client gives up after connect-timeout on a store it has never reached.
At farm start envoy has no builder yet (none joined, or not yet dialed and
health checked through its tunnel), answers with grpc-status 14, and a few
machine jobs failed on their first QueryMissing. Probe a health check through
the balancer until a builder answers it.
author
Hermes
committer
lassulus
date
(Oct 14, 2026, 12:13 PM CEST)
commit
47d593b9
47d593b9953b2ba46443e09d49eef5306da8ed07
parent
be84d47e
be84d47ebe849d0d941ff681d66ab92f1282b20b
+32
-5
+32
-5
tools/ci-farm/ci-farm.sh
+32
-5
tools/ci-farm/ci-farm.sh
···
140
140
done
141
141
}
142
142
143
+
# grpc.health.v1 Check on 127.0.0.1:PORT over the farm's TLS, prints the
144
+
# response headers and trailers. An empty HealthCheckRequest: 0 = uncompressed,
145
+
# 0 bytes.
146
+
health() {
147
+
printf '\0\0\0\0\0' | curl -s -o /dev/null -D - --max-time 10 --http2 \
148
+
-H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
149
+
--cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
150
+
"https://127.0.0.1:$1/grpc.health.v1.Health/Check"
151
+
}
152
+
143
153
# A dumbpipe dialer started before the farm is discoverable never reaches it,
144
154
# not even once the farm is online. Probe PORT with short-lived dialers to
145
155
# TICKET until it answers an HTTP/2 request over the farm's TLS, then leave
···
152
162
dumbpipe connect-tcp --addr "127.0.0.1:$port" "$ticket" 2>/dev/null &
153
163
probe=$!
154
164
sleep 3
155
-
# Empty grpc.health.v1.HealthCheckRequest: 0 = uncompressed, 0 bytes.
156
-
if printf '\0\0\0\0\0' | curl -s -o /dev/null --max-time 10 --http2 \
157
-
-H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
158
-
--cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
159
-
"https://127.0.0.1:$port/grpc.health.v1.Health/Check"; then
165
+
if health "$port" >/dev/null; then
160
166
kill "$probe" 2>/dev/null || true
161
167
return
162
168
fi
···
165
171
n=$((n + 1))
166
172
if [ $((n % 15)) -eq 0 ] && jobs_done coordinator; then
167
173
echo "ci-farm: the farm did not come up" >&2
174
+
exit 1
175
+
fi
176
+
done
177
+
}
178
+
179
+
# Envoy (via PORT) has a builder to send requests to only once one has joined
180
+
# the scheduler and the farm has dialed it and health checked it through the
181
+
# tunnel; until then it answers itself, with grpc-status 14 ("no healthy
182
+
# upstream", or "remote connection failure" for a builder not dialed yet).
183
+
# A client gives up on a store it has never reached, so wait until a health
184
+
# check through envoy reaches a builder (grpc-status 0). Gives up when the
185
+
# builder PID dies or the farm job has finished, like wait_farm.
186
+
wait_builder() {
187
+
local port=$1 pid=$2 n=0
188
+
until health "$port" | grep -qE '^grpc-status: 0\s*$'; do
189
+
kill -0 "$pid" 2>/dev/null || exit 1
190
+
sleep 5
191
+
n=$((n + 1))
192
+
if [ $((n % 24)) -eq 0 ] && jobs_done coordinator; then
193
+
echo "ci-farm: the farm has no builder" >&2
168
194
exit 1
169
195
fi
170
196
done
···
194
220
195
221
if wait "$eval"; then
196
222
echo "ci-farm: $name: $(<"$state/drv")"
223
+
wait_builder 5103 "$pid"
197
224
"$NIX_WITH_PLUGIN" build --no-link --print-out-paths --eval-store auto \
198
225
--store "grpc://127.0.0.1:5103?system=x86_64-linux&ca-cert=$FARM_CRT&client-cert=$FARM_CRT&client-key=$state/farm.key" \
199
226
"$(<"$state/drv")^*" || status=$?