This repository has no description
tools.ci-farm: keep the niks3 token when a refresh fails, drain the builder without one
author
Hermes
committer
lassulus
date
(Oct 15, 2026, 6:13 AM CEST)
commit
0c566f7b
0c566f7bca24484ff67bcece22fd2d1b60dd390f
parent
1bf00668
1bf00668f8ebe132cdf2732fb40450e91f6dd927
+32
-6
+32
-6
tools/ci-farm/ci-farm.sh
+32
-6
tools/ci-farm/ci-farm.sh
···
38
38
39
39
# The GitHub OIDC token of the job, refreshed in the background (niks3 re-reads
40
40
# the file, the daemon notices the new inode). Outside Actions, point
41
-
# NIKS3_TOKEN_FILE at a niks3 API token instead.
41
+
# NIKS3_TOKEN_FILE at a niks3 API token instead. A failed fetch keeps the
42
+
# current token: `set -e` doesn't apply in `niks3_token || ...`, so every step
43
+
# returns explicitly.
42
44
niks3_token() {
43
-
curl -fsS -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \
44
-
"$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" |
45
-
jq -er .value >"$state/niks3-token.new"
46
-
mv "$state/niks3-token.new" "$state/niks3-token"
45
+
local token
46
+
token=$(curl -fsS --retry 4 --retry-all-errors \
47
+
-H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \
48
+
"$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" | jq -er .value) || return 1
49
+
[ -n "$token" ] || return 1
50
+
printf '%s\n' "$token" >"$state/niks3-token.new" &&
51
+
mv "$state/niks3-token.new" "$state/niks3-token"
52
+
}
53
+
# Refreshes every 2 minutes, every 20 seconds after a failure. The token is
54
+
# valid for 5 minutes: once it is 4 minutes old a builder (machine writes
55
+
# builder.pid) could not publish its outputs anymore, so drain it, and the
56
+
# scheduler hands its builds to the others.
57
+
refresh_niks3_token() {
58
+
local fetched=$SECONDS wait=120
59
+
while sleep "$wait"; do
60
+
if niks3_token; then
61
+
fetched=$SECONDS wait=120
62
+
continue
63
+
fi
64
+
wait=20
65
+
echo "ci-farm: refreshing the niks3 token failed, it is $((SECONDS - fetched))s old" >&2
66
+
if [ $((SECONDS - fetched)) -ge 240 ] && [ -s "$state/builder.pid" ]; then
67
+
echo "ci-farm: no fresh niks3 token, draining the builder" >&2
68
+
kill -TERM "$(<"$state/builder.pid")" 2>/dev/null || true
69
+
rm -f "$state/builder.pid"
70
+
fi
71
+
done
47
72
}
48
73
if [ -n "${NIKS3_TOKEN_FILE:-}" ]; then
49
74
ln -sf "$NIKS3_TOKEN_FILE" "$state/niks3-token"
50
75
else
51
76
niks3_token
52
-
(while sleep 120; do niks3_token || echo "ci-farm: refreshing the niks3 token failed" >&2; done) &
77
+
refresh_niks3_token &
53
78
fi
54
79
55
80
daemon=(
···
217
242
"${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \
218
243
--advertise "127.0.0.1:$((5110 + i))" --worker-name "$name" &
219
244
pid=$!
245
+
echo "$pid" >"$state/builder.pid"
220
246
221
247
if wait "$eval"; then
222
248
echo "ci-farm: $name: $(<"$state/drv")"