This repository has no description
.github: build all machines with the farm in the ci check, one job per machine
ci.yml calls farm.yml (now reusable) instead of the eval-only nixos-*
checks, which are gone from flake.nix. The farm job is only the
coordinator (scheduler + envoy, which now requires the farm client
cert, reachable over iroh). Each machine job joins as a builder,
evaluates its machine, builds the toplevel through the farm and leaves
when done; its outputs are in niks3 by then.
ci runs for master pushes, PRs and dispatches; a newer run for the same
ref cancels the older one, farms of different refs queue FIFO
(queue: max). update-pkgs verifies PRs with ci.yml alone.
author
Hermes
committer
lassulus
date
(Oct 13, 2026, 7:57 PM CEST)
commit
d32ec1b5
d32ec1b59f4304ef7740565f9c0a02c3a6cd7f2f
parent
eb1f4244
eb1f4244acbcadec3bfd91aa6576ae37bb179ee0
+296
-235
+3
-2
AGENTS.md
+3
-2
AGENTS.md
···
93
93
```
94
94
95
95
`--rebase` keeps the individual commits on master. Only the user (repo admin)
96
-
bypasses the ruleset. The update-pkgs bot merges its PRs through it as well,
97
-
after dispatching farm.yml and ci.yml on them.
96
+
bypasses the ruleset. The `ci` check includes a farm build of all machines
97
+
(farm.yml, called from ci.yml). The update-pkgs bot merges its PRs through
98
+
the ruleset as well, after dispatching ci.yml on them.
98
99
99
100
## Machine Facts and Secrets via vars
100
101
+12
-27
flake.nix
+12
-27
flake.nix
···
260
260
legacyPackages.llm =
261
261
inputs.llm-agents.packages.${system}
262
262
// lib.optionalAttrs (self.packages.${system} ? orca) { inherit (self.packages.${system}) orca; };
263
-
checks =
264
-
let
265
-
# Check NixOS configurations can be evaluated (without building)
266
-
nixosChecks = lib.mapAttrs' (
267
-
name: config:
268
-
lib.nameValuePair "nixos-${name}" (
269
-
pkgs.runCommand "eval-${name}" { } ''
270
-
# Force evaluation of the toplevel path without building
271
-
echo "Evaluating ${name}: ${builtins.unsafeDiscardStringContext config.config.system.build.toplevel.drvPath}"
272
-
touch $out
273
-
''
274
-
)
275
-
) (lib.filterAttrs (_: config: config.pkgs.system == system) self.nixosConfigurations);
276
-
277
-
# Check Darwin configurations can be evaluated (without building)
278
-
darwinChecks = lib.mapAttrs' (
279
-
name: config:
280
-
lib.nameValuePair "darwin-${name}" (
281
-
pkgs.runCommand "eval-darwin-${name}" { } ''
282
-
# Force evaluation of the system path without building
283
-
echo "Evaluating ${name}: ${builtins.unsafeDiscardStringContext config.system.drvPath}"
284
-
touch $out
285
-
''
286
-
)
287
-
) (lib.filterAttrs (_: config: config.pkgs.system == system) self.darwinConfigurations);
288
-
in
289
-
nixosChecks // darwinChecks;
263
+
# Check Darwin configurations can be evaluated (without building).
264
+
# The NixOS machines are built by the farm (.github/workflows/farm.yml).
265
+
checks = lib.mapAttrs' (
266
+
name: config:
267
+
lib.nameValuePair "darwin-${name}" (
268
+
pkgs.runCommand "eval-darwin-${name}" { } ''
269
+
# Force evaluation of the system path without building
270
+
echo "Evaluating ${name}: ${builtins.unsafeDiscardStringContext config.system.drvPath}"
271
+
touch $out
272
+
''
273
+
)
274
+
) (lib.filterAttrs (_: config: config.pkgs.system == system) self.darwinConfigurations);
290
275
};
291
276
flake.lib = import ./lib { inherit (nixpkgs) lib; };
292
277
};
+24
-4
.github/workflows/ci.yml
+24
-4
.github/workflows/ci.yml
···
2
2
3
3
on:
4
4
push:
5
+
branches: [master]
5
6
pull_request:
6
7
# update-pkgs runs it on its PR branches: pushes and PRs made with
7
8
# GITHUB_TOKEN don't trigger workflows, and master requires the ci check.
8
9
workflow_dispatch:
9
10
10
-
# nix flake check evaluates every host in one process (~19 GB peak), more than
11
-
# the 16 GB RAM of ubuntu-latest. One job per check keeps the peak at the
12
-
# largest single host and shows failing hosts in the run overview.
11
+
# A newer run for the same ref replaces a running one, its farm included.
12
+
# Runs for different refs queue for the farm, see the farm job.
13
+
concurrency:
14
+
group: ci-${{ github.ref }}
15
+
cancel-in-progress: true
16
+
17
+
# One job per check shows failing checks in the run overview. The NixOS
18
+
# machines are built by the farm, one job per machine.
13
19
jobs:
14
20
list:
15
21
runs-on: ubuntu-latest
···
64
70
nix eval --raw ".#$output.x86_64-linux" --apply 'xs: builtins.deepSeq (builtins.mapAttrs (_: x: x.drvPath) xs) "${toString (builtins.length (builtins.attrNames xs))} evaluated\n"'
65
71
done
66
72
73
+
# Builds all NixOS machines (farm.yml). Its iroh endpoints are fixed, so one
74
+
# farm at a time across all runs; the others wait FIFO. Without `queue: max`
75
+
# GitHub keeps one pending run per group and cancels the one before.
76
+
farm:
77
+
uses: ./.github/workflows/farm.yml
78
+
secrets: inherit
79
+
permissions:
80
+
contents: read
81
+
id-token: write # niks3 push
82
+
actions: read # the jobs watch each other
83
+
concurrency:
84
+
group: farm
85
+
queue: max
86
+
67
87
# Fixed-name summary of the jobs above, the required status check for
68
88
# merging into master (the matrix job names change with the hosts).
69
89
ci:
70
-
needs: [list, check, outputs]
90
+
needs: [list, check, outputs, farm]
71
91
if: always()
72
92
runs-on: ubuntu-latest
73
93
steps:
+82
-53
.github/workflows/farm.yml
+82
-53
.github/workflows/farm.yml
···
1
1
name: farm
2
2
3
-
# On-demand nix-grpc-store farm (tools/ci-farm): one farm job (scheduler,
4
-
# balancer, client, builder) and a pool of worker jobs that build the
5
-
# requested machines together, each derivation once. Outputs land in
6
-
# cache.lassul.us via niks3.
3
+
# On-demand nix-grpc-store farm (tools/ci-farm) that builds NixOS machines,
4
+
# each derivation once. Outputs land in cache.lassul.us via niks3.
5
+
#
6
+
# - coordinator: scheduler and balancer, no builder; runs until all machine
7
+
# jobs have finished.
8
+
# - one job per machine: joins the farm as a builder, evaluates its machine
9
+
# and builds the toplevel through the farm, then leaves (its outputs are in
10
+
# niks3 by then). Red when its machine fails to evaluate or build.
11
+
#
12
+
# Called by ci.yml for pushes to master and PRs (its `ci` check requires this
13
+
# workflow), or dispatched by hand for some machines.
7
14
#
8
15
# Needs the FARM_KEY secret: the farm's TLS private key, matching
9
16
# tools/ci-farm/farm.crt (`pass show ci-farm/key`).
10
17
11
18
on:
19
+
workflow_call:
20
+
inputs:
21
+
machines:
22
+
description: NixOS machines to build, space separated (empty for all)
23
+
type: string
24
+
default: ""
12
25
workflow_dispatch:
13
26
inputs:
14
27
machines:
15
28
description: NixOS machines to build, space separated (empty for all)
16
29
default: ""
17
-
push:
18
-
branches: [master]
19
30
20
31
permissions:
21
32
contents: read
22
33
id-token: write # niks3 push
23
-
actions: read # workers watch the farm job
34
+
actions: read # the jobs watch each other
24
35
25
-
# The farm's iroh endpoint id is fixed, so only one run at a time. A push to
26
-
# master replaces a running build (an older master state or an update-pkgs
27
-
# verification, which retries). Dispatched runs (update-pkgs verifying a PR,
28
-
# manual runs) never cancel a running build; they wait as the group's pending
29
-
# run.
36
+
# The farm's iroh endpoints are fixed, so only one farm at a time, queued
37
+
# FIFO. Called from ci.yml the calling job holds the `farm` group, and the
38
+
# github context (github.workflow) is ci.yml's: a group of its own then, so
39
+
# the two don't wait for each other.
30
40
concurrency:
31
-
group: farm
32
-
cancel-in-progress: ${{ github.event_name == 'push' }}
33
-
34
-
env:
35
-
# must match the worker matrix below
36
-
FARM_WORKERS: 2
37
-
MACHINES: ${{ inputs.machines }}
41
+
group: ${{ github.workflow == 'farm' && 'farm' || format('farm-call-{0}', github.run_id) }}
42
+
queue: max
38
43
39
44
jobs:
40
-
farm:
41
-
name: farm
45
+
list:
42
46
runs-on: ubuntu-latest
47
+
outputs:
48
+
machines: ${{ steps.list.outputs.machines }}
49
+
steps:
50
+
- uses: actions/checkout@v4
51
+
- uses: cachix/install-nix-action@v31
52
+
- id: list
53
+
env:
54
+
MACHINES: ${{ inputs.machines }}
55
+
run: |
56
+
if [ -z "$MACHINES" ]; then
57
+
MACHINES=$(nix eval --raw .#nixosConfigurations --apply "c: builtins.concatStringsSep \" \" (builtins.attrNames c)")
58
+
fi
59
+
# index: the machine job's builder slot on the farm, the
60
+
# position in the coordinator's arguments
61
+
machines=$(jq -cn --arg m "$MACHINES" '$m | split(" ") | map(select(. != ""))
62
+
| to_entries | map({ index: (.key + 1), machine: .value })')
63
+
echo "machines=$machines" >> "$GITHUB_OUTPUT"
64
+
echo "$machines" | jq .
65
+
66
+
coordinator:
67
+
needs: list
68
+
runs-on: ubuntu-latest
69
+
steps:
70
+
- uses: actions/checkout@v4
71
+
- uses: cachix/install-nix-action@v31
72
+
with:
73
+
extra_nix_config: |
74
+
trusted-users = root runner
75
+
extra-substituters = https://cache.lassul.us
76
+
extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
77
+
- run: nix build -L .#ci-farm -o ci-farm
78
+
# so the next run substitutes it instead of compiling it on every runner
79
+
- env:
80
+
FARM_KEY: ${{ secrets.FARM_KEY }}
81
+
run: ./ci-farm/bin/ci-farm push ./ci-farm
82
+
- env:
83
+
FARM_KEY: ${{ secrets.FARM_KEY }}
84
+
GH_TOKEN: ${{ github.token }}
85
+
MACHINES: ${{ needs.list.outputs.machines }}
86
+
run: |
87
+
mapfile -t machines < <(jq -r '.[].machine' <<<"$MACHINES")
88
+
./ci-farm/bin/ci-farm farm "${machines[@]}"
89
+
90
+
machine:
91
+
needs: list
92
+
name: ${{ matrix.machine }}
93
+
runs-on: ubuntu-latest
94
+
strategy:
95
+
fail-fast: false
96
+
# GitHub's free plan runs 20 jobs at once. The machine jobs wait for the
97
+
# coordinator; leave it and other workflows room, so they can't take
98
+
# every slot while it is still queued.
99
+
max-parallel: 15
100
+
matrix:
101
+
include: ${{ fromJSON(needs.list.outputs.machines) }}
43
102
steps:
44
103
- uses: actions/checkout@v4
45
104
- uses: cachix/install-nix-action@v31
···
52
111
# a remembered miss (default: 1 h) fails the build instead.
53
112
narinfo-cache-negative-ttl = 0
54
113
- run: nix build -L .#ci-farm -o ci-farm
55
-
# so the next run substitutes it instead of compiling it on every runner
56
-
- env:
57
-
FARM_KEY: ${{ secrets.FARM_KEY }}
58
-
run: ./ci-farm/bin/ci-farm push ./ci-farm
59
-
- name: build ${{ env.MACHINES || 'all machines' }}
114
+
- name: build ${{ matrix.machine }}
60
115
env:
61
116
FARM_KEY: ${{ secrets.FARM_KEY }}
62
-
# MACHINES unquoted: one argument per machine
63
-
run: |
64
-
if [ -z "$MACHINES" ]; then
65
-
MACHINES=$(nix eval --raw .#nixosConfigurations --apply "c: builtins.concatStringsSep \" \" (builtins.attrNames c)")
66
-
fi
67
-
./ci-farm/bin/ci-farm farm $MACHINES
68
-
69
-
worker:
70
-
name: worker ${{ matrix.index }}
71
-
runs-on: ubuntu-latest
72
-
strategy:
73
-
matrix:
74
-
index: [1, 2] # FARM_WORKERS
75
-
steps:
76
-
- uses: actions/checkout@v4
77
-
- uses: cachix/install-nix-action@v31
78
-
with:
79
-
extra_nix_config: |
80
-
trusted-users = root runner
81
-
extra-substituters = https://cache.lassul.us
82
-
extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
83
-
# see the farm job
84
-
narinfo-cache-negative-ttl = 0
85
-
- run: nix build -L .#ci-farm -o ci-farm
86
-
- env:
87
-
FARM_KEY: ${{ secrets.FARM_KEY }}
88
117
GH_TOKEN: ${{ github.token }}
89
-
run: ./ci-farm/bin/ci-farm worker ${{ matrix.index }}
118
+
run: ./ci-farm/bin/ci-farm machine ${{ matrix.index }} ${{ matrix.machine }}
+18
-28
.github/workflows/update-pkgs.yml
+18
-28
.github/workflows/update-pkgs.yml
···
8
8
# packages on macOS), push update/<pkg> and open or refresh its PR.
9
9
# lock: `nix flake update`, pushed as update/flake-lock with its own PR.
10
10
# 2. verify: for every open update/* PR, one after another (flake.lock
11
-
# first), rebase it onto the current master, build all machines with the
12
-
# farm (farm.yml) and run ci.yml on that commit, and squash-merge the PR
13
-
# only if both succeeded (master's ruleset requires the ci check). A
14
-
# commit that already has finished runs is not checked again, so a
15
-
# failing PR stays open until its branch or master changes.
11
+
# first), rebase it onto the current master, run ci.yml on that commit
12
+
# (it builds all machines with the farm) and squash-merge the PR only if
13
+
# it succeeded (master's ruleset requires its ci check). A commit that
14
+
# already has a finished run is not checked again, so a failing PR stays
15
+
# open until its branch or master changes.
16
16
#
17
-
# One PR at a time because the farm has a fixed endpoint and runs in a single
18
-
# concurrency group, where GitHub keeps only one pending run.
17
+
# One PR at a time: the farm builds one commit at a time anyway, and each PR
18
+
# is rebased onto the master it gets merged into.
19
19
#
20
20
# Needs "Allow GitHub Actions to create and approve pull requests" in the repo
21
21
# settings. Merges made with GITHUB_TOKEN don't trigger push workflows, so the
22
-
# farm build of master after merging is dispatched here as well.
22
+
# ci run (farm build) of master after merging is dispatched here as well.
23
23
24
24
on:
25
25
schedule:
···
37
37
permissions:
38
38
contents: write
39
39
pull-requests: write
40
-
actions: write # dispatch and watch farm runs
40
+
actions: write # dispatch and watch ci runs
41
41
42
42
concurrency:
43
43
group: update-pkgs
···
210
210
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
211
211
212
212
# Dispatched run of a workflow for a head commit: "<id> <status>
213
-
# <conclusion>" or empty. Cancelled runs don't count: a push to
214
-
# master replaced them.
213
+
# <conclusion>" or empty. Cancelled runs don't count: a newer run
214
+
# for the branch replaced them.
215
215
dispatched_run() {
216
216
gh run list -R "$repo" -w "$1" -b "$2" -e workflow_dispatch -L 20 \
217
217
--json databaseId,headSha,status,conclusion \
···
242
242
gh run view "$id" -R "$repo" --json conclusion --jq .conclusion
243
243
}
244
244
245
-
# Rebase a PR branch onto master, farm-build that commit and run
246
-
# ci.yml on it. Prints "<conclusion> <sha>"; conclusion is the farm
247
-
# run's, then ci's as "ci-<conclusion>", or "conflict" / "nostart".
245
+
# Rebase a PR branch onto master and run ci.yml (with the farm
246
+
# build) on that commit. Prints "<conclusion> <sha>"; conclusion is
247
+
# the ci run's, or "conflict" / "nostart".
248
248
verify_pr() {
249
249
local branch=$1 sha conclusion
250
250
git fetch -q origin master "$branch"
···
258
258
if [ "$sha" != "$(git rev-parse "origin/$branch")" ]; then
259
259
git push -q --force-with-lease="$branch:origin/$branch" origin "$branch" >&2
260
260
fi
261
-
conclusion=$(run_workflow farm.yml "$branch" "$sha")
262
-
if [ "$conclusion" = success ]; then
263
-
conclusion=$(run_workflow ci.yml "$branch" "$sha")
264
-
[ "$conclusion" = success ] || conclusion="ci-$conclusion"
265
-
fi
261
+
conclusion=$(run_workflow ci.yml "$branch" "$sha")
266
262
echo "$conclusion $sha"
267
263
}
268
264
269
265
# flake.lock first: every package PR is rebased onto the master it
270
266
# gets merged into right before its build, so after the lock merge
271
267
# the packages are built once, against the new lock, and the final
272
-
# farm run on master only hits the cache.
268
+
# ci run on master only hits the cache.
273
269
prs=$(gh pr list -R "$repo" --state open --base master --limit 100 \
274
270
--json number,headRefName \
275
271
--jq '[.[] | select(.headRefName | startswith("update/"))]
···
279
275
while read -r pr branch <&3; do
280
276
[ -n "$pr" ] || continue
281
277
echo "::group::#$pr $branch"
282
-
# A push to master cancels a running verification; master has
283
-
# moved then, so rebase and build again.
284
-
for attempt in 1 2 3; do
285
-
read -r conclusion sha < <(verify_pr "$branch")
286
-
[ "$conclusion" = cancelled ] || break
287
-
echo "#$pr: farm run cancelled by a push to master (attempt $attempt)"
288
-
done
278
+
read -r conclusion sha < <(verify_pr "$branch")
289
279
echo "#$pr: $conclusion"
290
280
if [ "$conclusion" = success ] && [ "$AUTOMERGE" = true ]; then
291
281
gh pr merge "$pr" -R "$repo" --squash --delete-branch --match-head-commit "$sha"
···
296
286
297
287
echo "merged update PRs: $merged"
298
288
if [ "$merged" -gt 0 ]; then
299
-
gh workflow run farm.yml -R "$repo" --ref master
289
+
gh workflow run ci.yml -R "$repo" --ref master
300
290
fi
+107
-97
tools/ci-farm/ci-farm.sh
+107
-97
tools/ci-farm/ci-farm.sh
···
6
6
# Runners can't reach each other, so every hop goes through an iroh tunnel
7
7
# (dumbpipe) that ends on 127.0.0.1 on both sides:
8
8
#
9
-
# farm job worker job i
10
-
# client ─► envoy :50051 daemon :5100 (builder)
11
-
# ├─► daemon :50052 ◄─ iroh ─ :5101 scheduler session
12
-
# └─► :511i ─────────── iroh ─► :5100 builds
13
-
# daemon :50052 (scheduler + builder)
9
+
# farm job machine job i
10
+
# envoy :50051 ◄─────────────── iroh ─ :5103 ◄─ client (machine i)
11
+
# ├─► daemon :50052 ◄──────── iroh ─ :5101 scheduler session
12
+
# └─► :511i ───────────────── iroh ─► :5100 daemon (builder)
13
+
# daemon :50052 (scheduler)
14
14
#
15
15
# Every daemon and envoy present the same TLS identity (FARM_KEY with
16
16
# farm.crt, SAN 127.0.0.1) and accept only it, as `trusted`. The iroh
17
17
# endpoint keys are derived from FARM_KEY, so it is the only secret. Outputs
18
-
# go to niks3 with the job's GitHub OIDC token.
18
+
# go to niks3 with the job's GitHub OIDC token, before a build counts as
19
+
# done, so a machine job can leave once its own machine is built.
19
20
20
21
state=/tmp/ci-farm
21
22
niks3_url=https://niks3.lassul.us
···
74
75
exit 1
75
76
}
76
77
77
-
# Workers appear in envoy's clusters once their session with the scheduler is
78
-
# up. Their tunnels resolve the farm through iroh discovery, which takes a
79
-
# while after the farm comes online, so give them time before handing out
80
-
# work. Builds start anyway after the timeout, with whoever has joined.
81
-
wait_workers() {
82
-
local want=$1 have=0
83
-
for _ in $(seq 60); do
84
-
have=$(curl -fs http://127.0.0.1:9901/clusters |
85
-
grep -cE '::127\.0\.0\.1:511[0-9]+::health_flags::healthy$' || true)
86
-
[ "$have" -ge "$want" ] && break
87
-
sleep 5
78
+
# The jobs NAME... of this run have all finished. In a workflow called from
79
+
# another (ci.yml) the API names them "<caller job> / NAME". Outside Actions
80
+
# there are no jobs to watch: never.
81
+
jobs_done() {
82
+
local jobs name
83
+
[ -n "${GITHUB_RUN_ID:-}" ] || return 1
84
+
jobs=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs?per_page=100" \
85
+
--jq '.jobs[] | "\(.status) \(.name | sub("^.* / "; ""))"') || return 1
86
+
for name in "$@"; do
87
+
grep -qxF "completed $name" <<<"$jobs" || return 1
88
88
done
89
-
echo "ci-farm: $have of $want workers joined"
90
89
}
91
90
92
-
# ci-farm farm ARG...: ARG is a NixOS machine of this flake (its toplevel is
93
-
# built) or a /nix/store/….drv.
91
+
# ci-farm farm MACHINE...: scheduler and balancer for the machine jobs
92
+
# (builder i builds the i-th MACHINE), until they have all finished.
94
93
farm() {
95
-
local workers=${FARM_WORKERS:?FARM_WORKERS: number of worker jobs}
96
-
IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 &
97
-
for i in $(seq "$workers"); do
98
-
IROH_SECRET=$(derive "iroh-farm-dial-$i") \
99
-
dumbpipe connect-tcp --addr "127.0.0.1:$((5110 + i))" "$(ticket "worker-$i")" &
100
-
done
101
-
"${daemon[@]}" --role builder,scheduler --listen 127.0.0.1:50052 \
102
-
--advertise 127.0.0.1:50052 --worker-name farm &
94
+
local i port clusters
95
+
"${daemon[@]}" --role scheduler --listen 127.0.0.1:50052 2>"$state/daemon.log" &
96
+
local pid=$!
97
+
tail -f "$state/daemon.log" >&2 &
103
98
envoy -c "$ENVOY_CONFIG" --log-level warn &
104
99
wait_http http://127.0.0.1:9901/ready
105
-
wait_workers "$workers"
106
-
107
-
local store="grpc://127.0.0.1:50051?insecure=1&system=x86_64-linux"
108
-
declare -A builds=() evals=()
109
-
local failed=()
110
-
build() {
111
-
echo "ci-farm: $1: $2"
112
-
"$NIX_WITH_PLUGIN" build --no-link --print-out-paths --store "$store" --eval-store auto "$2^*" 2>&1 |
113
-
sed -u "s|^|[$1] |" &
114
-
builds[$1]=$!
115
-
}
116
-
# Waits for one eval, then starts its build.
117
-
eval_done() {
118
-
local pid status=0 arg
119
-
wait -n -p pid "${!evals[@]}" || status=$?
120
-
arg=${evals[$pid]}
121
-
unset "evals[$pid]"
122
-
if [ "$status" -eq 0 ]; then
123
-
build "$arg" "$(<"$state/$arg.drv")"
124
-
else
125
-
failed+=("$arg")
126
-
fi
127
-
}
128
-
for arg in "$@"; do
129
-
if [[ $arg == /nix/store/*.drv ]]; then
130
-
build "$arg" "$arg"
131
-
continue
132
-
fi
133
-
# One eval process per machine, at most 3 at a time: all hosts at once
134
-
# don't fit in 16 GB.
135
-
while [ ${#evals[@]} -ge 3 ]; do eval_done; done
136
-
nix eval --raw ".#nixosConfigurations.$arg.config.system.build.toplevel.drvPath" >"$state/$arg.drv" &
137
-
evals[$!]=$arg
100
+
# A daemon without the builder role schedules only once niks3 has made it
101
+
# the leader.
102
+
for _ in $(seq 60); do
103
+
grep -q event=scheduler_take_over "$state/daemon.log" && break
104
+
sleep 2
138
105
done
139
-
while [ ${#evals[@]} -gt 0 ]; do eval_done; done
140
-
141
-
for arg in "${!builds[@]}"; do
142
-
wait "${builds[$arg]}" || failed+=("$arg")
143
-
done
144
-
if [ ${#failed[@]} -gt 0 ]; then
145
-
echo "ci-farm: failed: ${failed[*]}" >&2
106
+
if ! grep -q event=scheduler_take_over "$state/daemon.log"; then
107
+
echo "ci-farm: the scheduler did not get the lead from niks3" >&2
146
108
exit 1
147
109
fi
148
-
}
110
+
IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 &
111
+
IROH_SECRET=$(derive iroh-lb) dumbpipe listen-tcp --host 127.0.0.1:50051 &
112
+
echo "ci-farm: the farm is up"
149
113
150
-
# Outside Actions there is no farm job to watch: run until killed.
151
-
farm_done() {
152
-
[ -n "${GITHUB_RUN_ID:-}" ] &&
153
-
[ "$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs" \
154
-
--jq '.jobs[] | select(.name == "farm") | .status')" = completed ]
114
+
# Builder i shows up in envoy's clusters once its scheduler session is up,
115
+
# i.e. its own iroh endpoint is online. Only then dial it (a dialer started
116
+
# before the endpoint is discoverable never reaches it), and dial again
117
+
# when the tunnel dies or envoy's health checks fail through it for a
118
+
# minute.
119
+
declare -A dialers=() since=()
120
+
until jobs_done "$@"; do
121
+
kill -0 "$pid" || exit 1
122
+
clusters=$(curl -fs http://127.0.0.1:9901/clusters) || clusters=
123
+
for i in $(seq $#); do
124
+
port=$((5110 + i))
125
+
grep -qF "::127.0.0.1:$port::health_flags::" <<<"$clusters" || continue
126
+
if [ -n "${dialers[$i]:-}" ] && kill -0 "${dialers[$i]}" 2>/dev/null; then
127
+
if grep -qxE ".*::127\.0\.0\.1:$port::health_flags::healthy" <<<"$clusters"; then
128
+
since[$i]=$SECONDS
129
+
continue
130
+
fi
131
+
[ $((SECONDS - since[$i])) -lt 60 ] && continue
132
+
kill "${dialers[$i]}" 2>/dev/null || true
133
+
fi
134
+
IROH_SECRET=$(derive "iroh-farm-dial-$i") \
135
+
dumbpipe connect-tcp --addr "127.0.0.1:$port" "$(ticket "worker-$i")" &
136
+
dialers[$i]=$!
137
+
since[$i]=$SECONDS
138
+
done
139
+
sleep 15
140
+
done
155
141
}
156
142
157
143
# A dumbpipe dialer started before the farm is discoverable never reaches it,
158
-
# not even once the farm is online. Probe with short-lived dialers until the
159
-
# farm daemon answers a gRPC health check, then start the long-lived one.
144
+
# not even once the farm is online. Probe PORT with short-lived dialers to
145
+
# TICKET until it answers an HTTP/2 request over the farm's TLS, then leave
146
+
# the long-lived dialer to the caller. Gives up once the farm job has
147
+
# finished (checked every 2 minutes, the API is rate limited).
160
148
wait_farm() {
161
-
for _ in $(seq 180); do
149
+
local port=$1 ticket=$2 probe n=0
150
+
while true; do
162
151
IROH_SECRET=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n') \
163
-
dumbpipe connect-tcp --addr 127.0.0.1:5102 "$1" 2>/dev/null &
164
-
local probe=$!
152
+
dumbpipe connect-tcp --addr "127.0.0.1:$port" "$ticket" 2>/dev/null &
153
+
probe=$!
165
154
sleep 3
166
155
# Empty grpc.health.v1.HealthCheckRequest: 0 = uncompressed, 0 bytes.
167
-
if printf '\0\0\0\0\0' | curl -fs -o /dev/null --max-time 10 --http2 \
156
+
if printf '\0\0\0\0\0' | curl -s -o /dev/null --max-time 10 --http2 \
168
157
-H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
169
158
--cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
170
-
https://127.0.0.1:5102/grpc.health.v1.Health/Check; then
171
-
kill "$probe"
159
+
"https://127.0.0.1:$port/grpc.health.v1.Health/Check"; then
160
+
kill "$probe" 2>/dev/null || true
172
161
return
173
162
fi
174
-
kill "$probe"
163
+
kill "$probe" 2>/dev/null || true
175
164
sleep 5
165
+
n=$((n + 1))
166
+
if [ $((n % 15)) -eq 0 ] && jobs_done coordinator; then
167
+
echo "ci-farm: the farm did not come up" >&2
168
+
exit 1
169
+
fi
176
170
done
177
-
echo "ci-farm: the farm did not come up" >&2
178
-
exit 1
179
171
}
180
172
181
-
worker() {
182
-
local i=${1:?worker index} farm
173
+
# ci-farm machine INDEX NAME: builder INDEX of the farm (NAME is the farm's
174
+
# INDEX-th MACHINE) while it builds the toplevel of NAME through the farm.
175
+
# Leaves once that is done: draining finishes its running builds, whose
176
+
# outputs are in niks3 then, and the scheduler hands the rest to the others.
177
+
machine() {
178
+
local i=${1:?machine index} name=${2:?machine name} farm lb eval pid status=0
183
179
farm=$(ticket farm)
180
+
lb=$(ticket lb)
181
+
nix eval --raw ".#nixosConfigurations.$name.config.system.build.toplevel.drvPath" >"$state/drv" &
182
+
eval=$!
184
183
IROH_SECRET=$(derive "iroh-worker-$i") dumbpipe listen-tcp --host 127.0.0.1:5100 &
185
-
wait_farm "$farm"
184
+
wait_farm 5102 "$farm"
185
+
wait_farm 5104 "$lb"
186
186
echo "ci-farm: the farm is up"
187
187
IROH_SECRET=$(derive "iroh-worker-dial-$i") \
188
188
dumbpipe connect-tcp --addr 127.0.0.1:5101 "$farm" &
189
+
IROH_SECRET=$(derive "iroh-worker-lb-$i") \
190
+
dumbpipe connect-tcp --addr 127.0.0.1:5103 "$lb" &
189
191
"${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \
190
-
--advertise "127.0.0.1:$((5110 + i))" --worker-name "worker-$i" &
191
-
local pid=$!
192
-
until farm_done; do
193
-
kill -0 "$pid" || exit 1
194
-
sleep 15
195
-
done
192
+
--advertise "127.0.0.1:$((5110 + i))" --worker-name "$name" &
193
+
pid=$!
194
+
195
+
if wait "$eval"; then
196
+
echo "ci-farm: $name: $(<"$state/drv")"
197
+
"$NIX_WITH_PLUGIN" build --no-link --print-out-paths --eval-store auto \
198
+
--store "grpc://127.0.0.1:5103?system=x86_64-linux&ca-cert=$FARM_CRT&client-cert=$FARM_CRT&client-key=$state/farm.key" \
199
+
"$(<"$state/drv")^*" || status=$?
200
+
else
201
+
status=$?
202
+
fi
203
+
[ "$status" -eq 0 ] || echo "ci-farm: $name failed" >&2
204
+
196
205
# Drains: running builds finish and publish first.
197
-
kill -TERM "$pid"
206
+
kill -TERM "$pid" 2>/dev/null || true
198
207
wait "$pid" || true
208
+
return "$status"
199
209
}
200
210
201
211
# ci-farm push PATH...: pushes the closures of PATH... to niks3, e.g. the
···
205
215
--ignore-upstream-key cache.nixos.org-1 "$@"
206
216
}
207
217
208
-
cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm worker INDEX | ci-farm push PATH...}
218
+
cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm machine INDEX NAME | ci-farm push PATH...}
209
219
shift
210
220
case $cmd in
211
221
farm) farm "$@" ;;
212
-
worker) worker "$@" ;;
222
+
machine) machine "$@" ;;
213
223
push) push "$@" ;;
214
224
*)
215
225
echo "ci-farm: unknown command $cmd" >&2
+50
-24
tools/ci-farm/flake-module.nix
+50
-24
tools/ci-farm/flake-module.nix
···
11
11
grpc = inputs.nix-grpc-store.packages.${system};
12
12
13
13
# Envoy as configured by nix-grpc-store's NixOS module, rendered for a
14
-
# plain Linux runner. Plaintext towards the local client, the farm
15
-
# identity towards the nodes (they verify it and grant it `trusted`, so
16
-
# it can read the builder list from the scheduler).
17
-
envoyConfig =
18
-
(pkgs.formats.json { }).generate "ci-farm-envoy.json"
19
-
(inputs.nixpkgs.lib.nixosSystem {
20
-
modules = [
21
-
inputs.nix-grpc-store.nixosModules.lb
22
-
{
23
-
nixpkgs.pkgs = pkgs;
24
-
services.nix-grpc-farm-lb = {
25
-
enable = true;
26
-
listen = "127.0.0.1:50051";
27
-
scheduler = "127.0.0.1:50052";
28
-
systems = [ system ];
29
-
# Written by ci-farm.sh. Envoy watches the directories of
30
-
# these files, so they must exist on the runner.
31
-
tls.upstream = {
14
+
# plain Linux runner. The farm identity on both sides: the clients in
15
+
# the machine jobs reach it over iroh, the nodes verify it and grant it
16
+
# `trusted` (so it can read the builder list from the scheduler).
17
+
envoySettings =
18
+
(inputs.nixpkgs.lib.nixosSystem {
19
+
modules = [
20
+
inputs.nix-grpc-store.nixosModules.lb
21
+
{
22
+
nixpkgs.pkgs = pkgs;
23
+
services.nix-grpc-farm-lb = {
24
+
enable = true;
25
+
listen = "127.0.0.1:50051";
26
+
scheduler = "127.0.0.1:50052";
27
+
systems = [ system ];
28
+
# Written by ci-farm.sh. Envoy watches the directories of
29
+
# these files, so they must exist on the runner.
30
+
tls = {
31
+
certFile = "/tmp/ci-farm/farm.crt";
32
+
keyFile = "/tmp/ci-farm/farm.key";
33
+
clientCaFile = "/tmp/ci-farm/farm.crt";
34
+
upstream = {
32
35
certFile = "/tmp/ci-farm/farm.crt";
33
36
keyFile = "/tmp/ci-farm/farm.key";
34
37
caFile = "/tmp/ci-farm/farm.crt";
35
38
};
36
39
};
40
+
};
41
+
}
42
+
];
43
+
}).config.services.envoy.settings;
44
+
45
+
# The module verifies a client certificate but doesn't require one, and
46
+
# passes requests on with the farm identity. Require it: the listener
47
+
# is reachable over iroh.
48
+
envoyConfig = (pkgs.formats.json { }).generate "ci-farm-envoy.json" (
49
+
envoySettings
50
+
// {
51
+
static_resources = envoySettings.static_resources // {
52
+
listeners = map (
53
+
listener:
54
+
listener
55
+
// {
56
+
filter_chains = map (
57
+
chain:
58
+
lib.recursiveUpdate chain { transport_socket.typed_config.require_client_certificate = true; }
59
+
) listener.filter_chains;
37
60
}
38
-
];
39
-
}).config.services.envoy.settings;
61
+
) envoySettings.static_resources.listeners;
62
+
};
63
+
}
64
+
);
40
65
in
41
66
{
42
-
# ci-farm farm MACHINE... scheduler + builder + balancer, builds the
43
-
# toplevels of MACHINE... through the farm
44
-
# ci-farm worker INDEX builder, until the farm job has finished
45
-
# ci-farm push PATH... pushes the closures of PATH... to niks3
67
+
# ci-farm farm MACHINE... scheduler + balancer until the machine
68
+
# jobs of MACHINE... have finished
69
+
# ci-farm machine INDEX NAME builder, builds the toplevel of NAME
70
+
# through the farm
71
+
# ci-farm push PATH... pushes the closures of PATH... to niks3
46
72
#
47
73
# Runs in .github/workflows/farm.yml. See ci-farm.sh for the layout.
48
74
packages = lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {