This repository has no description
remove the CI farm
ci.yml builds the machines with hestia now. Drops farm.yml, tools/ci-farm, neoprism's nix-grpc-daemon builder, barnacle's farm client, the nix-farm vars generators and the nix-grpc-store input. cache.lassul.us/niks3 moves to configs/nix-cache.nix.
author
Hermes
committer
lassulus
date
(Oct 16, 2026, 10:39 PM CEST)
commit
d48ef870
d48ef870c48d4c9098e3a2479747a7dd7969ebac
parent
a2b87573
a2b875732aa6932286c1e0af02e2646fce150f88
+147
-899
-25
flake.lock
-25
flake.lock
···
409
409
"type": "github"
410
410
}
411
411
},
412
-
"nix-grpc-store": {
413
-
"inputs": {
414
-
"niks3": [
415
-
"niks3"
416
-
],
417
-
"nixpkgs": [
418
-
"nixpkgs"
419
-
]
420
-
},
421
-
"locked": {
422
-
"lastModified": 1792066552,
423
-
"narHash": "sha256-Ovar8bsbICSlUzlZA9KpHmon6ZqSsjZv2Wbbjiep5Qk=",
424
-
"owner": "lassulus",
425
-
"repo": "nix-grpc-store",
426
-
"rev": "d91962e6c61ec4d7fbb52b77e7b698a443b6fbf1",
427
-
"type": "github"
428
-
},
429
-
"original": {
430
-
"owner": "lassulus",
431
-
"ref": "fix-drain-upload",
432
-
"repo": "nix-grpc-store",
433
-
"type": "github"
434
-
}
435
-
},
436
412
"nix-index-database": {
437
413
"inputs": {
438
414
"nixpkgs": [
···
679
655
"kartei": "kartei",
680
656
"llm-agents": "llm-agents",
681
657
"niks3": "niks3",
682
-
"nix-grpc-store": "nix-grpc-store",
683
658
"nix-index-database": "nix-index-database",
684
659
"nix-rosetta-builder": "nix-rosetta-builder",
685
660
"nixos-hardware": "nixos-hardware",
+3
-9
flake.nix
+3
-9
flake.nix
···
71
71
nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver";
72
72
nixos-mailserver.inputs.nixpkgs.follows = "nixpkgs";
73
73
74
-
# build farm (configs/nix-farm): remote builds over grpc:// ...
75
-
# Fork with the drain/upload race fix (Mic92/nix-grpc-store#204);
76
-
# back to upstream once that is merged.
77
-
nix-grpc-store.url = "github:lassulus/nix-grpc-store/fix-drain-upload";
78
-
nix-grpc-store.inputs.nixpkgs.follows = "nixpkgs";
79
-
nix-grpc-store.inputs.niks3.follows = "niks3";
80
-
81
-
# ... and the S3-backed binary cache the outputs land in. Fork for
82
-
# `push --ignore-upstream-key` (not upstream yet). It rebases onto
74
+
# S3-backed binary cache cache.lassul.us (configs/nix-cache.nix), CI
75
+
# pushes its builds there. Fork for `push --ignore-upstream-key` (not
76
+
# upstream yet). It rebases onto
83
77
# Mic92/niks3 daily (its .github/workflows/rebase-upstream.yml), so
84
78
# `nix flake update niks3` follows upstream.
85
79
niks3.url = "github:lassulus/niks3";
+2
-2
.github/niks3-push.sh
+2
-2
.github/niks3-push.sh
···
2
2
# Pushes the closures of PATH... to niks3.lassul.us, served as
3
3
# cache.lassul.us, with the job's GitHub OIDC token (needs id-token: write;
4
4
# niks3 on neoprism trusts tokens of lassulus/superconfig, see
5
-
# configs/nix-farm/cache.nix). cache.nixos.org keeps serving what it signed, those
6
-
# paths are left out. Needs niks3 in PATH.
5
+
# configs/nix-cache.nix). cache.nixos.org keeps serving what it signed,
6
+
# those paths are left out. Needs niks3 in PATH.
7
7
# Usage: niks3-push.sh PATH...
8
8
# niks3-push.sh --stdin the paths from stdin
9
9
# niks3-push.sh --token the token, as niks3's --auth-token-script
+1
-1
configs/default.nix
+1
-1
configs/default.nix
···
136
136
}
137
137
)
138
138
(
139
-
# binary cache of the neoprism build farm (configs/nix-farm). The key
139
+
# binary cache on neoprism, CI pushes there (configs/nix-cache.nix). The key
140
140
# only exists once `clan vars generate neoprism` has run; until then
141
141
# the cache is left out.
142
142
let
+139
configs/nix-cache.nix
+139
configs/nix-cache.nix
···
1
+
{
2
+
config,
3
+
pkgs,
4
+
self,
5
+
...
6
+
}:
7
+
let
8
+
# TODO: create this bucket (public) + an application key for it in B2
9
+
bucket = "lassulus-nix-cache";
10
+
region = "eu-central-003";
11
+
endpoint = "s3.${region}.backblazeb2.com";
12
+
13
+
vars = config.clan.core.vars.generators;
14
+
in
15
+
{
16
+
imports = [ self.inputs.niks3.nixosModules.niks3 ];
17
+
18
+
# B2 credentials (prompted), the niks3 API token (generated).
19
+
clan.core.vars.generators.niks3 = {
20
+
prompts.b2_key_id = {
21
+
description = "Backblaze B2 keyID with access to the ${bucket} bucket";
22
+
type = "line";
23
+
};
24
+
prompts.b2_application_key = {
25
+
description = "Backblaze B2 applicationKey for that keyID";
26
+
type = "hidden";
27
+
};
28
+
files."s3-access-key".owner = "niks3";
29
+
files."s3-secret-key".owner = "niks3";
30
+
files."api-token".owner = "niks3";
31
+
runtimeInputs = [
32
+
pkgs.coreutils
33
+
pkgs.openssl
34
+
];
35
+
script = ''
36
+
tr -d '\n' < "$prompts/b2_key_id" > "$out/s3-access-key"
37
+
tr -d '\n' < "$prompts/b2_application_key" > "$out/s3-secret-key"
38
+
openssl rand -hex 32 | tr -d '\n' > "$out/api-token"
39
+
'';
40
+
};
41
+
42
+
# Public half is read by configs/default.nix from the vars dir.
43
+
clan.core.vars.generators.nix-cache-signing = {
44
+
files."key.sec".owner = "niks3";
45
+
files."key.pub".secret = false;
46
+
runtimeInputs = [ pkgs.nix ];
47
+
script = ''
48
+
nix --extra-experimental-features nix-command key generate-secret \
49
+
--key-name cache.lassul.us-1 > "$out/key.sec"
50
+
nix --extra-experimental-features nix-command key convert-secret-to-public \
51
+
< "$out/key.sec" > "$out/key.pub"
52
+
'';
53
+
};
54
+
55
+
services.niks3 = {
56
+
enable = true;
57
+
s3 = {
58
+
inherit endpoint bucket region;
59
+
accessKeyFile = vars.niks3.files."s3-access-key".path;
60
+
secretKeyFile = vars.niks3.files."s3-secret-key".path;
61
+
};
62
+
apiTokenFile = vars.niks3.files."api-token".path;
63
+
signKeyFiles = [ vars.nix-cache-signing.files."key.sec".path ];
64
+
cacheUrl = "https://cache.lassul.us";
65
+
# Behind cache.nixos.org (40): nix asks upstream first, so B2 only sees
66
+
# lookups for our own paths. At niks3's default 30 every closure path of
67
+
# a CI build would hit B2 first, and B2 throttles that many misses with
68
+
# 503s.
69
+
priority = 50;
70
+
# CI (.github/workflows/ci.yml) pushes with the job's GitHub OIDC token.
71
+
# Fork PRs don't get one.
72
+
oidc.providers.github = {
73
+
issuer = "https://token.actions.githubusercontent.com";
74
+
audience = "https://niks3.lassul.us";
75
+
boundClaims.repository = [ "lassulus/superconfig" ];
76
+
};
77
+
nginx = {
78
+
enable = true;
79
+
domain = "niks3.lassul.us";
80
+
};
81
+
};
82
+
83
+
# Reads straight from the public bucket. niks3 uploads the metadata
84
+
# zstd-compressed with Content-Encoding: zstd; set it here so nix decodes
85
+
# it whether or not the bucket hands the header back. Same rules as
86
+
# clan-infra's Fastly config (modules/terranix/cache-new.nix): narinfo, .ls,
87
+
# realisations/ and log/. NARs (nar/*.nar.zst) are plain objects, nix
88
+
# decodes them via the narinfo's Compression field.
89
+
# `resolve`: looked up at runtime via resolved's stub, so an unresolvable
90
+
# B2 at startup doesn't keep nginx (and every other vhost) from starting.
91
+
# `keepalive` + HTTP/1.1 below: reuse TLS connections to B2. Without them
92
+
# nginx speaks HTTP/1.0 with a new handshake per request, and at hundreds
93
+
# of requests per second B2 closes connections mid-response.
94
+
# `max_fails=0`: B2 closes idle keepalive connections now and then. With the
95
+
# default max_fails=1 one such close marks that B2 address down for 10s;
96
+
# once all of them are, every request gets "no live upstreams" and a 502.
97
+
# A GET/HEAD on a closed reused connection is retried
98
+
# (default `proxy_next_upstream error timeout`) without using up a try, on
99
+
# the next address, or on the same one if it resolves to only one.
100
+
services.nginx.appendHttpConfig = ''
101
+
upstream cache_lassul_us_b2 {
102
+
zone cache_lassul_us_b2 64k;
103
+
resolver 127.0.0.53 ipv6=off;
104
+
server ${endpoint}:443 resolve max_fails=0;
105
+
keepalive 64;
106
+
keepalive_requests 10000;
107
+
keepalive_timeout 60s;
108
+
}
109
+
map $uri $cache_lassul_us_encoding {
110
+
default "";
111
+
~\.(narinfo|ls)$ zstd;
112
+
~^/realisations/ zstd;
113
+
~^/log/ zstd;
114
+
}
115
+
'';
116
+
services.nginx.virtualHosts."cache.lassul.us" = {
117
+
enableACME = true;
118
+
forceSSL = true;
119
+
locations."/".extraConfig = ''
120
+
proxy_pass https://cache_lassul_us_b2/${bucket}/;
121
+
proxy_set_header Host ${endpoint};
122
+
proxy_ssl_server_name on;
123
+
proxy_ssl_name ${endpoint};
124
+
proxy_http_version 1.1;
125
+
proxy_set_header Connection "";
126
+
# cache keys never have a query string; don't forward one to S3
127
+
set $args "";
128
+
# S3 answers 403 for keys missing from a bucket it won't list
129
+
proxy_intercept_errors on;
130
+
error_page 403 =404 @notfound;
131
+
132
+
gzip off;
133
+
proxy_hide_header Content-Encoding;
134
+
# only sent on 2xx/3xx, and not at all when the map yields ""
135
+
add_header Content-Encoding $cache_lassul_us_encoding;
136
+
'';
137
+
locations."@notfound".return = "404";
138
+
};
139
+
}
-118
.github/workflows/farm.yml
-118
.github/workflows/farm.yml
···
1
-
name: farm
2
-
3
-
# On-demand nix-grpc-store farm (tools/ci-farm) that builds NixOS machines,
4
-
# each derivation once. Outputs land in cache.lassul.us via niks3.
5
-
#
6
-
# - coordinator: scheduler and balancer, no builder; runs until all machine
7
-
# jobs have finished.
8
-
# - one job per machine: joins the farm as a builder, evaluates its machine
9
-
# and builds the toplevel through the farm, then leaves (its outputs are in
10
-
# niks3 by then). Red when its machine fails to evaluate or build.
11
-
#
12
-
# Called by ci.yml for pushes to master and PRs (its `ci` check requires this
13
-
# workflow), or dispatched by hand for some machines.
14
-
#
15
-
# Needs the FARM_KEY secret: the farm's TLS private key, matching
16
-
# tools/ci-farm/farm.crt (`pass show ci-farm/key`).
17
-
18
-
on:
19
-
workflow_call:
20
-
inputs:
21
-
machines:
22
-
description: NixOS machines to build, space separated (empty for all)
23
-
type: string
24
-
default: ""
25
-
workflow_dispatch:
26
-
inputs:
27
-
machines:
28
-
description: NixOS machines to build, space separated (empty for all)
29
-
default: ""
30
-
31
-
permissions:
32
-
contents: read
33
-
id-token: write # niks3 push
34
-
actions: read # the jobs watch each other
35
-
36
-
# The farm's iroh endpoints are fixed, so only one farm at a time, queued
37
-
# FIFO. Called from ci.yml the calling job holds the `farm` group, and the
38
-
# github context (github.workflow) is ci.yml's: a group of its own then, so
39
-
# the two don't wait for each other.
40
-
concurrency:
41
-
group: ${{ github.workflow == 'farm' && 'farm' || format('farm-call-{0}', github.run_id) }}
42
-
queue: max
43
-
44
-
jobs:
45
-
list:
46
-
runs-on: ubuntu-latest
47
-
outputs:
48
-
machines: ${{ steps.list.outputs.machines }}
49
-
steps:
50
-
- uses: actions/checkout@v4
51
-
- uses: cachix/install-nix-action@v31
52
-
- id: list
53
-
env:
54
-
MACHINES: ${{ inputs.machines }}
55
-
run: |
56
-
if [ -z "$MACHINES" ]; then
57
-
MACHINES=$(nix eval --raw .#nixosConfigurations --apply "c: builtins.concatStringsSep \" \" (builtins.attrNames c)")
58
-
fi
59
-
# index: the machine job's builder slot on the farm, the
60
-
# position in the coordinator's arguments
61
-
machines=$(jq -cn --arg m "$MACHINES" '$m | split(" ") | map(select(. != ""))
62
-
| to_entries | map({ index: (.key + 1), machine: .value })')
63
-
echo "machines=$machines" >> "$GITHUB_OUTPUT"
64
-
echo "$machines" | jq .
65
-
66
-
coordinator:
67
-
needs: list
68
-
runs-on: ubuntu-latest
69
-
steps:
70
-
- uses: actions/checkout@v4
71
-
- uses: cachix/install-nix-action@v31
72
-
with:
73
-
extra_nix_config: |
74
-
trusted-users = root runner
75
-
extra-substituters = https://cache.lassul.us
76
-
extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
77
-
- run: nix build -L .#ci-farm -o ci-farm
78
-
# so the next run substitutes it instead of compiling it on every runner
79
-
- env:
80
-
FARM_KEY: ${{ secrets.FARM_KEY }}
81
-
run: ./ci-farm/bin/ci-farm push ./ci-farm
82
-
- env:
83
-
FARM_KEY: ${{ secrets.FARM_KEY }}
84
-
GH_TOKEN: ${{ github.token }}
85
-
MACHINES: ${{ needs.list.outputs.machines }}
86
-
run: |
87
-
mapfile -t machines < <(jq -r '.[].machine' <<<"$MACHINES")
88
-
./ci-farm/bin/ci-farm farm "${machines[@]}"
89
-
90
-
machine:
91
-
needs: list
92
-
name: ${{ matrix.machine }}
93
-
runs-on: ubuntu-latest
94
-
strategy:
95
-
fail-fast: false
96
-
# GitHub's free plan runs 20 jobs at once. The machine jobs wait for the
97
-
# coordinator; leave it and other workflows room, so they can't take
98
-
# every slot while it is still queued.
99
-
max-parallel: 15
100
-
matrix:
101
-
include: ${{ fromJSON(needs.list.outputs.machines) }}
102
-
steps:
103
-
- uses: actions/checkout@v4
104
-
- uses: cachix/install-nix-action@v31
105
-
with:
106
-
extra_nix_config: |
107
-
trusted-users = root runner
108
-
extra-substituters = https://cache.lassul.us
109
-
extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
110
-
# Builders need each other's outputs as soon as they are pushed;
111
-
# a remembered miss (default: 1 h) fails the build instead.
112
-
narinfo-cache-negative-ttl = 0
113
-
- run: nix build -L .#ci-farm -o ci-farm
114
-
- name: build ${{ matrix.machine }}
115
-
env:
116
-
FARM_KEY: ${{ secrets.FARM_KEY }}
117
-
GH_TOKEN: ${{ github.token }}
118
-
run: ./ci-farm/bin/ci-farm machine ${{ matrix.index }} ${{ matrix.machine }}
-97
configs/nix-farm/builder.nix
-97
configs/nix-farm/builder.nix
···
1
-
{
2
-
config,
3
-
lib,
4
-
pkgs,
5
-
self,
6
-
...
7
-
}:
8
-
let
9
-
vars = config.clan.core.vars.generators;
10
-
cachePub = self + "/vars/per-machine/neoprism/nix-cache-signing/key.pub/value";
11
-
credDir = "/run/credentials/nix-grpc-daemon.service";
12
-
in
13
-
{
14
-
imports = [
15
-
self.inputs.nix-grpc-store.nixosModules.server
16
-
./ca.nix
17
-
];
18
-
19
-
# Server cert for the daemon, signed by the shared farm CA, plus a copy of
20
-
# the CA cert to verify clients against. Secret files for the same reason
21
-
# as in ca.nix. Root-owned: the daemon gets them via LoadCredential below.
22
-
clan.core.vars.generators.nix-farm-server = {
23
-
dependencies = [ "nix-farm-ca" ];
24
-
files."server.key" = { };
25
-
files."server.crt" = { };
26
-
files."ca.crt" = { };
27
-
runtimeInputs = [ pkgs.openssl ];
28
-
script = ''
29
-
openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
30
-
-keyout "$out/server.key" -out server.csr -subj "/CN=neoprism"
31
-
printf '%s\n' \
32
-
"basicConstraints=critical,CA:FALSE" \
33
-
"keyUsage=critical,digitalSignature" \
34
-
"extendedKeyUsage=serverAuth" \
35
-
"subjectAltName=DNS:neoprism.r,DNS:neoprism.lassul.us" > server.ext
36
-
openssl x509 -req -in server.csr \
37
-
-CA "$in/nix-farm-ca/ca.crt" -CAkey "$in/nix-farm-ca/ca.key" \
38
-
-CAcreateserial -days 3650 -extfile server.ext -out "$out/server.crt"
39
-
cp "$in/nix-farm-ca/ca.crt" "$out/ca.crt"
40
-
'';
41
-
};
42
-
43
-
# Standalone node: schedules onto itself, publishes outputs to niks3.
44
-
services.nix-grpc-daemon = {
45
-
enable = true;
46
-
roles = [
47
-
"builder"
48
-
"scheduler"
49
-
];
50
-
listen = "[::]:50051";
51
-
# gRPC's cert file watcher treats mtime 0 as a stat failure and never
52
-
# loads the key pair, and clan unpacks /run/secrets with mtime 0. systemd
53
-
# credentials are fresh copies with a real mtime.
54
-
tls = {
55
-
certFile = "${credDir}/server.crt";
56
-
keyFile = "${credDir}/server.key";
57
-
clientCaFile = "${credDir}/ca.crt";
58
-
};
59
-
# `nix build --store grpc://… --eval-store auto` needs write (README).
60
-
accessRules = [
61
-
{
62
-
cn = "lass";
63
-
role = "write";
64
-
}
65
-
];
66
-
niks3 = {
67
-
package = self.inputs.niks3.packages.${pkgs.stdenv.hostPlatform.system}.niks3;
68
-
# same box, skip nginx/ACME
69
-
url = "http://${config.services.niks3.httpAddr}";
70
-
tokenFile = vars.niks3.files."api-token-grpc".path;
71
-
cacheUrl = config.services.niks3.cacheUrl;
72
-
# empty until `clan vars generate neoprism` wrote the key
73
-
publicKeys = lib.optional (builtins.pathExists cachePub) (
74
-
lib.removeSuffix "\n" (builtins.readFile cachePub)
75
-
);
76
-
# cache.nixos.org keeps serving what it signed; only push our own paths
77
-
pushFlags = [
78
-
"--ignore-upstream-key"
79
-
"cache.nixos.org-1"
80
-
];
81
-
};
82
-
};
83
-
84
-
systemd.services.nix-grpc-daemon.serviceConfig = {
85
-
LoadCredential = map (f: "${f}:${vars.nix-farm-server.files.${f}.path}") [
86
-
"server.crt"
87
-
"server.key"
88
-
"ca.crt"
89
-
];
90
-
# Upstream's 2G default assumes the unit only proxies, but `niks3 push`
91
-
# runs in this cgroup too and got OOM-killed on large NARs (kernel
92
-
# modules), taking the whole daemon down with it.
93
-
MemoryMax = "8G";
94
-
};
95
-
96
-
networking.firewall.interfaces.retiolum.allowedTCPPorts = [ 50051 ];
97
-
}
-35
configs/nix-farm/ca.nix
-35
configs/nix-farm/ca.nix
···
1
-
{ pkgs, ... }:
2
-
# Private CA of the build farm plus the client cert lass uses towards it.
3
-
# Shared, so every machine importing this sees the same CA. Clan requires a
4
-
# shared generator to be defined identically on all machines, so nothing is
5
-
# deployed from here: per-machine generators (nix-farm-server in builder.nix,
6
-
# nix-farm-client in client.nix) depend on it and copy out the files they
7
-
# need with their own owner/mode. The certs are not secret, but as secret
8
-
# files their paths are known before `clan vars generate` ran, so the configs
9
-
# still evaluate until then.
10
-
{
11
-
clan.core.vars.generators.nix-farm-ca = {
12
-
share = true;
13
-
files."ca.key".deploy = false;
14
-
files."ca.crt".deploy = false;
15
-
files."client.key".deploy = false;
16
-
files."client.crt".deploy = false;
17
-
runtimeInputs = [ pkgs.openssl ];
18
-
script = ''
19
-
openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
20
-
-keyout "$out/ca.key" -out "$out/ca.crt" -days 3650 \
21
-
-subj "/CN=nix-farm-ca" \
22
-
-addext "basicConstraints=critical,CA:TRUE" \
23
-
-addext "keyUsage=critical,keyCertSign,cRLSign"
24
-
25
-
openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \
26
-
-keyout "$out/client.key" -out client.csr -subj "/CN=lass"
27
-
printf '%s\n' \
28
-
"basicConstraints=critical,CA:FALSE" \
29
-
"keyUsage=critical,digitalSignature" \
30
-
"extendedKeyUsage=clientAuth" > client.ext
31
-
openssl x509 -req -in client.csr -CA "$out/ca.crt" -CAkey "$out/ca.key" \
32
-
-CAcreateserial -days 3650 -extfile client.ext -out "$out/client.crt"
33
-
'';
34
-
};
35
-
}
-141
configs/nix-farm/cache.nix
-141
configs/nix-farm/cache.nix
···
1
-
{
2
-
config,
3
-
pkgs,
4
-
self,
5
-
...
6
-
}:
7
-
let
8
-
# TODO: create this bucket (public) + an application key for it in B2
9
-
bucket = "lassulus-nix-cache";
10
-
region = "eu-central-003";
11
-
endpoint = "s3.${region}.backblazeb2.com";
12
-
13
-
vars = config.clan.core.vars.generators;
14
-
in
15
-
{
16
-
imports = [ self.inputs.niks3.nixosModules.niks3 ];
17
-
18
-
# B2 credentials (prompted), the niks3 API token (generated; one copy per
19
-
# service user that reads it).
20
-
clan.core.vars.generators.niks3 = {
21
-
prompts.b2_key_id = {
22
-
description = "Backblaze B2 keyID with access to the ${bucket} bucket";
23
-
type = "line";
24
-
};
25
-
prompts.b2_application_key = {
26
-
description = "Backblaze B2 applicationKey for that keyID";
27
-
type = "hidden";
28
-
};
29
-
files."s3-access-key".owner = "niks3";
30
-
files."s3-secret-key".owner = "niks3";
31
-
files."api-token".owner = "niks3";
32
-
files."api-token-grpc".owner = "nix-grpc-daemon";
33
-
runtimeInputs = [
34
-
pkgs.coreutils
35
-
pkgs.openssl
36
-
];
37
-
script = ''
38
-
tr -d '\n' < "$prompts/b2_key_id" > "$out/s3-access-key"
39
-
tr -d '\n' < "$prompts/b2_application_key" > "$out/s3-secret-key"
40
-
openssl rand -hex 32 | tr -d '\n' > "$out/api-token"
41
-
cp "$out/api-token" "$out/api-token-grpc"
42
-
'';
43
-
};
44
-
45
-
# Public half is read by configs/default.nix from the vars dir.
46
-
clan.core.vars.generators.nix-cache-signing = {
47
-
files."key.sec".owner = "niks3";
48
-
files."key.pub".secret = false;
49
-
runtimeInputs = [ pkgs.nix ];
50
-
script = ''
51
-
nix --extra-experimental-features nix-command key generate-secret \
52
-
--key-name cache.lassul.us-1 > "$out/key.sec"
53
-
nix --extra-experimental-features nix-command key convert-secret-to-public \
54
-
< "$out/key.sec" > "$out/key.pub"
55
-
'';
56
-
};
57
-
58
-
services.niks3 = {
59
-
enable = true;
60
-
s3 = {
61
-
inherit endpoint bucket region;
62
-
accessKeyFile = vars.niks3.files."s3-access-key".path;
63
-
secretKeyFile = vars.niks3.files."s3-secret-key".path;
64
-
};
65
-
apiTokenFile = vars.niks3.files."api-token".path;
66
-
signKeyFiles = [ vars.nix-cache-signing.files."key.sec".path ];
67
-
cacheUrl = "https://cache.lassul.us";
68
-
# Behind cache.nixos.org (40): nix asks upstream first, so B2 only sees
69
-
# lookups for our own paths. At niks3's default 30 every closure path hit
70
-
# B2 first, and a farm run's ~1M misses got throttled with 503s.
71
-
priority = 50;
72
-
# The CI build farm (tools/ci-farm) pushes with the job's GitHub OIDC
73
-
# token. Fork PRs don't get one.
74
-
oidc.providers.github = {
75
-
issuer = "https://token.actions.githubusercontent.com";
76
-
audience = "https://niks3.lassul.us";
77
-
boundClaims.repository = [ "lassulus/superconfig" ];
78
-
};
79
-
nginx = {
80
-
enable = true;
81
-
domain = "niks3.lassul.us";
82
-
};
83
-
};
84
-
85
-
# Reads straight from the public bucket. niks3 uploads the metadata
86
-
# zstd-compressed with Content-Encoding: zstd; set it here so nix decodes
87
-
# it whether or not the bucket hands the header back. Same rules as
88
-
# clan-infra's Fastly config (modules/terranix/cache-new.nix): narinfo, .ls,
89
-
# realisations/ and log/. NARs (nar/*.nar.zst) are plain objects, nix
90
-
# decodes them via the narinfo's Compression field.
91
-
# `resolve`: looked up at runtime via resolved's stub, so an unresolvable
92
-
# B2 at startup doesn't keep nginx (and every other vhost) from starting.
93
-
# `keepalive` + HTTP/1.1 below: reuse TLS connections to B2. Without them
94
-
# nginx spoke HTTP/1.0 with a new handshake per request; at a farm run's
95
-
# ~400 req/s B2 started closing connections mid-response.
96
-
# `max_fails=0`: B2 closes idle keepalive connections now and then. With the
97
-
# default max_fails=1 one such close marked that B2 address down for 10s;
98
-
# once all of them were, every request got "no live upstreams" and a 502
99
-
# (failed a farm run). A GET/HEAD on a closed reused connection is retried
100
-
# (default `proxy_next_upstream error timeout`) without using up a try, on
101
-
# the next address, or on the same one if it resolves to only one.
102
-
services.nginx.appendHttpConfig = ''
103
-
upstream cache_lassul_us_b2 {
104
-
zone cache_lassul_us_b2 64k;
105
-
resolver 127.0.0.53 ipv6=off;
106
-
server ${endpoint}:443 resolve max_fails=0;
107
-
keepalive 64;
108
-
keepalive_requests 10000;
109
-
keepalive_timeout 60s;
110
-
}
111
-
map $uri $cache_lassul_us_encoding {
112
-
default "";
113
-
~\.(narinfo|ls)$ zstd;
114
-
~^/realisations/ zstd;
115
-
~^/log/ zstd;
116
-
}
117
-
'';
118
-
services.nginx.virtualHosts."cache.lassul.us" = {
119
-
enableACME = true;
120
-
forceSSL = true;
121
-
locations."/".extraConfig = ''
122
-
proxy_pass https://cache_lassul_us_b2/${bucket}/;
123
-
proxy_set_header Host ${endpoint};
124
-
proxy_ssl_server_name on;
125
-
proxy_ssl_name ${endpoint};
126
-
proxy_http_version 1.1;
127
-
proxy_set_header Connection "";
128
-
# cache keys never have a query string; don't forward one to S3
129
-
set $args "";
130
-
# S3 answers 403 for keys missing from a bucket it won't list
131
-
proxy_intercept_errors on;
132
-
error_page 403 =404 @notfound;
133
-
134
-
gzip off;
135
-
proxy_hide_header Content-Encoding;
136
-
# only sent on 2xx/3xx, and not at all when the map yields ""
137
-
add_header Content-Encoding $cache_lassul_us_encoding;
138
-
'';
139
-
locations."@notfound".return = "404";
140
-
};
141
-
}
-77
configs/nix-farm/client.nix
-77
configs/nix-farm/client.nix
···
1
-
{
2
-
config,
3
-
lib,
4
-
options,
5
-
self,
6
-
...
7
-
}:
8
-
# Client of the neoprism build farm (configs/nix-farm), for NixOS and
9
-
# nix-darwin. The plugin finds client.crt/client.key in
10
-
# /var/lib/nix-grpc-store by itself; the CA has to be passed explicitly:
11
-
# nix build --store 'grpc://neoprism.r:50051?ca-cert=/var/lib/nix-grpc-store/ca.crt' \
12
-
# --eval-store auto .#nixosConfigurations.<m>.config.system.build.toplevel
13
-
let
14
-
files = [
15
-
"ca.crt"
16
-
"client.crt"
17
-
"client.key"
18
-
];
19
-
client = config.clan.core.vars.generators.nix-farm-client.files;
20
-
in
21
-
{
22
-
imports = [
23
-
# darwinModules.default is only the daemon; the grpc:// plugin is in the
24
-
# client module, which only sets nix.settings.plugin-files and works on
25
-
# nix-darwin as is.
26
-
"${self.inputs.nix-grpc-store}/nixos/client.nix"
27
-
./ca.nix
28
-
];
29
-
30
-
config = lib.mkMerge [
31
-
{ programs.nix-grpc-store.enable = true; }
32
-
33
-
# Per-machine copy of the shared CA's client files (see ca.nix).
34
-
{
35
-
clan.core.vars.generators.nix-farm-client = {
36
-
dependencies = [ "nix-farm-ca" ];
37
-
files = lib.genAttrs files (_: { });
38
-
script = lib.concatMapStrings (f: ''
39
-
cp "$in/nix-farm-ca/${f}" "$out/${f}"
40
-
'') files;
41
-
};
42
-
}
43
-
44
-
# Branch on options, not pkgs: the latter recurses through the attr names.
45
-
(lib.optionalAttrs (options ? system.primaryUser) {
46
-
# clan's /run/secrets installer is Linux-only, so these come in as
47
-
# activation secrets (root-owned, below secretLocation) and are copied
48
-
# to where the plugin looks by default, readable by the user running nix.
49
-
clan.core.vars.generators.nix-farm-client.files = lib.genAttrs files (_: {
50
-
deploy = true;
51
-
neededFor = "activation";
52
-
});
53
-
# Skipped while the secrets have not been uploaded yet.
54
-
system.activationScripts.postActivation.text = ''
55
-
install -d -m 0755 /var/lib/nix-grpc-store
56
-
''
57
-
+ lib.concatMapStrings (f: ''
58
-
if [ -e ${client.${f}.path} ]; then
59
-
install -m 0400 -o ${config.system.primaryUser} ${client.${f}.path} /var/lib/nix-grpc-store/${f}
60
-
fi
61
-
'') files;
62
-
})
63
-
64
-
(lib.optionalAttrs (options ? systemd.tmpfiles) {
65
-
# Regular /run/secrets, readable by wheel (the users running nix) and
66
-
# root (nix-daemon's build hook), linked to where the plugin looks.
67
-
clan.core.vars.generators.nix-farm-client.files = lib.genAttrs files (_: {
68
-
deploy = true;
69
-
group = "wheel";
70
-
mode = "0440";
71
-
});
72
-
systemd.tmpfiles.rules = map (
73
-
f: "L+ /var/lib/nix-grpc-store/${f} - - - - ${client.${f}.path}"
74
-
) files;
75
-
})
76
-
];
77
-
}
-1
machines/barnacle/configuration.nix
-1
machines/barnacle/configuration.nix
···
10
10
self.inputs.nix-index-database.darwinModules.nix-index
11
11
self.inputs.nix-rosetta-builder.darwinModules.default
12
12
./retiolum.nix
13
-
../../configs/nix-farm/client.nix
14
13
];
15
14
clan.core.networking.targetHost = "root@localhost";
16
15
programs.nix-index-database.comma.enable = true;
+2
-4
machines/neoprism/config.nix
+2
-4
machines/neoprism/config.nix
···
118
118
# nixpkgs PR triage dashboard (review.lassul.us)
119
119
../../configs/nixpkgs-triage.nix
120
120
121
-
# build farm (grpc://neoprism.r:50051) + binary cache (cache.lassul.us,
122
-
# niks3.lassul.us)
123
-
../../configs/nix-farm/cache.nix
124
-
../../configs/nix-farm/builder.nix
121
+
# binary cache (cache.lassul.us, niks3.lassul.us)
122
+
../../configs/nix-cache.nix
125
123
];
126
124
127
125
# lassul.us shouldn't be the default vhost here (nginx.nix already sets one)
-281
tools/ci-farm/ci-farm.sh
-281
tools/ci-farm/ci-farm.sh
···
1
-
# shellcheck shell=bash
2
-
# (writeShellApplication adds the shebang and `set -euo pipefail`.)
3
-
4
-
# On-demand nix-grpc-store farm across GitHub Actions runners.
5
-
#
6
-
# Runners can't reach each other, so every hop goes through an iroh tunnel
7
-
# (dumbpipe) that ends on 127.0.0.1 on both sides:
8
-
#
9
-
# farm job machine job i
10
-
# envoy :50051 ◄─────────────── iroh ─ :5103 ◄─ client (machine i)
11
-
# ├─► daemon :50052 ◄──────── iroh ─ :5101 scheduler session
12
-
# └─► :511i ───────────────── iroh ─► :5100 daemon (builder)
13
-
# daemon :50052 (scheduler)
14
-
#
15
-
# Every daemon and envoy present the same TLS identity (FARM_KEY with
16
-
# farm.crt, SAN 127.0.0.1) and accept only it, as `trusted`. The iroh
17
-
# endpoint keys are derived from FARM_KEY, so it is the only secret. Outputs
18
-
# go to niks3 with the job's GitHub OIDC token, before a build counts as
19
-
# done, so a machine job can leave once its own machine is built.
20
-
21
-
state=/tmp/ci-farm
22
-
niks3_url=https://niks3.lassul.us
23
-
24
-
mkdir -p "$state"
25
-
chmod 700 "$state"
26
-
printf '%s\n' "${FARM_KEY:?FARM_KEY must hold the farm private key}" >"$state/farm.key"
27
-
chmod 600 "$state/farm.key"
28
-
unset FARM_KEY
29
-
# Where envoy's config (flake-module.nix) expects it, next to the key.
30
-
install -m 644 "$FARM_CRT" "$state/farm.crt"
31
-
FARM_CRT=$state/farm.crt
32
-
33
-
seed=$(sha256sum <"$state/farm.key" | cut -d' ' -f1)
34
-
derive() { printf '%s:%s' "$seed" "$1" | sha256sum | cut -d' ' -f1; }
35
-
ticket() { IROH_SECRET=$(derive "iroh-$1") dumbpipe generate-ticket; }
36
-
37
-
trap 'kill $(jobs -p) 2>/dev/null || true' EXIT
38
-
39
-
# The GitHub OIDC token of the job, refreshed in the background (niks3 re-reads
40
-
# the file, the daemon notices the new inode). Outside Actions, point
41
-
# NIKS3_TOKEN_FILE at a niks3 API token instead. A failed fetch keeps the
42
-
# current token: `set -e` doesn't apply in `niks3_token || ...`, so every step
43
-
# returns explicitly.
44
-
niks3_token() {
45
-
local token
46
-
token=$(curl -fsS --retry 4 --retry-all-errors \
47
-
-H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN:?needs id-token: write, or NIKS3_TOKEN_FILE}" \
48
-
"$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$niks3_url" | jq -er .value) || return 1
49
-
[ -n "$token" ] || return 1
50
-
printf '%s\n' "$token" >"$state/niks3-token.new" &&
51
-
mv "$state/niks3-token.new" "$state/niks3-token"
52
-
}
53
-
# Refreshes every 2 minutes, every 20 seconds after a failure. The token is
54
-
# valid for 5 minutes: once it is 4 minutes old a builder (machine writes
55
-
# builder.pid) could not publish its outputs anymore, so drain it, and the
56
-
# scheduler hands its builds to the others.
57
-
refresh_niks3_token() {
58
-
local fetched=$SECONDS wait=120
59
-
while sleep "$wait"; do
60
-
if niks3_token; then
61
-
fetched=$SECONDS wait=120
62
-
continue
63
-
fi
64
-
wait=20
65
-
echo "ci-farm: refreshing the niks3 token failed, it is $((SECONDS - fetched))s old" >&2
66
-
if [ $((SECONDS - fetched)) -ge 240 ] && [ -s "$state/builder.pid" ]; then
67
-
echo "ci-farm: no fresh niks3 token, draining the builder" >&2
68
-
kill -TERM "$(<"$state/builder.pid")" 2>/dev/null || true
69
-
rm -f "$state/builder.pid"
70
-
fi
71
-
done
72
-
}
73
-
if [ -n "${NIKS3_TOKEN_FILE:-}" ]; then
74
-
ln -sf "$NIKS3_TOKEN_FILE" "$state/niks3-token"
75
-
else
76
-
niks3_token
77
-
refresh_niks3_token &
78
-
fi
79
-
80
-
daemon=(
81
-
nix-grpc-daemon
82
-
--proxy-socket /nix/var/nix/daemon-socket/socket
83
-
--tls-cert "$FARM_CRT"
84
-
--tls-key "$state/farm.key"
85
-
--client-ca "$FARM_CRT"
86
-
--allow farm=trusted
87
-
--niks3 "$niks3_url"
88
-
--niks3-token-file "$state/niks3-token"
89
-
# cache.nixos.org keeps serving what it signed; only push our own paths
90
-
--niks3-push "niks3 push --ignore-upstream-key cache.nixos.org-1"
91
-
--min-free 2G
92
-
)
93
-
94
-
wait_http() {
95
-
for _ in $(seq 60); do
96
-
curl -fs -o /dev/null "$1" && return
97
-
sleep 1
98
-
done
99
-
echo "ci-farm: $1 did not come up" >&2
100
-
exit 1
101
-
}
102
-
103
-
# The jobs NAME... of this run have all finished. In a workflow called from
104
-
# another (ci.yml) the API names them "<caller job> / NAME". Outside Actions
105
-
# there are no jobs to watch: never.
106
-
jobs_done() {
107
-
local jobs name
108
-
[ -n "${GITHUB_RUN_ID:-}" ] || return 1
109
-
jobs=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs?per_page=100" \
110
-
--jq '.jobs[] | "\(.status) \(.name | sub("^.* / "; ""))"') || return 1
111
-
for name in "$@"; do
112
-
grep -qxF "completed $name" <<<"$jobs" || return 1
113
-
done
114
-
}
115
-
116
-
# ci-farm farm MACHINE...: scheduler and balancer for the machine jobs
117
-
# (builder i builds the i-th MACHINE), until they have all finished.
118
-
farm() {
119
-
local i port clusters
120
-
"${daemon[@]}" --role scheduler --listen 127.0.0.1:50052 2>"$state/daemon.log" &
121
-
local pid=$!
122
-
tail -f "$state/daemon.log" >&2 &
123
-
envoy -c "$ENVOY_CONFIG" --log-level warn &
124
-
wait_http http://127.0.0.1:9901/ready
125
-
# A daemon without the builder role schedules only once niks3 has made it
126
-
# the leader.
127
-
for _ in $(seq 60); do
128
-
grep -q event=scheduler_take_over "$state/daemon.log" && break
129
-
sleep 2
130
-
done
131
-
if ! grep -q event=scheduler_take_over "$state/daemon.log"; then
132
-
echo "ci-farm: the scheduler did not get the lead from niks3" >&2
133
-
exit 1
134
-
fi
135
-
IROH_SECRET=$(derive iroh-farm) dumbpipe listen-tcp --host 127.0.0.1:50052 &
136
-
IROH_SECRET=$(derive iroh-lb) dumbpipe listen-tcp --host 127.0.0.1:50051 &
137
-
echo "ci-farm: the farm is up"
138
-
139
-
# Builder i shows up in envoy's clusters once its scheduler session is up,
140
-
# i.e. its own iroh endpoint is online. Only then dial it (a dialer started
141
-
# before the endpoint is discoverable never reaches it), and dial again
142
-
# when the tunnel dies or envoy's health checks fail through it for a
143
-
# minute.
144
-
declare -A dialers=() since=()
145
-
until jobs_done "$@"; do
146
-
kill -0 "$pid" || exit 1
147
-
clusters=$(curl -fs http://127.0.0.1:9901/clusters) || clusters=
148
-
for i in $(seq $#); do
149
-
port=$((5110 + i))
150
-
grep -qF "::127.0.0.1:$port::health_flags::" <<<"$clusters" || continue
151
-
if [ -n "${dialers[$i]:-}" ] && kill -0 "${dialers[$i]}" 2>/dev/null; then
152
-
if grep -qxE ".*::127\.0\.0\.1:$port::health_flags::healthy" <<<"$clusters"; then
153
-
since[$i]=$SECONDS
154
-
continue
155
-
fi
156
-
[ $((SECONDS - since[$i])) -lt 60 ] && continue
157
-
kill "${dialers[$i]}" 2>/dev/null || true
158
-
fi
159
-
IROH_SECRET=$(derive "iroh-farm-dial-$i") \
160
-
dumbpipe connect-tcp --addr "127.0.0.1:$port" "$(ticket "worker-$i")" &
161
-
dialers[$i]=$!
162
-
since[$i]=$SECONDS
163
-
done
164
-
sleep 15
165
-
done
166
-
}
167
-
168
-
# grpc.health.v1 Check on 127.0.0.1:PORT over the farm's TLS, prints the
169
-
# response headers and trailers. An empty HealthCheckRequest: 0 = uncompressed,
170
-
# 0 bytes.
171
-
health() {
172
-
printf '\0\0\0\0\0' | curl -s -o /dev/null -D - --max-time 10 --http2 \
173
-
-H 'content-type: application/grpc' -H 'te: trailers' --data-binary @- \
174
-
--cacert "$FARM_CRT" --cert "$FARM_CRT" --key "$state/farm.key" \
175
-
"https://127.0.0.1:$1/grpc.health.v1.Health/Check"
176
-
}
177
-
178
-
# A dumbpipe dialer started before the farm is discoverable never reaches it,
179
-
# not even once the farm is online. Probe PORT with short-lived dialers to
180
-
# TICKET until it answers an HTTP/2 request over the farm's TLS, then leave
181
-
# the long-lived dialer to the caller. Gives up once the farm job has
182
-
# finished (checked every 2 minutes, the API is rate limited).
183
-
wait_farm() {
184
-
local port=$1 ticket=$2 probe n=0
185
-
while true; do
186
-
IROH_SECRET=$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n') \
187
-
dumbpipe connect-tcp --addr "127.0.0.1:$port" "$ticket" 2>/dev/null &
188
-
probe=$!
189
-
sleep 3
190
-
if health "$port" >/dev/null; then
191
-
kill "$probe" 2>/dev/null || true
192
-
return
193
-
fi
194
-
kill "$probe" 2>/dev/null || true
195
-
sleep 5
196
-
n=$((n + 1))
197
-
if [ $((n % 15)) -eq 0 ] && jobs_done coordinator; then
198
-
echo "ci-farm: the farm did not come up" >&2
199
-
exit 1
200
-
fi
201
-
done
202
-
}
203
-
204
-
# Envoy (via PORT) has a builder to send requests to only once one has joined
205
-
# the scheduler and the farm has dialed it and health checked it through the
206
-
# tunnel; until then it answers itself, with grpc-status 14 ("no healthy
207
-
# upstream", or "remote connection failure" for a builder not dialed yet).
208
-
# A client gives up on a store it has never reached, so wait until a health
209
-
# check through envoy reaches a builder (grpc-status 0). Gives up when the
210
-
# builder PID dies or the farm job has finished, like wait_farm.
211
-
wait_builder() {
212
-
local port=$1 pid=$2 n=0
213
-
until health "$port" | grep -qE '^grpc-status: 0\s*$'; do
214
-
kill -0 "$pid" 2>/dev/null || exit 1
215
-
sleep 5
216
-
n=$((n + 1))
217
-
if [ $((n % 24)) -eq 0 ] && jobs_done coordinator; then
218
-
echo "ci-farm: the farm has no builder" >&2
219
-
exit 1
220
-
fi
221
-
done
222
-
}
223
-
224
-
# ci-farm machine INDEX NAME: builder INDEX of the farm (NAME is the farm's
225
-
# INDEX-th MACHINE) while it builds the toplevel of NAME through the farm.
226
-
# Leaves once that is done: draining finishes its running builds, whose
227
-
# outputs are in niks3 then, and the scheduler hands the rest to the others.
228
-
machine() {
229
-
local i=${1:?machine index} name=${2:?machine name} farm lb eval pid status=0
230
-
farm=$(ticket farm)
231
-
lb=$(ticket lb)
232
-
nix eval --raw ".#nixosConfigurations.$name.config.system.build.toplevel.drvPath" >"$state/drv" &
233
-
eval=$!
234
-
IROH_SECRET=$(derive "iroh-worker-$i") dumbpipe listen-tcp --host 127.0.0.1:5100 &
235
-
wait_farm 5102 "$farm"
236
-
wait_farm 5104 "$lb"
237
-
echo "ci-farm: the farm is up"
238
-
IROH_SECRET=$(derive "iroh-worker-dial-$i") \
239
-
dumbpipe connect-tcp --addr 127.0.0.1:5101 "$farm" &
240
-
IROH_SECRET=$(derive "iroh-worker-lb-$i") \
241
-
dumbpipe connect-tcp --addr 127.0.0.1:5103 "$lb" &
242
-
"${daemon[@]}" --role builder --listen 127.0.0.1:5100 --scheduler 127.0.0.1:5101 \
243
-
--advertise "127.0.0.1:$((5110 + i))" --worker-name "$name" &
244
-
pid=$!
245
-
echo "$pid" >"$state/builder.pid"
246
-
247
-
if wait "$eval"; then
248
-
echo "ci-farm: $name: $(<"$state/drv")"
249
-
wait_builder 5103 "$pid"
250
-
"$NIX_WITH_PLUGIN" build --no-link --print-out-paths --eval-store auto \
251
-
--store "grpc://127.0.0.1:5103?system=x86_64-linux&ca-cert=$FARM_CRT&client-cert=$FARM_CRT&client-key=$state/farm.key" \
252
-
"$(<"$state/drv")^*" || status=$?
253
-
else
254
-
status=$?
255
-
fi
256
-
[ "$status" -eq 0 ] || echo "ci-farm: $name failed" >&2
257
-
258
-
# Drains: running builds finish and publish first.
259
-
kill -TERM "$pid" 2>/dev/null || true
260
-
wait "$pid" || true
261
-
return "$status"
262
-
}
263
-
264
-
# ci-farm push PATH...: pushes the closures of PATH... to niks3, e.g. the
265
-
# farm's own closure, which no other cache has.
266
-
push() {
267
-
niks3 push --server-url "$niks3_url" --auth-token-path "$state/niks3-token" \
268
-
--ignore-upstream-key cache.nixos.org-1 "$@"
269
-
}
270
-
271
-
cmd=${1:?usage: ci-farm farm MACHINE... | ci-farm machine INDEX NAME | ci-farm push PATH...}
272
-
shift
273
-
case $cmd in
274
-
farm) farm "$@" ;;
275
-
machine) machine "$@" ;;
276
-
push) push "$@" ;;
277
-
*)
278
-
echo "ci-farm: unknown command $cmd" >&2
279
-
exit 1
280
-
;;
281
-
esac
-12
tools/ci-farm/farm.crt
-12
tools/ci-farm/farm.crt
···
1
-
-----BEGIN CERTIFICATE-----
2
-
MIIBtTCCAVugAwIBAgIUf55GJik0D9Ps1ijr4MNBQhxPH30wCgYIKoZIzj0EAwIw
3
-
DzENMAsGA1UEAwwEZmFybTAeFw0yNjEwMTExNzA4MTNaFw0zNjEwMDgxNzA4MTNa
4
-
MA8xDTALBgNVBAMMBGZhcm0wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAStbhFn
5
-
CJvDygTYQUGc3Ck18WSNS2A5DhGvL0bT3BHpOpNTUSeZW+RWW3QemIHFkQjEfhYS
6
-
oTKieGTlWgtQIJepo4GUMIGRMB0GA1UdDgQWBBQ/TD12Xjs5LPh2YLGeL8CDhKut
7
-
LzAfBgNVHSMEGDAWgBQ/TD12Xjs5LPh2YLGeL8CDhKutLzAPBgNVHREECDAGhwR/
8
-
AAABMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgKEMB0GA1UdJQQWMBQG
9
-
CCsGAQUFBwMBBggrBgEFBQcDAjAKBggqhkjOPQQDAgNIADBFAiAvWiuqtBuhoDO1
10
-
6mlRKmqkwStUqhPDSTaQvaKOTAUVBgIhAMGDqzX7JCaZhEwdXwg9HfYGDQFqzseY
11
-
2o0he+UhHyTX
12
-
-----END CERTIFICATE-----
-96
tools/ci-farm/flake-module.nix
-96
tools/ci-farm/flake-module.nix
···
1
-
{ inputs, ... }:
2
-
{
3
-
perSystem =
4
-
{
5
-
lib,
6
-
pkgs,
7
-
system,
8
-
...
9
-
}:
10
-
let
11
-
grpc = inputs.nix-grpc-store.packages.${system};
12
-
13
-
# Envoy as configured by nix-grpc-store's NixOS module, rendered for a
14
-
# plain Linux runner. The farm identity on both sides: the clients in
15
-
# the machine jobs reach it over iroh, the nodes verify it and grant it
16
-
# `trusted` (so it can read the builder list from the scheduler).
17
-
envoySettings =
18
-
(inputs.nixpkgs.lib.nixosSystem {
19
-
modules = [
20
-
inputs.nix-grpc-store.nixosModules.lb
21
-
{
22
-
nixpkgs.pkgs = pkgs;
23
-
services.nix-grpc-farm-lb = {
24
-
enable = true;
25
-
listen = "127.0.0.1:50051";
26
-
scheduler = "127.0.0.1:50052";
27
-
systems = [ system ];
28
-
# Written by ci-farm.sh. Envoy watches the directories of
29
-
# these files, so they must exist on the runner.
30
-
tls = {
31
-
certFile = "/tmp/ci-farm/farm.crt";
32
-
keyFile = "/tmp/ci-farm/farm.key";
33
-
clientCaFile = "/tmp/ci-farm/farm.crt";
34
-
upstream = {
35
-
certFile = "/tmp/ci-farm/farm.crt";
36
-
keyFile = "/tmp/ci-farm/farm.key";
37
-
caFile = "/tmp/ci-farm/farm.crt";
38
-
};
39
-
};
40
-
};
41
-
}
42
-
];
43
-
}).config.services.envoy.settings;
44
-
45
-
# The module verifies a client certificate but doesn't require one, and
46
-
# passes requests on with the farm identity. Require it: the listener
47
-
# is reachable over iroh.
48
-
envoyConfig = (pkgs.formats.json { }).generate "ci-farm-envoy.json" (
49
-
envoySettings
50
-
// {
51
-
static_resources = envoySettings.static_resources // {
52
-
listeners = map (
53
-
listener:
54
-
listener
55
-
// {
56
-
filter_chains = map (
57
-
chain:
58
-
lib.recursiveUpdate chain { transport_socket.typed_config.require_client_certificate = true; }
59
-
) listener.filter_chains;
60
-
}
61
-
) envoySettings.static_resources.listeners;
62
-
};
63
-
}
64
-
);
65
-
in
66
-
{
67
-
# ci-farm farm MACHINE... scheduler + balancer until the machine
68
-
# jobs of MACHINE... have finished
69
-
# ci-farm machine INDEX NAME builder, builds the toplevel of NAME
70
-
# through the farm
71
-
# ci-farm push PATH... pushes the closures of PATH... to niks3
72
-
#
73
-
# Runs in .github/workflows/farm.yml. See ci-farm.sh for the layout.
74
-
packages = lib.optionalAttrs pkgs.stdenv.hostPlatform.isLinux {
75
-
ci-farm = pkgs.writeShellApplication {
76
-
name = "ci-farm";
77
-
runtimeInputs = [
78
-
grpc.default
79
-
inputs.niks3.packages.${system}.niks3
80
-
pkgs.coreutils
81
-
pkgs.curl
82
-
pkgs.dumbpipe
83
-
pkgs.envoy-bin
84
-
pkgs.gh
85
-
pkgs.jq
86
-
];
87
-
runtimeEnv = {
88
-
FARM_CRT = ./farm.crt;
89
-
ENVOY_CONFIG = envoyConfig;
90
-
NIX_WITH_PLUGIN = lib.getExe grpc.nix-with-plugin;
91
-
};
92
-
text = builtins.readFile ./ci-farm.sh;
93
-
};
94
-
};
95
-
};
96
-
}