This repository has no description
.github.ci: build the machines on a lazy /nix/store
A machine job substituted the whole input closure of the derivations it
builds, e.g. 3.4 GiB for neoprism, of which the builds open a few dozen
MiB. .github/lazy-store.sh now runs before the build: it takes what
`nix build --dry-run` would fetch, collects its closure from the
narinfos of cache.nixos.org and cache.lassul.us (tools.lazy-store),
mounts lazy-store on /nix/store, registers the paths with
`nix-store --load-db` and copies their signatures. The build itself,
ci-build.sh's niks3 push and hestia stay as they were: outputs are
written to the real store dir below the mount, and ci-build.sh's
before/after diff of the store starts after the registration.
If anything fails before the paths are registered, it logs an error,
undoes the mount and exits 0, so the build substitutes eagerly. The FUSE
log and the fetched NARs are printed by an always-run step after the
build.
The eval job builds lazy-store next to niks3 and pushes it, the machine
jobs substitute it. The eval and shared jobs keep the plain store: eval
builds nothing but niks3 and lazy-store, and shared builds package
derivations, which read most of their inputs anyway.
author
Hermes
committer
lassulus
date
(Oct 17, 2026, 1:19 AM CEST)
commit
f044be6c
f044be6cf9befe576fda3ab1b6a722a7b11d17ea
parent
319c4983
319c4983f7da466129c29d6f6ca0fca5f0b52be5
+113
-6
+91
.github/lazy-store.sh
+91
.github/lazy-store.sh
···
1
+
#!/usr/bin/env bash
2
+
# Mounts a lazy /nix/store for building INSTALLABLE... (the machine jobs of
3
+
# ci.yml): every store path the build would substitute from cache.nixos.org
4
+
# or cache.lassul.us is registered as valid from its narinfo and served by
5
+
# lazy-store (tools/lazy-store), which downloads a path's NAR only when a
6
+
# file in it is opened first. What nix builds, and what neither cache has,
7
+
# is built or substituted (also from hestia) as usual; writes go to the
8
+
# real store dir below the mount.
9
+
#
10
+
# Needs sudo, /dev/fuse and LAZY_STORE, the lazy-store package's out path
11
+
# (substitutable). If anything fails before the paths are registered, the
12
+
# mount is undone and the build substitutes eagerly: this script always
13
+
# exits 0. The FUSE log ends up in $RUNNER_TEMP/lazy-store/fuse.log.
14
+
# Usage: lazy-store.sh INSTALLABLE...
15
+
set -euo pipefail
16
+
L=$RUNNER_TEMP/lazy-store
17
+
mkdir -p "$L"
18
+
19
+
fallback() {
20
+
set +e
21
+
echo "::error::lazy store: $1; the build substitutes eagerly instead"
22
+
if [ -s "$L/fuse.log" ]; then
23
+
tail -n 50 "$L/fuse.log"
24
+
fi
25
+
if mountpoint -q /nix/store; then
26
+
sudo umount /nix/store || sudo umount -l /nix/store
27
+
fi
28
+
if mountpoint -q /lazy/nix; then
29
+
sudo umount -l /lazy/nix
30
+
fi
31
+
exit 0
32
+
}
33
+
trap 'fallback "line $LINENO failed"' ERR
34
+
35
+
nix-store --realise "$LAZY_STORE" >/dev/null || fallback "cannot get $LAZY_STORE"
36
+
lazy=$LAZY_STORE/bin/lazy-store
37
+
nixbin=$(dirname "$(readlink -f "$(command -v nix-store)")")
38
+
39
+
nix build --dry-run "$@" 2>"$L/plan" || fallback "nix build --dry-run failed: $(cat "$L/plan")"
40
+
# the path lines below the "will be fetched" header, up to the next header
41
+
awk '/^[^ ]/ { f = /will be fetched/; next } f && $1 ~ /^\/nix\/store\// { print $1 }' \
42
+
"$L/plan" >"$L/fetch"
43
+
if [ ! -s "$L/fetch" ]; then
44
+
echo "lazy store: nothing to substitute"
45
+
exit 0
46
+
fi
47
+
48
+
"$lazy" closure "$L" <"$L/fetch" >/dev/null || fallback "closure failed"
49
+
jq -r '"lazy store: \(.paths) of the \(.roots) paths to substitute and their closure (\(.file_size / 1048576 | floor) MiB to download eagerly); \(.missing | length) not in the caches, \(.incomplete) referring to those"' \
50
+
"$L/closure.json"
51
+
n=$(jq length "$L/paths.json")
52
+
if [ "$n" = 0 ]; then
53
+
exit 0
54
+
fi
55
+
56
+
sudo modprobe fuse || true
57
+
# The upper layer is the real store dir, reached through a private bind
58
+
# mount of /nix, so that the mount on /nix/store doesn't show up there.
59
+
sudo mkdir -p /lazy/nix
60
+
sudo mount --bind /nix /lazy/nix || fallback "bind mount of /nix failed"
61
+
sudo mount --make-private /lazy/nix || fallback "making /lazy/nix private failed"
62
+
# shellcheck disable=SC2024 # the log is ours, only the mount needs root
63
+
sudo "$lazy" mount "$L" /lazy/nix/store /nix/store >"$L/fuse.log" 2>&1 &
64
+
fuse=$!
65
+
# sudo runs as root (no kill -0) and stays a zombie until waited for
66
+
fuse_alive() {
67
+
local s
68
+
s=$(ps -o stat= -p "$fuse") && [[ $s != Z* ]]
69
+
}
70
+
# it fetches all .ls listings before mounting
71
+
until mountpoint -q /nix/store; do
72
+
fuse_alive || fallback "lazy-store exited before mounting /nix/store"
73
+
sleep 1
74
+
done
75
+
grep -m1 '^init: ' "$L/fuse.log" || true
76
+
77
+
some=$(jq -r 'keys[0]' "$L/paths.json")
78
+
[ -e "/nix/store/$some" ] || fallback "/nix/store/$some is not served"
79
+
# shellcheck disable=SC2024 # db.dump is ours, only nix-store needs root
80
+
sudo "$nixbin/nix-store" --load-db <"$L/db.dump" || fallback "registering the paths failed"
81
+
# registered: the mount has to stay
82
+
trap - ERR
83
+
set +e
84
+
echo "lazy store: mounted on /nix/store, $n paths registered"
85
+
# niks3 and hestia leave out what cache.nixos.org signed by the signatures
86
+
# in nix's database; without them they would upload (and so download)
87
+
# those paths
88
+
jq -r 'keys[] | "/nix/store/" + .' "$L/paths.json" |
89
+
sudo "$nixbin/nix" --extra-experimental-features nix-command \
90
+
store copy-sigs -s "file://$L/narinfo" --stdin ||
91
+
echo "::warning::lazy store: copying the signatures failed, uploads may download paths of cache.nixos.org"
+22
-6
.github/workflows/ci.yml
+22
-6
.github/workflows/ci.yml
···
74
74
# and lists the machines whose toplevel is in neither cache.
75
75
# - shared: builds the derivations needed by more than one of them once,
76
76
# so the machine jobs substitute them instead of each building them.
77
-
# - one job per listed machine, builds its toplevel by drv path.
77
+
# - one job per listed machine, builds its toplevel by drv path on a lazy
78
+
# /nix/store (.github/lazy-store.sh): the paths it would substitute are
79
+
# only downloaded once a file in them is opened.
78
80
#
79
81
# Every job pushes what it added to the store to cache.lassul.us
80
82
# (.github/ci-build.sh), for deploys.
···
88
90
matrix: ${{ steps.matrix.outputs.matrix }}
89
91
any-jobs: ${{ steps.matrix.outputs.any-jobs }}
90
92
manifest-version: ${{ steps.matrix.outputs.manifest-version }}
91
-
niks3: ${{ steps.niks3.outputs.path }}
93
+
niks3: ${{ steps.tools.outputs.niks3 }}
94
+
lazy-store: ${{ steps.tools.outputs.lazy-store }}
92
95
steps:
93
96
- uses: actions/checkout@v4
94
97
- uses: cachix/install-nix-action@v31
···
101
104
with:
102
105
upstream-cache-filter: true
103
106
filter-drv-closures: true
104
-
# the niks3 client of all jobs, the others substitute it
105
-
- id: niks3
107
+
# the niks3 client of all jobs and the lazy store of the machine jobs,
108
+
# the others substitute them
109
+
- id: tools
106
110
run: |
107
111
niks3=$(nix build --no-link --print-out-paths .#niks3)
108
-
echo "path=$niks3" >> "$GITHUB_OUTPUT"
109
-
PATH=$niks3/bin:$PATH .github/niks3-push.sh "$niks3"
112
+
lazy=$(nix build --no-link --print-out-paths .#lazy-store)
113
+
echo "niks3=$niks3" >> "$GITHUB_OUTPUT"
114
+
echo "lazy-store=$lazy" >> "$GITHUB_OUTPUT"
115
+
PATH=$niks3/bin:$PATH .github/niks3-push.sh "$niks3" "$lazy"
110
116
- id: matrix
111
117
uses: Mic92/hestia/matrix@v3
112
118
with:
···
209
215
- name: build ${{ matrix.name }}
210
216
env:
211
217
INSTALLABLES: ${{ matrix.installables }}
218
+
LAZY_STORE: ${{ needs.eval.outputs.lazy-store }}
212
219
run: |
213
220
read -ra installables <<<"$INSTALLABLES"
214
221
"$HESTIA_BIN" prefetch "${installables[@]}"
222
+
.github/lazy-store.sh "${installables[@]}"
215
223
.github/ci-build.sh "${installables[@]}"
224
+
- name: lazy store log
225
+
if: always()
226
+
run: |
227
+
dir=$RUNNER_TEMP/lazy-store
228
+
if [ -s "$dir/stats.json" ]; then
229
+
jq -r '"\(.nar_count // 0) NARs fetched, \((.nar_bytes // 0) / 1048576 | floor) MiB"' "$dir/stats.json"
230
+
fi
231
+
cat "$dir/fuse.log" 2>/dev/null || true
216
232
217
233
# Fixed-name summary of the jobs above, the required status check for
218
234
# merging into master (the matrix job names change with the hosts).