This repository has no description
9.5 kB
250 lines
1name: CI
2
3on:
4 push:
5 branches: [master]
6 pull_request:
7 workflow_dispatch:
8
9# A newer run for the same ref replaces a running one.
10concurrency:
11 group: ci-${{ github.ref }}
12 cancel-in-progress: true
13
14# One job per check shows failing checks in the run overview, the same for
15# the NixOS machines, see the eval job.
16jobs:
17 list:
18 runs-on: ubuntu-latest
19 outputs:
20 checks: ${{ steps.list.outputs.checks }}
21 steps:
22 - uses: actions/checkout@v4
23 with:
24 # Fetch all history for git-based operations
25 fetch-depth: 0
26
27 - uses: cachix/install-nix-action@v31
28
29 - id: list
30 run: |
31 checks=$(nix eval --json .#checks.x86_64-linux --apply builtins.attrNames)
32 echo "checks=$checks" >> "$GITHUB_OUTPUT"
33
34 check:
35 needs: list
36 name: ${{ matrix.check }}
37 runs-on: ubuntu-latest
38 strategy:
39 fail-fast: false
40 matrix:
41 check: ${{ fromJSON(needs.list.outputs.checks) }}
42 steps:
43 - uses: actions/checkout@v4
44 with:
45 # Fetch all history for git-based operations
46 fetch-depth: 0
47
48 - uses: cachix/install-nix-action@v31
49
50 - run: nix build -L --no-link '.#checks.x86_64-linux.${{ matrix.check }}'
51
52 outputs:
53 name: packages and devShells
54 runs-on: ubuntu-latest
55 steps:
56 - uses: actions/checkout@v4
57 with:
58 # Fetch all history for git-based operations
59 fetch-depth: 0
60
61 - uses: cachix/install-nix-action@v31
62
63 - run: |
64 for output in packages devShells; do
65 echo "$output:"
66 # shellcheck disable=SC2016 # ${...} is Nix interpolation
67 nix eval --raw ".#$output.x86_64-linux" --apply 'xs: builtins.deepSeq (builtins.mapAttrs (_: x: x.drvPath) xs) "${toString (builtins.length (builtins.attrNames xs))} evaluated\n"'
68 done
69
70 # The NixOS machines, built on plain runners with hestia (the GitHub Actions
71 # cache) as binary cache next to cache.lassul.us:
72 #
73 # - eval: evaluates ciToplevels once, uploads the drv closures to hestia
74 # and lists the machines whose toplevel is in neither cache.
75 # - shared: builds the derivations needed by more than one of them once,
76 # so the machine jobs substitute them instead of each building them.
77 # - one job per listed machine, builds its toplevel by drv path on a lazy
78 # /nix/store (.github/lazy-store.sh): the paths it would substitute are
79 # only downloaded once a file in them is opened.
80 #
81 # Every job pushes what it added to the store to cache.lassul.us
82 # (.github/ci-build.sh), for deploys.
83 eval:
84 runs-on: ubuntu-latest
85 permissions:
86 contents: read
87 actions: read # hestia: detect evicted cache entries upfront
88 id-token: write # niks3 push
89 outputs:
90 matrix: ${{ steps.matrix.outputs.matrix }}
91 any-jobs: ${{ steps.matrix.outputs.any-jobs }}
92 manifest-version: ${{ steps.matrix.outputs.manifest-version }}
93 niks3: ${{ steps.tools.outputs.niks3 }}
94 lazy-store: ${{ steps.tools.outputs.lazy-store }}
95 steps:
96 - uses: actions/checkout@v4
97 - uses: cachix/install-nix-action@v31
98 with:
99 extra_nix_config: |
100 trusted-users = root runner
101 extra-substituters = https://cache.lassul.us
102 extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
103 - uses: Mic92/hestia@v3
104 with:
105 upstream-cache-filter: true
106 upstream-cache-key-names: cache.nixos.org-1 cache.lassul.us-1
107 filter-drv-closures: true
108 # the niks3 client of all jobs and the lazy store of the machine jobs,
109 # the others substitute them
110 - id: tools
111 run: |
112 niks3=$(nix build --no-link --print-out-paths .#niks3)
113 lazy=$(nix build --no-link --print-out-paths .#lazy-store)
114 echo "niks3=$niks3" >> "$GITHUB_OUTPUT"
115 echo "lazy-store=$lazy" >> "$GITHUB_OUTPUT"
116 PATH=$niks3/bin:$PATH .github/niks3-push.sh "$niks3" "$lazy"
117 - id: matrix
118 uses: Mic92/hestia/matrix@v3
119 with:
120 flake: ".#ciToplevels"
121 nix-eval-jobs: "nix run nixpkgs#nix-eval-jobs -- --workers 4 --max-memory-size 3072"
122 - env:
123 MATRIX: ${{ steps.matrix.outputs.matrix }}
124 run: |
125 echo "not cached: $(jq -r '[.include[].name] | join(" ")' <<<"$MATRIX")" |
126 tee -a "$GITHUB_STEP_SUMMARY"
127
128 shared:
129 needs: eval
130 if: needs.eval.outputs.any-jobs == 'true'
131 runs-on: ubuntu-latest
132 permissions:
133 contents: read
134 actions: read # hestia: detect evicted cache entries upfront
135 id-token: write # niks3 push
136 outputs:
137 manifest-version: ${{ steps.build.outputs.manifest-version }}
138 steps:
139 - uses: actions/checkout@v4
140 - uses: cachix/install-nix-action@v31
141 with:
142 extra_nix_config: |
143 trusted-users = root runner
144 extra-substituters = https://cache.lassul.us
145 extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
146 - uses: Mic92/hestia@v3
147 with:
148 upstream-cache-filter: true
149 upstream-cache-key-names: cache.nixos.org-1 cache.lassul.us-1
150 wait-manifest-version: ${{ needs.eval.outputs.manifest-version }}
151 - env:
152 NIKS3: ${{ needs.eval.outputs.niks3 }}
153 run: |
154 nix-store --realise "$NIKS3"
155 echo "$NIKS3/bin" >> "$GITHUB_PATH"
156 - id: build
157 env:
158 MATRIX: ${{ needs.eval.outputs.matrix }}
159 EVAL_MANIFEST_VERSION: ${{ needs.eval.outputs.manifest-version }}
160 run: |
161 matrix=$RUNNER_TEMP/matrix.json
162 plans=$RUNNER_TEMP/plans
163 printf '%s' "$MATRIX" > "$matrix"
164 read -ra installables <<<"$(jq -r '[.include[].installables] | join(" ")' "$matrix")"
165 "$HESTIA_BIN" prefetch "${installables[@]}"
166 .github/ci-shared.sh "$matrix" "$plans"
167 mapfile -t shared < "$plans/shared.drvs"
168 version=$EVAL_MANIFEST_VERSION
169 if [ "${#shared[@]}" -gt 0 ]; then
170 # a failed one fails the jobs of the machines that need it
171 .github/ci-build.sh "${shared[@]/%/^*}" ||
172 echo "::warning::some shared derivations failed to build"
173 # upload now instead of in the post step, so the machine jobs can
174 # wait for the manifest version that contains the shared outputs
175 "$HESTIA_BIN" drain --socket "$HESTIA_SOCKET" 2>&1 | tee "$RUNNER_TEMP/drain.txt"
176 drained=$(sed -nE 's/.*; manifest m[0-9]+#([0-9]+)$/\1/p' "$RUNNER_TEMP/drain.txt")
177 version=${drained:-$version}
178 fi
179 echo "manifest-version=$version" >> "$GITHUB_OUTPUT"
180 {
181 for name in $(jq -r '.include[].name' "$matrix"); do
182 echo "- $name: $(wc -l < "$plans/$name.drvs") derivations to build"
183 done
184 echo "- shared (needed by >= 2 machines): ${#shared[@]}"
185 printf '%s\n' "${shared[@]}" | sed -E 's|^/nix/store/[^-]+-(.*)\.drv$| \1|'
186 } | tee -a "$GITHUB_STEP_SUMMARY"
187
188 build:
189 needs: [eval, shared]
190 if: needs.eval.outputs.any-jobs == 'true'
191 name: ${{ matrix.name }}
192 runs-on: ${{ matrix.os }}
193 permissions:
194 contents: read
195 actions: read # hestia: detect evicted cache entries upfront
196 id-token: write # niks3 push
197 strategy:
198 fail-fast: false
199 matrix: ${{ fromJSON(needs.eval.outputs.matrix) }}
200 steps:
201 - uses: actions/checkout@v4
202 - uses: cachix/install-nix-action@v31
203 with:
204 extra_nix_config: |
205 trusted-users = root runner
206 extra-substituters = https://cache.lassul.us
207 extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
208 - uses: Mic92/hestia@v3
209 with:
210 upstream-cache-filter: true
211 upstream-cache-key-names: cache.nixos.org-1 cache.lassul.us-1
212 wait-manifest-version: ${{ needs.shared.outputs.manifest-version }}
213 - env:
214 NIKS3: ${{ needs.eval.outputs.niks3 }}
215 run: |
216 nix-store --realise "$NIKS3"
217 echo "$NIKS3/bin" >> "$GITHUB_PATH"
218 - name: build ${{ matrix.name }}
219 env:
220 INSTALLABLES: ${{ matrix.installables }}
221 LAZY_STORE: ${{ needs.eval.outputs.lazy-store }}
222 run: |
223 read -ra installables <<<"$INSTALLABLES"
224 "$HESTIA_BIN" prefetch "${installables[@]}"
225 .github/lazy-store.sh "${installables[@]}"
226 .github/ci-build.sh "${installables[@]}"
227 - name: lazy store log
228 if: always()
229 run: |
230 dir=$RUNNER_TEMP/lazy-store
231 if [ -s "$dir/stats.json" ]; then
232 jq -r '"\(.nar_count // 0) NARs fetched, \((.nar_bytes // 0) / 1048576 | floor) MiB"' "$dir/stats.json"
233 fi
234 cat "$dir/fuse.log" 2>/dev/null || true
235
236 # Fixed-name summary of the jobs above, the required status check for
237 # merging into master (the matrix job names change with the hosts).
238 # Skipped: no machine to build, or a job it needs failed.
239 ci:
240 needs: [list, check, outputs, eval, shared, build]
241 if: always()
242 runs-on: ubuntu-latest
243 steps:
244 - env:
245 RESULTS: ${{ join(needs.*.result, ' ') }}
246 run: |
247 echo "results: $RESULTS"
248 for r in $RESULTS; do
249 [ "$r" = success ] || [ "$r" = skipped ] || exit 1
250 done