This repository has no description
11 kB
286 lines
1name: update-pkgs
2
3# Daily package and flake.lock updates. Nothing reaches master without a full
4# farm build:
5#
6# 1. update (one job per package): run its passthru.updateScript through
7# tools/update-pkgs, build the package on its platform (darwin-only
8# packages on macOS), push update/<pkg> and open or refresh its PR.
9# lock: `nix flake update`, pushed as update/flake-lock with its own PR.
10# 2. verify: for every open update/* PR, one after another (flake.lock
11# first), rebase it onto the current master, build all machines with the
12# farm (farm.yml) on that commit and squash-merge the PR only if the run
13# succeeded. A commit that already has a finished farm run is not built
14# again, so a failing PR stays open until its branch or master changes.
15#
16# One PR at a time because the farm has a fixed endpoint and runs in a single
17# concurrency group, where GitHub keeps only one pending run.
18#
19# Needs "Allow GitHub Actions to create and approve pull requests" in the repo
20# settings. Merges made with GITHUB_TOKEN don't trigger push workflows, so the
21# farm build of master after merging is dispatched here as well.
22
23on:
24 schedule:
25 - cron: "0 3 * * *" # daily 03:00 UTC
26 workflow_dispatch:
27 inputs:
28 packages:
29 description: Packages to update, space separated (empty for all)
30 default: ""
31 automerge:
32 description: Merge PRs whose farm build succeeded
33 type: boolean
34 default: true
35
36permissions:
37 contents: write
38 pull-requests: write
39 actions: write # dispatch and watch farm runs
40
41concurrency:
42 group: update-pkgs
43 cancel-in-progress: false
44
45env:
46 NIX_CONFIG: |
47 extra-substituters = https://cache.lassul.us
48 extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
49
50jobs:
51 list:
52 runs-on: ubuntu-latest
53 outputs:
54 matrix: ${{ steps.list.outputs.matrix }}
55 steps:
56 - uses: actions/checkout@v4
57 - uses: cachix/install-nix-action@v31
58 - id: list
59 env:
60 PACKAGES: ${{ inputs.packages }}
61 run: |
62 names() {
63 nix eval --json --impure --apply builtins.attrNames --expr \
64 "import ./tools/update-pkgs/scripts.nix { flake = toString ./.; host = \"$1\"; }"
65 }
66 # Linux packages update and build on ubuntu; the ones that only
67 # exist for darwin on macOS.
68 matrix=$(jq -cn --argjson linux "$(names x86_64-linux)" \
69 --argjson all "$(names aarch64-darwin)" --arg only "$PACKAGES" '
70 ($only | split(" ") | map(select(. != ""))) as $want
71 | [ $all[]
72 | select(($want | length) == 0 or (. as $p | $want | index($p)))
73 | if (. as $p | $linux | index($p))
74 then { pkg: ., os: "ubuntu-latest", system: "x86_64-linux" }
75 else { pkg: ., os: "macos-latest", system: "aarch64-darwin" }
76 end ]')
77 echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
78 echo "$matrix" | jq .
79
80 update:
81 needs: list
82 if: needs.list.outputs.matrix != '[]'
83 name: ${{ matrix.pkg }}
84 runs-on: ${{ matrix.os }}
85 strategy:
86 fail-fast: false
87 matrix:
88 include: ${{ fromJSON(needs.list.outputs.matrix) }}
89 steps:
90 - uses: actions/checkout@v4
91 with:
92 fetch-depth: 0
93 - uses: cachix/install-nix-action@v31
94 - env:
95 PKG: ${{ matrix.pkg }}
96 SYSTEM: ${{ matrix.system }}
97 GH_TOKEN: ${{ github.token }}
98 # nix-update and the update scripts query the GitHub API
99 GITHUB_TOKEN: ${{ github.token }}
100 run: |
101 version() { nix eval --raw ".#packages.$SYSTEM.$PKG.version" 2>/dev/null || echo unknown; }
102 old=$(version)
103
104 nix run .#update-pkgs -- "$PKG"
105 if [ -z "$(git status --porcelain)" ]; then
106 echo "$PKG: no update"
107 exit 0
108 fi
109
110 nix build -L --no-link ".#packages.$SYSTEM.$PKG"
111
112 new=$(version)
113 prefix=pkgs
114 [ -d "pkgs/$PKG" ] || prefix=tools
115 if [ "$old" != "$new" ]; then
116 msg="$prefix.$PKG: $old -> $new"
117 else
118 msg="$prefix.$PKG: update"
119 fi
120
121 branch="update/$PKG"
122 git config user.name "github-actions[bot]"
123 git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
124 git switch -c "$branch"
125 git add -A
126 git commit -m "$msg"
127
128 # Same change as the branch already carries (e.g. a PR whose farm
129 # build failed and upstream hasn't moved): keep its commit, so
130 # verify doesn't build it again.
131 if git fetch -q origin "$branch" 2>/dev/null &&
132 git diff --quiet FETCH_HEAD HEAD -- "$prefix/$PKG"; then
133 echo "$PKG: $branch already has this update"
134 else
135 git push --force origin "$branch"
136 fi
137
138 if [ -z "$(gh pr list --head "$branch" --state open --json number --jq '.[].number')" ]; then
139 gh pr create --base master --head "$branch" --title "$msg" \
140 --body "Automated update by the update-pkgs workflow (\`nix run .#update-pkgs -- $PKG\`), package built on $SYSTEM. Merged automatically once the farm has built all machines from this branch."
141 fi
142
143 lock:
144 # whole runs only, not manual runs for selected packages
145 if: inputs.packages == ''
146 runs-on: ubuntu-latest
147 steps:
148 - uses: actions/checkout@v4
149 with:
150 fetch-depth: 0
151 - uses: cachix/install-nix-action@v31
152 - env:
153 GH_TOKEN: ${{ github.token }}
154 GITHUB_TOKEN: ${{ github.token }}
155 run: |
156 export NIX_CONFIG="$NIX_CONFIG"$'\n'"access-tokens = github.com=$GITHUB_TOKEN"
157 nix flake update 2>&1 | tee "$RUNNER_TEMP/update.log"
158 if git diff --quiet -- flake.lock; then
159 echo "flake.lock: no update"
160 exit 0
161 fi
162
163 msg="flake.lock: update"
164 branch="update/flake-lock"
165 git config user.name "github-actions[bot]"
166 git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
167 git switch -c "$branch"
168 git commit -m "$msg" -- flake.lock
169
170 # Same lock as the branch already carries: keep its commit, so
171 # verify doesn't build it again.
172 if git fetch -q origin "$branch" 2>/dev/null &&
173 git diff --quiet FETCH_HEAD HEAD -- flake.lock; then
174 echo "flake.lock: $branch already has this update"
175 else
176 git push --force origin "$branch"
177 fi
178
179 {
180 echo "Automated \`nix flake update\` by the update-pkgs workflow. Merged automatically once the farm has built all machines from this branch."
181 echo
182 echo '```'
183 sed -n '/^• /,$p' "$RUNNER_TEMP/update.log"
184 echo '```'
185 } >"$RUNNER_TEMP/body.md"
186 pr=$(gh pr list --head "$branch" --state open --json number --jq '.[].number')
187 if [ -z "$pr" ]; then
188 gh pr create --base master --head "$branch" --title "$msg" --body-file "$RUNNER_TEMP/body.md"
189 else
190 gh pr edit "$pr" --body-file "$RUNNER_TEMP/body.md"
191 fi
192
193 verify:
194 needs: [update, lock]
195 if: always() && needs.update.result != 'cancelled' && needs.lock.result != 'cancelled'
196 runs-on: ubuntu-latest
197 timeout-minutes: 350
198 steps:
199 - uses: actions/checkout@v4
200 with:
201 fetch-depth: 0
202 - env:
203 GH_TOKEN: ${{ github.token }}
204 AUTOMERGE: ${{ github.event_name == 'schedule' || inputs.automerge }}
205 run: |
206 repo=$GITHUB_REPOSITORY
207 merged=0
208 git config user.name "github-actions[bot]"
209 git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
210
211 # Farm run for a head commit: "<id> <status> <conclusion>" or empty.
212 # Cancelled runs don't count: a push to master replaced them.
213 farm_run() {
214 gh run list -R "$repo" -w farm.yml -b "$1" -e workflow_dispatch -L 20 \
215 --json databaseId,headSha,status,conclusion \
216 --jq "[.[] | select(.headSha == \"$2\" and .conclusion != \"cancelled\")][0] // empty
217 | \"\(.databaseId) \(.status) \(.conclusion)\""
218 }
219
220 # Rebase a PR onto master, farm-build that commit and wait for the
221 # result. Prints "<conclusion> <sha>"; conclusion is the farm run's,
222 # or "conflict" / "nostart".
223 verify_pr() {
224 local pr=$1 branch=$2 sha run id
225 git fetch -q origin master "$branch"
226 git switch -q -C "$branch" "origin/$branch"
227 if ! git rebase -q origin/master >&2; then
228 git rebase --abort
229 echo "conflict -"
230 return
231 fi
232 sha=$(git rev-parse HEAD)
233 if [ "$sha" != "$(git rev-parse "origin/$branch")" ]; then
234 git push -q --force-with-lease="$branch:origin/$branch" origin "$branch" >&2
235 fi
236 run=$(farm_run "$branch" "$sha")
237 if [ -z "$run" ]; then
238 gh workflow run farm.yml -R "$repo" --ref "$branch" >&2
239 for _ in $(seq 60); do
240 sleep 10
241 run=$(farm_run "$branch" "$sha")
242 [ -n "$run" ] && break
243 done
244 fi
245 id=${run%% *}
246 if [ -z "$id" ]; then
247 echo "nostart $sha"
248 return
249 fi
250 echo "#$pr: farm run $id on $sha" >&2
251 gh run watch "$id" -R "$repo" --interval 60 >/dev/null 2>&1 </dev/null || true
252 echo "$(gh run view "$id" -R "$repo" --json conclusion --jq .conclusion) $sha"
253 }
254
255 # flake.lock first: every package PR is rebased onto the master it
256 # gets merged into right before its build, so after the lock merge
257 # the packages are built once, against the new lock, and the final
258 # farm run on master only hits the cache.
259 prs=$(gh pr list -R "$repo" --state open --base master --limit 100 \
260 --json number,headRefName \
261 --jq '[.[] | select(.headRefName | startswith("update/"))]
262 | sort_by(.headRefName != "update/flake-lock") | .[]
263 | "\(.number) \(.headRefName)"')
264
265 while read -r pr branch <&3; do
266 [ -n "$pr" ] || continue
267 echo "::group::#$pr $branch"
268 # A push to master cancels a running verification; master has
269 # moved then, so rebase and build again.
270 for attempt in 1 2 3; do
271 read -r conclusion sha < <(verify_pr "$pr" "$branch")
272 [ "$conclusion" = cancelled ] || break
273 echo "#$pr: farm run cancelled by a push to master (attempt $attempt)"
274 done
275 echo "#$pr: $conclusion"
276 if [ "$conclusion" = success ] && [ "$AUTOMERGE" = true ]; then
277 gh pr merge "$pr" -R "$repo" --squash --delete-branch --match-head-commit "$sha"
278 merged=$((merged + 1))
279 fi
280 echo "::endgroup::"
281 done 3<<<"$prs"
282
283 echo "merged update PRs: $merged"
284 if [ "$merged" -gt 0 ]; then
285 gh workflow run farm.yml -R "$repo" --ref master
286 fi