forked from
lass.pds.lassul.us/superconfig
This repository has no description
12 kB
290 lines
1name: update-pkgs
2
3# Daily package and flake.lock updates. Nothing reaches master without a full
4# farm build:
5#
6# 1. update (one job per package): run its passthru.updateScript through
7# tools/update-pkgs, build the package on its platform (darwin-only
8# packages on macOS), push update/<pkg> and open or refresh its PR.
9# lock: `nix flake update`, pushed as update/flake-lock with its own PR.
10# 2. verify: for every open update/* PR, one after another (flake.lock
11# first), rebase it onto the current master, run ci.yml on that commit
12# (it builds all machines with the farm) and merge the PR with a merge
13# commit only if it succeeded (master's ruleset requires its ci check). A
14# commit that already has a finished run is not checked again, so a
15# failing PR stays open until its branch or master changes.
16#
17# One PR at a time: the farm builds one commit at a time anyway, and each PR
18# is rebased onto the master it gets merged into.
19#
20# Needs "Allow GitHub Actions to create and approve pull requests" in the repo
21# settings. Merges made with GITHUB_TOKEN don't trigger push workflows, so the
22# ci run (farm build) of master after merging is dispatched here as well.
23
24on:
25 schedule:
26 - cron: "0 3 * * *" # daily 03:00 UTC
27 workflow_dispatch:
28 inputs:
29 packages:
30 description: Packages to update, space separated (empty for all)
31 default: ""
32 automerge:
33 description: Merge PRs whose farm build succeeded
34 type: boolean
35 default: true
36
37permissions:
38 contents: write
39 pull-requests: write
40 actions: write # dispatch and watch ci runs
41
42concurrency:
43 group: update-pkgs
44 cancel-in-progress: false
45
46env:
47 NIX_CONFIG: |
48 extra-substituters = https://cache.lassul.us
49 extra-trusted-public-keys = cache.lassul.us-1:aOMEO5TU6eLvhyPi0jmgjR2sVlGjMq3rK0MeACjTcQ0=
50
51jobs:
52 list:
53 runs-on: ubuntu-latest
54 outputs:
55 matrix: ${{ steps.list.outputs.matrix }}
56 steps:
57 - uses: actions/checkout@v4
58 - uses: cachix/install-nix-action@v31
59 - id: list
60 env:
61 PACKAGES: ${{ inputs.packages }}
62 run: |
63 names() {
64 nix eval --json --impure --apply builtins.attrNames --expr \
65 "import ./tools/update-pkgs/scripts.nix { flake = toString ./.; host = \"$1\"; }"
66 }
67 # Linux packages update and build on ubuntu; the ones that only
68 # exist for darwin on macOS.
69 matrix=$(jq -cn --argjson linux "$(names x86_64-linux)" \
70 --argjson all "$(names aarch64-darwin)" --arg only "$PACKAGES" '
71 ($only | split(" ") | map(select(. != ""))) as $want
72 | [ $all[]
73 | select(($want | length) == 0 or (. as $p | $want | index($p)))
74 | if (. as $p | $linux | index($p))
75 then { pkg: ., os: "ubuntu-latest", system: "x86_64-linux" }
76 else { pkg: ., os: "macos-latest", system: "aarch64-darwin" }
77 end ]')
78 echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
79 echo "$matrix" | jq .
80
81 update:
82 needs: list
83 if: needs.list.outputs.matrix != '[]'
84 name: ${{ matrix.pkg }}
85 runs-on: ${{ matrix.os }}
86 strategy:
87 fail-fast: false
88 matrix:
89 include: ${{ fromJSON(needs.list.outputs.matrix) }}
90 steps:
91 - uses: actions/checkout@v4
92 with:
93 fetch-depth: 0
94 - uses: cachix/install-nix-action@v31
95 - env:
96 PKG: ${{ matrix.pkg }}
97 SYSTEM: ${{ matrix.system }}
98 GH_TOKEN: ${{ github.token }}
99 # nix-update and the update scripts query the GitHub API
100 GITHUB_TOKEN: ${{ github.token }}
101 run: |
102 version() { nix eval --raw ".#packages.$SYSTEM.$PKG.version" 2>/dev/null || echo unknown; }
103 old=$(version)
104
105 nix run .#update-pkgs -- "$PKG"
106 if [ -z "$(git status --porcelain)" ]; then
107 echo "$PKG: no update"
108 exit 0
109 fi
110
111 nix build -L --no-link ".#packages.$SYSTEM.$PKG"
112
113 new=$(version)
114 prefix=pkgs
115 [ -d "pkgs/$PKG" ] || prefix=tools
116 if [ "$old" != "$new" ]; then
117 msg="$prefix.$PKG: $old -> $new"
118 else
119 msg="$prefix.$PKG: update"
120 fi
121
122 branch="update/$PKG"
123 git config user.name "github-actions[bot]"
124 git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
125 git switch -c "$branch"
126 git add -A
127 git commit -m "$msg"
128
129 # Same change as the branch already carries (e.g. a PR whose farm
130 # build failed and upstream hasn't moved): keep its commit, so
131 # verify doesn't build it again.
132 if git fetch -q origin "$branch" 2>/dev/null &&
133 git diff --quiet FETCH_HEAD HEAD -- "$prefix/$PKG"; then
134 echo "$PKG: $branch already has this update"
135 else
136 git push --force origin "$branch"
137 fi
138
139 if [ -z "$(gh pr list --head "$branch" --state open --json number --jq '.[].number')" ]; then
140 gh pr create --base master --head "$branch" --title "$msg" \
141 --body "Automated update by the update-pkgs workflow (\`nix run .#update-pkgs -- $PKG\`), package built on $SYSTEM. Merged automatically once the farm has built all machines from this branch."
142 fi
143
144 lock:
145 # whole runs only, not manual runs for selected packages
146 if: inputs.packages == ''
147 runs-on: ubuntu-latest
148 steps:
149 - uses: actions/checkout@v4
150 with:
151 fetch-depth: 0
152 - uses: cachix/install-nix-action@v31
153 - env:
154 GH_TOKEN: ${{ github.token }}
155 GITHUB_TOKEN: ${{ github.token }}
156 run: |
157 export NIX_CONFIG="$NIX_CONFIG"$'\n'"access-tokens = github.com=$GITHUB_TOKEN"
158 nix flake update 2>&1 | tee "$RUNNER_TEMP/update.log"
159 if git diff --quiet -- flake.lock; then
160 echo "flake.lock: no update"
161 exit 0
162 fi
163
164 msg="flake.lock: update"
165 branch="update/flake-lock"
166 git config user.name "github-actions[bot]"
167 git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
168 git switch -c "$branch"
169 git commit -m "$msg" -- flake.lock
170
171 # Same lock as the branch already carries: keep its commit, so
172 # verify doesn't build it again.
173 if git fetch -q origin "$branch" 2>/dev/null &&
174 git diff --quiet FETCH_HEAD HEAD -- flake.lock; then
175 echo "flake.lock: $branch already has this update"
176 else
177 git push --force origin "$branch"
178 fi
179
180 {
181 echo "Automated \`nix flake update\` by the update-pkgs workflow. Merged automatically once the farm has built all machines from this branch."
182 echo
183 echo '```'
184 sed -n '/^• /,$p' "$RUNNER_TEMP/update.log"
185 echo '```'
186 } >"$RUNNER_TEMP/body.md"
187 pr=$(gh pr list --head "$branch" --state open --json number --jq '.[].number')
188 if [ -z "$pr" ]; then
189 gh pr create --base master --head "$branch" --title "$msg" --body-file "$RUNNER_TEMP/body.md"
190 else
191 gh pr edit "$pr" --body-file "$RUNNER_TEMP/body.md"
192 fi
193
194 verify:
195 needs: [update, lock]
196 if: always() && needs.update.result != 'cancelled' && needs.lock.result != 'cancelled'
197 runs-on: ubuntu-latest
198 timeout-minutes: 350
199 steps:
200 - uses: actions/checkout@v4
201 with:
202 fetch-depth: 0
203 - env:
204 GH_TOKEN: ${{ github.token }}
205 AUTOMERGE: ${{ github.event_name == 'schedule' || inputs.automerge }}
206 run: |
207 repo=$GITHUB_REPOSITORY
208 merged=0
209 git config user.name "github-actions[bot]"
210 git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
211
212 # Dispatched run of a workflow for a head commit: "<id> <status>
213 # <conclusion>" or empty. Cancelled runs don't count: a newer run
214 # for the branch replaced them.
215 dispatched_run() {
216 gh run list -R "$repo" -w "$1" -b "$2" -e workflow_dispatch -L 20 \
217 --json databaseId,headSha,status,conclusion \
218 --jq "[.[] | select(.headSha == \"$3\" and .conclusion != \"cancelled\")][0] // empty
219 | \"\(.databaseId) \(.status) \(.conclusion)\""
220 }
221
222 # Dispatch a workflow on a branch unless its head commit already
223 # has a run, wait for it and print its conclusion, or "nostart".
224 run_workflow() {
225 local workflow=$1 branch=$2 sha=$3 run id
226 run=$(dispatched_run "$workflow" "$branch" "$sha")
227 if [ -z "$run" ]; then
228 gh workflow run "$workflow" -R "$repo" --ref "$branch" >&2
229 for _ in $(seq 60); do
230 sleep 10
231 run=$(dispatched_run "$workflow" "$branch" "$sha")
232 [ -n "$run" ] && break
233 done
234 fi
235 id=${run%% *}
236 if [ -z "$id" ]; then
237 echo nostart
238 return
239 fi
240 echo "$workflow run $id on $sha" >&2
241 gh run watch "$id" -R "$repo" --interval 60 >/dev/null 2>&1 </dev/null || true
242 gh run view "$id" -R "$repo" --json conclusion --jq .conclusion
243 }
244
245 # Rebase a PR branch onto master and run ci.yml (with the farm
246 # build) on that commit. Prints "<conclusion> <sha>"; conclusion is
247 # the ci run's, or "conflict" / "nostart".
248 verify_pr() {
249 local branch=$1 sha conclusion
250 git fetch -q origin master "$branch"
251 git switch -q -C "$branch" "origin/$branch"
252 if ! git rebase -q origin/master >&2; then
253 git rebase --abort
254 echo "conflict -"
255 return
256 fi
257 sha=$(git rev-parse HEAD)
258 if [ "$sha" != "$(git rev-parse "origin/$branch")" ]; then
259 git push -q --force-with-lease="$branch:origin/$branch" origin "$branch" >&2
260 fi
261 conclusion=$(run_workflow ci.yml "$branch" "$sha")
262 echo "$conclusion $sha"
263 }
264
265 # flake.lock first: every package PR is rebased onto the master it
266 # gets merged into right before its build, so after the lock merge
267 # the packages are built once, against the new lock, and the final
268 # ci run on master only hits the cache.
269 prs=$(gh pr list -R "$repo" --state open --base master --limit 100 \
270 --json number,headRefName \
271 --jq '[.[] | select(.headRefName | startswith("update/"))]
272 | sort_by(.headRefName != "update/flake-lock") | .[]
273 | "\(.number) \(.headRefName)"')
274
275 while read -r pr branch <&3; do
276 [ -n "$pr" ] || continue
277 echo "::group::#$pr $branch"
278 read -r conclusion sha < <(verify_pr "$branch")
279 echo "#$pr: $conclusion"
280 if [ "$conclusion" = success ] && [ "$AUTOMERGE" = true ]; then
281 gh pr merge "$pr" -R "$repo" --merge --delete-branch --match-head-commit "$sha"
282 merged=$((merged + 1))
283 fi
284 echo "::endgroup::"
285 done 3<<<"$prs"
286
287 echo "merged update PRs: $merged"
288 if [ "$merged" -gt 0 ]; then
289 gh workflow run ci.yml -R "$repo" --ref master
290 fi