This repository has no description
12 kB
266 lines
1{ self, inputs, ... }:
2{
3 perSystem =
4 {
5 pkgs,
6 system,
7 ...
8 }:
9 {
10 # deploy [--flake PATH] [--yes] <machine> [[user@]host] (host defaults to <machine>.r)
11 #
12 # Generates vars, copies the flake source to /var/lib/deploy/incoming
13 # on the target, builds it there in the systemd unit deploy-build
14 # (progress via nom), shows the source diff against the running system
15 # (/var/lib/deploy/src) and the closure diff, asks (--yes: does not
16 # ask and prints the diffs without a pager), then uploads
17 # secrets and switches in the unit deploy-switch, which records the
18 # deployed git revision in /var/lib/superconfig-revision. Both units
19 # survive the ssh connection dying; rerunning deploy attaches to a
20 # running one.
21 packages.deploy =
22 (pkgs.writeShellApplication {
23 name = "deploy";
24 runtimeInputs = [
25 self.packages.${system}.pass
26 inputs.clan-core.packages.${system}.clan-cli
27 pkgs.gnutar
28 pkgs.jq
29 pkgs.less
30 pkgs.nix-output-monitor
31 ];
32 text = ''
33 usage() {
34 echo "usage: deploy [--flake PATH] [--yes] <machine> [[user@]host] (host defaults to <machine>.r)" >&2
35 exit 1
36 }
37
38 flake=.
39 yes=0
40 while [ $# -gt 0 ]; do
41 case $1 in
42 --flake)
43 [ $# -ge 2 ] || usage
44 flake=$2
45 shift 2
46 ;;
47 --yes | -y)
48 yes=1
49 shift
50 ;;
51 -*) usage ;;
52 *) break ;;
53 esac
54 done
55 [ $# -eq 1 ] || [ $# -eq 2 ] || usage
56 machine=$1
57 target=''${2:-$machine.r}
58 [[ $target == *@* ]] || target=root@$target
59
60 state=/var/lib/deploy
61 nix="/run/current-system/sw/bin/nix --extra-experimental-features 'nix-command flakes'"
62
63 tmp=$(mktemp -d)
64 chmod 700 "$tmp"
65 # One ssh connection for the whole run, so the key signs only once.
66 ssh_opts=(-o ControlMaster=auto -o "ControlPath=$tmp/ssh" -o ControlPersist=yes)
67 cleanup() {
68 ssh "''${ssh_opts[@]}" -O exit "$target" 2>/dev/null || true
69 rm -rf "$tmp"
70 }
71 trap cleanup EXIT
72
73 remote() {
74 # shellcheck disable=SC2029 # the command line is meant for the target
75 ssh "''${ssh_opts[@]}" "$target" "$@"
76 }
77 step() {
78 printf '\n\033[1m== %s\033[0m\n' "$*" >&2
79 }
80 page() {
81 if [ -t 1 ] && [ "$yes" = 0 ]; then ''${PAGER:-less -RFX}; else cat; fi
82 }
83
84 # True while the unit's ExecStartPre or ExecStart is running.
85 # Finished units stay around (RemainAfterExit) until the next start,
86 # so their result can still be read after reconnecting.
87 in_progress() {
88 case $(remote systemctl show -p ActiveState -p SubState "$1") in
89 *ActiveState=activating* | *SubState=running*) return 0 ;;
90 esac
91 return 1
92 }
93
94 # start <unit> <systemd-run args...>: the arguments are quoted, so
95 # they reach systemd-run on the target unchanged. Units start with
96 # an empty environment; go through the daemon like a login shell
97 # (the store is read-only inside containers).
98 start() {
99 local unit=$1
100 shift
101 remote "systemctl stop $unit 2>/dev/null; systemctl reset-failed $unit 2>/dev/null;" \
102 "systemd-run --quiet --unit=$unit -p RemainAfterExit=yes" \
103 "-p StandardOutput=truncate:$state/$unit.log" \
104 "--setenv=NIX_REMOTE=daemon --setenv=PATH=/run/wrappers/bin:/run/current-system/sw/bin" \
105 "$(printf ' %q' "$@")"
106 }
107
108 # follow <unit>: stream its log until it finishes, fail if it did.
109 # tail can lag far behind the log when the reader (nom) is slow, so
110 # it must not be killed on a timer: --pid makes it read the log to
111 # the end once the waiter (unit finished) is gone, then exit, so
112 # no line is cut off. systemctl fails for a moment while a switch
113 # reexecutes systemd ("Failed to connect to system scope bus");
114 # that is retried, not taken for the unit having finished.
115 follow() {
116 ssh "''${ssh_opts[@]}" "$target" bash -s -- "$1" "$state" <<'EOF'
117 unit=$1 log=$2/$1.log
118 query() {
119 until systemctl show "$@" "$unit" 2>/dev/null; do sleep 1; done
120 }
121 while case $(query -p ActiveState -p SubState) in
122 *ActiveState=activating* | *SubState=running*) true ;;
123 *) false ;;
124 esac; do
125 sleep 1
126 done &
127 tail -n +1 -F --pid=$! "$log" 2>/dev/null
128 [ "$(query -p Result --value)" = success ] && exit 0
129 echo "deploy: $unit failed (exit status $(query -p ExecMainStatus --value)), log: $log" >&2
130 exit 1
131 EOF
132 }
133
134 # source_diff: per-file +/- summary, then the full diff, of the
135 # running system's source against the incoming one.
136 source_diff() {
137 ssh "''${ssh_opts[@]}" "$target" bash -s -- "$state" <<'EOF'
138 old=$1/src new=$1/incoming
139 if [ ! -d "$old" ]; then
140 echo "no source recorded for the running system"
141 exit 0
142 fi
143 # diff exits 1 when the trees differ, 2 on trouble.
144 d() { diff -ruN "$@" "$old" "$new" || [ $? -eq 1 ]; }
145 d | awk -v pre="$new/" '
146 /^diff -ruN / {
147 f = substr($0, index($0, " " pre) + 1 + length(pre))
148 files[++n] = f; hdr = 2; next
149 }
150 hdr > 0 { hdr--; next }
151 /^\+/ { add[f]++ }
152 /^-/ { del[f]++ }
153 END {
154 if (n == 0) { print "no source changes"; exit }
155 for (i = 1; i <= n; i++) {
156 f = files[i]
157 printf " %-60s \033[32m+%d\033[0m \033[31m-%d\033[0m\n", f, add[f], del[f]
158 }
159 printf "%d files changed\n\n", n
160 }'
161 d --color=always
162 EOF
163 }
164
165 step "connecting to $target"
166 remote true || { echo "deploy: cannot reach $target" >&2; exit 1; }
167 # shellcheck disable=SC2016 # expanded on the target
168 remote 'test -e /etc/NIXOS && command -v systemd-run >/dev/null' ||
169 { echo "deploy: $target is not a NixOS system with systemd" >&2; exit 1; }
170 if in_progress deploy-switch; then
171 step "a switch is still running on $target, attaching"
172 follow deploy-switch
173 exit
174 fi
175
176 # Decrypt the bulk key once (one Secure Enclave unlock); every
177 # later pass call from clan then uses it without prompting.
178 PASS_BULK_KEY_FILE="" pass show bulk-operations/age-key >"$tmp/bulk-key"
179 if [ ! -s "$tmp/bulk-key" ]; then
180 echo "deploy: could not extract the bulk key" >&2
181 exit 1
182 fi
183 export PASS_BULK_KEY_FILE="$tmp/bulk-key"
184
185 if in_progress deploy-build; then
186 step "a build is still running on $target, attaching"
187 else
188 step "generating vars for $machine"
189 pass git pull --rebase
190 clan vars generate --flake "$flake" "$machine"
191 pass git push
192
193 step "copying source to $target:$state/incoming"
194 # The flake source exactly as nix sees it: tracked files only.
195 meta=$(nix flake metadata --json "$flake")
196 src=$(jq -r .path <<<"$meta")
197 # Its git revision (-dirty with uncommitted changes), written to
198 # /var/lib/superconfig-revision once the switch succeeded.
199 revision=$(jq -r '.revision // .dirtyRevision // "unknown"' <<<"$meta")
200 tar -C "$src" --owner=0 --group=0 --numeric-owner -czf - . |
201 remote "rm -rf $state/incoming.new && mkdir -p $state/incoming.new &&" \
202 "tar -C $state/incoming.new -xzf - &&" \
203 "rm -rf $state/incoming && mv $state/incoming.new $state/incoming &&" \
204 "echo $revision >$state/incoming.revision"
205
206 step "source changes"
207 source_diff | page
208
209 step "building $machine on $target"
210 # nvd comes from the machine's own nixpkgs, so it is built for
211 # the target and usually substituted; out-links are result
212 # (system) and result-1 (nvd), in installable order.
213 start deploy-build \
214 /run/current-system/sw/bin/nix --extra-experimental-features "nix-command flakes" \
215 --log-format internal-json -v build --out-link "$state/result" \
216 "$state/incoming#nixosConfigurations.$machine.config.system.build.toplevel" \
217 "$state/incoming#nixosConfigurations.$machine.pkgs.nvd"
218 fi
219 follow deploy-build | nom --json
220 system=$(remote readlink -f "$state/result")
221
222 step "closure changes"
223 if [ "$(remote readlink -f /run/current-system)" = "$system" ]; then
224 echo "$system is already running"
225 else
226 remote "$state/result-1/bin/nvd diff /run/current-system $system"
227 fi
228
229 if [ "$yes" = 0 ]; then
230 read -r -p "switch $target to $system? [y/N] " answer </dev/tty
231 if [[ $answer != [yY]* ]]; then
232 echo "aborted, $target is unchanged" >&2
233 exit 1
234 fi
235 fi
236
237 step "uploading secrets"
238 secret_location=$(remote "$nix eval --raw $state/incoming#nixosConfigurations.$machine.config.clan.core.vars.password-store.secretLocation")
239 clan vars upload --flake "$flake" --directory "$tmp/vars" "$machine"
240 if [ -s "$tmp/vars/.pass_info" ] &&
241 [ "$(remote cat "$secret_location/.pass_info" 2>/dev/null)" = "$(cat "$tmp/vars/.pass_info")" ]; then
242 echo "secrets in $secret_location unchanged"
243 else
244 # Same layout and permissions as `clan vars upload`; swapped in
245 # with a rename so the directory is never empty.
246 find "$tmp/vars" -type d -exec chmod 700 {} +
247 find "$tmp/vars" -type f -exec chmod 400 {} +
248 # shellcheck disable=SC2016 # expanded on the target
249 tar -C "$tmp/vars" --owner=0 --group=0 --numeric-owner -czf - . |
250 remote "d=$secret_location;" \
251 'rm -rf "$d.new" "$d.old" && install -d -m 700 "$d.new" && tar -C "$d.new" -xzf - &&' \
252 '{ [ ! -e "$d" ] || mv "$d" "$d.old"; } && mv "$d.new" "$d" && rm -rf "$d.old"'
253 fi
254
255 step "switching $target"
256 start deploy-switch \
257 -p "ExecStartPre=/run/current-system/sw/bin/nix-env -p /nix/var/nix/profiles/system --set $system" \
258 -p "ExecStartPre=/bin/sh -c 'rm -rf $state/src && cp -a $state/incoming $state/src'" \
259 -p "ExecStartPost=/bin/sh -c '(cat $state/incoming.revision || echo unknown) >/var/lib/superconfig-revision.tmp && mv /var/lib/superconfig-revision.tmp /var/lib/superconfig-revision'" \
260 "$system/bin/switch-to-configuration" switch
261 follow deploy-switch
262 '';
263 }).overrideAttrs
264 { passthru.usage = builtins.readFile ./usage.kdl; };
265 };
266}