This repository has no description
machines.atlas: fetch nix store over http in netboot stage 1
author
lassulus
date
(Sep 13, 2026, 12:13 AM CEST)
commit
e334411d
e334411d50b23a3d18d2841a1b3cf99056f77dbf
parent
8e98e6fd
8e98e6fd827bf17d527872f4a8501360b4d2b105
+88
-10
+78
-7
machines/atlas/physical.nix
+78
-7
machines/atlas/physical.nix
···
47
47
image.fileName = lib.mkForce "atlas.iso";
48
48
};
49
49
50
-
# PXE/netboot: iPXE pulls bzImage + initrd, so nothing has to be flashed to
51
-
# iterate. The whole store rides inside the initrd (~3.8 GB), which is why
52
-
# it has to be served over HTTP rather than TFTP — see nat-share --pxe/--http.
53
-
image.modules.netboot = {
54
-
imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
55
-
netboot.squashfsCompression = "zstd -Xcompression-level 6";
56
-
};
50
+
# PXE/netboot: iPXE pulls only bzImage + a ~11 MB initrd, so nothing has to
51
+
# be flashed to iterate.
52
+
#
53
+
# The store is NOT embedded in the initrd the way upstream netboot.nix does
54
+
# it. iPXE has to hold whatever it downloads in EfiBootServicesData, and
55
+
# firmware caps that well below installed RAM — a 2.8 GB initrd died with
56
+
# "No space left on device" (ipxe.org/34182006) on a 16 GB machine. So stage
57
+
# 1 brings up the network itself and fetches the squashfs, which lands in
58
+
# kernel-managed memory where the whole 16 GB is actually usable.
59
+
#
60
+
# The file is fetched to exactly the path netboot.nix already mounts from
61
+
# ("../nix-store.squashfs", i.e. /nix-store.squashfs in the initrd root), so
62
+
# the mount itself is upstream's, unmodified. It survives switch_root
63
+
# because the loop device keeps the inode open after the old root is wiped.
64
+
image.modules.netboot =
65
+
{ config, lib, ... }:
66
+
{
67
+
imports = [ (modulesPath + "/installer/netboot/netboot.nix") ];
68
+
netboot.squashfsCompression = "zstd -Xcompression-level 6";
69
+
70
+
# Ship the initrd without the store; pxe-serve serves squashfsStore
71
+
# separately over HTTP.
72
+
system.build.netbootRamdisk = lib.mkForce config.system.build.initialRamdisk;
73
+
74
+
# DHCP in stage 1 (runs in preLVMCommands, before postDeviceCommands).
75
+
boot.initrd.network.enable = true;
76
+
boot.initrd.network.udhcpc.enable = true;
77
+
78
+
# hardware.enableAllHardware is storage-only — virtio_net is the single
79
+
# network driver in its 89 modules — so wired NICs have to be listed by
80
+
# hand. availableKernelModules only makes them available; udev loads just
81
+
# the ones matching present PCI/USB IDs, so unused entries cost bytes.
82
+
boot.initrd.availableKernelModules = [
83
+
"e1000e"
84
+
"e1000"
85
+
"igb"
86
+
"igc"
87
+
"ixgbe"
88
+
"r8169"
89
+
"alx"
90
+
"atl1c"
91
+
"tg3"
92
+
"bnx2"
93
+
"bnx2x"
94
+
"sky2"
95
+
"forcedeth"
96
+
"atlantic"
97
+
# USB NICs (docks, adapters)
98
+
"r8152"
99
+
"cdc_ether"
100
+
"cdc_ncm"
101
+
"asix"
102
+
"ax88179_178a"
103
+
"virtio_net"
104
+
];
105
+
106
+
# wget is busybox's, already present in the scripted initrd's
107
+
# extra-utils. The URL comes from the kernel command line rather than
108
+
# being baked in, so the image doesn't go stale when the serving address
109
+
# changes (pxe-serve passes store.url=).
110
+
boot.initrd.postDeviceCommands = ''
111
+
store_url=
112
+
for o in $(cat /proc/cmdline); do
113
+
case $o in
114
+
store.url=*) store_url=''${o#store.url=} ;;
115
+
esac
116
+
done
117
+
if [ -z "$store_url" ]; then
118
+
echo "netboot: no store.url= on the kernel command line"
119
+
fail
120
+
fi
121
+
echo "netboot: fetching nix store from $store_url"
122
+
if ! wget -O /nix-store.squashfs "$store_url"; then
123
+
echo "netboot: failed to fetch $store_url"
124
+
fail
125
+
fi
126
+
'';
127
+
};
57
128
58
129
nixpkgs.hostPlatform = "x86_64-linux";
59
130
+10
-3
tools/pxe-serve/pxe-serve.sh
+10
-3
tools/pxe-serve/pxe-serve.sh
···
129
129
INITRD=$(nix build --no-link --print-out-paths "${CFG}.system.build.netbootRamdisk")
130
130
TOPLEVEL=$(nix eval --raw "${CFG}.system.build.toplevel")
131
131
PARAMS=$(nix eval --raw --apply 'ps: builtins.concatStringsSep " " ps' "${CFG}.boot.kernelParams")
132
+
# Served separately from the initrd: iPXE can only hold so much in
133
+
# EfiBootServicesData, so stage 1 fetches this itself over HTTP.
134
+
SQUASHFS=$(nix build --no-link --print-out-paths "${CFG}.system.build.squashfsStore")
132
135
133
136
# iPXE gets the chain script baked in rather than asking DHCP for a filename.
134
137
# In proxy-DHCP mode dnsmasq answers through pxe-service, so a --dhcp-boot
···
173
176
174
177
ln -s "${KERNEL}/${KERNEL_FILE}" "$ROOT/bzImage"
175
178
ln -s "${INITRD}/initrd" "$ROOT/initrd"
179
+
ln -s "${SQUASHFS}" "$ROOT/nix-store.squashfs"
176
180
ln -s "${IPXE}/ipxe.efi" "$ROOT/ipxe.efi"
177
181
ln -s "${IPXE}/undionly.kpxe" "$ROOT/undionly.kpxe"
178
182
183
+
# store.url= is what stage 1 fetches the squashfs from; passing it on the
184
+
# command line instead of baking it into the image keeps the image valid when
185
+
# the serving address changes.
179
186
cat >"$ROOT/netboot.ipxe" <<EOF
180
187
#!ipxe
181
-
kernel http://${ADDR}:${HTTP_PORT}/bzImage init=${TOPLEVEL}/init initrd=initrd ${PARAMS}
188
+
kernel http://${ADDR}:${HTTP_PORT}/bzImage init=${TOPLEVEL}/init initrd=initrd ${PARAMS} store.url=http://${ADDR}:${HTTP_PORT}/nix-store.squashfs
182
189
initrd http://${ADDR}:${HTTP_PORT}/initrd
183
190
boot
184
191
EOF
185
192
186
-
echo "$PROG: serving ${MACHINE} on ${IFACE} (${ADDR}), kernel+initrd over http://${ADDR}:${HTTP_PORT}"
187
-
echo "$PROG: initrd is $(stat -Lc %s "$ROOT/initrd" | awk '{printf "%.1f GB", $1/1073741824}') — expect that much to cross the wire on every boot"
193
+
echo "$PROG: serving ${MACHINE} on ${IFACE} (${ADDR}) over http://${ADDR}:${HTTP_PORT}"
194
+
echo "$PROG: iPXE fetches $(stat -Lc %s "$ROOT/bzImage" | awk '{printf "%.0f MB", $1/1048576}') kernel + $(stat -Lc %s "$ROOT/initrd" | awk '{printf "%.0f MB", $1/1048576}') initrd; stage 1 then fetches $(stat -Lc %s "$ROOT/nix-store.squashfs" | awk '{printf "%.2f GB", $1/1073741824}') of store itself"
188
195
189
196
# exec, so there is no wrapper process between the terminal and dnsmasq: Ctrl+C
190
197
# reaches it directly and pxe-share's own trap does the privileged teardown