forked from
lass.pds.lassul.us/superconfig
This repository has no description
machines/xerxes: switch from ollama to llama-swap
author
lassulus
date
(Mar 18, 2026, 8:40 PM CET)
commit
9e663502
9e663502b50c7641bd7b4f75a902707eb1b461f0
parent
7f06b75c
7f06b75c53e41859a0a865fbe9a75af3706c1bfd
+23
-32
+23
-32
machines/xerxes/config.nix
+23
-32
machines/xerxes/config.nix
···
42
42
documentation.nixos.enable = true;
43
43
lass.workspace-manager.enable = true;
44
44
45
-
services.ollama =
45
+
# llama-swap: on-demand model hot-swapping proxy for llama.cpp
46
+
# Models are stored in /data/models/ as GGUF files
47
+
services.llama-swap =
46
48
let
47
-
gpuTargets = [ "gfx1100" ]; # Radeon 890M (gfx1150/RDNA 3.5) - only build for fallback target
48
-
customRocblas = pkgs.rocmPackages.rocblas.override {
49
-
withHipBlasLt = false; # hipblaslt takes extremely long to build; rocblas's own Tensile kernels work fine
50
-
inherit gpuTargets;
51
-
};
52
-
customRocsparse = pkgs.rocmPackages.rocsparse.override {
53
-
inherit gpuTargets;
54
-
};
55
-
customRocsolver = pkgs.rocmPackages.rocsolver.override {
56
-
inherit gpuTargets;
57
-
rocblas = customRocblas;
58
-
rocsparse = customRocsparse;
59
-
};
60
-
customHipblas = pkgs.rocmPackages.hipblas.override {
61
-
rocblas = customRocblas;
62
-
rocsolver = customRocsolver;
63
-
rocsparse = customRocsparse;
64
-
};
65
-
customRocmPackages = pkgs.rocmPackages // {
66
-
rocblas = customRocblas;
67
-
hipblas = customHipblas;
68
-
rocsolver = customRocsolver;
69
-
rocsparse = customRocsparse;
70
-
};
49
+
llama-server = "${pkgs.llama-cpp-rocm}/bin/llama-server";
71
50
in
72
51
{
73
52
enable = true;
74
-
package = pkgs.ollama-rocm.override {
75
-
rocmGpuTargets = gpuTargets;
76
-
rocmPackages = customRocmPackages;
77
-
};
78
-
environmentVariables = {
79
-
HSA_OVERRIDE_GFX_VERSION = "11.0.0"; # Radeon 890M (gfx1150/RDNA 3.5) - fallback to gfx1100
80
-
OLLAMA_NUM_CTX = "16384"; # 48 GiB RAM shared with GPU; 32k OOMs with 32B model
53
+
port = 11434;
54
+
settings = {
55
+
healthCheckTimeout = 300;
56
+
models = {
57
+
qwen3-32b = {
58
+
cmd = "${llama-server} --port \${PORT} --model /data/models/qwen3-32b.gguf --ctx-size 16384 --n-gpu-layers 99 --jinja";
59
+
ttl = 300;
60
+
};
61
+
qwen-coder-next = {
62
+
cmd = "${llama-server} --port \${PORT} --model /data/models/Qwen3-Coder-Next-Q4_K_M-00001-of-00004.gguf --ctx-size 16384 --n-gpu-layers 99 --jinja";
63
+
ttl = 300;
64
+
};
65
+
};
81
66
};
82
67
};
68
+
# ROCm GPU access and environment for llama-swap spawned llama-server processes
69
+
systemd.services.llama-swap.environment.HSA_OVERRIDE_GFX_VERSION = "11.0.0"; # Radeon 890M (gfx1150/RDNA 3.5) - fallback to gfx1100
70
+
systemd.services.llama-swap.serviceConfig.SupplementaryGroups = [
71
+
"render"
72
+
"video"
73
+
];
83
74
84
75
# extra-container for testing declarative containers without full rebuilds
85
76
programs.extra-container.enable = true;