From 327f6dc6f007a88bed1eaf7bacfaa1f81ed03bf5 Mon Sep 17 00:00:00 2001 From: Adrian Gunnar Lauterer Date: Mon, 2 Mar 2026 11:02:48 +0100 Subject: [PATCH] llama-cpp instead of swap, native swap functionality is coming --- hosts/galadriel/configuration.nix | 2 +- modules/llama-cpp.nix | 30 +++++++ modules/llama-swap.nix | 143 ------------------------------ 3 files changed, 31 insertions(+), 144 deletions(-) create mode 100644 modules/llama-cpp.nix delete mode 100644 modules/llama-swap.nix diff --git a/hosts/galadriel/configuration.nix b/hosts/galadriel/configuration.nix index cdb2761..c7be67d 100644 --- a/hosts/galadriel/configuration.nix +++ b/hosts/galadriel/configuration.nix @@ -38,7 +38,7 @@ ../../modules/miniflux.nix #../../modules/ollama.nix # replaced by llama-cpp + llama-swap #../../modules/openwebui.nix # using llama-cpp built-in UI instead - ../../modules/llama-swap.nix + ../../modules/llama-cpp.nix ../../modules/librechat.nix ../../modules/immich.nix diff --git a/modules/llama-cpp.nix b/modules/llama-cpp.nix new file mode 100644 index 0000000..515ec3f --- /dev/null +++ b/modules/llama-cpp.nix @@ -0,0 +1,30 @@ +{ + config, + pkgs, + lib, + unstable, + ... +}: + +let + hostname = config.networking.hostName; +in + +{ + environment.systemPackages = [ pkgs.unstable.ollama ]; + services.llama-cpp = { + enable = true; + host = "0.0.0.0"; + port = 11111; + package = pkgs.unstable.llama-cpp; + openFirewall = true; + model = "/var/lib/llama-cpp/models/Qwen3.5-35B-A3B-UD-Q3_K_M.gguf" ; + extraFlags = [ + "-c" "32000" + "-ngl" "38" + "--image-min-tokens" "1024" + #"--hf-repo" "unsloth/Qwen3.5-35B-A3B-GGUF:Q2_K_L" + "--mmproj" "/var/lib/llama-cpp/models/mmproj-F16.gguf" + ]; + }; +} diff --git a/modules/llama-swap.nix b/modules/llama-swap.nix deleted file mode 100644 index 1734f1b..0000000 --- a/modules/llama-swap.nix +++ /dev/null @@ -1,143 +0,0 @@ -{ - config, - pkgs, - lib, - ... -}: - -let - modelDir = "/var/lib/llama/models"; - legacyModelDir = "/var/lib/llama-cpp/models"; - - # llama-cpp with Vulkan for Intel Arc - llama-cpp-vulkan = pkgs.llama-cpp.override { - vulkanSupport = true; - cudaSupport = false; - rocmSupport = false; - openclSupport = false; - blasSupport = true; - }; - - llama-server = lib.getExe' llama-cpp-vulkan "llama-server"; - - models = { - "Ministral-3-8B-Reasoning-2512-Q4_K_M" = { - url = "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512-GGUF/resolve/main/Ministral-3-8B-Reasoning-2512-Q4_K_M.gguf"; - filename = "Ministral-3-8B-Reasoning-2512-Q4_K_M.gguf"; - }; - "Qwen3.5-35B-A3B-UD-Q3_K_M" = { - url = "https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/Qwen3.5-35B-A3B-UD-Q3_K_M.gguf"; - filename = "Qwen3.5-35B-A3B-UD-Q3_K_M.gguf"; - }; - "Qwen3.5-27B-UD-Q4_K_XL" = { - url = "https://huggingface.co/unsloth/Qwen3.5-27B-GGUF/resolve/main/Qwen3.5-27B-UD-Q4_K_XL.gguf"; - filename = "Qwen3.5-27B-UD-Q4_K_XL.gguf"; - }; - "LFM2-24B-A2B-Q4_K_M" = { - url = "https://huggingface.co/LiquidAI/LFM2-24B-A2B-GGUF/resolve/main/LFM2-24B-A2B-Q4_K_M.gguf"; - filename = "LFM2-24B-A2B-Q4_K_M.gguf"; - }; - }; - - downloadModelsScript = pkgs.writeShellScript "llama-swap-download-models" ( - '' - set -euo pipefail - - install -d -m 0755 -o llama-swap -g llama-swap "${modelDir}" - - if [ -d "${legacyModelDir}" ]; then - for f in "${legacyModelDir}"/*.gguf; do - [ -e "$f" ] || continue - base="$(basename "$f")" - if [ ! -e "${modelDir}/$base" ]; then - mv "$f" "${modelDir}/$base" - fi - done - fi - '' - + (lib.concatStringsSep "\n" ( - lib.mapAttrsToList (name: model: '' - MODEL_PATH="${modelDir}/${model.filename}" - if [ ! -f "$MODEL_PATH" ]; then - echo "Downloading ${name}..." - ${lib.getExe pkgs.curl} --fail -L --retry 5 --retry-delay 5 --retry-all-errors \ - -o "$MODEL_PATH.tmp" "${model.url}" - mv "$MODEL_PATH.tmp" "$MODEL_PATH" - fi - chown llama-swap:llama-swap "$MODEL_PATH" - chmod 0644 "$MODEL_PATH" - '') models - )) - ); -in -{ - environment.systemPackages = [ llama-cpp-vulkan ]; - - users.users.llama-swap = { - isSystemUser = true; - group = "llama-swap"; - home = "/var/lib/llama"; - description = "llama-swap service user"; - }; - users.groups.llama-swap = { }; - - systemd.tmpfiles.rules = [ - "d /var/lib/llama 0755 llama-swap llama-swap - -" - "d ${modelDir} 0755 llama-swap llama-swap - -" - ]; - - systemd.services.llama-swap-download-models = { - description = "Download llama-swap GGUF models from Hugging Face"; - after = [ "network-online.target" ]; - wants = [ "network-online.target" ]; - before = [ "llama-swap.service" ]; - wantedBy = [ "multi-user.target" ]; - - serviceConfig = { - Type = "oneshot"; - User = "root"; - Group = "root"; - ExecStart = downloadModelsScript; - }; - }; - - services.llama-swap = { - enable = true; - port = 11111; - openFirewall = true; - - settings = { - healthCheckTimeout = 120; - - models = { - "ministral-3-8b-reasoning" = { - cmd = "${llama-server} --port \${PORT} --host 127.0.0.1 -m ${modelDir}/Ministral-3-8B-Reasoning-2512-Q4_K_M.gguf -c 32768"; - aliases = [ - "ministral-3" - "ministral" - ]; - }; - "qwen3.5-35b-a3b" = { - cmd = "${llama-server} --port \${PORT} --host 127.0.0.1 -m ${modelDir}/Qwen3.5-35B-A3B-UD-Q3_K_M.gguf -c 32768"; - aliases = [ - "qwen3.5" - "qwen" - ]; - }; - "qwen3.5-27b" = { - cmd = "${llama-server} --port \${PORT} --host 127.0.0.1 -m ${modelDir}/Qwen3.5-27B-UD-Q4_K_XL.gguf -c 16384"; - aliases = [ - "qwen3.5-27b" - "qwen-27b" - ]; - }; - "lfm2-24b-a2b" = { - cmd = "${llama-server} --port \${PORT} --host 127.0.0.1 -m ${modelDir}/LFM2-24B-A2B-Q4_K_M.gguf -c 32768"; - aliases = [ - "lfm2" - ]; - }; - }; - }; - }; -}