llama-cpp instead of swap, native swap functionality is coming
This commit is contained in:
@@ -38,7 +38,7 @@
|
||||
../../modules/miniflux.nix
|
||||
#../../modules/ollama.nix # replaced by llama-cpp + llama-swap
|
||||
#../../modules/openwebui.nix # using llama-cpp built-in UI instead
|
||||
../../modules/llama-swap.nix
|
||||
../../modules/llama-cpp.nix
|
||||
../../modules/librechat.nix
|
||||
../../modules/immich.nix
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
unstable,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
hostname = config.networking.hostName;
|
||||
in
|
||||
|
||||
{
|
||||
environment.systemPackages = [ pkgs.unstable.ollama ];
|
||||
services.llama-cpp = {
|
||||
enable = true;
|
||||
host = "0.0.0.0";
|
||||
port = 11111;
|
||||
package = pkgs.unstable.llama-cpp;
|
||||
openFirewall = true;
|
||||
model = "/var/lib/llama-cpp/models/Qwen3.5-35B-A3B-UD-Q3_K_M.gguf" ;
|
||||
extraFlags = [
|
||||
"-c" "32000"
|
||||
"-ngl" "38"
|
||||
"--image-min-tokens" "1024"
|
||||
#"--hf-repo" "unsloth/Qwen3.5-35B-A3B-GGUF:Q2_K_L"
|
||||
"--mmproj" "/var/lib/llama-cpp/models/mmproj-F16.gguf"
|
||||
];
|
||||
};
|
||||
}
|
||||
@@ -1,143 +0,0 @@
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
modelDir = "/var/lib/llama/models";
|
||||
legacyModelDir = "/var/lib/llama-cpp/models";
|
||||
|
||||
# llama-cpp with Vulkan for Intel Arc
|
||||
llama-cpp-vulkan = pkgs.llama-cpp.override {
|
||||
vulkanSupport = true;
|
||||
cudaSupport = false;
|
||||
rocmSupport = false;
|
||||
openclSupport = false;
|
||||
blasSupport = true;
|
||||
};
|
||||
|
||||
llama-server = lib.getExe' llama-cpp-vulkan "llama-server";
|
||||
|
||||
models = {
|
||||
"Ministral-3-8B-Reasoning-2512-Q4_K_M" = {
|
||||
url = "https://huggingface.co/mistralai/Ministral-3-8B-Reasoning-2512-GGUF/resolve/main/Ministral-3-8B-Reasoning-2512-Q4_K_M.gguf";
|
||||
filename = "Ministral-3-8B-Reasoning-2512-Q4_K_M.gguf";
|
||||
};
|
||||
"Qwen3.5-35B-A3B-UD-Q3_K_M" = {
|
||||
url = "https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/Qwen3.5-35B-A3B-UD-Q3_K_M.gguf";
|
||||
filename = "Qwen3.5-35B-A3B-UD-Q3_K_M.gguf";
|
||||
};
|
||||
"Qwen3.5-27B-UD-Q4_K_XL" = {
|
||||
url = "https://huggingface.co/unsloth/Qwen3.5-27B-GGUF/resolve/main/Qwen3.5-27B-UD-Q4_K_XL.gguf";
|
||||
filename = "Qwen3.5-27B-UD-Q4_K_XL.gguf";
|
||||
};
|
||||
"LFM2-24B-A2B-Q4_K_M" = {
|
||||
url = "https://huggingface.co/LiquidAI/LFM2-24B-A2B-GGUF/resolve/main/LFM2-24B-A2B-Q4_K_M.gguf";
|
||||
filename = "LFM2-24B-A2B-Q4_K_M.gguf";
|
||||
};
|
||||
};
|
||||
|
||||
downloadModelsScript = pkgs.writeShellScript "llama-swap-download-models" (
|
||||
''
|
||||
set -euo pipefail
|
||||
|
||||
install -d -m 0755 -o llama-swap -g llama-swap "${modelDir}"
|
||||
|
||||
if [ -d "${legacyModelDir}" ]; then
|
||||
for f in "${legacyModelDir}"/*.gguf; do
|
||||
[ -e "$f" ] || continue
|
||||
base="$(basename "$f")"
|
||||
if [ ! -e "${modelDir}/$base" ]; then
|
||||
mv "$f" "${modelDir}/$base"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
''
|
||||
+ (lib.concatStringsSep "\n" (
|
||||
lib.mapAttrsToList (name: model: ''
|
||||
MODEL_PATH="${modelDir}/${model.filename}"
|
||||
if [ ! -f "$MODEL_PATH" ]; then
|
||||
echo "Downloading ${name}..."
|
||||
${lib.getExe pkgs.curl} --fail -L --retry 5 --retry-delay 5 --retry-all-errors \
|
||||
-o "$MODEL_PATH.tmp" "${model.url}"
|
||||
mv "$MODEL_PATH.tmp" "$MODEL_PATH"
|
||||
fi
|
||||
chown llama-swap:llama-swap "$MODEL_PATH"
|
||||
chmod 0644 "$MODEL_PATH"
|
||||
'') models
|
||||
))
|
||||
);
|
||||
in
|
||||
{
|
||||
environment.systemPackages = [ llama-cpp-vulkan ];
|
||||
|
||||
users.users.llama-swap = {
|
||||
isSystemUser = true;
|
||||
group = "llama-swap";
|
||||
home = "/var/lib/llama";
|
||||
description = "llama-swap service user";
|
||||
};
|
||||
users.groups.llama-swap = { };
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /var/lib/llama 0755 llama-swap llama-swap - -"
|
||||
"d ${modelDir} 0755 llama-swap llama-swap - -"
|
||||
];
|
||||
|
||||
systemd.services.llama-swap-download-models = {
|
||||
description = "Download llama-swap GGUF models from Hugging Face";
|
||||
after = [ "network-online.target" ];
|
||||
wants = [ "network-online.target" ];
|
||||
before = [ "llama-swap.service" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = "root";
|
||||
Group = "root";
|
||||
ExecStart = downloadModelsScript;
|
||||
};
|
||||
};
|
||||
|
||||
services.llama-swap = {
|
||||
enable = true;
|
||||
port = 11111;
|
||||
openFirewall = true;
|
||||
|
||||
settings = {
|
||||
healthCheckTimeout = 120;
|
||||
|
||||
models = {
|
||||
"ministral-3-8b-reasoning" = {
|
||||
cmd = "${llama-server} --port \${PORT} --host 127.0.0.1 -m ${modelDir}/Ministral-3-8B-Reasoning-2512-Q4_K_M.gguf -c 32768";
|
||||
aliases = [
|
||||
"ministral-3"
|
||||
"ministral"
|
||||
];
|
||||
};
|
||||
"qwen3.5-35b-a3b" = {
|
||||
cmd = "${llama-server} --port \${PORT} --host 127.0.0.1 -m ${modelDir}/Qwen3.5-35B-A3B-UD-Q3_K_M.gguf -c 32768";
|
||||
aliases = [
|
||||
"qwen3.5"
|
||||
"qwen"
|
||||
];
|
||||
};
|
||||
"qwen3.5-27b" = {
|
||||
cmd = "${llama-server} --port \${PORT} --host 127.0.0.1 -m ${modelDir}/Qwen3.5-27B-UD-Q4_K_XL.gguf -c 16384";
|
||||
aliases = [
|
||||
"qwen3.5-27b"
|
||||
"qwen-27b"
|
||||
];
|
||||
};
|
||||
"lfm2-24b-a2b" = {
|
||||
cmd = "${llama-server} --port \${PORT} --host 127.0.0.1 -m ${modelDir}/LFM2-24B-A2B-Q4_K_M.gguf -c 32768";
|
||||
aliases = [
|
||||
"lfm2"
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user