adriangl/nix-dotfiles-v2
1
1
Fork 0
Code Issues Pull Requests Releases Activity

format and llama tuning

Browse Source
This commit is contained in:
Adrian Gunnar Lauterer
2026-03-23 22:19:15 +01:00
parent 8173b617e7
commit 83fdef416e
10 changed files with 84 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
modules/llama-cpp.nix
View File

@@ -12,22 +12,30 @@ in
{
environment.systemPackages = [ pkgs.unstable.ollama ];
services.llama-cpp = {
enable = true;
host = "0.0.0.0";
port = 11111;
package = pkgs.unstable.llama-cpp-vulkan;
openFirewall = true;
model = "/var/lib/llama/models/Qwen3.5-35B-A3B-UD-Q2_K_XL.gguf" ;
extraFlags = [
"-c" "32000"
"-ngl" "41" # techincally entire qwen3.5
"--image-min-tokens" "1024"
"--image-max-tokens" "2048"
#"--hf-repo" "unsloth/Qwen3.5-35B-A3B-GGUF:Q2_K_L"
"--mmproj" "/var/lib/llama/models/mmproj-F16.gguf"
"-ctk" "q4_0" "-ctv" "q4_0" # quantisize kv cache.
"--no-mmap"
];
};
services.llama-cpp = {
enable = true;
host = "0.0.0.0";
port = 11111;
package = pkgs.unstable.llama-cpp-vulkan;
openFirewall = true;
model = "/var/lib/llama/models/Qwen3.5-35B-A3B-UD-Q2_K_XL.gguf";
extraFlags = [
"-c"
"32000"
"-ngl"
"41" # techincally entire qwen3.5
"--image-min-tokens"
"1024"
"--image-max-tokens"
"2048"
#"--hf-repo" "unsloth/Qwen3.5-35B-A3B-GGUF:Q2_K_L"
"--mmproj"
"/var/lib/llama/models/mmproj-F16.gguf"
"-ctk"
"q4_0"
"-ctv"
"q4_0" # quantisize kv cache.
"--no-mmap"
];
};
}