adriangl/nix-dotfiles-v2
1
1
Fork 0
Code Issues Pull Requests Releases Activity

llama swap update

Browse Source
This commit is contained in:
Adrian Gunnar Lauterer
2026-08-04 01:35:51 +02:00
parent ce6e393512
commit 489190baef
1 changed files with 5 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/llama-swap.nix
+5 -10
View File
@@ -48,7 +48,8 @@
batch = "-b 1024 -ub 1024"; # default 512 512
hf_repo = "";
image-tokens = "--image-min-tokens 256 --image-max-tokens 1536";
tools = "--tools 'all'";
#tools = "--tools 'all'";
tools = "";
thinking = "--reasoning on";
no-thinking = "--reasoning off";
};
@@ -79,18 +80,18 @@
};
"gemma4-26b-a4b" = {
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} -fa 0 -ngl $\{ngl\} --hf-repo $\{hf_repo\} $\{tools\}";
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} $\{kv_cache\} -ngl $\{ngl\} --hf-repo $\{hf_repo\} --no-mmap $\{tools\}";
aliases = [
"gemma4"
];
ttl = 900;
macros = {
hf_repo = "unsloth/gemma-4-26B-A4B-it-GGUF:UD-IQ3_XXS";
ctx = 36000;
ctx = 48000;
};
};
"gemma4-12b" = {
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} -fa 0 -ngl $\{ngl\} --hf-repo $\{hf_repo\} --spec-type draft-mtp $\{tools\}";
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} -fa 1 -ngl $\{ngl\} --hf-repo $\{hf_repo\} --spec-type draft-mtp --no-mmap $\{tools\}";
ttl = 900;
macros = {
hf_repo = "unsloth/gemma-4-12B-it-qat-GGUF:UD-Q4_K_XL";
@@ -116,12 +117,6 @@
};
};
"ministal-3-3b" = {
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} -ngl $\{ngl\} --hf-repo $\{hf_repo\} $\{tools\} ";
aliases = [ "ministral3-mini" ];
ttl = 900;
macros.hf_repo = "mistralai/Ministral-3-3B-Instruct-2512-GGUF";
};
"z-image-turbo" = {
cmd = "${sd-server} --listen-port $\{PORT\} --diffusion-model $\{diffusion_model\} --vae $\{vae\} --llm $\{llm\} --offload-to-cpu --cfg-scale 1.0 --height 1024 --width 1024 --steps 4";