31 lines
625 B
Nix
31 lines
625 B
Nix
{
|
|
config,
|
|
pkgs,
|
|
lib,
|
|
unstable,
|
|
...
|
|
}:
|
|
|
|
let
|
|
hostname = config.networking.hostName;
|
|
in
|
|
|
|
{
|
|
environment.systemPackages = [ pkgs.unstable.ollama ];
|
|
services.llama-cpp = {
|
|
enable = true;
|
|
host = "0.0.0.0";
|
|
port = 11111;
|
|
package = pkgs.unstable.llama-cpp;
|
|
openFirewall = true;
|
|
model = "/var/lib/llama-cpp/models/Qwen3.5-35B-A3B-UD-Q3_K_M.gguf" ;
|
|
extraFlags = [
|
|
"-c" "32000"
|
|
"-ngl" "38"
|
|
"--image-min-tokens" "1024"
|
|
#"--hf-repo" "unsloth/Qwen3.5-35B-A3B-GGUF:Q2_K_L"
|
|
"--mmproj" "/var/lib/llama-cpp/models/mmproj-F16.gguf"
|
|
];
|
|
};
|
|
}
|