254 lines
9.1 KiB
Nix
254 lines
9.1 KiB
Nix
{
|
|
config,
|
|
pkgs,
|
|
lib,
|
|
inputs,
|
|
system,
|
|
...
|
|
}:
|
|
{
|
|
|
|
environment.systemPackages = [
|
|
inputs.self.packages.${system}.llama-cpp-nightly-vulkan
|
|
pkgs.unstable.stable-diffusion-cpp-vulkan
|
|
pkgs.unstable.whisper-cpp-vulkan
|
|
inputs.self.packages.${system}.z-image-models
|
|
inputs.self.packages.${system}.whisper-models
|
|
#inputs.self.packages.${system}.fish-speech-models
|
|
];
|
|
|
|
services.llama-swap = {
|
|
enable = true;
|
|
package = inputs.self.packages.${system}.llama-swap;
|
|
port = 11112;
|
|
openFirewall = true;
|
|
|
|
settings =
|
|
let
|
|
llama-server = lib.getExe' inputs.self.packages.${system}.llama-cpp-nightly-vulkan "llama-server";
|
|
sd-server = lib.getExe' pkgs.unstable.stable-diffusion-cpp-vulkan "sd-server";
|
|
whisper-server = lib.getExe' pkgs.unstable.whisper-cpp-vulkan "whisper-server";
|
|
podman = lib.getExe pkgs.podman;
|
|
z-image-models = inputs.self.packages.${system}.z-image-models;
|
|
whisper-models = inputs.self.packages.${system}.whisper-models;
|
|
in
|
|
{
|
|
healthCheckTimeout = 180;
|
|
startPort = 12000;
|
|
globalTTL = 600;
|
|
logLevel = "info";
|
|
|
|
macros = {
|
|
ctx = 32000;
|
|
ngl = 99;
|
|
kv_cache = "-ctk q4_0 -ctv q4_0 -fa 1";
|
|
batch = "-b 1024 -ub 1024"; # default 512 512
|
|
hf_repo = "";
|
|
image-tokens = "--image-min-tokens 256 --image-max-tokens 1536";
|
|
qwen35-thinking = "--chat-template-kwargs '{\"enable_thinking\":true}'";
|
|
qwen35-no-thinking = "--chat-template-kwargs '{\"enable_thinking\":false}'";
|
|
};
|
|
|
|
models = {
|
|
|
|
"qwen3.5-35b-a3b" = {
|
|
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} -ngl $\{ngl\} $\{kv_cache\} $\{batch\} --hf-repo $\{hf_repo\} $\{image-tokens\} $\{qwen35-no-thinking\} ";
|
|
aliases = [ "qwen3.5" ];
|
|
ttl = 1800;
|
|
macros = {
|
|
ctx = 49152;
|
|
hf_repo = "unsloth/Qwen3.5-35B-A3B-GGUF:UD-IQ3_XXS";
|
|
};
|
|
};
|
|
"qwen3.5-9b" = {
|
|
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} $\{kv_cache\} -ngl $\{ngl\} --hf-repo $\{hf_repo\} $\{batch\} $\{image-tokens\} $\{qwen35-thinking\} ";
|
|
ttl = 900;
|
|
macros = {
|
|
hf_repo = "unsloth/Qwen3.5-9B-GGUF:UD-Q4_K_XL";
|
|
ctx = 128000;
|
|
};
|
|
};
|
|
"qwen3.5-2b" = {
|
|
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} $\{kv_cache\} $\{batch\} $\{image-tokens\} -ngl $\{ngl\} --hf-repo $\{hf_repo\} ";
|
|
ttl = 900;
|
|
macros = {
|
|
hf_repo = "unsloth/Qwen3.5-2B-GGUF:UD-Q8_K_XL";
|
|
ctx = 200000;
|
|
};
|
|
};
|
|
|
|
"glm4.7-flash" = {
|
|
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} -fa 0 -ngl $\{ngl\} --hf-repo $\{hf_repo\}";
|
|
ttl = 900;
|
|
aliases = [ "coder" ];
|
|
macros = {
|
|
hf_repo = "unsloth/GLM-4.7-Flash-REAP-23B-A3B-GGUF"; # Reap should allow more context in gpu mem
|
|
ctx = 32000;
|
|
};
|
|
};
|
|
|
|
"gemma4" = {
|
|
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} -fa 0 -ngl $\{ngl\} --hf-repo $\{hf_repo\}";
|
|
ttl = 900;
|
|
macros = {
|
|
hf_repo = "unsloth/gemma-4-26B-A4B-it-GGUF:UD-IQ3_XXS";
|
|
ctx = 36000;
|
|
};
|
|
};
|
|
|
|
"gemma4E4" = {
|
|
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} $\{kv_cache\} -ngl $\{ngl\} --hf-repo $\{hf_repo\}";
|
|
ttl = 900;
|
|
macros = {
|
|
hf_repo = "unsloth/gemma-4-E4B-it-GGUF";
|
|
ctx = 128000;
|
|
};
|
|
};
|
|
|
|
"ministal-3-8b-reasonning" = {
|
|
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} -ngl $\{ngl\} --hf-repo $\{hf_repo\}";
|
|
aliases = [ "ministral3" ];
|
|
ttl = 900;
|
|
macros.hf_repo = "mistralai/Ministral-3-8B-Reasoning-2512-GGUF";
|
|
};
|
|
"ministal-3-3b" = {
|
|
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} -ngl $\{ngl\} --hf-repo $\{hf_repo\}";
|
|
aliases = [ "ministral3-mini" ];
|
|
ttl = 900;
|
|
macros.hf_repo = "mistralai/Ministral-3-3B-Instruct-2512-GGUF";
|
|
};
|
|
|
|
"minicpm-o-4_5" = {
|
|
cmd = "${llama-server} --port $\{PORT\} --host 0.0.0.0 --ctx-size $\{ctx\} -ngl $\{ngl\} --hf-repo $\{hf_repo\} --mmproj-url https://huggingface.co/openbmb/MiniCPM-o-4_5-gguf/resolve/main/vision/MiniCPM-o-4_5-vision-F16.gguf";
|
|
aliases = [
|
|
"openbmb/MiniCPM-o-4_5-gguf"
|
|
"minicpm"
|
|
];
|
|
ttl = 900;
|
|
macros.hf_repo = "openbmb/MiniCPM-o-4_5-gguf";
|
|
};
|
|
"z-image-turbo" = {
|
|
cmd = "${sd-server} --listen-port $\{PORT\} --diffusion-model $\{diffusion_model\} --vae $\{vae\} --llm $\{llm\} --offload-to-cpu --cfg-scale 1.0 --height 1024 --width 1024 --steps 4";
|
|
checkEndpoint = "/";
|
|
ttl = 300;
|
|
aliases = [
|
|
"gpt-image-1"
|
|
"dall-e-3"
|
|
];
|
|
macros = {
|
|
diffusion_model = "${z-image-models}/models/z-image-turbo-Q4_K.gguf";
|
|
vae = "${z-image-models}/models/ae.safetensors";
|
|
llm = "${z-image-models}/models/Qwen3-4B-Instruct-2507-Q4_K_M.gguf";
|
|
};
|
|
};
|
|
"distil-whisper-v3.5" = {
|
|
cmd = "${whisper-server} --host 127.0.0.1 --port $\{PORT\} -m $\{model\} --request-path /v1/audio/transcriptions --inference-path \"\"";
|
|
checkEndpoint = "/v1/audio/transcriptions/";
|
|
ttl = 0;
|
|
aliases = [
|
|
"whisper"
|
|
"whisper-1"
|
|
];
|
|
macros.model = "${whisper-models}/models/distil-large-v3.5.bin";
|
|
};
|
|
"nb-whisper-small" = {
|
|
cmd = "${whisper-server} --host 127.0.0.1 --port $\{PORT\} -m $\{model\} --request-path /v1/audio/transcriptions --inference-path \"\" --language no";
|
|
checkEndpoint = "/v1/audio/transcriptions/";
|
|
ttl = 0;
|
|
aliases = [
|
|
"whisper-no"
|
|
"whisper-nb"
|
|
];
|
|
macros.model = "${whisper-models}/models/nb-whisper-small-q5_0.bin";
|
|
};
|
|
|
|
"omnivoice" = {
|
|
cmd = "${pkgs.bash}/bin/bash -c '${pkgs.systemd}/bin/systemctl start podman-omnivoice.service && exec sleep infinity'";
|
|
cmdStop = "${pkgs.systemd}/bin/systemctl stop podman-omnivoice.service";
|
|
checkEndpoint = "/v1/audio/voices";
|
|
proxy = "http://127.0.0.1:8091";
|
|
ttl = 900;
|
|
};
|
|
|
|
"kokoro" = {
|
|
cmd = "${podman} run --init --rm --name kokoro-tts -p $\{PORT\}:8880 ghcr.io/remsky/kokoro-fastapi-cpu:latest";
|
|
cmdStop = "${podman} stop kokoro-tts";
|
|
checkEndpoint = "/v1/audio/voices";
|
|
ttl = 900;
|
|
};
|
|
|
|
};
|
|
};
|
|
};
|
|
|
|
virtualisation.oci-containers.containers.omnivoice = {
|
|
image = "vllm/vllm-openai:nightly";
|
|
ports = [ "8091:8091" ];
|
|
environment = {
|
|
VLLM_DEVICE = "cpu";
|
|
};
|
|
cmd = [
|
|
"vllm"
|
|
"serve"
|
|
"k2-fsa/OmniVoice"
|
|
"--omni"
|
|
"--device"
|
|
"cpu"
|
|
"--port"
|
|
"8091"
|
|
"--trust-remote-code"
|
|
];
|
|
extraOptions = [ "--rm" ];
|
|
autoStart = false;
|
|
};
|
|
|
|
systemd.services.llama-swap = {
|
|
serviceConfig = {
|
|
StateDirectory = "llama-swap";
|
|
CacheDirectory = "llama-swap";
|
|
RuntimeDirectory = "llama-swap";
|
|
Environment = [
|
|
"HOME=/var/lib/llama-swap"
|
|
"XDG_CACHE_HOME=/var/cache/llama-swap"
|
|
"MESA_SHADER_CACHE_DIR=/var/cache/llama-swap/mesa"
|
|
];
|
|
|
|
DynamicUser = lib.mkForce false;
|
|
User = "root";
|
|
Group = "root";
|
|
|
|
PrivateDevices = lib.mkForce false;
|
|
PrivateTmp = lib.mkForce false;
|
|
PrivateMounts = lib.mkForce false;
|
|
PrivateUsers = lib.mkForce false;
|
|
ProtectClock = lib.mkForce false;
|
|
ProtectControlGroups = lib.mkForce false;
|
|
ProtectHome = lib.mkForce false;
|
|
ProtectKernelLogs = lib.mkForce false;
|
|
ProtectKernelModules = lib.mkForce false;
|
|
ProtectKernelTunables = lib.mkForce false;
|
|
ProtectSystem = lib.mkForce false;
|
|
ProtectHostname = lib.mkForce false;
|
|
ProtectProc = lib.mkForce "";
|
|
MemoryDenyWriteExecute = lib.mkForce false;
|
|
LockPersonality = lib.mkForce false;
|
|
NoNewPrivileges = lib.mkForce false;
|
|
RemoveIPC = lib.mkForce false;
|
|
RestrictNamespaces = lib.mkForce true;
|
|
RestrictRealtime = lib.mkForce false;
|
|
RestrictSUIDSGID = lib.mkForce false;
|
|
RestrictAddressFamilies = lib.mkForce [
|
|
"AF_INET"
|
|
"AF_INET6"
|
|
"AF_UNIX"
|
|
"AF_NETLINK"
|
|
];
|
|
CapabilityBoundingSet = lib.mkForce "";
|
|
SystemCallArchitectures = lib.mkForce "";
|
|
SystemCallFilter = lib.mkForce [ ];
|
|
SystemCallErrorNumber = lib.mkForce "";
|
|
ProcSubset = lib.mkForce "all";
|
|
};
|
|
};
|
|
}
|