:redid llama-swap

2026-03-02 15:51:40 +01:00
parent ca06437580
commit 62cedaa51c
2 changed files with 93 additions and 1 deletions
@@ -38,7 +38,7 @@
    ../../modules/miniflux.nix
    #../../modules/ollama.nix     # replaced by llama-cpp + llama-swap
    #../../modules/openwebui.nix  # using llama-cpp built-in UI instead
-    ../../modules/llama-cpp.nix
+    ../../modules/llama-swap.nix
    ../../modules/librechat.nix
    ../../modules/immich.nix

@@ -0,0 +1,92 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+{
+  environment.systemPackages = [ pkgs.unstable.llama-cpp-vulkan ];
+
+  services.llama-swap = {
+    enable = true;
+    port = 8085;
+    openFirewall = true;
+
+    settings =
+      let
+        #llama-server = "${lib.getExe  pkgs.unstable.llama-cpp-vulkan "llama-server"}";
+        llama-server = lib.getExe' pkgs.unstable.llama-cpp-vulkan "llama-server";
+      in
+      {
+        healthCheckTimeout = 180;
+        startPort = 12000;
+        globalTTL = 600;
+        logLevel = "info";
+
+        macros = {
+          ctx = 32768;
+
+          ngl = 99;
+
+          quant = "Q4_K_M";
+
+          kv_type_k = "q4_0";
+          kv_type_v = "q4_0";
+          kv_cache = "-ctk \${kv_type_k} -ctv \${kv_type_v}";
+
+          hf_repo = "";
+          hf_param = "--hf-repo \${hf_repo}:\${quant}";
+
+          llama-base = "${llama-server} --port \${PORT} --host 0.0.0.0 --ctx-size \${ctx} -ngl \${ngl} \${kv_cache} \${hf_param}";
+        };
+
+        models = {
+
+          "qwen3.5-35b-a3b" = {
+            macros = {
+              ngl = 40;
+              hf_repo = "unsloth/Qwen3.5-35B-A3B-GGUF";
+              quant = "Q2_K_L";
+            };
+            cmd = "\${llama-base}";
+            aliases = [ "qwen3.5" ];
+            ttl = 900;
+          };
+
+          "ministal-3-8b-reasonning" = {
+            macros = {
+              ngl = 99;
+              hf_repo = "mistralai/Ministral-3-8B-Reasoning-2512-GGUF";
+            };
+            cmd = "\${llama-base}";
+            aliases = [ "ministral" ];
+            ttl = 900;
+          };
+
+        };
+
+        peers = {
+          openrouter = {
+            proxy = "https://openrouter.ai/api";
+            apiKey = "\${env.OPENROUTER_API_KEY}";
+            models = [
+              "minimax/minimax-m2.5"
+              "z-ai/glm-5"
+              "qwen/qwen3-coder-next"
+              "moonshotai/kimi-k2.5"
+            ];
+            filters = {
+              stripParams = "temperature, top_p";
+              setParams = {
+                provider = {
+                  data_collection = "deny";
+                  zdr = true;
+                };
+              };
+            };
+          };
+        };
+
+      };
+  };
+}