llama swap updates

flake.nix

@@ -62,9 +62,10 @@
       system = "x86_64-linux";
     in
     {
-      packages.${system}.qwen-asr =
-        nixpkgs.legacyPackages.${system}.callPackage ./packages/qwen-asr.nix
-          { };
+      packages.${system} = {
+        qwen-asr = nixpkgs.legacyPackages.${system}.callPackage ./packages/qwen-asr.nix { };
+        llama-swap = nixpkgs.legacyPackages.${system}.callPackage ./packages/llama-swap { };
+      };
 
       # legolas
       nixosConfigurations.legolas = nixpkgs.lib.nixosSystem {

modules/llama-swap.nix

@@ -5,6 +5,16 @@
   ...
 }:
 {
+  sops.secrets = {
+    "ai/openai" = { };
+    "ai/openrouter" = { };
+    "ai/fireworks" = { };
+    "ai/cerebras" = { };
+    "ai/groq" = { };
+    "ai/mistral" = { };
+    "ai/zai" = { };
+  };
+
   environment.systemPackages = [ pkgs.unstable.llama-cpp-vulkan ];
 
   services.llama-swap = {
@@ -101,6 +111,21 @@
               };
             };
           };
+          fireworks = {
+            proxy = "https://api.fireworks.ai/inference/v1";
+            apiKey = "$\{env.FIREWORKS_API_KEY\}";
+            models = [
+              "accounts/fireworks/models/qwen3p5-397b-a17b"
+              "accounts/fireworks/models/minimax-m2p5"
+            ];
+          };
+          zai = {
+            proxy = "$\{env.ZAI_API_BASE\}";
+            apiKey = "$\{env.ZAI_API_KEY\}";
+            models = [
+              "glm-5"
+            ];
+          };
         };
 
       };
@@ -109,16 +134,29 @@
   # llama.cpp tries to create its cache under $HOME/.cache; when launched as a
   # system service HOME may default to "/" ("//.cache/..."), which is often
   # read-only. Give it a writable cache/state location.
-  systemd.services.llama-swap.serviceConfig = {
-    StateDirectory = "llama-swap";
-    CacheDirectory = "llama-swap";
-    Environment = [
-      "HOME=/var/lib/llama-swap"
-      "XDG_CACHE_HOME=/var/cache/llama-swap"
-      "MESA_SHADER_CACHE_DIR=/var/cache/llama-swap/mesa"
-      #"MESA_SHADER_CACHE_MAX_SIZE=1G"
-      #"GGML_VULKAN_MAX_NODES=16"
-      #"GGML_VK_RELAXED_SHAPES=0"
-    ];
+  systemd.services.llama-swap = {
+    preStart = ''
+      cat > /run/llama-swap/secrets.env <<EOF
+      OPENAI_API_KEY=$(cat ${config.sops.secrets."ai/openai".path})
+      OPENROUTER_API_KEY=$(cat ${config.sops.secrets."ai/openrouter".path})
+      FIREWORKS_API_KEY=$(cat ${config.sops.secrets."ai/fireworks".path})
+      CEREBRAS_API_KEY=$(cat ${config.sops.secrets."ai/cerebras".path})
+      GROQ_API_KEY=$(cat ${config.sops.secrets."ai/groq".path})
+      MISTRAL_API_KEY=$(cat ${config.sops.secrets."ai/mistral".path})
+      ZAI_API_KEY=$(cat ${config.sops.secrets."ai/zai".path})
+      ZAI_API_BASE=https://api.z.ai/api/coding/paas/v4
+      EOF
+    '';
+    serviceConfig = {
+      StateDirectory = "llama-swap";
+      CacheDirectory = "llama-swap";
+      RuntimeDirectory = "llama-swap";
+      EnvironmentFile = "/run/llama-swap/secrets.env";
+      Environment = [
+        "HOME=/var/lib/llama-swap"
+        "XDG_CACHE_HOME=/var/cache/llama-swap"
+        "MESA_SHADER_CACHE_DIR=/var/cache/llama-swap/mesa"
+      ];
+    };
   };
 }

packages/llama-swap/default.nix

@@ -0,0 +1 @@
+import ./package.nix

packages/llama-swap/package.nix

@@ -0,0 +1,144 @@
+{
+  lib,
+  stdenv,
+
+  buildGoModule,
+  fetchFromGitHub,
+  versionCheckHook,
+
+  callPackage,
+
+  nixosTests,
+  nix-update-script,
+}:
+
+let
+  canExecute = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+in
+buildGoModule (finalAttrs: {
+  pname = "llama-swap";
+  version = "198";
+
+  outputs = [
+    "out"
+    "wol"
+  ];
+
+  src = fetchFromGitHub {
+    owner = "mostlygeek";
+    repo = "llama-swap";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-7fZUKDCtj8RGca53CkLwVpvNWX6ryTbS02Uz/+uZpTs=";
+    leaveDotGit = true;
+    postFetch = ''
+      cd "$out"
+      git rev-parse HEAD > $out/COMMIT
+      date -u -d "@$(git log -1 --pretty=%ct)" "+'%Y-%m-%dT%H:%M:%SZ'" > $out/SOURCE_DATE_EPOCH
+      find "$out" -name .git -print0 | xargs -0 rm -rf
+    '';
+  };
+
+  vendorHash = "sha256-XiDYlw/byu8CWvg4KSPC7m8PGCZXtp08Y1velx4BR8U=";
+
+  passthru.ui = callPackage ./ui.nix { llama-swap = finalAttrs.finalPackage; };
+
+  nativeBuildInputs = [
+    versionCheckHook
+  ];
+
+  __darwinAllowLocalNetworking = true;
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X main.version=${finalAttrs.version}"
+  ];
+
+  preBuild = ''
+    ldflags+=" -X main.commit=$(cat COMMIT)"
+    ldflags+=" -X main.date=$(cat SOURCE_DATE_EPOCH)"
+    cp -r ${finalAttrs.passthru.ui}/ui_dist proxy/
+  '';
+
+  excludedPackages = [
+    "misc/process-cmd-test"
+    "misc/benchmark-chatcompletion"
+  ]
+  ++ lib.optionals (!canExecute) [
+    "misc/simple-responder"
+  ];
+
+  checkFlags =
+    let
+      skippedTests = lib.optionals (stdenv.isDarwin && stdenv.isx86_64) [
+        "TestProcess_AutomaticallyStartsUpstream"
+        "TestProcess_WaitOnMultipleStarts"
+        "TestProcess_BrokenModelConfig"
+        "TestProcess_UnloadAfterTTL"
+        "TestProcess_LowTTLValue"
+        "TestProcess_HTTPRequestsHaveTimeToFinish"
+        "TestProcess_SwapState"
+        "TestProcess_ShutdownInterruptsHealthCheck"
+        "TestProcess_ExitInterruptsHealthCheck"
+        "TestProcess_ConcurrencyLimit"
+        "TestProcess_StopImmediately"
+        "TestProcess_ForceStopWithKill"
+        "TestProcess_StopCmd"
+        "TestProcess_EnvironmentSetCorrectly"
+      ];
+    in
+    [ "-skip=^${builtins.concatStringsSep "$|^" skippedTests}$" ];
+
+  doCheck = canExecute;
+  preCheck = ''
+    mkdir build
+    ln -s "$GOPATH/bin/simple-responder" "./build/simple-responder_''${GOOS}_''${GOARCH}"
+  '';
+  postCheck = ''
+    rm "$GOPATH/bin/simple-responder"
+  '';
+
+  postInstall = ''
+    install -Dm444 -t "$out/share/llama-swap" config.example.yaml
+    mkdir -p "$wol/bin"
+    mv "$out/bin/wol-proxy" "$wol/bin/"
+  '';
+
+  doInstallCheck = true;
+  versionCheckProgramArg = "-version";
+
+  passthru.tests.nixos = nixosTests.llama-swap;
+  passthru.updateScript = nix-update-script {
+    extraArgs = [
+      "--subpackage"
+      "ui"
+    ];
+  };
+
+  meta = {
+    homepage = "https://github.com/mostlygeek/llama-swap";
+    changelog = "https://github.com/mostlygeek/llama-swap/releases/tag/${finalAttrs.src.tag}";
+    description = "Model swapping for llama.cpp (or any local OpenAPI compatible server)";
+    longDescription = ''
+      llama-swap is a light weight, transparent proxy server that provides
+      automatic model swapping to llama.cpp's server.
+
+      When a request is made to an OpenAI compatible endpoint, llama-swap will
+      extract the `model` value and load the appropriate server configuration to
+      serve it. If the wrong upstream server is running, it will be replaced
+      with the correct one. This is where the "swap" part comes in. The upstream
+      server is automatically swapped to the correct one to serve the request.
+
+      In the most basic configuration llama-swap handles one model at a time.
+      For more advanced use cases, the `groups` feature allows multiple models
+      to be loaded at the same time. You have complete control over how your
+      system resources are used.
+    '';
+    license = lib.licenses.mit;
+    mainProgram = "llama-swap";
+    maintainers = with lib.maintainers; [
+      jk
+      podium868909
+    ];
+  };
+})

packages/llama-swap/ui.nix

@@ -0,0 +1,35 @@
+{
+  llama-swap,
+
+  buildNpmPackage,
+  fetchFromGitHub,
+}:
+
+buildNpmPackage (finalAttrs: {
+  pname = "${llama-swap.pname}-ui";
+  inherit (llama-swap) version;
+
+  src = fetchFromGitHub {
+    owner = "mostlygeek";
+    repo = "llama-swap";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-FGrRwWgXTpH4h0MYCdMDagUskUZA+/s/dOjjkAigPQw=";
+  };
+
+  npmDepsHash = "sha256-gTDsuWPLCWsPltioziygFmSQFdLqjkZpmmVWIWoZwoc=";
+
+  postPatch = ''
+    substituteInPlace vite.config.ts \
+      --replace-fail "../proxy/ui_dist" "${placeholder "out"}/ui_dist"
+  '';
+
+  sourceRoot = "source/ui-svelte";
+
+  postInstall = ''
+    rm -rf $out/lib
+  '';
+
+  meta = (removeAttrs llama-swap.meta [ "mainProgram" ]) // {
+    description = "${llama-swap.meta.description} - UI";
+  };
+})