diff --git a/flake.nix b/flake.nix index dc58b83..a1c123a 100644 --- a/flake.nix +++ b/flake.nix @@ -62,9 +62,10 @@ system = "x86_64-linux"; in { - packages.${system}.qwen-asr = - nixpkgs.legacyPackages.${system}.callPackage ./packages/qwen-asr.nix - { }; + packages.${system} = { + qwen-asr = nixpkgs.legacyPackages.${system}.callPackage ./packages/qwen-asr.nix { }; + llama-swap = nixpkgs.legacyPackages.${system}.callPackage ./packages/llama-swap { }; + }; # legolas nixosConfigurations.legolas = nixpkgs.lib.nixosSystem { diff --git a/modules/llama-swap.nix b/modules/llama-swap.nix index 434238a..0ff49d6 100644 --- a/modules/llama-swap.nix +++ b/modules/llama-swap.nix @@ -5,6 +5,16 @@ ... }: { + sops.secrets = { + "ai/openai" = { }; + "ai/openrouter" = { }; + "ai/fireworks" = { }; + "ai/cerebras" = { }; + "ai/groq" = { }; + "ai/mistral" = { }; + "ai/zai" = { }; + }; + environment.systemPackages = [ pkgs.unstable.llama-cpp-vulkan ]; services.llama-swap = { @@ -101,6 +111,21 @@ }; }; }; + fireworks = { + proxy = "https://api.fireworks.ai/inference/v1"; + apiKey = "$\{env.FIREWORKS_API_KEY\}"; + models = [ + "accounts/fireworks/models/qwen3p5-397b-a17b" + "accounts/fireworks/models/minimax-m2p5" + ]; + }; + zai = { + proxy = "$\{env.ZAI_API_BASE\}"; + apiKey = "$\{env.ZAI_API_KEY\}"; + models = [ + "glm-5" + ]; + }; }; }; @@ -109,16 +134,29 @@ # llama.cpp tries to create its cache under $HOME/.cache; when launched as a # system service HOME may default to "/" ("//.cache/..."), which is often # read-only. Give it a writable cache/state location. - systemd.services.llama-swap.serviceConfig = { - StateDirectory = "llama-swap"; - CacheDirectory = "llama-swap"; - Environment = [ - "HOME=/var/lib/llama-swap" - "XDG_CACHE_HOME=/var/cache/llama-swap" - "MESA_SHADER_CACHE_DIR=/var/cache/llama-swap/mesa" - #"MESA_SHADER_CACHE_MAX_SIZE=1G" - #"GGML_VULKAN_MAX_NODES=16" - #"GGML_VK_RELAXED_SHAPES=0" - ]; + systemd.services.llama-swap = { + preStart = '' + cat > /run/llama-swap/secrets.env < $out/COMMIT + date -u -d "@$(git log -1 --pretty=%ct)" "+'%Y-%m-%dT%H:%M:%SZ'" > $out/SOURCE_DATE_EPOCH + find "$out" -name .git -print0 | xargs -0 rm -rf + ''; + }; + + vendorHash = "sha256-XiDYlw/byu8CWvg4KSPC7m8PGCZXtp08Y1velx4BR8U="; + + passthru.ui = callPackage ./ui.nix { llama-swap = finalAttrs.finalPackage; }; + + nativeBuildInputs = [ + versionCheckHook + ]; + + __darwinAllowLocalNetworking = true; + + ldflags = [ + "-s" + "-w" + "-X main.version=${finalAttrs.version}" + ]; + + preBuild = '' + ldflags+=" -X main.commit=$(cat COMMIT)" + ldflags+=" -X main.date=$(cat SOURCE_DATE_EPOCH)" + cp -r ${finalAttrs.passthru.ui}/ui_dist proxy/ + ''; + + excludedPackages = [ + "misc/process-cmd-test" + "misc/benchmark-chatcompletion" + ] + ++ lib.optionals (!canExecute) [ + "misc/simple-responder" + ]; + + checkFlags = + let + skippedTests = lib.optionals (stdenv.isDarwin && stdenv.isx86_64) [ + "TestProcess_AutomaticallyStartsUpstream" + "TestProcess_WaitOnMultipleStarts" + "TestProcess_BrokenModelConfig" + "TestProcess_UnloadAfterTTL" + "TestProcess_LowTTLValue" + "TestProcess_HTTPRequestsHaveTimeToFinish" + "TestProcess_SwapState" + "TestProcess_ShutdownInterruptsHealthCheck" + "TestProcess_ExitInterruptsHealthCheck" + "TestProcess_ConcurrencyLimit" + "TestProcess_StopImmediately" + "TestProcess_ForceStopWithKill" + "TestProcess_StopCmd" + "TestProcess_EnvironmentSetCorrectly" + ]; + in + [ "-skip=^${builtins.concatStringsSep "$|^" skippedTests}$" ]; + + doCheck = canExecute; + preCheck = '' + mkdir build + ln -s "$GOPATH/bin/simple-responder" "./build/simple-responder_''${GOOS}_''${GOARCH}" + ''; + postCheck = '' + rm "$GOPATH/bin/simple-responder" + ''; + + postInstall = '' + install -Dm444 -t "$out/share/llama-swap" config.example.yaml + mkdir -p "$wol/bin" + mv "$out/bin/wol-proxy" "$wol/bin/" + ''; + + doInstallCheck = true; + versionCheckProgramArg = "-version"; + + passthru.tests.nixos = nixosTests.llama-swap; + passthru.updateScript = nix-update-script { + extraArgs = [ + "--subpackage" + "ui" + ]; + }; + + meta = { + homepage = "https://github.com/mostlygeek/llama-swap"; + changelog = "https://github.com/mostlygeek/llama-swap/releases/tag/${finalAttrs.src.tag}"; + description = "Model swapping for llama.cpp (or any local OpenAPI compatible server)"; + longDescription = '' + llama-swap is a light weight, transparent proxy server that provides + automatic model swapping to llama.cpp's server. + + When a request is made to an OpenAI compatible endpoint, llama-swap will + extract the `model` value and load the appropriate server configuration to + serve it. If the wrong upstream server is running, it will be replaced + with the correct one. This is where the "swap" part comes in. The upstream + server is automatically swapped to the correct one to serve the request. + + In the most basic configuration llama-swap handles one model at a time. + For more advanced use cases, the `groups` feature allows multiple models + to be loaded at the same time. You have complete control over how your + system resources are used. + ''; + license = lib.licenses.mit; + mainProgram = "llama-swap"; + maintainers = with lib.maintainers; [ + jk + podium868909 + ]; + }; +}) diff --git a/packages/llama-swap/ui.nix b/packages/llama-swap/ui.nix new file mode 100644 index 0000000..a701eac --- /dev/null +++ b/packages/llama-swap/ui.nix @@ -0,0 +1,35 @@ +{ + llama-swap, + + buildNpmPackage, + fetchFromGitHub, +}: + +buildNpmPackage (finalAttrs: { + pname = "${llama-swap.pname}-ui"; + inherit (llama-swap) version; + + src = fetchFromGitHub { + owner = "mostlygeek"; + repo = "llama-swap"; + tag = "v${finalAttrs.version}"; + hash = "sha256-FGrRwWgXTpH4h0MYCdMDagUskUZA+/s/dOjjkAigPQw="; + }; + + npmDepsHash = "sha256-gTDsuWPLCWsPltioziygFmSQFdLqjkZpmmVWIWoZwoc="; + + postPatch = '' + substituteInPlace vite.config.ts \ + --replace-fail "../proxy/ui_dist" "${placeholder "out"}/ui_dist" + ''; + + sourceRoot = "source/ui-svelte"; + + postInstall = '' + rm -rf $out/lib + ''; + + meta = (removeAttrs llama-swap.meta [ "mainProgram" ]) // { + description = "${llama-swap.meta.description} - UI"; + }; +})