From 9e0e69c80dcd1379ef8d4ca5a911afb132ada094 Mon Sep 17 00:00:00 2001 From: Adrian Gunnar Lauterer Date: Fri, 10 Jan 2025 23:42:29 +0100 Subject: [PATCH 1/4] readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index f67f871..be3affb 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,6 @@ My nix dotfiles. Will not guarrante it to work as it is always a work in progres nix --extra-experimental-features "nix-command flakes" build ".#nixosConfigurations.galadriel.config.system.build.toplevel" nixos-rebuild switch --update-input nixpkgs --update-input unstable --no-write-lock-file --refresh --flake git+https://github.com/adrlau/nix-dotfiles.git --upgrade + + +nix why-depends .\#nixosConfigurations.eowyn.config.system.build.toplevel pkgs.python3.12-libarcus-4.12.0 --impure From f7f492adcca16d17e7d3caed3c2ac66f7520aabd Mon Sep 17 00:00:00 2001 From: Adrian Gunnar Lauterer Date: Thu, 16 Jan 2025 11:16:21 +0100 Subject: [PATCH 2/4] openwebui fixes --- services/ollama-webui.nix | 27 --------------------------- services/ollamaWebui.nix | 2 ++ 2 files changed, 2 insertions(+), 27 deletions(-) delete mode 100644 services/ollama-webui.nix diff --git a/services/ollama-webui.nix b/services/ollama-webui.nix deleted file mode 100644 index 6d91f02..0000000 --- a/services/ollama-webui.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ config, pkgs, lib, ... }: -let - openWebuiImage = "ghcr.io/open-webui/open-webui:main"; -in -{ - virtualisation.oci-containers = { - backend = { - image = openWebuiImage; - cmd = [ "-d" "--network=host" "-v" "open-webui:/app/backend/data" "--name" "open-webui" "--restart" "always" ]; - volumes = [ "open-webui:/app/backend/data" ]; - environment = { - OLLAMA_BASE_URL = "http://127.0.0.1:11434"; - }; - restart = "always"; - }; - }; - - services.nginx.virtualHosts."chat.${config.networking.hostName}.${config.networking.domain}" = { - forceSSL = true; - #useACMEHost = config.networking.domain; #not sure if this will work, unless - locations."/" = { - proxyWebsockets = true; - proxyPass = "http://${config.services.ollama.listenAddress}"; - }; - basicAuthFile = config.sops.secrets."nginx/defaultpass".path; - }; -} \ No newline at end of file diff --git a/services/ollamaWebui.nix b/services/ollamaWebui.nix index 92571bb..1edd898 100644 --- a/services/ollamaWebui.nix +++ b/services/ollamaWebui.nix @@ -2,6 +2,8 @@ { environment.systemPackages = [ pkgs.unstable.open-webui + pkgs.gvisor + ]; From 8ca7bb1890597cd5ee339cc2f8170a4bd00042fa Mon Sep 17 00:00:00 2001 From: Adrian Gunnar Lauterer Date: Wed, 22 Jan 2025 12:25:26 +0100 Subject: [PATCH 3/4] update ollama webui --- services/ollamaWebui.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/services/ollamaWebui.nix b/services/ollamaWebui.nix index 1edd898..25c672f 100644 --- a/services/ollamaWebui.nix +++ b/services/ollamaWebui.nix @@ -6,7 +6,11 @@ ]; + services.tika.enable=true; + services.tika.openFirewall=true; + services.tika.listenAddress = "localhost"; + services.open-webui = { enable = true; @@ -15,5 +19,12 @@ host = "0.0.0.0"; openFirewall = true; + enviroment = { + ANONYMIZED_TELEMETRY = "False"; + DO_NOT_TRACK = "True"; + SCARF_NO_ANALYTICS = "True"; + PDF_EXTRACT_IMAGES = "False"; + }; + }; } From e4cbc21843bdca537e41d06f9e22acd9f351f11b Mon Sep 17 00:00:00 2001 From: Adrian Gunnar Lauterer Date: Wed, 29 Jan 2025 12:51:19 +0100 Subject: [PATCH 4/4] ai --- machines/galadriel/nvidia.nix | 24 +++++++++++++++++--- services/ollama.nix | 41 +++++++++++++++++++++++++++-------- services/ollamaWebui.nix | 13 ++++++----- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/machines/galadriel/nvidia.nix b/machines/galadriel/nvidia.nix index eb7ca91..7f8dcd4 100644 --- a/machines/galadriel/nvidia.nix +++ b/machines/galadriel/nvidia.nix @@ -5,6 +5,11 @@ enable = true; enable32Bit = true; }; + + nixpkgs.config = { + allowUnfree = true; + cudaSupport = true; + }; # Load nvidia driver for Xorg and Wayland services.xserver.videoDrivers = ["nvidia"]; @@ -18,7 +23,7 @@ # Use the NVidia open source kernel module (not to be confused with the independent third-party "nouveau" open source driver). # Currently alpha-quality/buggy, so false is currently the recommended setting. - open = false; + open = false; #need proprietary for cuda. # Enable the Nvidia settings menu, accessible via `nvidia-settings`. #nvidiaSettings = true; @@ -29,11 +34,24 @@ # Enable the CUDA toolkit #install packages - environment.systemPackages = with pkgs; [ - cudaPackages.cudnn + environment.systemPackages = with pkgs; [ cudaPackages.cudatoolkit + cudaPackages.cudnn nvtopPackages.nvidia + gcc + cudaPackages.nccl + cmake + #llama-cpp + #python3Packages.pip + #cudaPackages.cuda_cudart + #xgboostWithCuda + #libxcrypt-legacy + #cudaPackages.setupCudaHook + #cudaPackages.markForCudatoolkitRootHook + #cudaPackages.cuda_cudart.static + pkgs.cudaPackages.libcublas #cudaPackages.tensorrt_8_6_0 #needs to be added manually, to the store and is a pain because of the license agreement and garbage collection + ]; nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ diff --git a/services/ollama.nix b/services/ollama.nix index 62a0fe8..96ae8b2 100644 --- a/services/ollama.nix +++ b/services/ollama.nix @@ -4,21 +4,44 @@ pkgs.unstable.ollama ]; - services.ollama.enable = true; - services.ollama.package = pkgs.unstable.ollama; - services.ollama.host = "0.0.0.0"; - services.ollama.port = 11434; - services.ollama.models = "/var/lib/ollama/models"; - services.ollama.home = "/var/lib/ollama"; - - #possibly a flawed idea. + services.ollama = { + enable = true; + package = pkgs.unstable.ollama; + host = "0.0.0.0"; + openFirewall = true; + port = 11434; + home = "/var/lib/ollama"; + + loadModels = [ + "llama3.2" + "gemma2:2b" + "qwen2.5:3b" + + "llama3.2-vision" + "llava-phi3" + "llava-llama3" + "moondream" + "minicpm-v" + + "llama3.1" + "mistral-nemo" + "phi4" + + "zylonai/multilingual-e5-large" + "nomic-embed-text" + "snowflake-arctic-embed" + + ]; + }; + + #possibly a flawed idea, should just set cudaSupport and rocm support. services.ollama.acceleration = lib.mkDefault ( let hostname = config.networking.hostName; in if hostname == "galadriel" then "cuda" else if hostname == "aragorn" then "rocm" else null); - + services.nginx.virtualHosts."ollama.${config.networking.hostName}.${config.networking.domain}" = { forceSSL = true; #useACMEHost = config.networking.domain; #not sure if this will work, unless diff --git a/services/ollamaWebui.nix b/services/ollamaWebui.nix index 25c672f..ddf0064 100644 --- a/services/ollamaWebui.nix +++ b/services/ollamaWebui.nix @@ -3,13 +3,16 @@ environment.systemPackages = [ pkgs.unstable.open-webui pkgs.gvisor + pkgs.bash ]; - services.tika.enable=true; - services.tika.openFirewall=true; - services.tika.listenAddress = "localhost"; - + services.tika = { + enable=true; + openFirewall=true; + listenAddress = "localhost"; + enableOcr = true; + }; services.open-webui = { enable = true; @@ -19,7 +22,7 @@ host = "0.0.0.0"; openFirewall = true; - enviroment = { + environment = { ANONYMIZED_TELEMETRY = "False"; DO_NOT_TRACK = "True"; SCARF_NO_ANALYTICS = "True";