From 27e67cc1929e7a05856f17b1782188c8b7a24b58 Mon Sep 17 00:00:00 2001 From: Peder Bergebakken Sundt Date: Tue, 16 Apr 2024 06:10:04 +0200 Subject: [PATCH] wip remote-builders rework --- hosts/bolle/default.nix | 2 +- hosts/brumlebasse/default.nix | 1 + hosts/eple/default.nix | 2 +- hosts/garp/default.nix | 1 + hosts/known-hosts.toml | 109 ++++++++++++++++++++++++++++++++++ hosts/nord/default.nix | 2 +- hosts/noximilien/default.nix | 2 +- hosts/sopp/default.nix | 2 +- profiles/remote-builders.nix | 77 ++++++++++++++++++++++++ 9 files changed, 193 insertions(+), 5 deletions(-) create mode 100644 hosts/known-hosts.toml create mode 100644 profiles/remote-builders.nix diff --git a/hosts/bolle/default.nix b/hosts/bolle/default.nix index 350d187..ab03cc4 100644 --- a/hosts/bolle/default.nix +++ b/hosts/bolle/default.nix @@ -16,7 +16,7 @@ ../../profiles/domeneshop-dyndns #../../profiles/code-remote - #../../profiles/remote-builders + ../../profiles/remote-builders.nix #../../profiles/autossh-reverse-tunnels ]; services.domeneshop-updater.targets = [ config.networking.fqdn ]; diff --git a/hosts/brumlebasse/default.nix b/hosts/brumlebasse/default.nix index c8fb051..6502cae 100644 --- a/hosts/brumlebasse/default.nix +++ b/hosts/brumlebasse/default.nix @@ -19,6 +19,7 @@ ../../profiles/shell.nix #../../profiles/domeneshop-dyndns + ../../profiles/remote-builders.nix ]; #services.domeneshop-updater.targets = [ config.networking.fqdn ]; diff --git a/hosts/eple/default.nix b/hosts/eple/default.nix index 93acebd..77802bb 100644 --- a/hosts/eple/default.nix +++ b/hosts/eple/default.nix @@ -17,7 +17,7 @@ ../../profiles/domeneshop-dyndns #../../profiles/code-remote - #../../profiles/remote-builders + ../../profiles/remote-builders.nix #../../profiles/autossh-reverse-tunnels ]; services.domeneshop-updater.targets = [ config.networking.fqdn ]; diff --git a/hosts/garp/default.nix b/hosts/garp/default.nix index 3024d06..e447f4f 100644 --- a/hosts/garp/default.nix +++ b/hosts/garp/default.nix @@ -38,6 +38,7 @@ #../../profiles/desktop/sound/pipewire.nix ../../profiles/domeneshop-dyndns + ../../profiles/remote-builders.nix ]; services.domeneshop-updater.targets = [ config.networking.fqdn ]; diff --git a/hosts/known-hosts.toml b/hosts/known-hosts.toml new file mode 100644 index 0000000..022ca51 --- /dev/null +++ b/hosts/known-hosts.toml @@ -0,0 +1,109 @@ +#primarily user for remote builders + +#["host"] +# https://search.nixos.org/options?query=nix.buildMachine +#systems +#maxJobs +#speedFactor +#supportedFeatures +#mandatoryFeatures +#ssh.user +#ssh.port +#ssh.protocol +#ssh.proxyJump +#ssh.publicKeyListen # cat /etc/ssh/ssh_host_ed25519_key.pub || ssh-keyscan {{fqdn}} +#ssh.publicKeyUser # sudo ssh-keygen -t ed25519 && sudo cat /root/.ssh/id_ed25519.pub + +[default] +systems = ["x86_64-linux"] +maxJobs = 0 # not a builder +speedFactor = 1 +supportedFeatures = [] +mandatoryFeatures = [] +ssh.user = "nixbld-remote" # "pbsds" +ssh.port = 22 +ssh.protocol = "ssh" # "ssh-ng" + +["bjarte"] +ssh.publicKeyUser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID7Ftu1LP+p+D6YWIo32V9w6ckHCIbrQWPyCNU4rBAbl root@bjarte" + +# in general: one job per 4 threads and 8GB RAM + +["bolle.pbsds.net"] +maxJobs = 3 # 12 threads 32GB +speedFactor = 5 +supportedFeatures = ["kvm","big-parallel","nixos-test"] +ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILeOB/57N1fQPVorIUlkkJZaQduBo+4+km2Qbj4ebd/k" +ssh.proxyJump = "microbel.pvv.ntnu.no" + +["eple.pbsds.net"] +maxJobs = 3 # 12 threads 32GB +speedFactor = 5 +supportedFeatures = ["kvm","big-parallel","nixos-test"] +ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH03MEINNnjBvtmvN2QsCDCLkvF9ow5FQJp9uiyQ1Iwi" +ssh.proxyJump = "microbel.pvv.ntnu.no" + +["garp.pbsds.net"] +maxJobs = 2 # 8 threads 32GB +speedFactor = 4 +supportedFeatures = ["kvm","big-parallel","nixos-test"] +ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOkcZ3cUAKk8uUvZPsX7PDBInkb3Eps3Xh+xVrhPY+sx" +ssh.proxyJump = "microbel.pvv.ntnu.no" + +["noximilien.pbsds.net"] +#maxJobs = 1 # 8 threads 8GB +speedFactor = 2 +ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ3QhTGS03Sqm6OeCEz5AIGqJnBttKaBqMgNXp3Md7t4" +ssh.publicKeyUser = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC7fYndgIXJM+tLSfkbprWc8ClOI58wlaZCg6I+wMYINeOwxLU24BmIyQAhNeqhHYBdXiyIAl5KN3+YajN1nx6zq2XPXLut31Xtf+0yMdRMX4rXgqOnsBeG4eTfNsPx+v7VNANth8dIADpk59Y9ioWB6JI6NF0wfkqrCSTpt2q9gpTA35MBe41hlaxqxYGq+PlfZyJbN4TJCORZROkjw1P6K+EoYUHTHmduMZSAnpzx5bTHL2r1VK1jLRL4q2O1LP9G7eVYUsZKxKznJqtAeoOGBL4OX2JeIXT51/pXTW0NNyVPELD6aUUZjK8aVK2JDXupXegYO8cHqwLaz7rZj3G8evGamSlGvAYR4Gwvvp4Du8ZRZVM3Gt1allhPMTLnm/gy9Lta35D8SHH0IUKWD3buo5HZliZgSMAvoSrT03vpuGILLoWEkTjpPT0qKIlBd/qlACBzKC9Wwmda5WWgMsfe0zP4zNLVdves5nkMrbY91TYSFM0FuDCaRsK5Mrhx7i0= root@noximilien" + +["sopp.pbsds.net"] +#maxJobs = 4 # 8 threads 32GB +speedFactor = 3 +supportedFeatures = ["kvm","big-parallel","nixos-test"] +ssh.port = 26 +ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDYB9H1pHB1vTBiGhO/GCQjn70BtVdQuJyXx38zN2CDj" +ssh.publicKeyUser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6eTQkxO/1XflHpGf3478+Z7HFYYaf1d4M6mvSK2nAU root@sopp" + +["nord.pbsds.net"] +maxJobs = 1 # 4 threads 32GB +speedFactor = 3 +supportedFeatures = ["kvm","big-parallel","nixos-test"] +ssh.port = 24 +ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIBSdIUtUfAxnVbPDmDDFdP2S3Wd3+CC8IfZAANJ76oh" +ssh.publicKeyUser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINnS1TmV9q7n+s7+RouuB6vQllnhqNCE1RqPmTMJ2/29 root@nord" + +["rocm.pbsds.net"] +maxJobs = 4 # 16 threads 32GB +speedFactor = 5 +supportedFeatures = ["kvm","big-parallel"] +ssh.user = "pbsds" +ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGDuWdqEQ5mmVjuKi6f/Q2PFxuqB3URpgTHid06Vw7we" + +["isvegg.pvv.ntnu.no"] +maxJobs = 1 # 4 threads 16GB +speedFactor = 2 +ssh.user = "pederbs" +ssh.publicKeyListen = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGurF7rdnrDP/VgIK2Tx38of+bX/QGCGL+alrWnZ1Ca5llGneMulUt1RB9xZzNLHiaWIE+HOP0i4spEaeZhilfU=" + +["eirin.pvv.ntnu.no"] +maxJobs = 2 # 8 threads 16GB +speedFactor = 2 +ssh.user = "pederbs" +ssh.publicKeyListen = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBILGULKEzYe5kPorM0rWATv10qq6debfCuYUYqw3HWZm4Y5Pi7mVKcf8lKFNPc1DxT/dStfxxtHj/2fbezaxElk=" + +["demiurgen.pvv.ntnu.no"] +maxJobs = 2 # 8 threads 16GB +speedFactor = 2 +ssh.user = "pederbs" +ssh.publicKeyListen = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKw92q3eB5HZbKJN3p+80MtirqcXPu01USE9LnoGYJuDvko1udjIy4UR0wAwELqgs+r7mJyuQPeXmOZKwjHP6tM=" + +["hildring.pvv.ntnu.no"] +ssh.user = "pederbs" +ssh.publicKeyListen = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGurF7rdnrDP/VgIK2Tx38of+bX/QGCGL+alrWnZ1Ca5llGneMulUt1RB9xZzNLHiaWIE+HOP0i4spEaeZhilfU=" + +["microbel.pvv.ntnu.no"] +ssh.user = "pederbs" +ssh.publicKeyListen = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEq0yasKP0mH6PI6ypmuzPzMnbHELo9k+YB5yW534aKudKZS65YsHJKQ9vapOtmegrn5MQbCCgrshf+/XwZcjbM=" + +#["bob.pvv.ntnu.no"] +#maxJobs = 10 # 40 threads diff --git a/hosts/nord/default.nix b/hosts/nord/default.nix index e114bf1..d6e3dcc 100644 --- a/hosts/nord/default.nix +++ b/hosts/nord/default.nix @@ -35,7 +35,7 @@ ../../profiles/desktop/steam.nix ../../profiles/desktop/flatpak.nix - ../../profiles/remote-builders + ../../profiles/remote-builders.nix #../../profiles/autossh-reverse-tunnels #../../profiles/domeneshop-dyndns # handled by noximilien ]; diff --git a/hosts/noximilien/default.nix b/hosts/noximilien/default.nix index 92a46d4..b01c9a0 100644 --- a/hosts/noximilien/default.nix +++ b/hosts/noximilien/default.nix @@ -82,7 +82,7 @@ #../../profiles/code-remote # TODO: move into web? services? ../../profiles/domeneshop-dyndns - ../../profiles/remote-builders + ../../profiles/remote-builders.nix ../../profiles/autossh-reverse-tunnels #../../profiles/xrdp ]; diff --git a/hosts/sopp/default.nix b/hosts/sopp/default.nix index 90ea571..d89299f 100644 --- a/hosts/sopp/default.nix +++ b/hosts/sopp/default.nix @@ -45,7 +45,7 @@ ../../profiles/desktop/lutris.nix ../../profiles/desktop/flatpak.nix - ../../profiles/remote-builders + ../../profiles/remote-builders.nix #../../profiles/autossh-reverse-tunnels #../../profiles/domeneshop-dyndns # handled by noximilien ]; diff --git a/profiles/remote-builders.nix b/profiles/remote-builders.nix new file mode 100644 index 0000000..edc1a11 --- /dev/null +++ b/profiles/remote-builders.nix @@ -0,0 +1,77 @@ +{} +/** / +{ config, lib, ... }: + +# TODO: make a remote-build user on nixos boxes, instead of giving access to pbsds +# TODO: https://exozy.me/quickstart +# TODO: https://github.com/winterqt/darwin-build-box + +let + inherit (builtins) map fromTOML readFile elem attrNames; + inherit (lib) mkIf; + + hosts' = fromTOML (readFile ../../hosts/known-hosts.toml); # eww + hosts = lib.pipe hosts' [ + (lib.filterAttrs (name: host: name != "default")) + (lib.mapAttrs (name: host: + lib.recursiveUpdate (hosts'."default" or {}) host + )) + ]; + hostNames = attrNames hosts; + thisHost = hosts.${config.networking.fqdn}; + thisHostIsBuilder = thisHost.maxJobs > 0; + + mkRemoteConfig = fqdn: let + host = hosts.${fqdn}; + jump = hosts.${host.ssh.proxyJump}; + buildMachine = (lib.filterAttrs (key: _: !elem key ["ssh"]) host) // { + hostName = fqdn; + sshUser = fqdn.ssh.user; + }; + isBuilder = host.maxJobs > 0; + isConsumer = host.ssh ? publicKeyUser && thisHostIsBuilder; + isThis = fqdn == config.networking.fqdn; + in mkIf (!isThis) { + + # out + nix.buildMachines = mkIf isBuilder [ buildMachine ]; + programs.ssh.knownHosts.${fqdn}.publicKey = mkIf isBuilder host.ssh.publicKeyListen; + + # timeout is great when remote is unresponsive. nix doesn't care + programs.ssh.extraConfig = '' + Host ${fqdn} + ConnectTimeout 3 + Port ${builtins.toString (host.ssh.port or 22)} + ${lib.optionalString (host.ssh ? proxyJump) '' + ProxyJump ${host.ssh.proxyJump} + ''} + ''; + + # in + users = mkIf isConsumer { + users.${thisHost.ssh.user} = { + isSystemUser = lib.mkDefault (!config.users.users.${thisHost.ssh.user}.isNormalUser); + openssh.authorizedKeys.keys = [ + host.ssh.publicKeyUser + ]; + group = lib.mkDefault "nogroup"; + }; + }; + nix.settings.allowed-users = mkIf isConsumer [ thisHost.ssh.user ]; + nix.settings.trusted-users = mkIf isConsumer [ thisHost.ssh.user ]; + }; + +in { + + nix.distributedBuilds = true; + + # TODO: Allow setting speedFactor for local builds, as local is currently fixed to 0 + # https://github.com/NixOS/nix/issues/2457 + + # useful when the builder has a faster internet connection than i do + nix.settings.builders-use-substitutes = true; + + imports = lib.forEach hostNames mkRemoteConfig; + +} +/**/