From 93bfd533bf979c58ec90a3ffa3fc42b3dc5d301c Mon Sep 17 00:00:00 2001 From: Peder Bergebakken Sundt Date: Fri, 4 Oct 2024 08:30:58 +0200 Subject: [PATCH] refactor remote builders --- hosts/known-hosts.toml | 238 ++++++++++++++++++----------------- profiles/remote-builders.nix | 57 ++++----- 2 files changed, 151 insertions(+), 144 deletions(-) diff --git a/hosts/known-hosts.toml b/hosts/known-hosts.toml index ac6d34d..7fb94e1 100644 --- a/hosts/known-hosts.toml +++ b/hosts/known-hosts.toml @@ -2,11 +2,11 @@ #["host.name"] # https://search.nixos.org/options?query=nix.buildMachine -#systems -#maxJobs -#speedFactor -#supportedFeatures -#mandatoryFeatures +#buildMachine.systems +#buildMachine.maxJobs +#buildMachine.speedFactor +#buildMachine.supportedFeatures +#buildMachine.mandatoryFeatures #ssh.listenUser #ssh.listenPort #ssh.listenPublicKey # cat /etc/ssh/ssh_host_ed25519_key.pub || ssh-keyscan {{fqdn}} @@ -15,12 +15,18 @@ #ssh.proxyJump #ssh.userPrivateKey # IdentityFile to use +# buildMachine.supportedFeatures: +# - "kvm" - has hypervisor +# - "nixos-test" - the same as ^? nixos? +# - "benchmark" - has "equal" performance +# - "big-parallel" - is beefy, for stuff like llvm + [__default__] -systems = ["x86_64-linux"] -maxJobs = 0 # not a builder -speedFactor = 1 -supportedFeatures = [] -mandatoryFeatures = [] +buildMachine.systems = ["x86_64-linux"] +buildMachine.maxJobs = 0 # not a builder +buildMachine.speedFactor = 1 +buildMachine.supportedFeatures = [] +buildMachine.mandatoryFeatures = [] ssh.listenUser = "pbsds" # TODO: change # ssh.listenUser = "nixbld-remote" ssh.listenPort = 22 @@ -31,51 +37,51 @@ ssh.protocol = "ssh" # "ssh-ng" # graphical: one job ["bolle.pbsds.net"] -maxJobs = 3 # 12 threads 32GB -speedFactor = 4 # ??? -supportedFeatures = ["kvm","big-parallel","nixos-test"] +buildMachine.maxJobs = 3 # 12 threads 32GB +buildMachine.speedFactor = 4 # ??? +buildMachine.supportedFeatures = ["kvm", "big-parallel", "nixos-test"] ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILeOB/57N1fQPVorIUlkkJZaQduBo+4+km2Qbj4ebd/k" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["eple.pbsds.net"] # r9 290x -maxJobs = 3 # 12 threads 32GB -speedFactor = 3 # i7-5820K -supportedFeatures = ["kvm","big-parallel","nixos-test"] +buildMachine.maxJobs = 3 # 12 threads 32GB +buildMachine.speedFactor = 3 # i7-5820K +buildMachine.supportedFeatures = ["kvm", "big-parallel", "nixos-test"] ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH03MEINNnjBvtmvN2QsCDCLkvF9ow5FQJp9uiyQ1Iwi" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["garp.pbsds.net"] # gtx 1080 -maxJobs = 2 # 8 threads 32GB -speedFactor = 4 # i7-6700 -supportedFeatures = ["kvm","big-parallel","nixos-test","cuda"] +buildMachine.maxJobs = 2 # 8 threads 32GB +buildMachine.speedFactor = 4 # i7-6700 +buildMachine.supportedFeatures = ["kvm", "big-parallel", "nixos-test", "cuda"] ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOkcZ3cUAKk8uUvZPsX7PDBInkb3Eps3Xh+xVrhPY+sx" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["noximilien.pbsds.net"] -#maxJobs = 1 # 8 threads 8GB -speedFactor = 1 # i7-3770S +#buildMachine.maxJobs = 1 # 8 threads 8GB +buildMachine.speedFactor = 1 # i7-3770S ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ3QhTGS03Sqm6OeCEz5AIGqJnBttKaBqMgNXp3Md7t4" ssh.userPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC7fYndgIXJM+tLSfkbprWc8ClOI58wlaZCg6I+wMYINeOwxLU24BmIyQAhNeqhHYBdXiyIAl5KN3+YajN1nx6zq2XPXLut31Xtf+0yMdRMX4rXgqOnsBeG4eTfNsPx+v7VNANth8dIADpk59Y9ioWB6JI6NF0wfkqrCSTpt2q9gpTA35MBe41hlaxqxYGq+PlfZyJbN4TJCORZROkjw1P6K+EoYUHTHmduMZSAnpzx5bTHL2r1VK1jLRL4q2O1LP9G7eVYUsZKxKznJqtAeoOGBL4OX2JeIXT51/pXTW0NNyVPELD6aUUZjK8aVK2JDXupXegYO8cHqwLaz7rZj3G8evGamSlGvAYR4Gwvvp4Du8ZRZVM3Gt1allhPMTLnm/gy9Lta35D8SHH0IUKWD3buo5HZliZgSMAvoSrT03vpuGILLoWEkTjpPT0qKIlBd/qlACBzKC9Wwmda5WWgMsfe0zP4zNLVdves5nkMrbY91TYSFM0FuDCaRsK5Mrhx7i0= root@noximilien" ["sopp.pbsds.net"] # gtx 1080 -#maxJobs = 2 # 8 threads 32GB -speedFactor = 2 # i7-4790K -supportedFeatures = ["kvm","nixos-test","cuda"] +#buildMachine.maxJobs = 2 # 8 threads 32GB +buildMachine.speedFactor = 2 # i7-4790K +buildMachine.supportedFeatures = ["kvm", "nixos-test", "cuda"] ssh.listenPort = 26 ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDYB9H1pHB1vTBiGhO/GCQjn70BtVdQuJyXx38zN2CDj" ssh.userPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6eTQkxO/1XflHpGf3478+Z7HFYYaf1d4M6mvSK2nAU root@sopp" ["nord.pbsds.net"] # rx 580 -#maxJobs = 1 # 4 threads 32GB -speedFactor = 1 # i5-2500 -supportedFeatures = ["kvm","nixos-test"] +#buildMachine.maxJobs = 1 # 4 threads 32GB +buildMachine.speedFactor = 1 # i5-2500 +buildMachine.supportedFeatures = ["kvm", "nixos-test"] ssh.listenPort = 24 ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIBSdIUtUfAxnVbPDmDDFdP2S3Wd3+CC8IfZAANJ76oh" ssh.userPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINnS1TmV9q7n+s7+RouuB6vQllnhqNCE1RqPmTMJ2/29 root@nord" ["rocm.pbsds.net"] # gtx 3070 laptop edition -#maxJobs = 1 # 16 threads 32GB -speedFactor = 5 # i7-11800H +#buildMachine.maxJobs = 1 # 16 threads 32GB +buildMachine.speedFactor = 5 # i7-11800H ssh.listenUser = "pbsds" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGDuWdqEQ5mmVjuKi6f/Q2PFxuqB3URpgTHid06Vw7we" @@ -89,34 +95,34 @@ ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILD7+tE6rm742fz+nqa3+ ssh.userPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJVohqGMKp/UEZtb71RSBBXOEGX4o3lN5GYBlP7HEKbs root@brumlebasse" ["isvegg.pvv.ntnu.no"] -# maxJobs = 1 # 4 threads 16GB -speedFactor = 1 # i5-3570 +# buildMachine.maxJobs = 1 # 4 threads 16GB +buildMachine.speedFactor = 1 # i5-3570 ssh.listenUser = "pederbs" ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGurF7rdnrDP/VgIK2Tx38of+bX/QGCGL+alrWnZ1Ca5llGneMulUt1RB9xZzNLHiaWIE+HOP0i4spEaeZhilfU=" ["eirin.pvv.ntnu.no"] -maxJobs = 2 # 8 threads 16GB -speedFactor = 1 # i7-3770S +buildMachine.maxJobs = 2 # 8 threads 16GB +buildMachine.speedFactor = 1 # i7-3770S ssh.listenUser = "pederbs" ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBILGULKEzYe5kPorM0rWATv10qq6debfCuYUYqw3HWZm4Y5Pi7mVKcf8lKFNPc1DxT/dStfxxtHj/2fbezaxElk=" ["demiurgen.pvv.ntnu.no"] -maxJobs = 2 # 8 threads 16GB -speedFactor = 1 # i7-3770S +buildMachine.maxJobs = 2 # 8 threads 16GB +buildMachine.speedFactor = 1 # i7-3770S ssh.listenUser = "pederbs" ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKw92q3eB5HZbKJN3p+80MtirqcXPu01USE9LnoGYJuDvko1udjIy4UR0wAwELqgs+r7mJyuQPeXmOZKwjHP6tM=" ["orchid.pvv.ntnu.no"] # GTX 1060 -maxJobs = 2 # 12 threads 16GB -speedFactor = 2 # Ryzen 5 1600 -supportedFeatures = ["cuda"] +buildMachine.maxJobs = 2 # 12 threads 16GB +buildMachine.speedFactor = 2 # Ryzen 5 1600 +buildMachine.supportedFeatures = ["cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJXBrUdX0iFCkADXvqrcljEu0gxFOX+uOy6kqNx15WcNI+vt/YI7e/K4COf6LxuLsplLrrymcPdqsteAeWnPszs=" ["wegonke.pvv.ntnu.no"] # GTX 1080 -maxJobs = 1 # 4 threads 16GB -speedFactor = 2 # ??? -supportedFeatures = ["cuda"] +buildMachine.maxJobs = 1 # 4 threads 16GB +buildMachine.speedFactor = 2 # ??? +buildMachine.supportedFeatures = ["cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBswbjEdrTjExfZD0PwVbN5Wfu0UkPbl2M6rgreEPzStBwVdAjv979Y6TrZNLve/TYiIGRaX2OdAX6lt/aS/cLo=" @@ -130,206 +136,206 @@ ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIb #["heid.idi.ntnu.no"] -#maxJobs = 24 # 96 threads 1.4TB -#supportedFeatures = [ "big-parallel" ]; +#buildMachine.maxJobs = 24 # 96 threads 1.4TB +#buildMachine.supportedFeatures = [ "big-parallel" ]; #ssh.listenUser = "pederbs" #ssh.listenPublicKey = "TODO" #ssh.proxyJump = "isvegg.pvv.ntnu.no" #["bob.pvv.ntnu.no"] -#maxJobs = 10 # 40 threads +#buildMachine.maxJobs = 10 # 40 threads #ssh.listenUser = "pederbs" #ssh.listenPublicKey = "TODO" ["darwin-build-box.nix-community.org"] # https://nix-community.org/community-builder/ -systems = [ "aarch64-darwin", "x86_64-darwin" ] # macOS 14.6.1 -maxJobs = 1 # 8 cores 24GB - M2 -supportedFeatures = [ "big-parallel" ] +buildMachine.systems = [ "aarch64-darwin", "x86_64-darwin" ] # macOS 14.6.1 +buildMachine.maxJobs = 1 # 8 cores 24GB - M2 +buildMachine.supportedFeatures = [ "big-parallel" ] ssh.listenUser = "pbsds" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFz8FXSVEdf8FvDMfboxhB5VjSe7y2WgSa09q1L4t099" ssh.userPrivateKey = "/run/secrets/nix-community-builders-ssh-key" #["aarch64.nixos.community"] # https://github.com/NixOS/aarch64-build-box -#systems = [ "aarch64-linux" ] -# maxJobs = 1 # 64 threads? -#supportedFeatures = [ "big-parallel" ] +#buildMachine.systems = [ "aarch64-linux" ] +# buildMachine.maxJobs = 1 # 64 threads? +#buildMachine.supportedFeatures = [ "big-parallel" ] #ssh.listenUser = "TODO" #ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMUTz5i9u5H2FHNAmZJyoJfIGyUm/HfGhfwnc142L3ds" #ssh.userPrivateKey = "/run/secrets/nix-community-builders-ssh-key" ["clab01.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJfJV5Ov3D0qErVnbQZ3oxhA3i0zuAmjmVUf3JV08aSg" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab02.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHNhQPotOGWZdFeW4B3eDYGcaF/2xB56hNL+x3QEURa6" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab03.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH5srnYPuULchLvlCOlWOwrhQEBznQn61kj0Oawnp44Y" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab04.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICgMxLYYiYb/6IAH6nyc9eGXASgDPTE2JcRZ9ODjhQt5" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab05.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHw4h4dH689bLYWjrhhsvfljyWfUEClPa1Kb0cYxLRaD" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab06.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDVZ394P3124lSxkzVodFqbindIvCB3kcn4YcgbaPrAs" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab07.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILKQfcOmWC73bmE2mlWEcXFHiDUhsYWA7Xy9Dtq8kKmn" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab08.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKlyZq3uTBCgkvPgs6nWRzsdhHmXHph14dmYWgt1vuBx" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab09.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAj4eXT/k7iiUYx+CXq5ShLWm1N6SNO23EIs4xYEaQaW" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab10.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG5xEUkiwXWaUCA+QfMDq2vHfXKzcpXlrHpJMNQ8EU+K" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab11.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFlKZhdQBrjafzzwdRR3arem3TXnnPucQskd7RWW9L5V" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab12.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDEr2zGzev8JffE67Hkb3Qli7K0kzVdu8VXxJW47PK7m" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab13.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIZgIhgpMCdegJaW6Huad7Dj4YfyR8Zhi1UmDsgcJYK2" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab14.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL3pAYx5rtbaUCf4xsiy+7/qKqnGMnSa9KCp42j+XmEh" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab15.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP7XwwhLJGwK+a7zShr2Ok9f2GlvPkP+FxKdbGYsNHtd" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab16.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJKAaMXBAYsDd2QQOAQhXAAJCejbylQNLI9KsN3/EsY+" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab20.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEGBbcKU5uDTgaQoREjaNuzQkCKNm5wlnhln6ZNiL3o2" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab22.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMHlaYq184VDBoEOtaIIu2jnuBihhWiGPlyku0SMKORG" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab23.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAsIRgqoFF900olTCy6DSrFMpZyRmtK6aVP2oYQhNi8g" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab24.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICqh/Sp13OcUnZ8gVgiylcLsqAgIw+twQG92GyZK3FBZ" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab25.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOdL5coXj0geu9O1cMLdYuUE0TWlIkKLNj71/XF0e8eg" ssh.proxyJump = "isvegg.pvv.ntnu.no" ["clab26.idi.ntnu.no"] # gtx 4090 -#maxJobs = 1 # 24 threads 64GB -speedFactor = 5 # AMD Ryzen 9 7900X -supportedFeatures = ["cuda"] +#buildMachine.maxJobs = 1 # 24 threads 64GB +buildMachine.speedFactor = 5 # AMD Ryzen 9 7900X +buildMachine.supportedFeatures = ["big-parallel", "cuda"] ssh.listenUser = "pederbs" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINqqxg0hVT/gPBM1xqrR9QtMRHVBZDYWZ3pzbJv9MHUG" ssh.proxyJump = "isvegg.pvv.ntnu.no" diff --git a/profiles/remote-builders.nix b/profiles/remote-builders.nix index 7cac1de..8be2374 100644 --- a/profiles/remote-builders.nix +++ b/profiles/remote-builders.nix @@ -8,63 +8,67 @@ let inherit (builtins) map fromTOML readFile elem attrNames attrValues; inherit (lib) mkIf; - hosts' = fromTOML (readFile ../hosts/known-hosts.toml); # TODO: eww - hosts = lib.pipe hosts' [ + known-hosts' = fromTOML (readFile ../hosts/known-hosts.toml); # TODO: eww + known-hosts = lib.pipe known-hosts' [ (lib.filterAttrs (name: host: name != "__default__")) (lib.mapAttrs (name: host: - lib.recursiveUpdate (hosts'."__default__" or {}) host + lib.recursiveUpdate (known-hosts'."__default__" or {}) host )) ]; - hostNames = attrNames hosts; - thisHost = hosts.${config.networking.fqdn}; + hostNames = attrNames known-hosts; + thisHost = known-hosts.${config.networking.fqdn}; thisHostIsBuilder = thisHost.maxJobs > 0; - thisHostIsHopHost = builtins.elem config.networking.fqdn (lib.forEach (attrValues hosts) (host: host.ssh.proxyJump or null)); + thisHostIsHopHost = builtins.elem config.networking.fqdn (lib.forEach (attrValues known-hosts) (host: host.ssh.proxyJump or null)); thisHostIsConsumer = thisHost.ssh ? userPublicKey; mkRemoteConfig = fqdn: let - host = hosts.${fqdn}; - jump = hosts.${host.ssh.proxyJump}; - buildMachine = (lib.filterAttrs (key: _: !elem key ["ssh"]) host) // { + thatHost = known-hosts.${fqdn}; + thatJump = known-hosts.${thatHost.ssh.proxyJump}; + buildMachine = thatHost.buildMachine // { hostName = fqdn; - sshUser = host.ssh.listenUser; + sshUser = thatHost.ssh.listenUser; }; - isBuilder = host.maxJobs > 0; - isConsumer = host.ssh ? userPublicKey && thisHostIsBuilder; - isThis = fqdn == config.networking.fqdn; - in mkIf (!isThis) ( lib.mkMerge [ + thatHostIsBuilder = thatHost.buildMachine.maxJobs > 0; + thatHostIsConsumer = thatHost.ssh ? userPublicKey && thisHostIsBuilder; + thatHostIsThis = fqdn == config.networking.fqdn; + in mkIf (!thatHostIsThis) ( lib.mkMerge [ # out - (lib.mkIf (thisHostIsConsumer && isBuilder) { + (lib.mkIf (thisHostIsConsumer && thatHostIsBuilder) { nix.buildMachines = [ buildMachine ]; }) # out or jump - (lib.mkIf (thisHostIsConsumer && host.ssh ? listenPublicKey) { - programs.ssh.knownHosts.${fqdn}.publicKey = host.ssh.listenPublicKey; + (lib.mkIf (thisHostIsConsumer && thatHost.ssh ? listenPublicKey) { + programs.ssh.knownHosts.${fqdn}.publicKey = thatHost.ssh.listenPublicKey; # TODO: use nix.buildMachines.*.publicHostKey ? # timeouts are great when remote is unresponsive. nix doesn't care, lix is way and tests each remote only once programs.ssh.extraConfig = '' Host ${fqdn} ConnectTimeout 3 - Port ${builtins.toString host.ssh.listenPort} - ${lib.optionalString (host.ssh ? proxyJump) '' - ProxyJump ${jump.ssh.listenUser}@${host.ssh.proxyJump}:${builtins.toString jump.ssh.listenPort} + Port ${builtins.toString thatHost.ssh.listenPort} + ${lib.optionalString (thatHost.ssh ? proxyJump) '' + ProxyJump ${thatJump.ssh.listenUser}@${thatHost.ssh.proxyJump}:${builtins.toString thatJump.ssh.listenPort} ''} - ${lib.optionalString (host.ssh ? userPrivateKey) '' - IdentityFile ${host.ssh.userPrivateKey} + ${lib.optionalString (thatHost.ssh ? userPrivateKey) '' + IdentityFile ${thatHost.ssh.userPrivateKey} ''} ''; + + sops.secrets = lib.mkIf (lib.hasPrefix "/run/secrets/" (thatHost.ssh.userPrivateKey or "")) { + "${lib.removePrefix "/run/secrets/" thatHost.ssh.userPrivateKey}" = {}; + }; }) # in - (mkIf ((thisHostIsBuilder || thisHostIsHopHost) && isConsumer) { + (mkIf ((thisHostIsBuilder || thisHostIsHopHost) && thatHostIsConsumer) { users.users.${thisHost.ssh.listenUser} = { isSystemUser = lib.mkDefault (!config.users.users.${thisHost.ssh.listenUser}.isNormalUser); - openssh.authorizedKeys.keys = [ host.ssh.userPublicKey ]; + openssh.authorizedKeys.keys = [ thatHost.ssh.userPublicKey ]; group = lib.mkOptionDefault "nogroup"; }; }) - (mkIf (thisHostIsBuilder && isConsumer) { + (mkIf (thisHostIsBuilder && thatHostIsConsumer) { nix.settings.allowed-users = [ thisHost.ssh.listenUser ]; nix.settings.trusted-users = [ thisHost.ssh.listenUser ]; }) @@ -82,7 +86,4 @@ in { imports = lib.forEach hostNames mkRemoteConfig; - # TODO: derive this one from known-hosts.toml - sops.secrets.nix-community-builders-ssh-key = {}; - }