wip remote-builders rework
This commit is contained in:
parent
6bb8dc8567
commit
27e67cc192
|
@ -16,7 +16,7 @@
|
|||
|
||||
../../profiles/domeneshop-dyndns
|
||||
#../../profiles/code-remote
|
||||
#../../profiles/remote-builders
|
||||
../../profiles/remote-builders.nix
|
||||
#../../profiles/autossh-reverse-tunnels
|
||||
];
|
||||
services.domeneshop-updater.targets = [ config.networking.fqdn ];
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
../../profiles/shell.nix
|
||||
|
||||
#../../profiles/domeneshop-dyndns
|
||||
../../profiles/remote-builders.nix
|
||||
];
|
||||
#services.domeneshop-updater.targets = [ config.networking.fqdn ];
|
||||
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
../../profiles/domeneshop-dyndns
|
||||
#../../profiles/code-remote
|
||||
#../../profiles/remote-builders
|
||||
../../profiles/remote-builders.nix
|
||||
#../../profiles/autossh-reverse-tunnels
|
||||
];
|
||||
services.domeneshop-updater.targets = [ config.networking.fqdn ];
|
||||
|
|
|
@ -38,6 +38,7 @@
|
|||
#../../profiles/desktop/sound/pipewire.nix
|
||||
|
||||
../../profiles/domeneshop-dyndns
|
||||
../../profiles/remote-builders.nix
|
||||
];
|
||||
services.domeneshop-updater.targets = [ config.networking.fqdn ];
|
||||
|
||||
|
|
|
@ -0,0 +1,109 @@
|
|||
#primarily user for remote builders
|
||||
|
||||
#["host"]
|
||||
# https://search.nixos.org/options?query=nix.buildMachine
|
||||
#systems
|
||||
#maxJobs
|
||||
#speedFactor
|
||||
#supportedFeatures
|
||||
#mandatoryFeatures
|
||||
#ssh.user
|
||||
#ssh.port
|
||||
#ssh.protocol
|
||||
#ssh.proxyJump
|
||||
#ssh.publicKeyListen # cat /etc/ssh/ssh_host_ed25519_key.pub || ssh-keyscan {{fqdn}}
|
||||
#ssh.publicKeyUser # sudo ssh-keygen -t ed25519 && sudo cat /root/.ssh/id_ed25519.pub
|
||||
|
||||
[default]
|
||||
systems = ["x86_64-linux"]
|
||||
maxJobs = 0 # not a builder
|
||||
speedFactor = 1
|
||||
supportedFeatures = []
|
||||
mandatoryFeatures = []
|
||||
ssh.user = "nixbld-remote" # "pbsds"
|
||||
ssh.port = 22
|
||||
ssh.protocol = "ssh" # "ssh-ng"
|
||||
|
||||
["bjarte"]
|
||||
ssh.publicKeyUser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID7Ftu1LP+p+D6YWIo32V9w6ckHCIbrQWPyCNU4rBAbl root@bjarte"
|
||||
|
||||
# in general: one job per 4 threads and 8GB RAM
|
||||
|
||||
["bolle.pbsds.net"]
|
||||
maxJobs = 3 # 12 threads 32GB
|
||||
speedFactor = 5
|
||||
supportedFeatures = ["kvm","big-parallel","nixos-test"]
|
||||
ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILeOB/57N1fQPVorIUlkkJZaQduBo+4+km2Qbj4ebd/k"
|
||||
ssh.proxyJump = "microbel.pvv.ntnu.no"
|
||||
|
||||
["eple.pbsds.net"]
|
||||
maxJobs = 3 # 12 threads 32GB
|
||||
speedFactor = 5
|
||||
supportedFeatures = ["kvm","big-parallel","nixos-test"]
|
||||
ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIH03MEINNnjBvtmvN2QsCDCLkvF9ow5FQJp9uiyQ1Iwi"
|
||||
ssh.proxyJump = "microbel.pvv.ntnu.no"
|
||||
|
||||
["garp.pbsds.net"]
|
||||
maxJobs = 2 # 8 threads 32GB
|
||||
speedFactor = 4
|
||||
supportedFeatures = ["kvm","big-parallel","nixos-test"]
|
||||
ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOkcZ3cUAKk8uUvZPsX7PDBInkb3Eps3Xh+xVrhPY+sx"
|
||||
ssh.proxyJump = "microbel.pvv.ntnu.no"
|
||||
|
||||
["noximilien.pbsds.net"]
|
||||
#maxJobs = 1 # 8 threads 8GB
|
||||
speedFactor = 2
|
||||
ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ3QhTGS03Sqm6OeCEz5AIGqJnBttKaBqMgNXp3Md7t4"
|
||||
ssh.publicKeyUser = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC7fYndgIXJM+tLSfkbprWc8ClOI58wlaZCg6I+wMYINeOwxLU24BmIyQAhNeqhHYBdXiyIAl5KN3+YajN1nx6zq2XPXLut31Xtf+0yMdRMX4rXgqOnsBeG4eTfNsPx+v7VNANth8dIADpk59Y9ioWB6JI6NF0wfkqrCSTpt2q9gpTA35MBe41hlaxqxYGq+PlfZyJbN4TJCORZROkjw1P6K+EoYUHTHmduMZSAnpzx5bTHL2r1VK1jLRL4q2O1LP9G7eVYUsZKxKznJqtAeoOGBL4OX2JeIXT51/pXTW0NNyVPELD6aUUZjK8aVK2JDXupXegYO8cHqwLaz7rZj3G8evGamSlGvAYR4Gwvvp4Du8ZRZVM3Gt1allhPMTLnm/gy9Lta35D8SHH0IUKWD3buo5HZliZgSMAvoSrT03vpuGILLoWEkTjpPT0qKIlBd/qlACBzKC9Wwmda5WWgMsfe0zP4zNLVdves5nkMrbY91TYSFM0FuDCaRsK5Mrhx7i0= root@noximilien"
|
||||
|
||||
["sopp.pbsds.net"]
|
||||
#maxJobs = 4 # 8 threads 32GB
|
||||
speedFactor = 3
|
||||
supportedFeatures = ["kvm","big-parallel","nixos-test"]
|
||||
ssh.port = 26
|
||||
ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDYB9H1pHB1vTBiGhO/GCQjn70BtVdQuJyXx38zN2CDj"
|
||||
ssh.publicKeyUser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6eTQkxO/1XflHpGf3478+Z7HFYYaf1d4M6mvSK2nAU root@sopp"
|
||||
|
||||
["nord.pbsds.net"]
|
||||
maxJobs = 1 # 4 threads 32GB
|
||||
speedFactor = 3
|
||||
supportedFeatures = ["kvm","big-parallel","nixos-test"]
|
||||
ssh.port = 24
|
||||
ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIBSdIUtUfAxnVbPDmDDFdP2S3Wd3+CC8IfZAANJ76oh"
|
||||
ssh.publicKeyUser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINnS1TmV9q7n+s7+RouuB6vQllnhqNCE1RqPmTMJ2/29 root@nord"
|
||||
|
||||
["rocm.pbsds.net"]
|
||||
maxJobs = 4 # 16 threads 32GB
|
||||
speedFactor = 5
|
||||
supportedFeatures = ["kvm","big-parallel"]
|
||||
ssh.user = "pbsds"
|
||||
ssh.publicKeyListen = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGDuWdqEQ5mmVjuKi6f/Q2PFxuqB3URpgTHid06Vw7we"
|
||||
|
||||
["isvegg.pvv.ntnu.no"]
|
||||
maxJobs = 1 # 4 threads 16GB
|
||||
speedFactor = 2
|
||||
ssh.user = "pederbs"
|
||||
ssh.publicKeyListen = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGurF7rdnrDP/VgIK2Tx38of+bX/QGCGL+alrWnZ1Ca5llGneMulUt1RB9xZzNLHiaWIE+HOP0i4spEaeZhilfU="
|
||||
|
||||
["eirin.pvv.ntnu.no"]
|
||||
maxJobs = 2 # 8 threads 16GB
|
||||
speedFactor = 2
|
||||
ssh.user = "pederbs"
|
||||
ssh.publicKeyListen = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBILGULKEzYe5kPorM0rWATv10qq6debfCuYUYqw3HWZm4Y5Pi7mVKcf8lKFNPc1DxT/dStfxxtHj/2fbezaxElk="
|
||||
|
||||
["demiurgen.pvv.ntnu.no"]
|
||||
maxJobs = 2 # 8 threads 16GB
|
||||
speedFactor = 2
|
||||
ssh.user = "pederbs"
|
||||
ssh.publicKeyListen = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKw92q3eB5HZbKJN3p+80MtirqcXPu01USE9LnoGYJuDvko1udjIy4UR0wAwELqgs+r7mJyuQPeXmOZKwjHP6tM="
|
||||
|
||||
["hildring.pvv.ntnu.no"]
|
||||
ssh.user = "pederbs"
|
||||
ssh.publicKeyListen = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGurF7rdnrDP/VgIK2Tx38of+bX/QGCGL+alrWnZ1Ca5llGneMulUt1RB9xZzNLHiaWIE+HOP0i4spEaeZhilfU="
|
||||
|
||||
["microbel.pvv.ntnu.no"]
|
||||
ssh.user = "pederbs"
|
||||
ssh.publicKeyListen = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEq0yasKP0mH6PI6ypmuzPzMnbHELo9k+YB5yW534aKudKZS65YsHJKQ9vapOtmegrn5MQbCCgrshf+/XwZcjbM="
|
||||
|
||||
#["bob.pvv.ntnu.no"]
|
||||
#maxJobs = 10 # 40 threads
|
|
@ -35,7 +35,7 @@
|
|||
../../profiles/desktop/steam.nix
|
||||
../../profiles/desktop/flatpak.nix
|
||||
|
||||
../../profiles/remote-builders
|
||||
../../profiles/remote-builders.nix
|
||||
#../../profiles/autossh-reverse-tunnels
|
||||
#../../profiles/domeneshop-dyndns # handled by noximilien
|
||||
];
|
||||
|
|
|
@ -82,7 +82,7 @@
|
|||
|
||||
#../../profiles/code-remote # TODO: move into web? services?
|
||||
../../profiles/domeneshop-dyndns
|
||||
../../profiles/remote-builders
|
||||
../../profiles/remote-builders.nix
|
||||
../../profiles/autossh-reverse-tunnels
|
||||
#../../profiles/xrdp
|
||||
];
|
||||
|
|
|
@ -45,7 +45,7 @@
|
|||
../../profiles/desktop/lutris.nix
|
||||
../../profiles/desktop/flatpak.nix
|
||||
|
||||
../../profiles/remote-builders
|
||||
../../profiles/remote-builders.nix
|
||||
#../../profiles/autossh-reverse-tunnels
|
||||
#../../profiles/domeneshop-dyndns # handled by noximilien
|
||||
];
|
||||
|
|
|
@ -0,0 +1,77 @@
|
|||
{}
|
||||
/** /
|
||||
{ config, lib, ... }:
|
||||
|
||||
# TODO: make a remote-build user on nixos boxes, instead of giving access to pbsds
|
||||
# TODO: https://exozy.me/quickstart
|
||||
# TODO: https://github.com/winterqt/darwin-build-box
|
||||
|
||||
let
|
||||
inherit (builtins) map fromTOML readFile elem attrNames;
|
||||
inherit (lib) mkIf;
|
||||
|
||||
hosts' = fromTOML (readFile ../../hosts/known-hosts.toml); # eww
|
||||
hosts = lib.pipe hosts' [
|
||||
(lib.filterAttrs (name: host: name != "default"))
|
||||
(lib.mapAttrs (name: host:
|
||||
lib.recursiveUpdate (hosts'."default" or {}) host
|
||||
))
|
||||
];
|
||||
hostNames = attrNames hosts;
|
||||
thisHost = hosts.${config.networking.fqdn};
|
||||
thisHostIsBuilder = thisHost.maxJobs > 0;
|
||||
|
||||
mkRemoteConfig = fqdn: let
|
||||
host = hosts.${fqdn};
|
||||
jump = hosts.${host.ssh.proxyJump};
|
||||
buildMachine = (lib.filterAttrs (key: _: !elem key ["ssh"]) host) // {
|
||||
hostName = fqdn;
|
||||
sshUser = fqdn.ssh.user;
|
||||
};
|
||||
isBuilder = host.maxJobs > 0;
|
||||
isConsumer = host.ssh ? publicKeyUser && thisHostIsBuilder;
|
||||
isThis = fqdn == config.networking.fqdn;
|
||||
in mkIf (!isThis) {
|
||||
|
||||
# out
|
||||
nix.buildMachines = mkIf isBuilder [ buildMachine ];
|
||||
programs.ssh.knownHosts.${fqdn}.publicKey = mkIf isBuilder host.ssh.publicKeyListen;
|
||||
|
||||
# timeout is great when remote is unresponsive. nix doesn't care
|
||||
programs.ssh.extraConfig = ''
|
||||
Host ${fqdn}
|
||||
ConnectTimeout 3
|
||||
Port ${builtins.toString (host.ssh.port or 22)}
|
||||
${lib.optionalString (host.ssh ? proxyJump) ''
|
||||
ProxyJump ${host.ssh.proxyJump}
|
||||
''}
|
||||
'';
|
||||
|
||||
# in
|
||||
users = mkIf isConsumer {
|
||||
users.${thisHost.ssh.user} = {
|
||||
isSystemUser = lib.mkDefault (!config.users.users.${thisHost.ssh.user}.isNormalUser);
|
||||
openssh.authorizedKeys.keys = [
|
||||
host.ssh.publicKeyUser
|
||||
];
|
||||
group = lib.mkDefault "nogroup";
|
||||
};
|
||||
};
|
||||
nix.settings.allowed-users = mkIf isConsumer [ thisHost.ssh.user ];
|
||||
nix.settings.trusted-users = mkIf isConsumer [ thisHost.ssh.user ];
|
||||
};
|
||||
|
||||
in {
|
||||
|
||||
nix.distributedBuilds = true;
|
||||
|
||||
# TODO: Allow setting speedFactor for local builds, as local is currently fixed to 0
|
||||
# https://github.com/NixOS/nix/issues/2457
|
||||
|
||||
# useful when the builder has a faster internet connection than i do
|
||||
nix.settings.builders-use-substitutes = true;
|
||||
|
||||
imports = lib.forEach hostNames mkRemoteConfig;
|
||||
|
||||
}
|
||||
/**/
|
Loading…
Reference in New Issue