pederbs/config
pederbs
/
config
2
1
Fork 0
Code Issues Pull Requests Releases Activity

lets test it, remote-builders

This commit is contained in:
Peder Bergebakken Sundt 2024-07-10 00:34:38 +02:00
parent 8b67ff20b2
commit c4d9d0df85
7 changed files with 66 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hosts/bolle/default.nix
View File

@ -18,8 +18,8 @@
../../profiles/shell.nix ../../profiles/shell.nix
../../profiles/domeneshop-dyndns ../../profiles/domeneshop-dyndns
#../../profiles/code-remote
../../profiles/remote-builders.nix ../../profiles/remote-builders.nix
#../../profiles/code-remote
#../../profiles/autossh-reverse-tunnels #../../profiles/autossh-reverse-tunnels
]; ];
services.domeneshop-updater.targets = [ config.networking.fqdn ]; services.domeneshop-updater.targets = [ config.networking.fqdn ];

2
hosts/brumlebasse/default.nix
View File

@ -19,7 +19,7 @@
../../profiles/shell.nix ../../profiles/shell.nix
#../../profiles/domeneshop-dyndns #../../profiles/domeneshop-dyndns
../../profiles/remote-builders.nix /* ../../profiles/remote-builders.nix */
]; ];
#services.domeneshop-updater.targets = [ config.networking.fqdn ]; #services.domeneshop-updater.targets = [ config.networking.fqdn ];

49
hosts/known-hosts.toml
View File

@ -1,4 +1,4 @@
#primarily user for remote builders # primarily used for remote builders, but also configures known public keys
#["host.name"] #["host.name"]
# https://search.nixos.org/options?query=nix.buildMachine # https://search.nixos.org/options?query=nix.buildMachine
@ -14,19 +14,17 @@
#ssh.protocol #ssh.protocol
#ssh.proxyJump #ssh.proxyJump
[default] [__default__]
systems = ["x86_64-linux"] systems = ["x86_64-linux"]
maxJobs = 0 # not a builder maxJobs = 0 # not a builder
speedFactor = 1 speedFactor = 1
supportedFeatures = [] supportedFeatures = []
mandatoryFeatures = [] mandatoryFeatures = []
ssh.listenUser = "nixbld-remote" # "pbsds" ssh.listenUser = "pbsds" # TODO: change
# ssh.listenUser = "nixbld-remote"
ssh.listenPort = 22 ssh.listenPort = 22
ssh.protocol = "ssh" # "ssh-ng" ssh.protocol = "ssh" # "ssh-ng"
["bjarte.pbsds.net"]
ssh.publicKeyUser = "TODO"
# in general: # in general:
# headless: one job per 4 threads and 8GB RAM # headless: one job per 4 threads and 8GB RAM
# graphical: one job # graphical: one job
@ -56,33 +54,35 @@ ssh.proxyJump = "isvegg.pvv.ntnu.no"
#maxJobs = 1 # 8 threads 8GB #maxJobs = 1 # 8 threads 8GB
speedFactor = 2 speedFactor = 2
ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ3QhTGS03Sqm6OeCEz5AIGqJnBttKaBqMgNXp3Md7t4" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ3QhTGS03Sqm6OeCEz5AIGqJnBttKaBqMgNXp3Md7t4"
ssh.publicKeyUser = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC7fYndgIXJM+tLSfkbprWc8ClOI58wlaZCg6I+wMYINeOwxLU24BmIyQAhNeqhHYBdXiyIAl5KN3+YajN1nx6zq2XPXLut31Xtf+0yMdRMX4rXgqOnsBeG4eTfNsPx+v7VNANth8dIADpk59Y9ioWB6JI6NF0wfkqrCSTpt2q9gpTA35MBe41hlaxqxYGq+PlfZyJbN4TJCORZROkjw1P6K+EoYUHTHmduMZSAnpzx5bTHL2r1VK1jLRL4q2O1LP9G7eVYUsZKxKznJqtAeoOGBL4OX2JeIXT51/pXTW0NNyVPELD6aUUZjK8aVK2JDXupXegYO8cHqwLaz7rZj3G8evGamSlGvAYR4Gwvvp4Du8ZRZVM3Gt1allhPMTLnm/gy9Lta35D8SHH0IUKWD3buo5HZliZgSMAvoSrT03vpuGILLoWEkTjpPT0qKIlBd/qlACBzKC9Wwmda5WWgMsfe0zP4zNLVdves5nkMrbY91TYSFM0FuDCaRsK5Mrhx7i0= root@noximilien" ssh.userPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC7fYndgIXJM+tLSfkbprWc8ClOI58wlaZCg6I+wMYINeOwxLU24BmIyQAhNeqhHYBdXiyIAl5KN3+YajN1nx6zq2XPXLut31Xtf+0yMdRMX4rXgqOnsBeG4eTfNsPx+v7VNANth8dIADpk59Y9ioWB6JI6NF0wfkqrCSTpt2q9gpTA35MBe41hlaxqxYGq+PlfZyJbN4TJCORZROkjw1P6K+EoYUHTHmduMZSAnpzx5bTHL2r1VK1jLRL4q2O1LP9G7eVYUsZKxKznJqtAeoOGBL4OX2JeIXT51/pXTW0NNyVPELD6aUUZjK8aVK2JDXupXegYO8cHqwLaz7rZj3G8evGamSlGvAYR4Gwvvp4Du8ZRZVM3Gt1allhPMTLnm/gy9Lta35D8SHH0IUKWD3buo5HZliZgSMAvoSrT03vpuGILLoWEkTjpPT0qKIlBd/qlACBzKC9Wwmda5WWgMsfe0zP4zNLVdves5nkMrbY91TYSFM0FuDCaRsK5Mrhx7i0= root@noximilien"
["sopp.pbsds.net"] ["sopp.pbsds.net"]
#maxJobs = 1 # 8 threads 32GB #maxJobs = 2 # 8 threads 32GB
speedFactor = 3 speedFactor = 3
supportedFeatures = ["kvm","big-parallel","nixos-test"] supportedFeatures = ["kvm","nixos-test"]
ssh.listenPort = 26 ssh.listenPort = 26
ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDYB9H1pHB1vTBiGhO/GCQjn70BtVdQuJyXx38zN2CDj" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDYB9H1pHB1vTBiGhO/GCQjn70BtVdQuJyXx38zN2CDj"
ssh.publicKeyUser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6eTQkxO/1XflHpGf3478+Z7HFYYaf1d4M6mvSK2nAU root@sopp" ssh.userPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6eTQkxO/1XflHpGf3478+Z7HFYYaf1d4M6mvSK2nAU root@sopp"
["nord.pbsds.net"] ["nord.pbsds.net"]
maxJobs = 1 # 4 threads 32GB #maxJobs = 1 # 4 threads 32GB
speedFactor = 3 speedFactor = 2
supportedFeatures = ["kvm","big-parallel","nixos-test"] supportedFeatures = ["kvm","nixos-test"]
ssh.listenPort = 24 ssh.listenPort = 24
ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIBSdIUtUfAxnVbPDmDDFdP2S3Wd3+CC8IfZAANJ76oh" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIBSdIUtUfAxnVbPDmDDFdP2S3Wd3+CC8IfZAANJ76oh"
ssh.publicKeyUser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINnS1TmV9q7n+s7+RouuB6vQllnhqNCE1RqPmTMJ2/29 root@nord" ssh.userPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINnS1TmV9q7n+s7+RouuB6vQllnhqNCE1RqPmTMJ2/29 root@nord"
["rocm.pbsds.net"] ["rocm.pbsds.net"]
maxJobs = 1 # 16 threads 32GB #maxJobs = 1 # 16 threads 32GB
speedFactor = 5 speedFactor = 5
supportedFeatures = ["kvm","big-parallel"]
ssh.listenUser = "pbsds" ssh.listenUser = "pbsds"
ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGDuWdqEQ5mmVjuKi6f/Q2PFxuqB3URpgTHid06Vw7we" ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGDuWdqEQ5mmVjuKi6f/Q2PFxuqB3URpgTHid06Vw7we"
["bjarte.pbsds.net"]
ssh.userPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF4v1+FbiEa6Mohpf3/Una5ahKeKSG9yZ9iU5TC7ddL5 root@bjarte"
["isvegg.pvv.ntnu.no"] ["isvegg.pvv.ntnu.no"]
maxJobs = 1 # 4 threads 16GB # maxJobs = 1 # 4 threads 16GB
speedFactor = 2 speedFactor = 2
ssh.listenUser = "pederbs" ssh.listenUser = "pederbs"
ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGurF7rdnrDP/VgIK2Tx38of+bX/QGCGL+alrWnZ1Ca5llGneMulUt1RB9xZzNLHiaWIE+HOP0i4spEaeZhilfU=" ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGurF7rdnrDP/VgIK2Tx38of+bX/QGCGL+alrWnZ1Ca5llGneMulUt1RB9xZzNLHiaWIE+HOP0i4spEaeZhilfU="
@ -107,11 +107,18 @@ ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIb
ssh.listenUser = "pederbs" ssh.listenUser = "pederbs"
ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEq0yasKP0mH6PI6ypmuzPzMnbHELo9k+YB5yW534aKudKZS65YsHJKQ9vapOtmegrn5MQbCCgrshf+/XwZcjbM=" ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEq0yasKP0mH6PI6ypmuzPzMnbHELo9k+YB5yW534aKudKZS65YsHJKQ9vapOtmegrn5MQbCCgrshf+/XwZcjbM="
["bob.pvv.ntnu.no"]
#maxJobs = 10 # 40 threads
ssh.listenUser = "pederbs"
#ssh.listenPublicKey = ""
#["heid.idi.ntnu.no"]
#maxJobs = 24 # 96 threads 1.4TB
#supportedFeatures = [ "big-parallel" ];
#ssh.listenUser = "pederbs"
#ssh.listenPublicKey = "TODO"
#ssh.proxyJump = "isvegg.pvv.ntnu.no"
#["bob.pvv.ntnu.no"]
#maxJobs = 10 # 40 threads
#ssh.listenUser = "pederbs"
#ssh.listenPublicKey = "TODO"
#["darwin-build-box.winter.cafe"] #["darwin-build-box.winter.cafe"]
#systems = [ "aarch64-darwin", "x86_64-darwin" ]; #systems = [ "aarch64-darwin", "x86_64-darwin" ];

2
hosts/nord/default.nix
View File

@ -37,7 +37,7 @@
../../profiles/desktop/steam.nix ../../profiles/desktop/steam.nix
../../profiles/desktop/flatpak.nix ../../profiles/desktop/flatpak.nix
../../profiles/remote-builders.nix /* ../../profiles/remote-builders.nix */
#../../profiles/autossh-reverse-tunnels #../../profiles/autossh-reverse-tunnels
#../../profiles/domeneshop-dyndns # handled by noximilien #../../profiles/domeneshop-dyndns # handled by noximilien
]; ];

2
hosts/noximilien/default.nix
View File

@ -89,7 +89,7 @@
#../../profiles/code-remote # TODO: move into web? services? #../../profiles/code-remote # TODO: move into web? services?
../../profiles/domeneshop-dyndns ../../profiles/domeneshop-dyndns
../../profiles/remote-builders.nix /* ../../profiles/remote-builders.nix */
../../profiles/autossh-reverse-tunnels ../../profiles/autossh-reverse-tunnels
#../../profiles/xrdp #../../profiles/xrdp
]; ];

2
hosts/sopp/default.nix
View File

@ -47,7 +47,7 @@
../../profiles/desktop/lutris.nix ../../profiles/desktop/lutris.nix
../../profiles/desktop/flatpak.nix ../../profiles/desktop/flatpak.nix
../../profiles/remote-builders.nix /* ../../profiles/remote-builders.nix */
#../../profiles/autossh-reverse-tunnels #../../profiles/autossh-reverse-tunnels
#../../profiles/domeneshop-dyndns # handled by noximilien #../../profiles/domeneshop-dyndns # handled by noximilien
]; ];

51
profiles/remote-builders.nix
View File

@ -1,5 +1,3 @@
{}
/** /
{ config, lib, ... }: { config, lib, ... }:
# TODO: make a remote-build user on nixos boxes, instead of giving access to pbsds # TODO: make a remote-build user on nixos boxes, instead of giving access to pbsds
@ -10,16 +8,17 @@ let
inherit (builtins) map fromTOML readFile elem attrNames; inherit (builtins) map fromTOML readFile elem attrNames;
inherit (lib) mkIf; inherit (lib) mkIf;
hosts' = fromTOML (readFile ../../hosts/known-hosts.toml); # eww hosts' = fromTOML (readFile ../hosts/known-hosts.toml); # TODO: eww
hosts = lib.pipe hosts' [ hosts = lib.pipe hosts' [
(lib.filterAttrs (name: host: name != "default")) (lib.filterAttrs (name: host: name != "__default__"))
(lib.mapAttrs (name: host: (lib.mapAttrs (name: host:
lib.recursiveUpdate (hosts'."default" or {}) host lib.recursiveUpdate (hosts'."__default__" or {}) host
)) ))
]; ];
hostNames = attrNames hosts; hostNames = attrNames hosts;
thisHost = hosts.${config.networking.fqdn}; thisHost = hosts.${config.networking.fqdn};
thisHostIsBuilder = thisHost.maxJobs > 0; thisHostIsBuilder = thisHost.maxJobs > 0;
thisHostIsHopHost = builtins.elem config.networking.fqdn (lib.forEach hosts (host: host.ssh.proxyJump or null));
mkRemoteConfig = fqdn: let mkRemoteConfig = fqdn: let
host = hosts.${fqdn}; host = hosts.${fqdn};
@ -29,38 +28,43 @@ let
sshUser = host.ssh.listenUser; sshUser = host.ssh.listenUser;
}; };
isBuilder = host.maxJobs > 0; isBuilder = host.maxJobs > 0;
isConsumer = host.ssh ? publicKeyUser && thisHostIsBuilder; isConsumer = host.ssh ? userPublicKey && thisHostIsBuilder;
isThis = fqdn == config.networking.fqdn; isThis = fqdn == config.networking.fqdn;
in mkIf (!isThis) { in mkIf (!isThis) ( lib.mkMerge [
# out # out
nix.buildMachines = mkIf isBuilder [ buildMachine ]; (lib.mkIf isBuilder {
programs.ssh.knownHosts.${fqdn}.publicKey = mkIf isBuilder host.ssh.listenPublicKey;
# timeout is great when remote is unresponsive. nix doesn't care nix.buildMachines = [ buildMachine ];
})
# out or jump
(lib.mkIf (host.ssh ? listenPublicKey) {
programs.ssh.knownHosts.${fqdn}.publicKey = host.ssh.listenPublicKey;
# timeouts are great when remote is unresponsive. nix doesn't care
programs.ssh.extraConfig = '' programs.ssh.extraConfig = ''
Host ${fqdn} Host ${fqdn}
ConnectTimeout 3 ConnectTimeout 3
Port ${builtins.toString (host.ssh.listenPort or 22)} Port ${builtins.toString host.ssh.listenPort}
${lib.optionalString (host.ssh ? proxyJump) '' ${lib.optionalString (host.ssh ? proxyJump) ''
ProxyJump ${host.ssh.proxyJump} ProxyJump ${host.ssh.proxyJump}
''} ''}
''; '';
})
# in # in
users = mkIf isConsumer { (mkIf (isConsumer && (thisHostIsBuilder || thisHostIsHopHost) ) {
users.${thisHost.ssh.listenUser} = {
nix.settings.allowed-users = [ thisHost.ssh.listenUser ];
nix.settings.trusted-users = [ thisHost.ssh.listenUser ];
users.users.${thisHost.ssh.listenUser} = {
isSystemUser = lib.mkDefault (!config.users.users.${thisHost.ssh.listenUser}.isNormalUser); isSystemUser = lib.mkDefault (!config.users.users.${thisHost.ssh.listenUser}.isNormalUser);
openssh.authorizedKeys.keys = [ openssh.authorizedKeys.keys = [ host.ssh.userPublicKey ];
host.ssh.userPublicKey group = lib.mkOptionDefault "nogroup";
];
group = lib.mkDefault "nogroup";
};
};
nix.settings.allowed-users = mkIf isConsumer [ thisHost.ssh.listenUser ];
nix.settings.trusted-users = mkIf isConsumer [ thisHost.ssh.listenUser ];
}; };
})
]);
in { in {
nix.distributedBuilds = true; nix.distributedBuilds = true;
@ -74,4 +78,3 @@ in {
imports = lib.forEach hostNames mkRemoteConfig; imports = lib.forEach hostNames mkRemoteConfig;
} }
/**/