lets test it, remote-builders

hosts/bolle/default.nix

@@ -18,8 +18,8 @@
     ../../profiles/shell.nix
 
     ../../profiles/domeneshop-dyndns
-    #../../profiles/code-remote
     ../../profiles/remote-builders.nix
+    #../../profiles/code-remote
     #../../profiles/autossh-reverse-tunnels
   ];
   services.domeneshop-updater.targets = [ config.networking.fqdn ];

hosts/brumlebasse/default.nix

@@ -19,7 +19,7 @@
     ../../profiles/shell.nix
 
     #../../profiles/domeneshop-dyndns
-    ../../profiles/remote-builders.nix
+    /* ../../profiles/remote-builders.nix */
   ];
   #services.domeneshop-updater.targets = [ config.networking.fqdn ];
 

hosts/known-hosts.toml

@@ -1,4 +1,4 @@
-#primarily user for remote builders
+# primarily used for remote builders, but also configures known public keys
 
 #["host.name"]
 # https://search.nixos.org/options?query=nix.buildMachine
@@ -14,19 +14,17 @@
 #ssh.protocol
 #ssh.proxyJump
 
-[default]
+[__default__]
 systems = ["x86_64-linux"]
 maxJobs = 0 # not a builder
 speedFactor = 1
 supportedFeatures = []
 mandatoryFeatures = []
-ssh.listenUser = "nixbld-remote" # "pbsds"
+ssh.listenUser = "pbsds" # TODO: change
+# ssh.listenUser = "nixbld-remote"
 ssh.listenPort = 22
 ssh.protocol = "ssh" # "ssh-ng"
 
-["bjarte.pbsds.net"]
-ssh.publicKeyUser = "TODO"
-
 # in general:
 #   headless:  one job per 4 threads and 8GB RAM
 #   graphical: one job
@@ -56,33 +54,35 @@ ssh.proxyJump = "isvegg.pvv.ntnu.no"
 #maxJobs = 1 # 8 threads 8GB
 speedFactor = 2
 ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ3QhTGS03Sqm6OeCEz5AIGqJnBttKaBqMgNXp3Md7t4"
-ssh.publicKeyUser = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC7fYndgIXJM+tLSfkbprWc8ClOI58wlaZCg6I+wMYINeOwxLU24BmIyQAhNeqhHYBdXiyIAl5KN3+YajN1nx6zq2XPXLut31Xtf+0yMdRMX4rXgqOnsBeG4eTfNsPx+v7VNANth8dIADpk59Y9ioWB6JI6NF0wfkqrCSTpt2q9gpTA35MBe41hlaxqxYGq+PlfZyJbN4TJCORZROkjw1P6K+EoYUHTHmduMZSAnpzx5bTHL2r1VK1jLRL4q2O1LP9G7eVYUsZKxKznJqtAeoOGBL4OX2JeIXT51/pXTW0NNyVPELD6aUUZjK8aVK2JDXupXegYO8cHqwLaz7rZj3G8evGamSlGvAYR4Gwvvp4Du8ZRZVM3Gt1allhPMTLnm/gy9Lta35D8SHH0IUKWD3buo5HZliZgSMAvoSrT03vpuGILLoWEkTjpPT0qKIlBd/qlACBzKC9Wwmda5WWgMsfe0zP4zNLVdves5nkMrbY91TYSFM0FuDCaRsK5Mrhx7i0= root@noximilien"
+ssh.userPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC7fYndgIXJM+tLSfkbprWc8ClOI58wlaZCg6I+wMYINeOwxLU24BmIyQAhNeqhHYBdXiyIAl5KN3+YajN1nx6zq2XPXLut31Xtf+0yMdRMX4rXgqOnsBeG4eTfNsPx+v7VNANth8dIADpk59Y9ioWB6JI6NF0wfkqrCSTpt2q9gpTA35MBe41hlaxqxYGq+PlfZyJbN4TJCORZROkjw1P6K+EoYUHTHmduMZSAnpzx5bTHL2r1VK1jLRL4q2O1LP9G7eVYUsZKxKznJqtAeoOGBL4OX2JeIXT51/pXTW0NNyVPELD6aUUZjK8aVK2JDXupXegYO8cHqwLaz7rZj3G8evGamSlGvAYR4Gwvvp4Du8ZRZVM3Gt1allhPMTLnm/gy9Lta35D8SHH0IUKWD3buo5HZliZgSMAvoSrT03vpuGILLoWEkTjpPT0qKIlBd/qlACBzKC9Wwmda5WWgMsfe0zP4zNLVdves5nkMrbY91TYSFM0FuDCaRsK5Mrhx7i0= root@noximilien"
 
 ["sopp.pbsds.net"]
-#maxJobs = 1 # 8 threads 32GB
+#maxJobs = 2 # 8 threads 32GB
 speedFactor = 3
-supportedFeatures = ["kvm","big-parallel","nixos-test"]
+supportedFeatures = ["kvm","nixos-test"]
 ssh.listenPort = 26
 ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDYB9H1pHB1vTBiGhO/GCQjn70BtVdQuJyXx38zN2CDj"
-ssh.publicKeyUser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6eTQkxO/1XflHpGf3478+Z7HFYYaf1d4M6mvSK2nAU root@sopp"
+ssh.userPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6eTQkxO/1XflHpGf3478+Z7HFYYaf1d4M6mvSK2nAU root@sopp"
 
 ["nord.pbsds.net"]
-maxJobs = 1 # 4 threads 32GB
-speedFactor = 3
-supportedFeatures = ["kvm","big-parallel","nixos-test"]
+#maxJobs = 1 # 4 threads 32GB
+speedFactor = 2
+supportedFeatures = ["kvm","nixos-test"]
 ssh.listenPort = 24
 ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIBSdIUtUfAxnVbPDmDDFdP2S3Wd3+CC8IfZAANJ76oh"
-ssh.publicKeyUser = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINnS1TmV9q7n+s7+RouuB6vQllnhqNCE1RqPmTMJ2/29 root@nord"
+ssh.userPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINnS1TmV9q7n+s7+RouuB6vQllnhqNCE1RqPmTMJ2/29 root@nord"
 
 ["rocm.pbsds.net"]
-maxJobs = 1 # 16 threads 32GB
+#maxJobs = 1 # 16 threads 32GB
 speedFactor = 5
-supportedFeatures = ["kvm","big-parallel"]
 ssh.listenUser = "pbsds"
 ssh.listenPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGDuWdqEQ5mmVjuKi6f/Q2PFxuqB3URpgTHid06Vw7we"
 
+["bjarte.pbsds.net"]
+ssh.userPublicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF4v1+FbiEa6Mohpf3/Una5ahKeKSG9yZ9iU5TC7ddL5 root@bjarte"
+
 ["isvegg.pvv.ntnu.no"]
-maxJobs = 1 # 4 threads 16GB
+# maxJobs = 1 # 4 threads 16GB
 speedFactor = 2
 ssh.listenUser = "pederbs"
 ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGurF7rdnrDP/VgIK2Tx38of+bX/QGCGL+alrWnZ1Ca5llGneMulUt1RB9xZzNLHiaWIE+HOP0i4spEaeZhilfU="
@@ -107,11 +107,18 @@ ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIb
 ssh.listenUser = "pederbs"
 ssh.listenPublicKey = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEq0yasKP0mH6PI6ypmuzPzMnbHELo9k+YB5yW534aKudKZS65YsHJKQ9vapOtmegrn5MQbCCgrshf+/XwZcjbM="
 
-["bob.pvv.ntnu.no"]
-#maxJobs = 10 # 40 threads
-ssh.listenUser = "pederbs"
-#ssh.listenPublicKey = ""
 
+#["heid.idi.ntnu.no"]
+#maxJobs = 24 # 96 threads 1.4TB
+#supportedFeatures = [ "big-parallel" ];
+#ssh.listenUser = "pederbs"
+#ssh.listenPublicKey = "TODO"
+#ssh.proxyJump = "isvegg.pvv.ntnu.no"
+
+#["bob.pvv.ntnu.no"]
+#maxJobs = 10 # 40 threads
+#ssh.listenUser = "pederbs"
+#ssh.listenPublicKey = "TODO"
 
 #["darwin-build-box.winter.cafe"]
 #systems = [ "aarch64-darwin", "x86_64-darwin" ];

hosts/nord/default.nix

@@ -37,7 +37,7 @@
     ../../profiles/desktop/steam.nix
     ../../profiles/desktop/flatpak.nix
 
-    ../../profiles/remote-builders.nix
+    /* ../../profiles/remote-builders.nix */
     #../../profiles/autossh-reverse-tunnels
     #../../profiles/domeneshop-dyndns # handled by noximilien
   ];

hosts/noximilien/default.nix

@@ -89,7 +89,7 @@
 
     #../../profiles/code-remote # TODO: move into web? services?
     ../../profiles/domeneshop-dyndns
-    ../../profiles/remote-builders.nix
+    /* ../../profiles/remote-builders.nix */
     ../../profiles/autossh-reverse-tunnels
     #../../profiles/xrdp
   ];

hosts/sopp/default.nix

@@ -47,7 +47,7 @@
     ../../profiles/desktop/lutris.nix
     ../../profiles/desktop/flatpak.nix
 
-    ../../profiles/remote-builders.nix
+    /* ../../profiles/remote-builders.nix */
     #../../profiles/autossh-reverse-tunnels
     #../../profiles/domeneshop-dyndns # handled by noximilien
   ];

profiles/remote-builders.nix

@@ -1,5 +1,3 @@
-{}
-/** /
 { config, lib, ... }:
 
 # TODO: make a remote-build user on nixos boxes, instead of giving access to pbsds
@@ -10,16 +8,17 @@ let
   inherit (builtins) map fromTOML readFile elem attrNames;
   inherit (lib) mkIf;
 
-  hosts' = fromTOML (readFile ../../hosts/known-hosts.toml); # eww
+  hosts' = fromTOML (readFile ../hosts/known-hosts.toml); # TODO: eww
   hosts = lib.pipe hosts' [
-    (lib.filterAttrs (name: host: name != "default"))
+    (lib.filterAttrs (name: host: name != "__default__"))
     (lib.mapAttrs (name: host:
-      lib.recursiveUpdate (hosts'."default" or {}) host
+      lib.recursiveUpdate (hosts'."__default__" or {}) host
     ))
   ];
   hostNames = attrNames hosts;
   thisHost = hosts.${config.networking.fqdn};
   thisHostIsBuilder = thisHost.maxJobs > 0;
+  thisHostIsHopHost = builtins.elem config.networking.fqdn (lib.forEach hosts (host: host.ssh.proxyJump or null));
 
   mkRemoteConfig = fqdn: let
     host = hosts.${fqdn};
@@ -29,38 +28,43 @@ let
       sshUser = host.ssh.listenUser;
     };
     isBuilder = host.maxJobs > 0;
-    isConsumer = host.ssh ? publicKeyUser && thisHostIsBuilder;
+    isConsumer = host.ssh ? userPublicKey && thisHostIsBuilder;
     isThis = fqdn == config.networking.fqdn;
-  in mkIf (!isThis) {
-
+  in mkIf (!isThis) ( lib.mkMerge [
     # out
-    nix.buildMachines = mkIf isBuilder [ buildMachine ];
-    programs.ssh.knownHosts.${fqdn}.publicKey = mkIf isBuilder host.ssh.listenPublicKey;
+    (lib.mkIf isBuilder {
 
-    # timeout is great when remote is unresponsive. nix doesn't care
+      nix.buildMachines = [ buildMachine ];
+
+    })
+    # out or jump
+    (lib.mkIf (host.ssh ? listenPublicKey) {
+      programs.ssh.knownHosts.${fqdn}.publicKey = host.ssh.listenPublicKey;
+
+      # timeouts are great when remote is unresponsive. nix doesn't care
       programs.ssh.extraConfig = ''
         Host ${fqdn}
           ConnectTimeout 3
-        Port ${builtins.toString (host.ssh.listenPort or 22)}
+          Port ${builtins.toString host.ssh.listenPort}
           ${lib.optionalString (host.ssh ? proxyJump) ''
             ProxyJump ${host.ssh.proxyJump}
           ''}
       '';
-
+    })
     # in
-    users = mkIf isConsumer {
-      users.${thisHost.ssh.listenUser} = {
+    (mkIf (isConsumer && (thisHostIsBuilder || thisHostIsHopHost) ) {
+
+      nix.settings.allowed-users = [ thisHost.ssh.listenUser ];
+      nix.settings.trusted-users = [ thisHost.ssh.listenUser ];
+      users.users.${thisHost.ssh.listenUser} = {
         isSystemUser = lib.mkDefault (!config.users.users.${thisHost.ssh.listenUser}.isNormalUser);
-        openssh.authorizedKeys.keys = [
-          host.ssh.userPublicKey
-        ];
-        group = lib.mkDefault "nogroup";
-      };
-    };
-    nix.settings.allowed-users = mkIf isConsumer [ thisHost.ssh.listenUser ];
-    nix.settings.trusted-users = mkIf isConsumer [ thisHost.ssh.listenUser ];
+        openssh.authorizedKeys.keys = [ host.ssh.userPublicKey ];
+        group = lib.mkOptionDefault "nogroup";
       };
 
+    })
+  ]);
+
 in {
 
   nix.distributedBuilds = true;
@@ -74,4 +78,3 @@ in {
   imports = lib.forEach hostNames mkRemoteConfig;
 
 }
-/**/