From da815061d139cd0fe2668901c75af5012b622a17 Mon Sep 17 00:00:00 2001 From: Peder Bergebakken Sundt Date: Wed, 31 Jan 2024 22:17:55 +0100 Subject: [PATCH] wip: nspawn --- flake.lock | 17 ++++++++ flake.nix | 36 ++++++++++------ hosts/brumlebasse/default.nix | 36 +++++----------- hosts/brumlebasse/hardware-configuration.nix | 38 ----------------- pkgs/mk-nspawn-deployer/default.nix | 29 +++++++++++++ pkgs/mk-nspawn-deployer/setup-nspawn.sh | 44 ++++++++++++++++++++ users/pbsds/home/profiles/ssh.nix | 2 + 7 files changed, 127 insertions(+), 75 deletions(-) delete mode 100644 hosts/brumlebasse/hardware-configuration.nix create mode 100644 pkgs/mk-nspawn-deployer/default.nix create mode 100644 pkgs/mk-nspawn-deployer/setup-nspawn.sh diff --git a/flake.lock b/flake.lock index 0f8689d..debf28a 100644 --- a/flake.lock +++ b/flake.lock @@ -135,6 +135,22 @@ "type": "github" } }, + "nixos-nspawn": { + "flake": false, + "locked": { + "lastModified": 1705399691, + "narHash": "sha256-NEasvnjAi1pqkFtVbKiHL+HjkgMa72yqdZQQskxb3lg=", + "owner": "tfc", + "repo": "nspawn-nixos", + "rev": "7fe5a42c6f9116402a68abc81410a59d18fd48c2", + "type": "github" + }, + "original": { + "owner": "tfc", + "repo": "nspawn-nixos", + "type": "github" + } + }, "nixpkgs-1909": { "flake": false, "locked": { @@ -352,6 +368,7 @@ "home-manager-edge": "home-manager-edge", "nixos-generators-2311": "nixos-generators-2311", "nixos-hardware": "nixos-hardware", + "nixos-nspawn": "nixos-nspawn", "nixpkgs-1909": "nixpkgs-1909", "nixpkgs-2003": "nixpkgs-2003", "nixpkgs-2009": "nixpkgs-2009", diff --git a/flake.nix b/flake.nix index 067e3c8..111877a 100644 --- a/flake.nix +++ b/flake.nix @@ -26,6 +26,10 @@ # https://github.com/NixOS/nixos-hardware nixos-hardware.url = "github:NixOS/nixos-hardware"; + # https://github.com/NixOS/nixos-hardware + nixos-nspawn.url = "github:tfc/nspawn-nixos"; + nixos-nspawn.flake = false; # we don't use it /shrug + # https://github.com/wamserma/flake-programs-sqlite flake-programs-sqlite-2311.url = "github:wamserma/flake-programs-sqlite"; flake-programs-sqlite-2311.inputs.nixpkgs.follows = "nixpkgs-2311"; @@ -74,6 +78,7 @@ outputs = { self, nixos-hardware, + nixos-nspawn, nixos-generators-2311, ... } @ inputs': @@ -129,7 +134,7 @@ #"riscv64-linux" ]; - mkModule = domain: system: inputs: stateVersion: modules: hostname: ({ lib, ... }: { + mkModule = extra-modules: domain: system: inputs: stateVersion: modules: hostname: ({ lib, ... }: { system.stateVersion = lib.mkDefault stateVersion; # TODO: home-manager imports = let ifExists = p: if builtins.pathExists p then p else {}; in [ @@ -137,7 +142,7 @@ (ifExists "${self}/hosts/${hostname}") inputs.sops-nix.nixosModules.sops inputs.home-manager.nixosModule - ] ++ modules; + ] ++ modules ++ extra-modules; #++ inputs.flake-programs-sqlite.nixosModules.programs-sqlite; # TODO: make work sops.defaultSopsFile = lib.mkIf (builtins.pathExists ./secrets/${hostname}.yaml) @@ -183,16 +188,16 @@ "nixpkgs-git=github:NixOS/nixpkgs/nixos-unstable-small" ]; }); - mkConfig = domain: system: inputs: stateVersion: modules: hostname: inputs.nixpkgs.lib.nixosSystem { + mkConfig = extra-modules: domain: system: inputs: stateVersion: modules: hostname: inputs.nixpkgs.lib.nixosSystem { inherit system; specialArgs = { inherit inputs; flakes = mkFlakeView inputs system; }; - modules = [ (mkModule domain system inputs stateVersion modules hostname) ]; + modules = [ (mkModule extra-modules domain system inputs stateVersion modules hostname) ]; }; - mkReport = domain: system: inputs: stateVersion: modules: hostname: let - nixos = mkConfig domain system inputs stateVersion modules hostname; + mkReport = extra-modules: domain: system: inputs: stateVersion: modules: hostname: let + nixos = mkConfig extra-modules domain system inputs stateVersion modules hostname; cfg = nixos.config; inherit (nixos.pkgs) lib; in { @@ -221,12 +226,13 @@ cuda = ls [ ./hardware/gpu/cuda.nix hw.common-gpu-nvidia-nonprime ]; cuda-prime = ls [ ./hardware/gpu/cuda.nix hw.common-gpu-nvidia ]; rocm = ls [ ./hardware/gpu/rocm.nix hw.common-gpu-amd ]; + nspawn = ls [ "${nixos-nspawn}/nspawn-image.nix" { boot.isContainer = true; } ]; hidpi = hw.common-hidpi; p1005 = ./hardware/printer/hp-laserjet-p1005.nix; in builtins.mapAttrs (hostname: curried: curried hostname) { #hostname "domain" "system" inputs "state" [ modules ... ] noximilien = mk "pbsds.net" "x86_64-linux" inputs-2311 "22.11" [ intel ]; - brumlebasse = mk "pbsds.net" "x86_64-linux" inputs-2311 "23.11" [ amd ]; + brumlebasse = mk "pbsds.net" "x86_64-linux" inputs-2311 "23.11" [ amd nspawn ]; nord = mk "pbsds.net" "x86_64-linux" inputs-2311 "23.11" [ intel-novga hw.common-cpu-intel-sandy-bridge rocm hidpi ]; sopp = mk "pbsds.net" "x86_64-linux" inputs-2311 "23.11" [ intel cuda p1005 ]; bolle = mk "pbsds.net" "x86_64-linux" inputs-2311 "23.11" [ intel ]; @@ -247,21 +253,27 @@ flakes-2305 = mkFlakeView inputs-2305 system; }); - nixosModules = mkHosts mkModule; - nixosConfigurations = mkHosts mkConfig; - nixosReports = mkHosts mkReport; + nixosModules = mkHosts (mkModule []); + nixosConfigurations = mkHosts (mkConfig []); + nixosReports = mkHosts (mkReport []); - packages = forAllSystems ({ inputs, pkgs, lib, flakes, ... }: { + packages = forAllSystems ({ inputs, pkgs, lib, flakes, ... }: let + mk-nspawn-deployer = hostname: + (pkgs.callPackage ./pkgs/mk-nspawn-deployer {}) + (mkHosts (mkConfig [ "${nixos-nspawn}/nspawn-tarball.nix" ])).${hostname}; + in { # TODO: get faketty to work ${expect}/bin/unbuffer is bad nixos-rebuild-nom = pkgs.writeScriptBin "nixos-rebuild" '' exec ${lib.getExe pkgs.nixos-rebuild} "$@" |& ${lib.getExe pkgs.nix-output-monitor} ''; + nspawn-setup-brumlebasse = mk-nspawn-deployer "brumlebasse"; + # nixos-generators images image-brumlebasse-openstack = nixos-generators-2311.nixosGenerate { system = "x86_64-linux"; specialArgs = { inherit inputs flakes; }; - modules = [ (mkHosts mkModule).brumlebasse ]; + modules = [ (mkHosts (mkModule [])).brumlebasse ]; format = "openstack"; }; }); diff --git a/hosts/brumlebasse/default.nix b/hosts/brumlebasse/default.nix index a715dc0..d5996b7 100644 --- a/hosts/brumlebasse/default.nix +++ b/hosts/brumlebasse/default.nix @@ -1,13 +1,14 @@ { config, pkgs, lib, ... }: { # Bootloader + #N/A - # gated on formatAttr which is set by nixos-generators - boot = lib.mkIf ((config.formatAttr or null) == null) { - loader.systemd-boot.enable = true; - loader.efi.canTouchEfiVariables = true; - loader.efi.efiSysMountPoint = "/boot/efi"; - }; + ## gated on formatAttr which is set by nixos-generators + #boot = lib.mkIf ((config.formatAttr or null) == null) { + # loader.systemd-boot.enable = true; + # loader.efi.canTouchEfiVariables = true; + # loader.efi.efiSysMountPoint = "/boot/efi"; + #}; imports = [ #./hardware-configuration.nix @@ -20,33 +21,18 @@ ../../profiles/shell/base.nix ../../profiles/shell/archives.nix ../../profiles/shell/nix-utils.nix - #../../profiles/shell/binfmt-emu.nix # qemu won't compile... #../../profiles/domeneshop-dyndns ]; #services.domeneshop-updater.targets = [ config.networking.fqdn ]; + services.openssh.enable = true; + services.openssh.ports = lib.mkForce [ 2222 ]; + networking.firewall.allowedTCPPorts = [ 2222 ]; + # Networking networking.networkmanager.enable = true; - /** / - #networking.wireless.enable = true; # Enables wireless support via wpa_supplicant. - #networking.iwd.enable = true - networking.interfaces.eno1.ipv4.addresses = [ - { address = "129.241.105.252"; prefixLength = 23; } - ]; - networking.defaultGateway.address = "192.241.104.1"; - networking.defaultGateway.interface = "eno1"; - networking.nameservers = [ - "129.241.0.200" - "129.241.0.201" - #"2001:700:300::200" - #"2001:700:300::201" - "8.8.8.8" - "1.1.1.1" - ]; - /**/ - # TODO: remove? Move? programs.dconf.enable = true; } diff --git a/hosts/brumlebasse/hardware-configuration.nix b/hosts/brumlebasse/hardware-configuration.nix deleted file mode 100644 index 0ac171e..0000000 --- a/hosts/brumlebasse/hardware-configuration.nix +++ /dev/null @@ -1,38 +0,0 @@ -# Do not modify this file! It was generated by ‘nixos-generate-config’ -# and may be overwritten by future invocations. Please make changes -# to /etc/nixos/configuration.nix instead. -{ config, lib, pkgs, modulesPath, ... }: - -{ - imports = - [ (modulesPath + "/installer/scan/not-detected.nix") - ]; - - boot.initrd.availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usbhid" "usb_storage" "sd_mod" ]; - boot.initrd.kernelModules = [ ]; - boot.kernelModules = [ "kvm-intel" ]; - boot.extraModulePackages = [ ]; - - fileSystems."/" = - { device = "/dev/disk/by-uuid/a489fe59-1f67-46a0-8c7a-91adbac021e0"; - fsType = "ext4"; - }; - - fileSystems."/boot" = - { device = "/dev/disk/by-uuid/2663-B2BA"; - fsType = "vfat"; - }; - - swapDevices = [ ]; - - # Enables DHCP on each ethernet and wireless interface. In case of scripted networking - # (the default) this is the recommended approach. When using systemd-networkd it's - # still possible to use this option, but it's recommended to use it in conjunction - # with explicit per-interface declarations with `networking.interfaces..useDHCP`. - networking.useDHCP = lib.mkDefault true; - # networking.interfaces.enp3s0.useDHCP = lib.mkDefault true; - - nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; - powerManagement.cpuFreqGovernor = lib.mkDefault "powersave"; - hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware; -} diff --git a/pkgs/mk-nspawn-deployer/default.nix b/pkgs/mk-nspawn-deployer/default.nix new file mode 100644 index 0000000..7b25901 --- /dev/null +++ b/pkgs/mk-nspawn-deployer/default.nix @@ -0,0 +1,29 @@ +{ lib +, pkgs +}: + +# assumes nspawn-tarball.nix is mixed into it +nixosConfiguration: + +let + + hostname = nixosConfiguration.config.networking.hostName; + + setup = pkgs.substituteAll { + src = ./setup-nspawn.sh; + isExecutable = true; + inherit hostname; + }; + + inherit (nixosConfiguration.config.system.build) tarball; + +in + +pkgs.runCommandNoCC "nspawn-setup-${hostname}.sh" { + nativeBuildInputs = with pkgs; [ makeself ]; +} '' + mkdir -p archive/ + ln -s ${setup} archive/setup.sh + ln -s ${tarball}/* archive/nixos-${hostname}.tar + makeself --follow archive/ $out setup-nixos-nspawn-${hostname} ./setup.sh +'' diff --git a/pkgs/mk-nspawn-deployer/setup-nspawn.sh b/pkgs/mk-nspawn-deployer/setup-nspawn.sh new file mode 100644 index 0000000..2b18dd0 --- /dev/null +++ b/pkgs/mk-nspawn-deployer/setup-nspawn.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# TODO: assert correct system + +NSPAWN=nixos-@hostname@ + +TARBALL=./"$NSPAWN".tar #"https://github.com/tfc/nspawn-nixos/releases/download/v1.0/nixos-system-x86_64-linux.tar.xz" + +test $(id -u) -eq 0 || { + >&2 echo you must run this as root + exit 1 +} + +install_pkg() { + # TODO: use bash hashmaps to map from apt to other package managers + # * [x] apt + # * [ ] apk + # * [ ] pacman + # * [ ] dnf + DEBIAN_FRONTEND=noninteractive apt install -y "$@" +} + + +if ! >/dev/null command -v systemd-nspawn; then + # TODO: support more than ubuntu + install_pkg systemd-container +fi + +machinectl remove "$NSPAWN" || true # TODO: interactive? +#machinectl pull-tar "$TARBALL" "$NSPAWN" --verify=no +machinectl import-tar "$TARBALL" "$NSPAWN" + +# use host network +cat <<"EOF" > /etc/systemd/nspawn/"$NSPAWN".nspawn +[Network] +VirtualEthernet=no +EOF + +machinectl enable "$NSPAWN" +machinectl start "$NSPAWN" +echo Setting root password... +machinectl shell "$NSPAWN" /usr/bin/env passwd + +machinectl status "$NSPAWN" diff --git a/users/pbsds/home/profiles/ssh.nix b/users/pbsds/home/profiles/ssh.nix index 1617621..b82eb30 100644 --- a/users/pbsds/home/profiles/ssh.nix +++ b/users/pbsds/home/profiles/ssh.nix @@ -35,6 +35,8 @@ "garp.pbsds.net".proxyJump = "microbel.pvv.ntnu.no"; "eple.pbsds.net".proxyJump = "microbel.pvv.ntnu.no"; "bolle.pbsds.net".proxyJump = "microbel.pvv.ntnu.no"; + "pederbs.idi.ntnu.no" = {}; + "brumlebasse.pbsds.net".port = 2222; "knut.pbsds.net".port = 23; "nord.pbsds.net".port = 24; "sopp.pbsds.net".port = 26;