qemu systemd not virtd

flake.nix

@@ -14,8 +14,8 @@
     # stylix.inputs.nixpkgs.follows = "nixpkgs";
     # stylix.inputs.home-manager.follows = "home-manager";
 
-    NixVirt.url = "https://flakehub.com/f/AshleyYakeley/NixVirt/*.tar.gz";
+    # NixVirt.url = "https://flakehub.com/f/AshleyYakeley/NixVirt/*.tar.gz";
-    NixVirt.inputs.nixpkgs.follows = "nixpkgs";
+    # NixVirt.inputs.nixpkgs.follows = "nixpkgs";
 
 
     ozai.url = "git+https://git.pvv.ntnu.no/Projects/ozai.git";
@@ -46,7 +46,6 @@
     , nixpkgs
     , sops-nix
     , nixos-hardware
-    , NixVirt
     , unstable
   , ... }@inputs:
     let
@@ -117,43 +116,10 @@
             inherit inputs;
           };
           modules = [
-            NixVirt.nixosModules.default
             ./machines/gandalf/configuration.nix
             ({ config, pkgs, ... }: { nixpkgs.overlays = [ overlay-unstable ]; })
             sops-nix.nixosModules.sops
 
-            ({ config, pkgs, ... }: {
-            # Your VM configuration here
-            virtualisation.libvirt.enable = true;
-            virtualisation.libvirt.connections."qemu:///system".domains = [
-              {
-                definition = NixVirt.lib.domain.writeXML (NixVirt.lib.domain.templates.q35 {
-                  name = "gandalf-grey";
-                  uuid = "a1db010b-4ad3-436a-bd99-f290f5ac8806"; # Replace with a generated UUID
-                  memory = { count = 4; unit = "GiB"; };
-                  vcpu = { value = 2; }; # Number of CPU cores
-                  storage_vol = "/vm-images/OPNsense-24.7-nano-amd64.img"; # Path to your storage image file
-                  install_vol = null; # No installation volume since we're using an existing image
-                  virtio_net = true;
-                  virtio_video = true;
-                  virtio_drive = true;
-                  devices = [
-                    { hostdev = "/sys/bus/pci/devices/0000:04:00.0"; }
-                    { hostdev = "/sys/bus/pci/devices/0000:05:00.0"; }
-                    { hostdev = "/sys/bus/pci/devices/0000:06:00.0"; }
-                    { hostdev = "/sys/bus/pci/devices/0000:07:00.0"; }
-                    { hostdev = "/sys/bus/pci/devices/0000:08:00.0"; }
-                    { hostdev = "/sys/bus/pci/devices/0000:0b:00.0"; }
-                    { hostdev = "/sys/bus/pci/devices/0000:0b:00.1"; }
-                    { hostdev = "/sys/bus/pci/devices/0000:0c:00.0"; }
-                    { hostdev = "/sys/bus/pci/devices/0000:0c:00.1"; }
-                  ];
-                });
-                active = true;
-              }
-            ];
-          })
-
           ];
         };
 

machines/gandalf/bootvm-daemon.sh

@@ -0,0 +1,30 @@
+qemu-system-x86_64 \
+  -m 4G \
+  -cpu host \
+  -smp 2 \
+  -enable-kvm \
+  -device vfio-pci,host=04:00.0 \
+  -device vfio-pci,host=05:00.0 \
+  -device vfio-pci,host=06:00.0 \
+  -device vfio-pci,host=07:00.0 \
+  -device vfio-pci,host=08:00.0 \
+  -device vfio-pci,host=0b:00.0 \
+  -device vfio-pci,host=0b:00.1 \
+  -device vfio-pci,host=0c:00.0 \
+  -device vfio-pci,host=0c:00.1 \
+  -drive file=/vm-images/OPNsense-24.7-nano-amd64.img,format=raw \
+  -vga virtio \
+  -netdev user,id=net0 -device virtio-net,netdev=net0 \
+  -display none \
+  -daemonize \
+
+#-net none
+# -device hostdev,/sys/bus/pci/devices/0000:04:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:05:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:06:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:07:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:08:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:0b:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:0b:00.1\
+#  -device hostdev,/sys/bus/pci/devices/0000:0c:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:0c:00.1\ #

machines/gandalf/bootvm.sh

@@ -5,11 +5,25 @@ qemu-system-x86_64 \
   -enable-kvm \
   -device vfio-pci,host=04:00.0 \
   -device vfio-pci,host=05:00.0 \
+  -device vfio-pci,host=06:00.0 \
+  -device vfio-pci,host=07:00.0 \
+  -device vfio-pci,host=08:00.0 \
+  -device vfio-pci,host=0c:00.1 \
+  -device vfio-pci,host=08:00.0 \
+  -device vfio-pci,host=08:00.0 \
+  -device vfio-pci,host=08:00.0 \
   -drive file=/vm-images/OPNsense-24.7-nano-amd64.img,format=raw \
   -vga virtio \
   -netdev user,id=net0 -device virtio-net,netdev=net0 \
-  -daemonize \
+  -nographic \
-  -vnc
 
-#-nographic \
 #-net none
+# -device hostdev,/sys/bus/pci/devices/0000:04:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:05:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:06:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:07:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:08:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:0b:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:0b:00.1\
+#  -device hostdev,/sys/bus/pci/devices/0000:0c:00.0\
+#  -device hostdev,/sys/bus/pci/devices/0000:0c:00.1\ #

machines/gandalf/configuration.nix

@@ -9,7 +9,7 @@
       ./hardware-configuration.nix
       ../../profiles/base.nix
         ../../services/virt.nix
-       # ./nixvirt.nix
+       ./qemu.nix
     ];
 
   # Use the systemd-boot EFI boot loader.

machines/gandalf/qemu.nix

@@ -0,0 +1,55 @@
+{ pkgs, ... }:
+{
+
+  users.users.qemu = {
+    isSystemUser = true;
+    description = "QEMU User";
+    home = "/var/lib/qemu";
+    createHome = true;
+    group = "qemu";
+    extraGroups = [ "vfio" ];  # Add qemu to vfio group
+  };
+  users.groups.qemu = {};
+
+services.udev.extraRules = ''
+  # Set proper permissions for VFIO devices
+  SUBSYSTEM=="vfio", GROUP="vfio", MODE="0660"
+'';
+
+
+  environment.systemPackages = with pkgs; [
+    qemu_kvm
+  ];
+
+  systemd.services.qemu-vm = {
+    description = "QEMU VM Service";
+    #wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      #can add ,rombar=0 to vfio devices, but it does at least run now.
+      ExecStart = ''
+        ${pkgs.qemu_kvm}/bin/qemu-system-x86_64 \
+        -m 4G \
+        -cpu host \
+        -smp 2 \
+        -enable-kvm \
+        -device vfio-pci,host=04:00.0 \
+        -device vfio-pci,host=05:00.0 \
+        -device vfio-pci,host=06:00.0 \
+        -device vfio-pci,host=07:00.0 \
+        -device vfio-pci,host=08:00.0 \
+        -device vfio-pci,host=0b:00.0 \
+        -device vfio-pci,host=0b:00.1 \
+        -device vfio-pci,host=0c:00.0 \
+        -device vfio-pci,host=0c:00.1 \
+        -drive file=/vm-images/OPNsense-24.7-nano-amd64.img,format=raw \
+        -vga virtio \
+        -netdev user,id=net0 -device virtio-net,netdev=net0 \
+        -display none \
+        #-daemonize
+      '';
+      Restart = "on-failure";
+      User = "root";  # Run as the qemu user
+    };
+  };
+}
+