treewide: add missing systemd ordering directives

In particular, we were missing `sops-install-secrets.service` in a ton of places, and `network-online.target` for some homebrewn services. This has caused failure of some services on machine startup.
2026-07-22 17:56:06 +09:00
40 changed files with 259 additions and 468 deletions
@@ -30,9 +30,9 @@
    ./services/journald-upload.nix
    ./services/logrotate.nix
    ./services/nginx.nix
-    ./services/nullmailer.nix
    ./services/openssh.nix
    ./services/polkit.nix
+    ./services/postfix.nix
    ./services/prometheus-flake-input-exporter.nix
    ./services/prometheus-node-exporter.nix
    ./services/prometheus-systemd-exporter.nix
@@ -1,15 +0,0 @@
-{ config, lib, ... }:
-{
-  services.nullmailer = {
-    enable = true;
-
-    config = {
-      adminaddr = "root@pvv.ntnu.no";
-      defaultdomain = "pvv.ntnu.no";
-      defaulthost = "pvv.ntnu.no";
-
-      me = lib.mkDefault config.networking.fqdn;
-      remotes = lib.mkDefault "smtp.pvv.ntnu.no smtp port=465 tls";
-    };
-  };
-}
@@ -10,18 +10,12 @@
      Match All
    '';
    settings.PermitRootLogin = "yes";
-    settings.AcceptEnv = [
-      "LANG"
-      "LC_*"
-      "COLORTERM"
-      "GIT_PROTOCOL"
-    ];
+
  };
+    users.users."root".openssh.authorizedKeys.keys = [
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqVt4LCe0YIttr9swFxjkjn37ZDY9JxwVC+2gvfSINDJorOCtqPjDOTD2fTS1Gz08QCwpnLWq2kyvRchu6WgriAbSACpbZZBgxRaF/FVh3oiMVFGnNKGnv6/fdo/vZtu8mUVuqtmTrgLYpZdbR4oD3XiBlDKs7Cv5hPqt95lnP6MNFvE8mICCfd1PwhsABd2IQ5laz3u77/RXhNFJL0Kf2/+6gk9awcLuwHrPdvq7c3BxRHbc9UMRQENyjyQPa7aLe+uJBFLKP51I8VBuDpDacuibQx7nMt6N2UJ2KWI0JxRMHuJNq4S5jidR82aOw9gzGbTv30SKNLMqsZ0xj4LtdqCXDiZF6Lr09PsJYsvnBUFWa14HGcThKDtgwQwBryNViYmfv//0h9+RLZiU0ab+NEwSs7Zh5iAD+vhx64QqNX3tR7Le4SWXh8W0eShU9N78qYdSkiC3Ui7htxeqOocXM/P4AwbnHsLELIvkHdvgchCPvl8ygZa4WJTEWv16+ICskJcAKWGuqjvXAFuwjJJmPp9xLW9O0DFfQhMELiGamQR9wK07yYQVr34iah6qZO7cwhSKyEPFrVPIaNtfDhsjED639F7vmktf26SWNJHWfW0wOHILjI6TgqUvy0JDd8W8w0CHlAfz6Fs2l99NNgNF8dB3vBASbxS0hu/y0PVu/xQ== openstack-sleipner"

-  users.users."root".openssh.authorizedKeys.keys = [
-    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqVt4LCe0YIttr9swFxjkjn37ZDY9JxwVC+2gvfSINDJorOCtqPjDOTD2fTS1Gz08QCwpnLWq2kyvRchu6WgriAbSACpbZZBgxRaF/FVh3oiMVFGnNKGnv6/fdo/vZtu8mUVuqtmTrgLYpZdbR4oD3XiBlDKs7Cv5hPqt95lnP6MNFvE8mICCfd1PwhsABd2IQ5laz3u77/RXhNFJL0Kf2/+6gk9awcLuwHrPdvq7c3BxRHbc9UMRQENyjyQPa7aLe+uJBFLKP51I8VBuDpDacuibQx7nMt6N2UJ2KWI0JxRMHuJNq4S5jidR82aOw9gzGbTv30SKNLMqsZ0xj4LtdqCXDiZF6Lr09PsJYsvnBUFWa14HGcThKDtgwQwBryNViYmfv//0h9+RLZiU0ab+NEwSs7Zh5iAD+vhx64QqNX3tR7Le4SWXh8W0eShU9N78qYdSkiC3Ui7htxeqOocXM/P4AwbnHsLELIvkHdvgchCPvl8ygZa4WJTEWv16+ICskJcAKWGuqjvXAFuwjJJmPp9xLW9O0DFfQhMELiGamQR9wK07yYQVr34iah6qZO7cwhSKyEPFrVPIaNtfDhsjED639F7vmktf26SWNJHWfW0wOHILjI6TgqUvy0JDd8W8w0CHlAfz6Fs2l99NNgNF8dB3vBASbxS0hu/y0PVu/xQ== openstack-sleipner"
-
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICCbgJ0Uwh9VSVhfId7l9i5/jk4CvAK5rbkiab8R+moF root@sleipner"
-  ];
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICCbgJ0Uwh9VSVhfId7l9i5/jk4CvAK5rbkiab8R+moF root@sleipner"
+    ];
 }

@@ -0,0 +1,22 @@
+{ config, pkgs, lib, ... }:
+let
+  cfg = config.services.postfix;
+in
+{
+  services.postfix = {
+    enable = true;
+
+    settings.main = {
+      myhostname = "${config.networking.hostName}.pvv.ntnu.no";
+      mydomain = "pvv.ntnu.no";
+
+      # Nothing should be delivered to this machine
+      mydestination = [ ];
+
+      relayhost = [ "smtp.pvv.ntnu.no:465" ];
+
+      smtp_tls_wrappermode = "yes";
+      smtp_tls_security_level = "encrypt";
+    };
+  };
+}
@@ -379,11 +379,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1784979981,
-        "narHash": "sha256-OnvjEXYU/InjJYH2+iIrvHWIjHnc5J9f84qWhrgAWlY=",
+        "lastModified": 1782759909,
+        "narHash": "sha256-gktjBeZyoRvVBkm2cO1tD99fdQ34iUDyB6iecRdorm4=",
        "ref": "main",
-        "rev": "ffb8ed0c9df2444727e49f16f7e43ea70184dc4b",
-        "revCount": 590,
+        "rev": "ad6c79fb713884a4a2df8aab30914cd0c1c2e6cb",
+        "revCount": 587,
        "type": "git",
        "url": "https://git.pvv.ntnu.no/Projects/nettsiden.git"
      },
@@ -268,7 +268,6 @@
          modules = [
            inputs.nix-gitea-themes.nixosModules.default
            inputs.disko.nixosModules.disko
-            self.nixosModules.robots-txt
          ];
        };

@@ -177,6 +177,11 @@ in
      };
    };

+    systemd.services."phpfpm-idp" = {
+      after = [ "sops-install-secrets.service" ];
+      requires = [ "sops-install-secrets.service" ];
+    };
+
    services.nginx.virtualHosts."idp.pvv.ntnu.no" = {
      forceSSL = true;
      enableACME = true;
@@ -287,6 +287,7 @@ in {

  systemd.services.mediawiki-init = lib.mkIf cfg.enable {
    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
    serviceConfig = {
      UMask = lib.mkForce "0007";
    };
@@ -294,6 +295,7 @@ in {

  systemd.services.phpfpm-mediawiki = lib.mkIf cfg.enable {
    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
    serviceConfig = {
      UMask = lib.mkForce "0007";
    };
@@ -61,6 +61,11 @@ in {
    };
  };

+  systemd.services.vaultwarden = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
  services.nginx.virtualHosts."${domain}" = {
    forceSSL = true;
    enableACME = true;
@@ -49,6 +49,11 @@ in
    '';
  };

+  systemd.services."phpfpm-roundcube" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
  # TODO: move this back to `webmail.pvv.ntnu.no/roundcube` subpath

  services.nginx.virtualHosts.${domain} = {
@@ -116,6 +116,11 @@ in {
    ];
  };

+  systemd.services."phpfpm-pvv-nettsiden" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
  services.nginx.virtualHosts."pvv.ntnu.no" = {
    globalRedirect = cfg.domainName;
    redirectCode = 307;
@@ -35,4 +35,9 @@ in {
      onCalendar = "*-*-* 09:00:00";
    };
  };
+
+  systemd.services.pvv-calendar-bot = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
 }
@@ -20,6 +20,8 @@ in
      lfs = false;
    };

+    # TODO: add a module setting to specify additional systemd dependencies
+    #       and then mark "sops-install-secrets.service" as a dependency here
    instances = let
      defaultGithubConfig = {
        settings.token_file = config.sops.secrets."gickup/github-token".path;
@@ -55,6 +55,11 @@
  # It needs this to be allowed to access the files with the acme group
  systemd.services.coturn.serviceConfig.PrivateUsers = lib.mkForce false;

+  systemd.services.coturn = {
+    requires = [ "sops-install-secrets.service" ];
+    after = [ "sops-install-secrets.service" ];
+  };
+
  systemd.services."acme-${config.services.coturn.realm}".serviceConfig = {
    AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
  };
@@ -39,6 +39,8 @@ in
  };

  systemd.services.mx-puppet-discord = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
    serviceConfig.SupplementaryGroups = [
      config.users.groups.keys-matrix-registrations.name
    ];
@@ -183,6 +183,7 @@ in
  };

  services.matrix-hookshot.serviceDependencies = [
+    "sops-install-secrets.service"
    "matrix-synapse.target"
    "nginx.service"
  ];
@@ -43,7 +43,12 @@ in
    keyFile = config.sops.templates."matrix-livekit-keyfile".path;
  };

-  systemd.services.lk-jwt-service.environment.LIVEKIT_FULL_ACCESS_HOMESERVERS = lib.mkIf cfg.enable (builtins.concatStringsSep "," [ "pvv.ntnu.no" "dodsorf.as" ]);
+  systemd.services.lk-jwt-service = lib.mkIf cfg.enable {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+
+    environment.LIVEKIT_FULL_ACCESS_HOMESERVERS = builtins.concatStringsSep "," [ "pvv.ntnu.no" "dodsorf.as" ];
+  };

  services.nginx.virtualHosts.${matrixDomain} = lib.mkIf cfg.enable {
    locations."^~ /livekit/jwt/" = {
@@ -55,52 +55,57 @@
    pantalaimon.username = "bot_admin";
  };

-  systemd.services.mjolnir.serviceConfig = {
-    DynamicUser = true;
-    RuntimeDirectory = [ "mjolnir/root-mnt" ];
-    RootDirectory = "/run/mjolnir/root-mnt";
-    BindReadOnlyPaths = [
-      config.sops.secrets."matrix/mjolnir/access_token".path
-      builtins.storeDir
-      "/etc"
-      "/run/nscd"
-      "/var/run/nscd"
-    ];
+  systemd.services.mjolnir = {
+    requires = [ "sops-install-secrets.service" ];
+    after = [ "sops-install-secrets.service" ];

-    AmbientCapabilities = "";
-    CapabilityBoundingSet = "";
-    LockPersonality = true;
-    MemoryDenyWriteExecute = false; # node needs this
-    NoNewPrivileges = true;
-    PrivateDevices = true;
-    PrivateMounts = true;
-    PrivateTmp = true;
-    PrivateUsers = true;
-    ProcSubset = "pid";
-    ProtectClock = true;
-    ProtectControlGroups = true;
-    ProtectHome = true;
-    ProtectHostname = true;
-    ProtectKernelLogs = true;
-    ProtectKernelModules = true;
-    ProtectKernelTunables = true;
-    ProtectProc = "invisible";
-    ProtectSystem = "strict";
-    RemoveIPC = true;
-    RestrictAddressFamilies = [
-      "AF_INET"
-      "AF_INET6"
-      "AF_UNIX"
-    ];
-    RestrictNamespaces = true;
-    RestrictRealtime = true;
-    RestrictSUIDSGID = true;
-    SystemCallArchitectures = "native";
-    SystemCallFilter = [
-      "@system-service"
-      "~@privileged"
-      "~@resources"
-    ];
-    UMask = "0077";
+    serviceConfig = {
+      DynamicUser = true;
+      RuntimeDirectory = [ "mjolnir/root-mnt" ];
+      RootDirectory = "/run/mjolnir/root-mnt";
+      BindReadOnlyPaths = [
+        config.sops.secrets."matrix/mjolnir/access_token".path
+        builtins.storeDir
+        "/etc"
+        "/run/nscd"
+        "/var/run/nscd"
+      ];
+
+      AmbientCapabilities = "";
+      CapabilityBoundingSet = "";
+      LockPersonality = true;
+      MemoryDenyWriteExecute = false; # node needs this
+      NoNewPrivileges = true;
+      PrivateDevices = true;
+      PrivateMounts = true;
+      PrivateTmp = true;
+      PrivateUsers = true;
+      ProcSubset = "pid";
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = true;
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      ProtectProc = "invisible";
+      ProtectSystem = "strict";
+      RemoveIPC = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+        "AF_UNIX"
+      ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [
+        "@system-service"
+        "~@privileged"
+        "~@resources"
+      ];
+      UMask = "0077";
+    };
  };
 }
@@ -56,7 +56,15 @@ in
    enableSynapseIntegration = false;
  };

+  systemd.services."matrix-ooye-pre-start" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
  systemd.services."matrix-ooye" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+
    serviceConfig = {
      RuntimeDirectory = [ "matrix-ooye/root-mnt" ];
      RootDirectory = "/run/matrix-ooye/root-mnt";
@@ -44,6 +44,11 @@ in {
    };
  };

+  systemd.targets."matrix-synapse" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
  services.matrix-synapse-next = {
    enable = true;

@@ -74,7 +79,7 @@ in {
        name = "psycopg2";
        args = {
          host = "/var/run/postgresql";
-          database = "synapse";
+          dbname = "synapse";
          user = "matrix-synapse";
          cp_min = 1;
          cp_max = 5;
@@ -23,6 +23,15 @@ in
  };

  systemd.services.minecraft-heatmap-ingest-logs = lib.mkIf cfg.enable {
+    after = [
+      "sops-install-secrets.service"
+      "network-online.target"
+    ];
+    requires = [
+      "sops-install-secrets.service"
+      "network-online.target"
+    ];
+
    serviceConfig = {
      LoadCredential = [
        "sshkey:${config.sops.secrets."minecraft-heatmap/ssh-key/private".path}"
@@ -4,10 +4,7 @@ let
  dataDir = "/data/mysql";
 in
 {
-  imports = [
-    ./backup.nix
-    ./timed-jobs.nix
-  ];
+  imports = [ ./backup.nix ];

  sops.secrets."mysql/password" = {
    owner = "mysql";
@@ -57,26 +54,17 @@ in
    mode = "0700";
  };

-  fileSystems.${dataDir} = lib.mkIf cfg.enable {
-    device = dataDir;
-    fsType = "none";
-    options = [
-      "bind"
-      "noatime"
-      "noauto"
-      "x-systemd.requires=systemd-tmpfiles-setup.service"
-      "x-systemd.requires=systemd-tmpfiles-resetup.service"
-    ];
-  };
-
  systemd.services.mysql = lib.mkIf cfg.enable {
    after = [
+      "sops-install-secrets.service"
      "systemd-tmpfiles-setup.service"
      "systemd-tmpfiles-resetup.service"
    ];
+    requires = [
+      "sops-install-secrets.service"
+    ];

    serviceConfig = {
-      RequiresMountsFor = [ dataDir ];
      BindPaths = [ "${dataDir}:${cfg.dataDir}" ];

      LogsDirectory = "mysql";
@@ -1,48 +0,0 @@
-{ config, lib, pkgs, ... }:
-let
-  cfg = config.services.mysql;
-in
-{
-  config = lib.mkIf cfg.enable {
-    systemd.services = {
-      mysql-analyze = {
-        requires = [ "mysql.service" ];
-        after = [ "mysql.service" ];
-        description = "Refresh MariaDB optimizer statistics for all databases";
-        startAt = "Mon 05:00:00";
-        serviceConfig = {
-          Type = "oneshot";
-          User = cfg.user;
-          Group = cfg.group;
-
-          Nice = 19;
-          IOSchedulingClass = "best-effort";
-          IOSchedulingPriority = 7;
-
-          ExecStart = "${lib.getExe' cfg.package "mariadb-check"} --all-databases --analyze";
-        };
-      };
-
-      mysql-optimize = {
-        requires = [ "mysql.service" ];
-        after = [ "mysql.service" ];
-        description = "Check, repair and optimize all MariaDB databases";
-        startAt = "*-*-01 04:00:00";
-        serviceConfig = {
-          Type = "oneshot";
-          User = cfg.user;
-          Group = cfg.group;
-
-          Nice = 19;
-          IOSchedulingClass = "best-effort";
-          IOSchedulingPriority = 7;
-
-          ExecStart = [
-            "${lib.getExe' cfg.package "mariadb-check"} --all-databases --auto-repair"
-            "${lib.getExe' cfg.package "mariadb-check"} --all-databases --optimize"
-          ];
-        };
-      };
-    };
-  };
-}
@@ -99,18 +99,6 @@ in
    mode = "0700";
  };

-  fileSystems."/data/postgresql" = lib.mkIf cfg.enable {
-    device = "/data/postgresql";
-    fsType = "none";
-    options = [
-      "bind"
-      "noatime"
-      "noauto"
-      "x-systemd.requires=systemd-tmpfiles-setup.service"
-      "x-systemd.requires=systemd-tmpfiles-resetup.service"
-    ];
-  };
-
  systemd.services.postgresql-setup = lib.mkIf cfg.enable {
    after = [
      "systemd-tmpfiles-setup.service"
@@ -122,7 +110,6 @@ in
        "key:/etc/certs/postgres.key"
      ];

-      RequiresMountsFor = [ "/data/postgresql" ];
      BindPaths = [ "/data/postgresql:/var/lib/postgresql" ];
    };
  };
@@ -138,7 +125,6 @@ in
        "key:/etc/certs/postgres.key"
      ];

-      RequiresMountsFor = [ "/data/postgresql" ];
      BindPaths = [ "/data/postgresql:/var/lib/postgresql" ];
    };
  };
@@ -150,6 +136,7 @@ in
  };

  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ 5432 ];
+  networking.firewall.allowedUDPPorts = lib.mkIf cfg.enable [ 5432 ];

  environment.systemPackages = [
    (pkgs.writeShellApplication {
@@ -82,6 +82,15 @@ in {
  };

  systemd.services."render-bluemap-maps" = {
+    after = [
+      "sops-install-secrets.service"
+      "network-online.target"
+    ];
+    requires = [
+      "sops-install-secrets.service"
+      "network-online.target"
+    ];
+
    serviceConfig = {
      StateDirectory = [ "bluemap/world" ];
      ExecStartPre = let
@@ -5,8 +5,6 @@ in
 {
  imports = [
    ./minecraft-checker.nix
-    ./postgres-checker.nix
-    ./mariadb-checker.nix
  ];

  services.gatus = {
@@ -135,18 +133,6 @@ in
            "[BODY].ok == true"
          ];
        })
-        (mkService "PostgreSQL" "http://localhost:1338" // {
-          conditions = [
-            "[STATUS] == 200"
-            "[BODY].ok == true"
-          ];
-        })
-        (mkService "MariaDB" "http://localhost:1339" // {
-          conditions = [
-            "[STATUS] == 200"
-            "[BODY].ok == true"
-          ];
-        })
        (mkService "Email (SMTP)" "starttls://mail.pvv.ntnu.no:587")
        (mkService "Email (POP3)" "tls://mail.pvv.ntnu.no:995")
        (mkService "Email (IMAP)" "tls://mail.pvv.ntnu.no:993")
@@ -1,79 +0,0 @@
-{ config, pkgs, ... }:
-{
-  sops.secrets."keys/gatus/mariadb" = {
-    restartUnits = [ "phh-gatus-mariadb-checker.service" ];
-  };
-
-  sops.templates."gatus-mariadb-checker.env" = {
-    restartUnits = [ "phh-gatus-mariadb-checker.service" ];
-    content = ''
-      MYSQL_HOST=mysql.pvv.ntnu.no
-      MYSQL_PORT=3306
-      MYSQL_DATABASE=mysql
-      MYSQL_USER=gatus_healthcheck
-      MYSQL_PASSWORD=${config.sops.placeholder."keys/gatus/mariadb"}
-    '';
-  };
-
-  services.python-http-handlers."gatus-mariadb-checker" = {
-    listenStreams = [ "127.0.0.1:1339" ];
-    libraries = with pkgs.python3Packages; [
-      pymysql
-    ];
-    serviceConfig = {
-      EnvironmentFile = config.sops.templates."gatus-mariadb-checker.env".path;
-    };
-    handler = ''
-      import os
-      import json
-      import pymysql
-
-      class Handler(BaseHTTPRequestHandler):
-          def do_GET(self):
-              try:
-                  conn = pymysql.connect(
-                      host=os.environ["MYSQL_HOST"],
-                      port=int(os.environ.get("MYSQL_PORT", "3306")),
-                      user=os.environ["MYSQL_USER"],
-                      password=os.environ["MYSQL_PASSWORD"],
-                      database=os.environ.get("MYSQL_DATABASE", "mysql"),
-                      connect_timeout=5,
-                  )
-
-                  try:
-                      with conn.cursor() as cur:
-                          cur.execute("SELECT VERSION();")
-                          (version,) = cur.fetchone()
-
-                          cur.execute("SHOW STATUS LIKE 'Threads_connected';")
-                          (_, connections) = cur.fetchone()
-
-                          cur.execute("SHOW DATABASES;")
-                          databases = cur.fetchall()
-                  finally:
-                      conn.close()
-
-                  body = {
-                      "ok": True,
-                      "version": version,
-                      "connections": int(connections),
-                      "databases": len(databases),
-                  }
-                  data = json.dumps(body).encode()
-
-                  self.send_response(200)
-                  self.send_header("Content-Type", "application/json")
-                  self.send_header("Content-Length", str(len(data)))
-                  self.end_headers()
-                  self.wfile.write(data)
-
-              except Exception as e:
-                  data = json.dumps({"ok": False, "error": str(e)}).encode()
-                  self.send_response(500)
-                  self.send_header("Content-Type", "application/json")
-                  self.send_header("Content-Length", str(len(data)))
-                  self.end_headers()
-                  self.wfile.write(data)
-    '';
-  };
-}
@@ -1,79 +0,0 @@
-{ config, pkgs, ... }:
-{
-  sops.secrets."keys/gatus/postgres" = {
-    restartUnits = [ "phh-gatus-postgres-checker.service" ];
-  };
-
-  sops.templates."gatus-postgres-checker.env" = {
-    restartUnits = [ "phh-gatus-postgres-checker.service" ];
-    content = ''
-      PGHOST=postgres.pvv.ntnu.no
-      PGPORT=5432
-      PGDATABASE=postgres
-      PGUSER=gatus_healthcheck
-      PGPASSWORD=${config.sops.placeholder."keys/gatus/postgres"}
-    '';
-  };
-
-  services.python-http-handlers."gatus-postgres-checker" = {
-    listenStreams = [ "127.0.0.1:1338" ];
-    libraries = with pkgs.python3Packages; [
-      psycopg2
-    ];
-    serviceConfig = {
-      EnvironmentFile = config.sops.templates."gatus-postgres-checker.env".path;
-    };
-    handler = ''
-      import os
-      import json
-      import psycopg2
-
-      class Handler(BaseHTTPRequestHandler):
-          def do_GET(self):
-              try:
-                  conn = psycopg2.connect(
-                      host=os.environ["PGHOST"],
-                      port=os.environ.get("PGPORT", "5432"),
-                      dbname=os.environ.get("PGDATABASE", "postgres"),
-                      user=os.environ["PGUSER"],
-                      password=os.environ["PGPASSWORD"],
-                      connect_timeout=5,
-                  )
-
-                  try:
-                      with conn.cursor() as cur:
-                          cur.execute("SELECT version();")
-                          (version,) = cur.fetchone()
-
-                          cur.execute("SELECT count(*) FROM pg_stat_activity;")
-                          (connections,) = cur.fetchone()
-
-                          cur.execute("SELECT count(*) FROM pg_database WHERE NOT datistemplate;")
-                          (databases,) = cur.fetchone()
-                  finally:
-                      conn.close()
-
-                  body = {
-                      "ok": True,
-                      "version": version,
-                      "connections": connections,
-                      "databases": databases,
-                  }
-                  data = json.dumps(body).encode()
-
-                  self.send_response(200)
-                  self.send_header("Content-Type", "application/json")
-                  self.send_header("Content-Length", str(len(data)))
-                  self.end_headers()
-                  self.wfile.write(data)
-
-              except Exception as e:
-                  data = json.dumps({"ok": False, "error": str(e)}).encode()
-                  self.send_response(500)
-                  self.send_header("Content-Type", "application/json")
-                  self.send_header("Content-Length", str(len(data)))
-                  self.end_headers()
-                  self.wfile.write(data)
-    '';
-  };
-}
@@ -79,6 +79,11 @@ in {
    };
  };

+  systemd.services.grafana = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
  services.nginx.virtualHosts.${cfg.settings.server.domain} = {
    enableACME = true;
    forceSSL = true;
@@ -37,9 +37,15 @@ in {
    };
  };

-  systemd.services.prometheus-postgres-exporter-knakelibrak.serviceConfig = let
-    localCfg = config.services.prometheus.exporters.postgres;
-  in lib.recursiveUpdate config.systemd.services.prometheus-postgres-exporter.serviceConfig {
+  systemd.services.prometheus-postgres-exporter-knakelibrak = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+
+    # TODO: is it really necessary to use recursiveUpdate on the entire attrset here?
+    #       Why don't we just update the relevant attrs directly?
+    serviceConfig = let
+      localCfg = config.services.prometheus.exporters.postgres;
+    in lib.recursiveUpdate config.systemd.services.prometheus-postgres-exporter.serviceConfig {
      EnvironmentFile = config.sops.secrets."keys/postgres/postgres_exporter_knakelibrak_env".path;
      ExecStart = ''
        ${pkgs.prometheus-postgres-exporter}/bin/postgres_exporter \
@@ -48,4 +54,5 @@ in {
          ${lib.concatStringsSep " \\\n  " localCfg.extraFlags}
      '';
    };
+  };
 }
@@ -154,8 +154,14 @@ in {
  environment.systemPackages = [ cfg.package ];

  systemd.services.gitea = lib.mkIf cfg.enable {
-    wants = [ "redis-gitea.service" ];
-    after = [ "redis-gitea.service" ];
+    after = [
+      "sops-install-secrets.service"
+      "redis-gitea.service"
+    ];
+    requires = [
+      "sops-install-secrets.service"
+      "redis-gitea.service"
+    ];

    serviceConfig = {
      CPUSchedulingPolicy = "batch";
@@ -196,108 +202,6 @@ in {
    };
  };

-  environment.robots-txt."gitea" = {
-    virtualHost = domain;
-    rules = [
-      {
-        pre_comment = ''
-          Gitea internals
-
-          See these for more information:
-          - https://gitea.com/robots.txt
-          - https://codeberg.org/robots.txt
-        '';
-        User-agent = "*";
-        Disallow = [
-          "/api/*"
-          "/avatars"
-          "/*/*/src/commit/*"
-          "/*/*/commit/*"
-          "/*/*/*/refs/*"
-          "/*/*/*/star"
-          "/*/*/*/watch"
-          "/*/*/labels"
-          "/*/*/activity/*"
-          "/vendor/*"
-          "/swagger.*.json"
-          "/repo/create"
-          "/repo/migrate"
-          "/org/create"
-          "/*/*/fork"
-          "/*/*/watchers"
-          "/*/*/stargazers"
-          "/*/*/forks"
-          "*/.git/"
-          "/*.git"
-          "/*.atom"
-          "/*.rss"
-        ];
-        Crawl-delay = "2";
-      }
-      {
-        pre_comment = "Language Spam";
-        Disallow = "/*?lang=";
-      }
-      {
-        pre_comment = ''
-          AI bots
-
-          Sourced from:
-          - https://www.vg.no/robots.txt
-          - https://codeberg.org/robots.txt
-        '';
-        User-agent = [
-          "AI2Bot"
-          "Ai2Bot-Dolma"
-          "Amazonbot"
-          "Applebot-Extended"
-          "Bytespider"
-          "CCBot"
-          "ChatGPT-User"
-          "Claude-Web"
-          "ClaudeBot"
-          "Crawlspace"
-          "Diffbot"
-          "FacebookBot"
-          "FriendlyCrawler"
-          "GPTBot"
-          "Google-Extended"
-          "ICC-Crawler"
-          "ImagesiftBot"
-          "Kangaroo Bot"
-          "Meta-ExternalAgent"
-          "OAI-SearchBot"
-          "Omgili"
-          "Omgilibot"
-          "PanguBot"
-          "PerplexityBot"
-          "PetalBot"
-          "Scrapy"
-          "SemrushBot-OCOB"
-          "Sidetrade indexer bot"
-          "Timpibot"
-          "VelenPublicWebCrawler"
-          "Webzio-Extended"
-          "YouBot"
-          "anthropic-ai"
-          "cohere-ai"
-          "cohere-training-data-crawler"
-          "facebookexternalhit"
-          "iaskspider/2.0"
-          "img2dataset"
-          "meta-externalagent"
-          "omgili"
-          "omgilibot"
-        ];
-        Disallow = "/";
-        Crawl-delay = "2";
-      }
-      {
-        Sitemap = "https://${domain}/sitemap.xml";
-      }
-    ];
-  };
-
  networking.firewall.allowedTCPPorts = [ sshPort ];

  services.rsync-pull-targets = {
@@ -318,6 +222,9 @@ in {
  };

  systemd.services.gitea-dump = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+
    serviceConfig.ExecStart = let
      args = lib.cli.toCommandLineShellGNU { } {
        type = cfg.dump.type;
@@ -32,7 +32,11 @@ in

  systemd.services.gitea-ensure-gnupg-homedir = {
    description = "Import gpg key for gitea";
+
    before = [ "gitea.service" ];
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+
    environment = { inherit GNUPGHOME; };
    serviceConfig = {
      Type = "oneshot";
@@ -12,6 +12,16 @@ in
  systemd.services.gitea-import-users = lib.mkIf cfg.enable {
    enable = true;
    environment.PASSWD_FILE_PATH = "/run/gitea-import-users/passwd";
+    after = [
+      "sops-install-secrets.service"
+      "gitea.service"
+      "network-online.target"
+    ];
+    requires = [
+      "sops-install-secrets.service"
+      "gitea.service"
+      "network-online.target"
+    ];
    serviceConfig = {
      ExecStartPre = ''${pkgs.rsync}/bin/rsync -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=$CREDENTIALS_DIRECTORY/ssh-known-hosts -i $CREDENTIALS_DIRECTORY/sshkey" -a pvv@smtp.pvv.ntnu.no:/etc/passwd /run/gitea-import-users/passwd'';
      ExecStart = pkgs.writers.writePython3 "gitea-import-users" {
@@ -48,7 +48,16 @@ in
  # %d - secrets directory
  systemd.services."gitea-web-secret-provider@" = {
    description = "Ensure all repos in %i has an SSH key to push web content";
-    requires = [ "gitea.service" "network.target" ];
+    after = [
+      "sops-install-secrets.service"
+      "gitea.service"
+      "network-online.target"
+    ];
+    requires = [
+      "sops-install-secrets.service"
+      "gitea.service"
+      "network-online.target"
+    ];
    serviceConfig = {
      Slice = "system-giteaweb.slice";
      Type = "oneshot";
@@ -94,6 +94,11 @@ in
    };
  };

+  systemd.services."openvpn-ov-firewall" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
  networking.firewall = {
    allowedTCPPorts = [ 1194 ];
    allowedUDPPorts = [ 1194 ];
@@ -53,5 +53,10 @@
    };
  };

+  systemd.services.drumknotty-screen-session = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
  services.roowho2.settings.rwhod.ignoreUsers = [ "drumknotty" ];
 }
@@ -1,13 +1,14 @@
 { config, lib, pkgs, ... }:
 {
-  assertions = [{
-    assertion = config.services.nullmailer.enable;
-    message = ''
-      Expected nullmailer to be enabled for temmie userweb.
+  services.postfix.enable = lib.mkForce false;

-      If you change the default sendmail implementation, please make sure that temmie userweb works correctly!
-    '';
-  }];
+  services.nullmailer = {
+    enable = true;
+    config = {
+      me = config.networking.fqdn;
+      remotes = "mail.pvv.ntnu.no smtp --port=25";
+    };
+  };

  services.bro = {
    enable = true;
@@ -29,9 +29,15 @@ in
    systemd.services."httpd-passwd-sync" = {
      requiredBy = [ "userweb.target" ];
      after = [
+        "network-online.target"
+        "sops-install-secrets.service"
        "systemd-tmpfiles-setup.service"
        "systemd-tmpfiles-resetup.service"
      ];
+      requires = [
+        "network-online.target"
+        "sops-install-secrets.service"
+      ];
      serviceConfig = {
        Type = "oneshot";
        Slice = "system-userweb.slice";
@@ -159,7 +159,6 @@ in
        description = "Out of Your Element - a Discord bridge for Matrix.";

        wants = [
-          "network-online.target"
          "matrix-synapse.service"
          "conduit.service"
          "dendrite.service"
@@ -168,7 +167,10 @@ in
          "matrix-ooye-pre-start.service"
          "network-online.target"
        ];
-        requires = [ "matrix-ooye-pre-start.service" ];
+        requires = [
+          "network-online.target"
+          "matrix-ooye-pre-start.service"
+        ];
        wantedBy = [ "multi-user.target" ];

        startLimitIntervalSec = 5;
@@ -1,9 +1,6 @@
 config:
    mysqld_exporter_password: ENC[AES256_GCM,data:I9K+QMqaN3FOOVKzeOR9Q6UERStXX0P8WEHyN1jzzbM=,iv:UxvIdlfAyJvNuxPkU4+guKPa0fiD0vVLzHOTYktcmso=,tag:ltnIqEwESYx9HBu8UN0ZLw==,type:str]
 keys:
-    gatus:
-        mariadb: ENC[AES256_GCM,data:U7CGtjHacJlKcReEw3JO86okCVgLAkP92iX1bHGNw8qeImpvk5mNw77Ox7tbsChvozrJWa/Npu2pTBsNnVYYYA==,iv:hWa+KPiBc3AnH1tHCGJa8LhPXZ1o8sxufpGe7nUjV0E=,tag:mso5HEfmWAOGw350408X7g==,type:str]
-        postgres: ENC[AES256_GCM,data:7fI0JeZDK1MX5poXWaMo67PXbjdjvM+xPKioHSHmGdS8iNDGe5Ym6LlS/hOnPvvDuNFyHjKKizqDst0227U2gg==,iv:D3bz/xyFrnQsUCiMEFq8JOjSudyiw+idEMWQ1e6Uk3k=,tag:8XqadKVHK9NMWehK7V5J0Q==,type:str]
    grafana:
        secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
        admin_password: ENC[AES256_GCM,data:ttKwfC4WuXeL/6x4,iv:x1X+e3z08CR992GzC62YnFIN7SGrE81/nDNrgcgVzx0=,tag:YajUoy61kYbpeGeC7yNrXQ==,type:str]
@@ -12,7 +9,8 @@ keys:
        postgres_exporter_knakelibrak_env: ENC[AES256_GCM,data:xjC7DGXrW2GIJq8XioIZb+jSe/Hzcz0tv9cUHmX/n1nhI+D64lYt+EKnq1+RX/vJzU4sTaKjveKBh88Qqnv6RQm+MZC//dIxcvnnAdl50qnHZyBCaFFEzSNI8I8vGyArMk8Ja72clBq3kMpUz/pLBP0qDrjblKDoWkU=,iv:ZW98hJy8A5t4Oxtu17R3tM7gou183VLbgBsHA8LFuJY=,tag:VMOvQz3X/XDylV1YFg2Jsg==,type:str]
 sops:
    age:
-        - enc: |
+        - recipient: age102e6y8gah0ntr6fxqnkpepc8ar29p6ls7ks9ka7v8w87q8scm9yqmc2u8d
+          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyTWRSM3IwMmxtTmZVcCsw
            OUhlakxHZzgrSEhEdUZFTXE1anNjQ2wvdkZFCnB6S1l3TXQ3ZGFYWmtYM1cwMFZT
@@ -20,8 +18,8 @@ sops:
            YlltQ3FBU3RBYUx4TnNPRk1SUWNqZG8KAJjc09x553ncaWduGLsnIHdroaOmMasP
            /fq0GzW6UNfmE2rQ6qrQti21B37/sN0WMLCSPLUPG45kBgx20GG4hQ==
            -----END AGE ENCRYPTED FILE-----
-          recipient: age102e6y8gah0ntr6fxqnkpepc8ar29p6ls7ks9ka7v8w87q8scm9yqmc2u8d
-        - enc: |
+        - recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
+          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTOWFvcFFzc1lmNVdmV3lX
            cVVLNHowcGdENzI5UUJLZTNSMHhjNlI2c0FnCmdwOS9oL1kwTnhwbXRodWxxWVE3
@@ -29,8 +27,8 @@ sops:
            eDVwd2dKMG9FRW1OY1pyUkhLeWw3b0EKtJpsQ/Ss39ZLiRNqUhn8sdB3hpQy7Syv
            ererqhMkqmDugGEHPk6KpZuj7DVSK1di7JgA2qZOUPzI7UpxjaC0Kg==
            -----END AGE ENCRYPTED FILE-----
-          recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
-        - enc: |
+        - recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
+          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOQU1iVXRkQmo4b3F1Vngz
            S0pkNkVFR1FUb1djdmI0eHh3V3BBTDJTSTJZClA0S0Z0cTdFRmRaOEZQdHQzdGZ3
@@ -38,8 +36,8 @@ sops:
            REdjRFZyY2pNdEd6cmgvQisyVDhLUEkKRItJ0CGbzlEB5RNAyem4feMVhTfcLef3
            QIqltZ2l4LLexnkECi3FCJZHxrbUa+/RF6p1DsueUw7LLUnOcphB9A==
            -----END AGE ENCRYPTED FILE-----
-          recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
-        - enc: |
+        - recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
+          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvZTBpaWM0c1hmODBaK0Iw
            bmp1NzRacklXMHU0K2J2d0g5ajBiNlRNWGhRCm1DOWI0cm5BdTdlNmFzM2JVekNk
@@ -47,8 +45,8 @@ sops:
            U0ZCT2toZ1ZMZ3E0bXRhSTQvNGFWNVkKhxfQDIDe2LQW7OMBJv0J267AW1wI32df
            ZQxd657TEqzm7i19azrCS0jyRbfj2MYzEJAtTGiGZaNC9uKDFzBhKw==
            -----END AGE ENCRYPTED FILE-----
-          recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
-        - enc: |
+        - recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
+          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmRlJROHRKb21YUnlicmc1
            MlptQllEcXFhajNKS1krMDdUMWk5QWo4eVJBCndGSlhXS1Vaa2RSTllIcmF1ZVpl
@@ -56,8 +54,8 @@ sops:
            TE41aFdjU0h0ekQ2Zjg4Z3VQVjFWcnMK6zjSalqeYjyc4NH6nOeghlhYJydrz4pM
            N5ZcXjRbrIVFdhbYnvQGKvGKZm0kK6vjzBjdT7BM6ctr8cq/qrz1xQ==
            -----END AGE ENCRYPTED FILE-----
-          recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
-        - enc: |
+        - recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
+          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIRXpBa2tYc2xEeldub0VK
            bkFwOFdlUGRZM0FVT0tyUW1RWnl0TjRUTUh3CnZlMC92MU1hRW1yZU1NSUdoUEZh
@@ -65,8 +63,8 @@ sops:
            aXFQWlNVQ2laVm1ETStRemNZRXc3TUEKlPYSU3gp67dsPfbEJkru4ieMvspC7+pu
            rfp315HLyj1FGhrA8f2qOxE/PYI2rn0yKm80KffWBV7ylX/uonm4Fg==
            -----END AGE ENCRYPTED FILE-----
-          recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
-        - enc: |
+        - recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
+          enc: |
            -----BEGIN AGE ENCRYPTED FILE-----
            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsaTEzOFBEeG9LVThSVmQ5
            VXBoaFpueFRCbFJ1akE4RWc1aE1HUmVGcXdZCjFnbU0wd2drazNsTmNBMHNuOFhO
@@ -74,9 +72,8 @@ sops:
            dC9meDZlc3d3aUJEVjc4REF0Y1BLcGcK79LbJzc5KVgEgyJR11crGuX8YcVoJBbT
            Fin7Zoon06L7qx0Zw5u27wV7RKMnYT7hOMiWs6660ZTLcYJ5M1aEZQ==
            -----END AGE ENCRYPTED FILE-----
-          recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
-    lastmodified: "2026-07-21T09:40:54Z"
-    mac: ENC[AES256_GCM,data:FLpmsW9I21qNMFs6Tn/1AzFo+RcmCwtVdAaaGIrGlUi2l3gzb//ZK6ruI/VYx1R5DoX1OhD3EIg6aLMSOT6jiHf+8GwfhdijpayG1nrgeZW3SuFbjVlCYNMPH3DOJJB82U3T+ozSbpnZEtdUwmNWcZJt5F2hOEnoaNa+jLEKDao=,iv:qyQuyWuu+w2ynRvhIx0QVtw+tieO3dbQsUxEB5Brzp0=,tag:qYAj4h6foWm16YqIdZ6xbQ==,type:str]
+    lastmodified: "2025-03-16T20:08:18Z"
+    mac: ENC[AES256_GCM,data:C2tpWppc13jKJq5d4nmAKQOaNWHm27TKwxAxm1fi2lejN1lqUaoz5bHfTBA7MfaWvuP5uZnfbtG32eeu48mnlWpo58XRUFFecAhb9JUpW9s5IR3/nbzLNkGU7H5C0oWPrxI4thd+bAVduIgBjjFyGj1pe6J9db3c0yUWRwNlwGU=,iv:YpoQ4psiFYOWLGipxv1QvRvr034XFsyn2Bhyy39HmOo=,tag:ByiCWygFC/VokVTbdLoLgg==,type:str]
    pgp:
        - created_at: "2026-05-20T17:35:58Z"
          enc: |-
@@ -99,4 +96,4 @@ sops:
            -----END PGP MESSAGE-----
          fp: F7D37890228A907440E1FD4846B9228E814A2AAC
    unencrypted_suffix: _unencrypted
-    version: 3.13.2
+    version: 3.9.4