mitigations: patch matrix-synapse

This commit extends the python http handler template with the following
things:

- Integration with systemd notify
- Named systemd sockets, to avoid depending on expected FD number
- Graceful and customizable handling of SIGHUP reloads
- Graceful shutdown on SIGTERM with timeout
- Multithreaded HTTP server, one thread per request
- Better error message on undefined handler

base/default.nix

@@ -30,9 +30,9 @@
     ./services/journald-upload.nix
     ./services/logrotate.nix
     ./services/nginx.nix
+    ./services/nullmailer.nix
     ./services/openssh.nix
     ./services/polkit.nix
-    ./services/postfix.nix
     ./services/prometheus-flake-input-exporter.nix
     ./services/prometheus-node-exporter.nix
     ./services/prometheus-systemd-exporter.nix

base/mitigations.nix

@@ -21,4 +21,18 @@ in
   boot.blacklistedKernelModules = modulesToBan;
 
   boot.extraModprobeConfig = lib.concatMapStringsSep "\n" (mod: "install ${mod} ${lib.getExe' pkgs.coreutils "false"}") modulesToBan;
+
+  nixpkgs.overlays = [
+    (final: prev: {
+      matrix-synapse-unwrapped = prev.matrix-synapse-unwrapped.overrideAttrs (old: rec {
+        version = "1.157.2";
+        src = final.fetchFromGitHub {
+          owner = "element-hq";
+          repo = "synapse";
+          rev = "v${version}";
+          hash = "sha256-TUHcNAXrV43+J7jqfstlYdrZrwL5kDCh03yxO+vL/gw=";
+        };
+      });
+    })
+  ];
 }

base/services/journald-upload.nix

@@ -9,7 +9,7 @@ in
       URL = "https://journald.pvv.ntnu.no:${toString config.services.journald.remote.port}";
       ServerKeyFile = "-";
       ServerCertificateFile = "-";
-      TrustedCertificateFile = "-";
+      TrustedCertificateFile = config.security.pki.caBundle;
     };
   };
 

base/services/nullmailer.nix

@@ -0,0 +1,15 @@
+{ config, lib, ... }:
+{
+  services.nullmailer = {
+    enable = true;
+
+    config = {
+      adminaddr = "root@pvv.ntnu.no";
+      defaultdomain = "pvv.ntnu.no";
+      defaulthost = "pvv.ntnu.no";
+
+      me = lib.mkDefault config.networking.fqdn;
+      remotes = lib.mkDefault "smtp.pvv.ntnu.no smtp port=465 tls";
+    };
+  };
+}

base/services/openssh.nix

@@ -10,12 +10,18 @@
       Match All
     '';
     settings.PermitRootLogin = "yes";
-
+    settings.AcceptEnv = [
-  };
+      "LANG"
-    users.users."root".openssh.authorizedKeys.keys = [
+      "LC_*"
-      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqVt4LCe0YIttr9swFxjkjn37ZDY9JxwVC+2gvfSINDJorOCtqPjDOTD2fTS1Gz08QCwpnLWq2kyvRchu6WgriAbSACpbZZBgxRaF/FVh3oiMVFGnNKGnv6/fdo/vZtu8mUVuqtmTrgLYpZdbR4oD3XiBlDKs7Cv5hPqt95lnP6MNFvE8mICCfd1PwhsABd2IQ5laz3u77/RXhNFJL0Kf2/+6gk9awcLuwHrPdvq7c3BxRHbc9UMRQENyjyQPa7aLe+uJBFLKP51I8VBuDpDacuibQx7nMt6N2UJ2KWI0JxRMHuJNq4S5jidR82aOw9gzGbTv30SKNLMqsZ0xj4LtdqCXDiZF6Lr09PsJYsvnBUFWa14HGcThKDtgwQwBryNViYmfv//0h9+RLZiU0ab+NEwSs7Zh5iAD+vhx64QqNX3tR7Le4SWXh8W0eShU9N78qYdSkiC3Ui7htxeqOocXM/P4AwbnHsLELIvkHdvgchCPvl8ygZa4WJTEWv16+ICskJcAKWGuqjvXAFuwjJJmPp9xLW9O0DFfQhMELiGamQR9wK07yYQVr34iah6qZO7cwhSKyEPFrVPIaNtfDhsjED639F7vmktf26SWNJHWfW0wOHILjI6TgqUvy0JDd8W8w0CHlAfz6Fs2l99NNgNF8dB3vBASbxS0hu/y0PVu/xQ== openstack-sleipner"
+      "COLORTERM"
-
+      "GIT_PROTOCOL"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICCbgJ0Uwh9VSVhfId7l9i5/jk4CvAK5rbkiab8R+moF root@sleipner"
     ];
+  };
+
+  users.users."root".openssh.authorizedKeys.keys = [
+    "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqVt4LCe0YIttr9swFxjkjn37ZDY9JxwVC+2gvfSINDJorOCtqPjDOTD2fTS1Gz08QCwpnLWq2kyvRchu6WgriAbSACpbZZBgxRaF/FVh3oiMVFGnNKGnv6/fdo/vZtu8mUVuqtmTrgLYpZdbR4oD3XiBlDKs7Cv5hPqt95lnP6MNFvE8mICCfd1PwhsABd2IQ5laz3u77/RXhNFJL0Kf2/+6gk9awcLuwHrPdvq7c3BxRHbc9UMRQENyjyQPa7aLe+uJBFLKP51I8VBuDpDacuibQx7nMt6N2UJ2KWI0JxRMHuJNq4S5jidR82aOw9gzGbTv30SKNLMqsZ0xj4LtdqCXDiZF6Lr09PsJYsvnBUFWa14HGcThKDtgwQwBryNViYmfv//0h9+RLZiU0ab+NEwSs7Zh5iAD+vhx64QqNX3tR7Le4SWXh8W0eShU9N78qYdSkiC3Ui7htxeqOocXM/P4AwbnHsLELIvkHdvgchCPvl8ygZa4WJTEWv16+ICskJcAKWGuqjvXAFuwjJJmPp9xLW9O0DFfQhMELiGamQR9wK07yYQVr34iah6qZO7cwhSKyEPFrVPIaNtfDhsjED639F7vmktf26SWNJHWfW0wOHILjI6TgqUvy0JDd8W8w0CHlAfz6Fs2l99NNgNF8dB3vBASbxS0hu/y0PVu/xQ== openstack-sleipner"
+
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICCbgJ0Uwh9VSVhfId7l9i5/jk4CvAK5rbkiab8R+moF root@sleipner"
+  ];
 }
 

base/services/postfix.nix

@@ -1,22 +0,0 @@
-{ config, pkgs, lib, ... }:
-let
-  cfg = config.services.postfix;
-in
-{
-  services.postfix = {
-    enable = true;
-
-    settings.main = {
-      myhostname = "${config.networking.hostName}.pvv.ntnu.no";
-      mydomain = "pvv.ntnu.no";
-
-      # Nothing should be delivered to this machine
-      mydestination = [ ];
-
-      relayhost = [ "smtp.pvv.ntnu.no:465" ];
-
-      smtp_tls_wrappermode = "yes";
-      smtp_tls_security_level = "encrypt";
-    };
-  };
-}

flake.lock

@@ -126,11 +126,11 @@
         "rust-overlay": "rust-overlay_2"
       },
       "locked": {
-        "lastModified": 1784553736,
+        "lastModified": 1784709844,
-        "narHash": "sha256-W1pPlTHmp+EvONwf8Qa5NvCiuyV4LC/yNDdc27RSpuA=",
+        "narHash": "sha256-5PW5+UEFFjsru5FiDg9ZRUjQyo7AVyc+nesQlBSqM6c=",
         "ref": "main",
-        "rev": "e06daaef1f48160254c202c59b060ac6f1da9f38",
+        "rev": "a8fb5df19d1cdcc3ac29cdb8ad192c1d578761f0",
-        "revCount": 62,
+        "revCount": 63,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Grzegorz/greg-ng.git"
       },
@@ -291,11 +291,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1784526640,
+        "lastModified": 1785211367,
-        "narHash": "sha256-UDnuu3uM64qjfVvsGSislJgP1j15FiFiDiKOjbGTv9Q=",
+        "narHash": "sha256-LCMjGYtOaXMLCj5aRf/YHGPF32t5MhOqfwEuLbDo1sc=",
-        "rev": "036c2421db75172aeef4bb83921988891d4773c9",
+        "rev": "3770c18e5fe45a754c62d4d5081126195d9ab2ee",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/26.05-small/nixos-26.05.5692.036c2421db75/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/26.05-small/nixos-26.05.6345.3770c18e5fe4/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -319,11 +319,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1784536202,
+        "lastModified": 1785210918,
-        "narHash": "sha256-7CmHmK10lEXBgymS0DeYj4/4STLpzRWCc/5BfoIxHQo=",
+        "narHash": "sha256-gnVHwvnrk8Rh/44ryN50iuqoNzLqpNvSw9ZSNC9C4XM=",
-        "rev": "94f12be79cb7f961726a0157b4168bc722befeac",
+        "rev": "29a3e341634615ff16f700e9a1fdbc9c42f0198c",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/unstable-small/nixos-26.11pre1037880.94f12be79cb7/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/unstable-small/nixos-26.11pre1042951.29a3e3416346/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -379,11 +379,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1782759909,
+        "lastModified": 1784979981,
-        "narHash": "sha256-gktjBeZyoRvVBkm2cO1tD99fdQ34iUDyB6iecRdorm4=",
+        "narHash": "sha256-OnvjEXYU/InjJYH2+iIrvHWIjHnc5J9f84qWhrgAWlY=",
         "ref": "main",
-        "rev": "ad6c79fb713884a4a2df8aab30914cd0c1c2e6cb",
+        "rev": "ffb8ed0c9df2444727e49f16f7e43ea70184dc4b",
-        "revCount": 587,
+        "revCount": 590,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/nettsiden.git"
       },
@@ -447,17 +447,17 @@
         "rust-overlay": "rust-overlay_4"
       },
       "locked": {
-        "lastModified": 1784550731,
+        "lastModified": 1785060737,
-        "narHash": "sha256-KDrBGT98fQWlnEBmTvIijPq4n4sVLcETBu8EuL486SM=",
+        "narHash": "sha256-tlOCUK+X1Um1y/pP/vToGrygn1wmrtUDKrvQquPLVfA=",
         "ref": "main",
-        "rev": "c0c0e47c8a45673c62462cadf38a489b72fd39ec",
+        "rev": "35fe97baa795d1564c79b013d8a36ee01698f47b",
-        "revCount": 112,
+        "revCount": 126,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/roowho2.git"
       },
       "original": {
         "ref": "main",
-        "rev": "c0c0e47c8a45673c62462cadf38a489b72fd39ec",
+        "rev": "35fe97baa795d1564c79b013d8a36ee01698f47b",
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/roowho2.git"
       }
@@ -575,11 +575,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1780762731,
+        "lastModified": 1785100355,
-        "narHash": "sha256-EuDaLnasWN0mpi995n+fAQQfPGBhqNW4fNjlQRpHt58=",
+        "narHash": "sha256-wBGnYrF7Kfs4vk3djspT3RdQqsYiqm60fKn7/J3vH4I=",
         "ref": "main",
-        "rev": "931bd2d63e285e767c8c81c103dda3c0a63f2965",
+        "rev": "611f6ac1a304cc319fc85876d3f0301f49ce458e",
-        "revCount": 108,
+        "revCount": 120,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/worblehat.git"
       },

flake.nix

@@ -35,7 +35,7 @@
     minecraft-heatmap.url = "git+https://git.pvv.ntnu.no/Projects/minecraft-heatmap.git?ref=main";
     minecraft-heatmap.inputs.nixpkgs.follows = "nixpkgs";
 
-    roowho2.url = "git+https://git.pvv.ntnu.no/Projects/roowho2.git?ref=main&rev=c0c0e47c8a45673c62462cadf38a489b72fd39ec";
+    roowho2.url = "git+https://git.pvv.ntnu.no/Projects/roowho2.git?ref=main&rev=35fe97baa795d1564c79b013d8a36ee01698f47b";
     roowho2.inputs.nixpkgs.follows = "nixpkgs";
 
     greg-ng.url = "git+https://git.pvv.ntnu.no/Grzegorz/greg-ng.git?ref=main";
@@ -268,6 +268,7 @@
           modules = [
             inputs.nix-gitea-themes.nixosModules.default
             inputs.disko.nixosModules.disko
+            self.nixosModules.robots-txt
           ];
         };
 

hosts/bekkalokk/configuration.nix

@@ -13,6 +13,7 @@
     ./services/mediawiki
     ./services/nginx.nix
     ./services/phpfpm.nix
+    ./services/prometheus-phpfpm-exporter.nix
     ./services/vaultwarden.nix
     ./services/webmail
     ./services/website

hosts/bekkalokk/services/prometheus-phpfpm-exporter.nix

@@ -0,0 +1,54 @@
+{ config, lib, values, ... }:
+let
+  cfg = config.services.prometheus.exporters.php-fpm;
+  pools = [
+    "idp"
+    "mediawiki"
+    "pvv-nettsiden"
+    "roundcube"
+    "snappymail"
+  ];
+in
+{
+  services.phpfpm.pools = lib.genAttrs pools (_: {
+    settings."pm.status_path" = "/status";
+  });
+
+  services.prometheus.exporters.php-fpm = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    extraFlags = [
+      "--phpfpm.scrape-uri=${lib.concatMapStringsSep "," (name: "unix://${config.services.phpfpm.pools.${name}.socket};/status") pools}"
+    ];
+  };
+
+  systemd.services.prometheus-php-fpm-exporter.serviceConfig = {
+    Slice = "system-monitoring.slice";
+    SupplementaryGroups = [ config.services.nginx.group ];
+    RestrictAddressFamilies = lib.mkForce [
+      "AF_INET"
+      "AF_INET6"
+      "AF_UNIX"
+    ];
+  };
+
+  services.nginx = lib.mkIf cfg.enable {
+    virtualHosts."www.pvv.ntnu.no" = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-php-fpm-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
+}

hosts/bekkalokk/services/well-known/root/autoconfig/mail/config-v1.1.xml

@@ -16,8 +16,8 @@
 
     <outgoingServer type="smtp">
       <hostname>smtp.pvv.ntnu.no</hostname>
-      <port>587</port>
+      <port>465</port>
-      <socketType>STARTTLS</socketType>
+      <socketType>SSL</socketType>
       <username>%EMAILLOCALPART%</username>
       <authentication>password-cleartext</authentication>
       <useGlobalPreferredServer>true</useGlobalPreferredServer>

hosts/bicep/services/mysql/default.nix

@@ -4,7 +4,10 @@ let
   dataDir = "/data/mysql";
 in
 {
-  imports = [ ./backup.nix ];
+  imports = [
+    ./backup.nix
+    ./timed-jobs.nix
+  ];
 
   sops.secrets."mysql/password" = {
     owner = "mysql";
@@ -54,6 +57,18 @@ in
     mode = "0700";
   };
 
+  fileSystems.${dataDir} = lib.mkIf cfg.enable {
+    device = dataDir;
+    fsType = "none";
+    options = [
+      "bind"
+      "noatime"
+      "noauto"
+      "x-systemd.requires=systemd-tmpfiles-setup.service"
+      "x-systemd.requires=systemd-tmpfiles-resetup.service"
+    ];
+  };
+
   systemd.services.mysql = lib.mkIf cfg.enable {
     after = [
       "systemd-tmpfiles-setup.service"
@@ -61,6 +76,7 @@ in
     ];
 
     serviceConfig = {
+      RequiresMountsFor = [ dataDir ];
       BindPaths = [ "${dataDir}:${cfg.dataDir}" ];
 
       LogsDirectory = "mysql";

hosts/bicep/services/mysql/timed-jobs.nix

@@ -0,0 +1,48 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.mysql;
+in
+{
+  config = lib.mkIf cfg.enable {
+    systemd.services = {
+      mysql-analyze = {
+        requires = [ "mysql.service" ];
+        after = [ "mysql.service" ];
+        description = "Refresh MariaDB optimizer statistics for all databases";
+        startAt = "Mon 05:00:00";
+        serviceConfig = {
+          Type = "oneshot";
+          User = cfg.user;
+          Group = cfg.group;
+
+          Nice = 19;
+          IOSchedulingClass = "best-effort";
+          IOSchedulingPriority = 7;
+
+          ExecStart = "${lib.getExe' cfg.package "mariadb-check"} --all-databases --analyze";
+        };
+      };
+
+      mysql-optimize = {
+        requires = [ "mysql.service" ];
+        after = [ "mysql.service" ];
+        description = "Check, repair and optimize all MariaDB databases";
+        startAt = "*-*-01 04:00:00";
+        serviceConfig = {
+          Type = "oneshot";
+          User = cfg.user;
+          Group = cfg.group;
+
+          Nice = 19;
+          IOSchedulingClass = "best-effort";
+          IOSchedulingPriority = 7;
+
+          ExecStart = [
+            "${lib.getExe' cfg.package "mariadb-check"} --all-databases --auto-repair"
+            "${lib.getExe' cfg.package "mariadb-check"} --all-databases --optimize"
+          ];
+        };
+      };
+    };
+  };
+}

hosts/bicep/services/postgresql/default.nix

@@ -99,6 +99,18 @@ in
     mode = "0700";
   };
 
+  fileSystems."/data/postgresql" = lib.mkIf cfg.enable {
+    device = "/data/postgresql";
+    fsType = "none";
+    options = [
+      "bind"
+      "noatime"
+      "noauto"
+      "x-systemd.requires=systemd-tmpfiles-setup.service"
+      "x-systemd.requires=systemd-tmpfiles-resetup.service"
+    ];
+  };
+
   systemd.services.postgresql-setup = lib.mkIf cfg.enable {
     after = [
       "systemd-tmpfiles-setup.service"
@@ -110,6 +122,7 @@ in
         "key:/etc/certs/postgres.key"
       ];
 
+      RequiresMountsFor = [ "/data/postgresql" ];
       BindPaths = [ "/data/postgresql:/var/lib/postgresql" ];
     };
   };
@@ -125,6 +138,7 @@ in
         "key:/etc/certs/postgres.key"
       ];
 
+      RequiresMountsFor = [ "/data/postgresql" ];
       BindPaths = [ "/data/postgresql:/var/lib/postgresql" ];
     };
   };
@@ -136,7 +150,6 @@ in
   };
 
   networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ 5432 ];
-  networking.firewall.allowedUDPPorts = lib.mkIf cfg.enable [ 5432 ];
 
   environment.systemPackages = [
     (pkgs.writeShellApplication {

hosts/ildkule/services/monitoring/gatus/default.nix

@@ -5,6 +5,8 @@ in
 {
   imports = [
     ./minecraft-checker.nix
+    ./postgres-checker.nix
+    ./mariadb-checker.nix
   ];
 
   services.gatus = {
@@ -133,6 +135,18 @@ in
             "[BODY].ok == true"
           ];
         })
+        (mkService "PostgreSQL" "http://localhost:1338" // {
+          conditions = [
+            "[STATUS] == 200"
+            "[BODY].ok == true"
+          ];
+        })
+        (mkService "MariaDB" "http://localhost:1339" // {
+          conditions = [
+            "[STATUS] == 200"
+            "[BODY].ok == true"
+          ];
+        })
         (mkService "Email (SMTP)" "starttls://mail.pvv.ntnu.no:587")
         (mkService "Email (POP3)" "tls://mail.pvv.ntnu.no:995")
         (mkService "Email (IMAP)" "tls://mail.pvv.ntnu.no:993")

hosts/ildkule/services/monitoring/gatus/mariadb-checker.nix

@@ -0,0 +1,79 @@
+{ config, pkgs, ... }:
+{
+  sops.secrets."keys/gatus/mariadb" = {
+    restartUnits = [ "phh-gatus-mariadb-checker.service" ];
+  };
+
+  sops.templates."gatus-mariadb-checker.env" = {
+    restartUnits = [ "phh-gatus-mariadb-checker.service" ];
+    content = ''
+      MYSQL_HOST=mysql.pvv.ntnu.no
+      MYSQL_PORT=3306
+      MYSQL_DATABASE=mysql
+      MYSQL_USER=gatus_healthcheck
+      MYSQL_PASSWORD=${config.sops.placeholder."keys/gatus/mariadb"}
+    '';
+  };
+
+  services.python-http-handlers."gatus-mariadb-checker" = {
+    listenStreams = [ "127.0.0.1:1339" ];
+    libraries = with pkgs.python3Packages; [
+      pymysql
+    ];
+    serviceConfig = {
+      EnvironmentFile = config.sops.templates."gatus-mariadb-checker.env".path;
+    };
+    handler = ''
+      import os
+      import json
+      import pymysql
+
+      class Handler(BaseHTTPRequestHandler):
+          def do_GET(self):
+              try:
+                  conn = pymysql.connect(
+                      host=os.environ["MYSQL_HOST"],
+                      port=int(os.environ.get("MYSQL_PORT", "3306")),
+                      user=os.environ["MYSQL_USER"],
+                      password=os.environ["MYSQL_PASSWORD"],
+                      database=os.environ.get("MYSQL_DATABASE", "mysql"),
+                      connect_timeout=5,
+                  )
+
+                  try:
+                      with conn.cursor() as cur:
+                          cur.execute("SELECT VERSION();")
+                          (version,) = cur.fetchone()
+
+                          cur.execute("SHOW STATUS LIKE 'Threads_connected';")
+                          (_, connections) = cur.fetchone()
+
+                          cur.execute("SHOW DATABASES;")
+                          databases = cur.fetchall()
+                  finally:
+                      conn.close()
+
+                  body = {
+                      "ok": True,
+                      "version": version,
+                      "connections": int(connections),
+                      "databases": len(databases),
+                  }
+                  data = json.dumps(body).encode()
+
+                  self.send_response(200)
+                  self.send_header("Content-Type", "application/json")
+                  self.send_header("Content-Length", str(len(data)))
+                  self.end_headers()
+                  self.wfile.write(data)
+
+              except Exception as e:
+                  data = json.dumps({"ok": False, "error": str(e)}).encode()
+                  self.send_response(500)
+                  self.send_header("Content-Type", "application/json")
+                  self.send_header("Content-Length", str(len(data)))
+                  self.end_headers()
+                  self.wfile.write(data)
+    '';
+  };
+}

hosts/ildkule/services/monitoring/gatus/postgres-checker.nix

@@ -0,0 +1,79 @@
+{ config, pkgs, ... }:
+{
+  sops.secrets."keys/gatus/postgres" = {
+    restartUnits = [ "phh-gatus-postgres-checker.service" ];
+  };
+
+  sops.templates."gatus-postgres-checker.env" = {
+    restartUnits = [ "phh-gatus-postgres-checker.service" ];
+    content = ''
+      PGHOST=postgres.pvv.ntnu.no
+      PGPORT=5432
+      PGDATABASE=postgres
+      PGUSER=gatus_healthcheck
+      PGPASSWORD=${config.sops.placeholder."keys/gatus/postgres"}
+    '';
+  };
+
+  services.python-http-handlers."gatus-postgres-checker" = {
+    listenStreams = [ "127.0.0.1:1338" ];
+    libraries = with pkgs.python3Packages; [
+      psycopg2
+    ];
+    serviceConfig = {
+      EnvironmentFile = config.sops.templates."gatus-postgres-checker.env".path;
+    };
+    handler = ''
+      import os
+      import json
+      import psycopg2
+
+      class Handler(BaseHTTPRequestHandler):
+          def do_GET(self):
+              try:
+                  conn = psycopg2.connect(
+                      host=os.environ["PGHOST"],
+                      port=os.environ.get("PGPORT", "5432"),
+                      dbname=os.environ.get("PGDATABASE", "postgres"),
+                      user=os.environ["PGUSER"],
+                      password=os.environ["PGPASSWORD"],
+                      connect_timeout=5,
+                  )
+
+                  try:
+                      with conn.cursor() as cur:
+                          cur.execute("SELECT version();")
+                          (version,) = cur.fetchone()
+
+                          cur.execute("SELECT count(*) FROM pg_stat_activity;")
+                          (connections,) = cur.fetchone()
+
+                          cur.execute("SELECT count(*) FROM pg_database WHERE NOT datistemplate;")
+                          (databases,) = cur.fetchone()
+                  finally:
+                      conn.close()
+
+                  body = {
+                      "ok": True,
+                      "version": version,
+                      "connections": connections,
+                      "databases": databases,
+                  }
+                  data = json.dumps(body).encode()
+
+                  self.send_response(200)
+                  self.send_header("Content-Type", "application/json")
+                  self.send_header("Content-Length", str(len(data)))
+                  self.end_headers()
+                  self.wfile.write(data)
+
+              except Exception as e:
+                  data = json.dumps({"ok": False, "error": str(e)}).encode()
+                  self.send_response(500)
+                  self.send_header("Content-Type", "application/json")
+                  self.send_header("Content-Length", str(len(data)))
+                  self.end_headers()
+                  self.wfile.write(data)
+    '';
+  };
+}

hosts/ildkule/services/monitoring/prometheus/default.nix

@@ -7,6 +7,7 @@ in {
     ./machines.nix
     ./matrix-synapse.nix
     ./mysqld.nix
+    ./phpfpm.nix
     ./postgres.nix
   ];
 

hosts/ildkule/services/monitoring/prometheus/phpfpm.nix

@@ -0,0 +1,15 @@
+{ ... }:
+{
+  services.prometheus.scrapeConfigs = [{
+    job_name = "phpfpm";
+    scheme = "https";
+    metrics_path = "/prometheus-php-fpm-exporter/metrics";
+
+    static_configs = [
+      {
+        labels.hostname = "bekkalokk";
+        targets = [ "www.pvv.ntnu.no:443" ];
+      }
+    ];
+  }];
+}

hosts/kommode/services/gitea/default.nix

@@ -196,6 +196,108 @@ in {
     };
   };
 
+  environment.robots-txt."gitea" = {
+    virtualHost = domain;
+    rules = [
+      {
+        pre_comment = ''
+          Gitea internals
+
+          See these for more information:
+          - https://gitea.com/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = "*";
+        Disallow = [
+          "/api/*"
+          "/avatars"
+          "/*/*/src/commit/*"
+          "/*/*/commit/*"
+          "/*/*/*/refs/*"
+          "/*/*/*/star"
+          "/*/*/*/watch"
+          "/*/*/labels"
+          "/*/*/activity/*"
+          "/vendor/*"
+          "/swagger.*.json"
+          "/repo/create"
+          "/repo/migrate"
+          "/org/create"
+          "/*/*/fork"
+          "/*/*/watchers"
+          "/*/*/stargazers"
+          "/*/*/forks"
+          "*/.git/"
+          "/*.git"
+          "/*.atom"
+          "/*.rss"
+        ];
+        Crawl-delay = "2";
+      }
+      {
+        pre_comment = "Language Spam";
+        Disallow = "/*?lang=";
+      }
+      {
+        pre_comment = ''
+          AI bots
+
+          Sourced from:
+          - https://www.vg.no/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = [
+          "AI2Bot"
+          "Ai2Bot-Dolma"
+          "Amazonbot"
+          "Applebot-Extended"
+          "Bytespider"
+          "CCBot"
+          "ChatGPT-User"
+          "Claude-Web"
+          "ClaudeBot"
+          "Crawlspace"
+          "Diffbot"
+          "FacebookBot"
+          "FriendlyCrawler"
+          "GPTBot"
+          "Google-Extended"
+          "ICC-Crawler"
+          "ImagesiftBot"
+          "Kangaroo Bot"
+          "Meta-ExternalAgent"
+          "OAI-SearchBot"
+          "Omgili"
+          "Omgilibot"
+          "PanguBot"
+          "PerplexityBot"
+          "PetalBot"
+          "Scrapy"
+          "SemrushBot-OCOB"
+          "Sidetrade indexer bot"
+          "Timpibot"
+          "VelenPublicWebCrawler"
+          "Webzio-Extended"
+          "YouBot"
+          "anthropic-ai"
+          "cohere-ai"
+          "cohere-training-data-crawler"
+          "facebookexternalhit"
+          "iaskspider/2.0"
+          "img2dataset"
+          "meta-externalagent"
+          "omgili"
+          "omgilibot"
+        ];
+        Disallow = "/";
+        Crawl-delay = "2";
+      }
+      {
+        Sitemap = "https://${domain}/sitemap.xml";
+      }
+    ];
+  };
+
   networking.firewall.allowedTCPPorts = [ sshPort ];
 
   services.rsync-pull-targets = {

hosts/temmie/services/userweb/mail.nix

@@ -1,14 +1,13 @@
 { config, lib, pkgs, ... }:
 {
-  services.postfix.enable = lib.mkForce false;
+  assertions = [{
+    assertion = config.services.nullmailer.enable;
+    message = ''
+      Expected nullmailer to be enabled for temmie userweb.
 
-  services.nullmailer = {
+      If you change the default sendmail implementation, please make sure that temmie userweb works correctly!
-    enable = true;
+    '';
-    config = {
+  }];
-      me = config.networking.fqdn;
-      remotes = "mail.pvv.ntnu.no smtp --port=25";
-    };
-  };
 
   services.bro = {
     enable = true;

modules/python-http-handlers.nix

@@ -36,6 +36,9 @@ in
           description = ''
             For each item in this list, a `ListenStream`
             option in the `[Socket]` section will be created.
+
+            Only a single `ListenStream` is currently supported by the handler script; if
+            you need more than one, you'll have to adjust {option}`handler` accordingly.
           '';
         };
 
@@ -98,8 +101,10 @@ in
             "E303" # too many blank lines
             "E305" # expected 2 blank lines after end of function or class
             "E306" # expected 1 blank line before a nested definition
+            "E402" # module level import not at top of file
             "E501" # max line length
             "E704" # multiple statements on one line (def)
+            "F811" # redefined while unused
           ];
           description = ''
             A list of flake8 rules to ignore while linting the python code.
@@ -122,6 +127,11 @@ in
                     self.send_header("Content-Length", str(len(data)))
                     self.end_headers()
                     self.wfile.write(data)
+
+            def on_reload():
+                # This function is called when the service receives SIGHUP (e.g. via `systemctl reload`).
+                # You can use it to clear caches or re-read state. Completely optional
+                pass
           '';
           description = ''
             Python code including the HTTP handler for the server.
@@ -145,6 +155,7 @@ in
           inherit (v) listenStreams;
           socketConfig = {
             Accept = false;
+            FileDescriptorName = v.name;
           } // v.socketConfig;
         };
       }))
@@ -157,30 +168,126 @@ in
         inherit (v) name;
         value = {
           serviceConfig = {
-            Type = "simple";
+            Type = "notify-reload";
+            NotifyAccess = "main";
             DynamicUser = true;
+            TimeoutStopSec = "35s";
 
             ExecStart = let
               package = pkgs.writers.writePython3Bin "${v.name}-bin" {
                 inherit (v) libraries flakeIgnore;
               } ''
+                import os
+                import signal
                 import socket
+                import socketserver
+                import threading
+                import time
                 from http.server import HTTPServer, BaseHTTPRequestHandler
 
-                ${v.handler}
+                SOCKET_NAME = "${v.name}"
+                SHUTDOWN_TIMEOUT = 30
+
+                def sd_notify(message: str):
+                    addr = os.environ.get("NOTIFY_SOCKET")
+                    if not addr:
+                        return
+                    if addr[0] == "@":
+                        addr = "\0" + addr[1:]
+                    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_CLOEXEC) as sock:
+                        sock.connect(addr)
+                        sock.sendall(message.encode())
+
+                def sd_listen_fd(name: str) -> int:
+                    if os.environ.get("LISTEN_PID") != str(os.getpid()):
+                        raise RuntimeError("No sockets were passed to this service by systemd")
+
+                    try:
+                        count = int(os.environ.get("LISTEN_FDS", "0"))
+                    except ValueError:
+                        count = 0
+
+                    raw_names = os.environ.get("LISTEN_FDNAMES")
+                    names = raw_names.split(":") if raw_names else []
+
+                    for i in range(count):
+                        if i < len(names) and names[i] == name:
+                            return 3 + i
+
+                    raise RuntimeError(
+                        "No systemd socket named %r was passed to this service; check the "
+                        "FileDescriptorName= of the corresponding .socket unit" % name
+                    )
+
+                class Server(socketserver.ThreadingMixIn, HTTPServer):
+                    daemon_threads = True
 
-                class NoBindHTTPServer(HTTPServer):
                     def server_bind(): pass
                     def server_activate(): pass
 
+                    def __init__(self, *args, **kwargs):
+                        super().__init__(*args, **kwargs)
+                        self._request_threads = []
+                        self._request_threads_lock = threading.Lock()
+
+                    def process_request(self, request, client_address):
+                        thread = threading.Thread(
+                            target=self.process_request_thread,
+                            args=(request, client_address),
+                        )
+                        thread.daemon = self.daemon_threads
+                        with self._request_threads_lock:
+                            self._request_threads.append(thread)
+                        thread.start()
+
+                    def join_request_threads(self, timeout):
+                        deadline = time.monotonic() + timeout
+                        with self._request_threads_lock:
+                            threads = list(self._request_threads)
+                        for thread in threads:
+                            thread.join(max(deadline - time.monotonic(), 0))
+
+                def handle_reload(signum, frame):
+                    monotonic_usec = time.clock_gettime_ns(time.CLOCK_MONOTONIC) // 1000
+                    sd_notify("RELOADING=1\nMONOTONIC_USEC=%d" % monotonic_usec)
+
+                    on_reload = globals().get("on_reload")
+                    if callable(on_reload):
+                        on_reload()
+
+                    sd_notify("READY=1")
+
+                shutdown_requested = threading.Event()
+
+                def handle_sigterm(signum, frame):
+                    sd_notify("STOPPING=1")
+                    shutdown_requested.set()
+
+                ${v.handler}
+
+                assert "Handler" in globals(), "You must define a class Handler(BaseHTTPRequestHandler) in the handler code"
+
                 def main():
-                    httpd = NoBindHTTPServer(
+                    signal.signal(signal.SIGHUP, handle_reload)
-                        ("", 0),
+                    signal.signal(signal.SIGTERM, handle_sigterm)
-                        Handler,
+
-                        bind_and_activate=False,
+                    fd = sd_listen_fd(SOCKET_NAME)
-                    )
+
-                    httpd.socket = socket.fromfd(3, socket.AF_INET, socket.SOCK_STREAM)
+                    httpd = Server(("", 0), Handler, bind_and_activate=False)
-                    httpd.serve_forever()
+                    httpd.socket = socket.socket(fileno=fd)
+
+                    server_thread = threading.Thread(target=httpd.serve_forever, name="http-server", daemon=True)
+                    server_thread.start()
+
+                    sd_notify("READY=1")
+                    shutdown_requested.wait()
+
+                    deadline = time.monotonic() + SHUTDOWN_TIMEOUT
+
+                    httpd.shutdown()
+                    server_thread.join(max(deadline - time.monotonic(), 0))
+                    httpd.join_request_threads(max(deadline - time.monotonic(), 0))
+                    httpd.server_close()
 
                 if __name__ == '__main__':
                     main()

secrets/ildkule/ildkule.yaml

@@ -1,6 +1,9 @@
 config:
     mysqld_exporter_password: ENC[AES256_GCM,data:I9K+QMqaN3FOOVKzeOR9Q6UERStXX0P8WEHyN1jzzbM=,iv:UxvIdlfAyJvNuxPkU4+guKPa0fiD0vVLzHOTYktcmso=,tag:ltnIqEwESYx9HBu8UN0ZLw==,type:str]
 keys:
+    gatus:
+        mariadb: ENC[AES256_GCM,data:U7CGtjHacJlKcReEw3JO86okCVgLAkP92iX1bHGNw8qeImpvk5mNw77Ox7tbsChvozrJWa/Npu2pTBsNnVYYYA==,iv:hWa+KPiBc3AnH1tHCGJa8LhPXZ1o8sxufpGe7nUjV0E=,tag:mso5HEfmWAOGw350408X7g==,type:str]
+        postgres: ENC[AES256_GCM,data:7fI0JeZDK1MX5poXWaMo67PXbjdjvM+xPKioHSHmGdS8iNDGe5Ym6LlS/hOnPvvDuNFyHjKKizqDst0227U2gg==,iv:D3bz/xyFrnQsUCiMEFq8JOjSudyiw+idEMWQ1e6Uk3k=,tag:8XqadKVHK9NMWehK7V5J0Q==,type:str]
     grafana:
         secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
         admin_password: ENC[AES256_GCM,data:ttKwfC4WuXeL/6x4,iv:x1X+e3z08CR992GzC62YnFIN7SGrE81/nDNrgcgVzx0=,tag:YajUoy61kYbpeGeC7yNrXQ==,type:str]
@@ -9,8 +12,7 @@ keys:
         postgres_exporter_knakelibrak_env: ENC[AES256_GCM,data:xjC7DGXrW2GIJq8XioIZb+jSe/Hzcz0tv9cUHmX/n1nhI+D64lYt+EKnq1+RX/vJzU4sTaKjveKBh88Qqnv6RQm+MZC//dIxcvnnAdl50qnHZyBCaFFEzSNI8I8vGyArMk8Ja72clBq3kMpUz/pLBP0qDrjblKDoWkU=,iv:ZW98hJy8A5t4Oxtu17R3tM7gou183VLbgBsHA8LFuJY=,tag:VMOvQz3X/XDylV1YFg2Jsg==,type:str]
 sops:
     age:
-        - recipient: age102e6y8gah0ntr6fxqnkpepc8ar29p6ls7ks9ka7v8w87q8scm9yqmc2u8d
+        - enc: |
-          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAyTWRSM3IwMmxtTmZVcCsw
             OUhlakxHZzgrSEhEdUZFTXE1anNjQ2wvdkZFCnB6S1l3TXQ3ZGFYWmtYM1cwMFZT
@@ -18,8 +20,8 @@ sops:
             YlltQ3FBU3RBYUx4TnNPRk1SUWNqZG8KAJjc09x553ncaWduGLsnIHdroaOmMasP
             /fq0GzW6UNfmE2rQ6qrQti21B37/sN0WMLCSPLUPG45kBgx20GG4hQ==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
+          recipient: age102e6y8gah0ntr6fxqnkpepc8ar29p6ls7ks9ka7v8w87q8scm9yqmc2u8d
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTOWFvcFFzc1lmNVdmV3lX
             cVVLNHowcGdENzI5UUJLZTNSMHhjNlI2c0FnCmdwOS9oL1kwTnhwbXRodWxxWVE3
@@ -27,8 +29,8 @@ sops:
             eDVwd2dKMG9FRW1OY1pyUkhLeWw3b0EKtJpsQ/Ss39ZLiRNqUhn8sdB3hpQy7Syv
             ererqhMkqmDugGEHPk6KpZuj7DVSK1di7JgA2qZOUPzI7UpxjaC0Kg==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
+          recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOQU1iVXRkQmo4b3F1Vngz
             S0pkNkVFR1FUb1djdmI0eHh3V3BBTDJTSTJZClA0S0Z0cTdFRmRaOEZQdHQzdGZ3
@@ -36,8 +38,8 @@ sops:
             REdjRFZyY2pNdEd6cmgvQisyVDhLUEkKRItJ0CGbzlEB5RNAyem4feMVhTfcLef3
             QIqltZ2l4LLexnkECi3FCJZHxrbUa+/RF6p1DsueUw7LLUnOcphB9A==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
+          recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvZTBpaWM0c1hmODBaK0Iw
             bmp1NzRacklXMHU0K2J2d0g5ajBiNlRNWGhRCm1DOWI0cm5BdTdlNmFzM2JVekNk
@@ -45,8 +47,8 @@ sops:
             U0ZCT2toZ1ZMZ3E0bXRhSTQvNGFWNVkKhxfQDIDe2LQW7OMBJv0J267AW1wI32df
             ZQxd657TEqzm7i19azrCS0jyRbfj2MYzEJAtTGiGZaNC9uKDFzBhKw==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
+          recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBmRlJROHRKb21YUnlicmc1
             MlptQllEcXFhajNKS1krMDdUMWk5QWo4eVJBCndGSlhXS1Vaa2RSTllIcmF1ZVpl
@@ -54,8 +56,8 @@ sops:
             TE41aFdjU0h0ekQ2Zjg4Z3VQVjFWcnMK6zjSalqeYjyc4NH6nOeghlhYJydrz4pM
             N5ZcXjRbrIVFdhbYnvQGKvGKZm0kK6vjzBjdT7BM6ctr8cq/qrz1xQ==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
+          recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIRXpBa2tYc2xEeldub0VK
             bkFwOFdlUGRZM0FVT0tyUW1RWnl0TjRUTUh3CnZlMC92MU1hRW1yZU1NSUdoUEZh
@@ -63,8 +65,8 @@ sops:
             aXFQWlNVQ2laVm1ETStRemNZRXc3TUEKlPYSU3gp67dsPfbEJkru4ieMvspC7+pu
             rfp315HLyj1FGhrA8f2qOxE/PYI2rn0yKm80KffWBV7ylX/uonm4Fg==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
+          recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsaTEzOFBEeG9LVThSVmQ5
             VXBoaFpueFRCbFJ1akE4RWc1aE1HUmVGcXdZCjFnbU0wd2drazNsTmNBMHNuOFhO
@@ -72,8 +74,9 @@ sops:
             dC9meDZlc3d3aUJEVjc4REF0Y1BLcGcK79LbJzc5KVgEgyJR11crGuX8YcVoJBbT
             Fin7Zoon06L7qx0Zw5u27wV7RKMnYT7hOMiWs6660ZTLcYJ5M1aEZQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-03-16T20:08:18Z"
+          recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
-    mac: ENC[AES256_GCM,data:C2tpWppc13jKJq5d4nmAKQOaNWHm27TKwxAxm1fi2lejN1lqUaoz5bHfTBA7MfaWvuP5uZnfbtG32eeu48mnlWpo58XRUFFecAhb9JUpW9s5IR3/nbzLNkGU7H5C0oWPrxI4thd+bAVduIgBjjFyGj1pe6J9db3c0yUWRwNlwGU=,iv:YpoQ4psiFYOWLGipxv1QvRvr034XFsyn2Bhyy39HmOo=,tag:ByiCWygFC/VokVTbdLoLgg==,type:str]
+    lastmodified: "2026-07-21T09:40:54Z"
+    mac: ENC[AES256_GCM,data:FLpmsW9I21qNMFs6Tn/1AzFo+RcmCwtVdAaaGIrGlUi2l3gzb//ZK6ruI/VYx1R5DoX1OhD3EIg6aLMSOT6jiHf+8GwfhdijpayG1nrgeZW3SuFbjVlCYNMPH3DOJJB82U3T+ozSbpnZEtdUwmNWcZJt5F2hOEnoaNa+jLEKDao=,iv:qyQuyWuu+w2ynRvhIx0QVtw+tieO3dbQsUxEB5Brzp0=,tag:qYAj4h6foWm16YqIdZ6xbQ==,type:str]
     pgp:
         - created_at: "2026-05-20T17:35:58Z"
           enc: |-
@@ -96,4 +99,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.9.4
+    version: 3.13.2