mitigations: patch matrix-synapse

This commit extends the python http handler template with the following
things:

- Integration with systemd notify
- Named systemd sockets, to avoid depending on expected FD number
- Graceful and customizable handling of SIGHUP reloads
- Graceful shutdown on SIGTERM with timeout
- Multithreaded HTTP server, one thread per request
- Better error message on undefined handler

base/mitigations.nix

@@ -21,4 +21,18 @@ in
   boot.blacklistedKernelModules = modulesToBan;
 
   boot.extraModprobeConfig = lib.concatMapStringsSep "\n" (mod: "install ${mod} ${lib.getExe' pkgs.coreutils "false"}") modulesToBan;
+
+  nixpkgs.overlays = [
+    (final: prev: {
+      matrix-synapse-unwrapped = prev.matrix-synapse-unwrapped.overrideAttrs (old: rec {
+        version = "1.157.2";
+        src = final.fetchFromGitHub {
+          owner = "element-hq";
+          repo = "synapse";
+          rev = "v${version}";
+          hash = "sha256-TUHcNAXrV43+J7jqfstlYdrZrwL5kDCh03yxO+vL/gw=";
+        };
+      });
+    })
+  ];
 }

base/services/journald-upload.nix

@@ -9,7 +9,7 @@ in
       URL = "https://journald.pvv.ntnu.no:${toString config.services.journald.remote.port}";
       ServerKeyFile = "-";
       ServerCertificateFile = "-";
-      TrustedCertificateFile = "-";
+      TrustedCertificateFile = config.security.pki.caBundle;
     };
   };
 

flake.lock

@@ -126,11 +126,11 @@
         "rust-overlay": "rust-overlay_2"
       },
       "locked": {
-        "lastModified": 1784553736,
+        "lastModified": 1784709844,
-        "narHash": "sha256-W1pPlTHmp+EvONwf8Qa5NvCiuyV4LC/yNDdc27RSpuA=",
+        "narHash": "sha256-5PW5+UEFFjsru5FiDg9ZRUjQyo7AVyc+nesQlBSqM6c=",
         "ref": "main",
-        "rev": "e06daaef1f48160254c202c59b060ac6f1da9f38",
+        "rev": "a8fb5df19d1cdcc3ac29cdb8ad192c1d578761f0",
-        "revCount": 62,
+        "revCount": 63,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Grzegorz/greg-ng.git"
       },
@@ -291,11 +291,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1784526640,
+        "lastModified": 1785211367,
-        "narHash": "sha256-UDnuu3uM64qjfVvsGSislJgP1j15FiFiDiKOjbGTv9Q=",
+        "narHash": "sha256-LCMjGYtOaXMLCj5aRf/YHGPF32t5MhOqfwEuLbDo1sc=",
-        "rev": "036c2421db75172aeef4bb83921988891d4773c9",
+        "rev": "3770c18e5fe45a754c62d4d5081126195d9ab2ee",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/26.05-small/nixos-26.05.5692.036c2421db75/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/26.05-small/nixos-26.05.6345.3770c18e5fe4/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -319,11 +319,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1784536202,
+        "lastModified": 1785210918,
-        "narHash": "sha256-7CmHmK10lEXBgymS0DeYj4/4STLpzRWCc/5BfoIxHQo=",
+        "narHash": "sha256-gnVHwvnrk8Rh/44ryN50iuqoNzLqpNvSw9ZSNC9C4XM=",
-        "rev": "94f12be79cb7f961726a0157b4168bc722befeac",
+        "rev": "29a3e341634615ff16f700e9a1fdbc9c42f0198c",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/unstable-small/nixos-26.11pre1037880.94f12be79cb7/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/unstable-small/nixos-26.11pre1042951.29a3e3416346/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -379,11 +379,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1782759909,
+        "lastModified": 1784979981,
-        "narHash": "sha256-gktjBeZyoRvVBkm2cO1tD99fdQ34iUDyB6iecRdorm4=",
+        "narHash": "sha256-OnvjEXYU/InjJYH2+iIrvHWIjHnc5J9f84qWhrgAWlY=",
         "ref": "main",
-        "rev": "ad6c79fb713884a4a2df8aab30914cd0c1c2e6cb",
+        "rev": "ffb8ed0c9df2444727e49f16f7e43ea70184dc4b",
-        "revCount": 587,
+        "revCount": 590,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/nettsiden.git"
       },
@@ -447,17 +447,17 @@
         "rust-overlay": "rust-overlay_4"
       },
       "locked": {
-        "lastModified": 1784550731,
+        "lastModified": 1785060737,
-        "narHash": "sha256-KDrBGT98fQWlnEBmTvIijPq4n4sVLcETBu8EuL486SM=",
+        "narHash": "sha256-tlOCUK+X1Um1y/pP/vToGrygn1wmrtUDKrvQquPLVfA=",
         "ref": "main",
-        "rev": "c0c0e47c8a45673c62462cadf38a489b72fd39ec",
+        "rev": "35fe97baa795d1564c79b013d8a36ee01698f47b",
-        "revCount": 112,
+        "revCount": 126,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/roowho2.git"
       },
       "original": {
         "ref": "main",
-        "rev": "c0c0e47c8a45673c62462cadf38a489b72fd39ec",
+        "rev": "35fe97baa795d1564c79b013d8a36ee01698f47b",
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/roowho2.git"
       }
@@ -575,11 +575,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1780762731,
+        "lastModified": 1785100355,
-        "narHash": "sha256-EuDaLnasWN0mpi995n+fAQQfPGBhqNW4fNjlQRpHt58=",
+        "narHash": "sha256-wBGnYrF7Kfs4vk3djspT3RdQqsYiqm60fKn7/J3vH4I=",
         "ref": "main",
-        "rev": "931bd2d63e285e767c8c81c103dda3c0a63f2965",
+        "rev": "611f6ac1a304cc319fc85876d3f0301f49ce458e",
-        "revCount": 108,
+        "revCount": 120,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/worblehat.git"
       },

flake.nix

@@ -35,7 +35,7 @@
     minecraft-heatmap.url = "git+https://git.pvv.ntnu.no/Projects/minecraft-heatmap.git?ref=main";
     minecraft-heatmap.inputs.nixpkgs.follows = "nixpkgs";
 
-    roowho2.url = "git+https://git.pvv.ntnu.no/Projects/roowho2.git?ref=main&rev=c0c0e47c8a45673c62462cadf38a489b72fd39ec";
+    roowho2.url = "git+https://git.pvv.ntnu.no/Projects/roowho2.git?ref=main&rev=35fe97baa795d1564c79b013d8a36ee01698f47b";
     roowho2.inputs.nixpkgs.follows = "nixpkgs";
 
     greg-ng.url = "git+https://git.pvv.ntnu.no/Grzegorz/greg-ng.git?ref=main";

hosts/bekkalokk/configuration.nix

@@ -13,6 +13,7 @@
     ./services/mediawiki
     ./services/nginx.nix
     ./services/phpfpm.nix
+    ./services/prometheus-phpfpm-exporter.nix
     ./services/vaultwarden.nix
     ./services/webmail
     ./services/website

hosts/bekkalokk/services/prometheus-phpfpm-exporter.nix

@@ -0,0 +1,54 @@
+{ config, lib, values, ... }:
+let
+  cfg = config.services.prometheus.exporters.php-fpm;
+  pools = [
+    "idp"
+    "mediawiki"
+    "pvv-nettsiden"
+    "roundcube"
+    "snappymail"
+  ];
+in
+{
+  services.phpfpm.pools = lib.genAttrs pools (_: {
+    settings."pm.status_path" = "/status";
+  });
+
+  services.prometheus.exporters.php-fpm = {
+    enable = true;
+    listenAddress = "127.0.0.1";
+    extraFlags = [
+      "--phpfpm.scrape-uri=${lib.concatMapStringsSep "," (name: "unix://${config.services.phpfpm.pools.${name}.socket};/status") pools}"
+    ];
+  };
+
+  systemd.services.prometheus-php-fpm-exporter.serviceConfig = {
+    Slice = "system-monitoring.slice";
+    SupplementaryGroups = [ config.services.nginx.group ];
+    RestrictAddressFamilies = lib.mkForce [
+      "AF_INET"
+      "AF_INET6"
+      "AF_UNIX"
+    ];
+  };
+
+  services.nginx = lib.mkIf cfg.enable {
+    virtualHosts."www.pvv.ntnu.no" = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-php-fpm-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
+}

hosts/bekkalokk/services/well-known/root/autoconfig/mail/config-v1.1.xml

@@ -16,8 +16,8 @@
 
     <outgoingServer type="smtp">
       <hostname>smtp.pvv.ntnu.no</hostname>
-      <port>587</port>
+      <port>465</port>
-      <socketType>STARTTLS</socketType>
+      <socketType>SSL</socketType>
       <username>%EMAILLOCALPART%</username>
       <authentication>password-cleartext</authentication>
       <useGlobalPreferredServer>true</useGlobalPreferredServer>

hosts/bicep/services/postgresql/default.nix

@@ -150,7 +150,6 @@ in
   };
 
   networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ 5432 ];
-  networking.firewall.allowedUDPPorts = lib.mkIf cfg.enable [ 5432 ];
 
   environment.systemPackages = [
     (pkgs.writeShellApplication {

hosts/ildkule/services/monitoring/prometheus/default.nix

@@ -7,6 +7,7 @@ in {
     ./machines.nix
     ./matrix-synapse.nix
     ./mysqld.nix
+    ./phpfpm.nix
     ./postgres.nix
   ];
 

hosts/ildkule/services/monitoring/prometheus/phpfpm.nix

@@ -0,0 +1,15 @@
+{ ... }:
+{
+  services.prometheus.scrapeConfigs = [{
+    job_name = "phpfpm";
+    scheme = "https";
+    metrics_path = "/prometheus-php-fpm-exporter/metrics";
+
+    static_configs = [
+      {
+        labels.hostname = "bekkalokk";
+        targets = [ "www.pvv.ntnu.no:443" ];
+      }
+    ];
+  }];
+}

modules/python-http-handlers.nix

@@ -36,6 +36,9 @@ in
           description = ''
             For each item in this list, a `ListenStream`
             option in the `[Socket]` section will be created.
+
+            Only a single `ListenStream` is currently supported by the handler script; if
+            you need more than one, you'll have to adjust {option}`handler` accordingly.
           '';
         };
 
@@ -98,8 +101,10 @@ in
             "E303" # too many blank lines
             "E305" # expected 2 blank lines after end of function or class
             "E306" # expected 1 blank line before a nested definition
+            "E402" # module level import not at top of file
             "E501" # max line length
             "E704" # multiple statements on one line (def)
+            "F811" # redefined while unused
           ];
           description = ''
             A list of flake8 rules to ignore while linting the python code.
@@ -122,6 +127,11 @@ in
                     self.send_header("Content-Length", str(len(data)))
                     self.end_headers()
                     self.wfile.write(data)
+
+            def on_reload():
+                # This function is called when the service receives SIGHUP (e.g. via `systemctl reload`).
+                # You can use it to clear caches or re-read state. Completely optional
+                pass
           '';
           description = ''
             Python code including the HTTP handler for the server.
@@ -145,6 +155,7 @@ in
           inherit (v) listenStreams;
           socketConfig = {
             Accept = false;
+            FileDescriptorName = v.name;
           } // v.socketConfig;
         };
       }))
@@ -157,30 +168,126 @@ in
         inherit (v) name;
         value = {
           serviceConfig = {
-            Type = "simple";
+            Type = "notify-reload";
+            NotifyAccess = "main";
             DynamicUser = true;
+            TimeoutStopSec = "35s";
 
             ExecStart = let
               package = pkgs.writers.writePython3Bin "${v.name}-bin" {
                 inherit (v) libraries flakeIgnore;
               } ''
+                import os
+                import signal
                 import socket
+                import socketserver
+                import threading
+                import time
                 from http.server import HTTPServer, BaseHTTPRequestHandler
 
-                ${v.handler}
+                SOCKET_NAME = "${v.name}"
+                SHUTDOWN_TIMEOUT = 30
+
+                def sd_notify(message: str):
+                    addr = os.environ.get("NOTIFY_SOCKET")
+                    if not addr:
+                        return
+                    if addr[0] == "@":
+                        addr = "\0" + addr[1:]
+                    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_CLOEXEC) as sock:
+                        sock.connect(addr)
+                        sock.sendall(message.encode())
+
+                def sd_listen_fd(name: str) -> int:
+                    if os.environ.get("LISTEN_PID") != str(os.getpid()):
+                        raise RuntimeError("No sockets were passed to this service by systemd")
+
+                    try:
+                        count = int(os.environ.get("LISTEN_FDS", "0"))
+                    except ValueError:
+                        count = 0
+
+                    raw_names = os.environ.get("LISTEN_FDNAMES")
+                    names = raw_names.split(":") if raw_names else []
+
+                    for i in range(count):
+                        if i < len(names) and names[i] == name:
+                            return 3 + i
+
+                    raise RuntimeError(
+                        "No systemd socket named %r was passed to this service; check the "
+                        "FileDescriptorName= of the corresponding .socket unit" % name
+                    )
+
+                class Server(socketserver.ThreadingMixIn, HTTPServer):
+                    daemon_threads = True
 
-                class NoBindHTTPServer(HTTPServer):
                     def server_bind(): pass
                     def server_activate(): pass
 
+                    def __init__(self, *args, **kwargs):
+                        super().__init__(*args, **kwargs)
+                        self._request_threads = []
+                        self._request_threads_lock = threading.Lock()
+
+                    def process_request(self, request, client_address):
+                        thread = threading.Thread(
+                            target=self.process_request_thread,
+                            args=(request, client_address),
+                        )
+                        thread.daemon = self.daemon_threads
+                        with self._request_threads_lock:
+                            self._request_threads.append(thread)
+                        thread.start()
+
+                    def join_request_threads(self, timeout):
+                        deadline = time.monotonic() + timeout
+                        with self._request_threads_lock:
+                            threads = list(self._request_threads)
+                        for thread in threads:
+                            thread.join(max(deadline - time.monotonic(), 0))
+
+                def handle_reload(signum, frame):
+                    monotonic_usec = time.clock_gettime_ns(time.CLOCK_MONOTONIC) // 1000
+                    sd_notify("RELOADING=1\nMONOTONIC_USEC=%d" % monotonic_usec)
+
+                    on_reload = globals().get("on_reload")
+                    if callable(on_reload):
+                        on_reload()
+
+                    sd_notify("READY=1")
+
+                shutdown_requested = threading.Event()
+
+                def handle_sigterm(signum, frame):
+                    sd_notify("STOPPING=1")
+                    shutdown_requested.set()
+
+                ${v.handler}
+
+                assert "Handler" in globals(), "You must define a class Handler(BaseHTTPRequestHandler) in the handler code"
+
                 def main():
-                    httpd = NoBindHTTPServer(
+                    signal.signal(signal.SIGHUP, handle_reload)
-                        ("", 0),
+                    signal.signal(signal.SIGTERM, handle_sigterm)
-                        Handler,
+
-                        bind_and_activate=False,
+                    fd = sd_listen_fd(SOCKET_NAME)
-                    )
+
-                    httpd.socket = socket.fromfd(3, socket.AF_INET, socket.SOCK_STREAM)
+                    httpd = Server(("", 0), Handler, bind_and_activate=False)
-                    httpd.serve_forever()
+                    httpd.socket = socket.socket(fileno=fd)
+
+                    server_thread = threading.Thread(target=httpd.serve_forever, name="http-server", daemon=True)
+                    server_thread.start()
+
+                    sd_notify("READY=1")
+                    shutdown_requested.wait()
+
+                    deadline = time.monotonic() + SHUTDOWN_TIMEOUT
+
+                    httpd.shutdown()
+                    server_thread.join(max(deadline - time.monotonic(), 0))
+                    httpd.join_request_threads(max(deadline - time.monotonic(), 0))
+                    httpd.server_close()
 
                 if __name__ == '__main__':
                     main()