WIP: create tests

MIGRATIONS.MD

@@ -2,9 +2,9 @@
 
 This is a best effort document descibing neccecary changes you might have to do when updating
 
-## 0.5.0
+## 0.5.0 UNRELEASED
+
+The module has been renamed from `synapse` to `default`
 
-* The module has been renamed from `synapse` to `default`
-* The synapse module now expects a wrapper-style package. This means the module is now incompatible with nixpkgs < 23.11.
 
 

flake.lock

@@ -2,22 +2,38 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1706098335,
-        "narHash": "sha256-r3dWjT8P9/Ah5m5ul4WqIWD8muj5F+/gbCdjiNVBKmU=",
+        "lastModified": 1690789960,
+        "narHash": "sha256-3K+2HuyGTiJUSZNJxXXvc0qj4xFx1FHC/ItYtEa7/Xs=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a77ab169a83a4175169d78684ddd2e54486ac651",
+        "rev": "fb942492b7accdee4e6d17f5447091c65897dde4",
         "type": "github"
       },
       "original": {
         "id": "nixpkgs",
-        "ref": "nixos-23.11",
+        "ref": "nixos-unstable",
         "type": "indirect"
       }
     },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1673743903,
+        "narHash": "sha256-sloY6KYyVOozJ1CkbgJPpZ99TKIjIvM+04V48C04sMQ=",
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "rev": "7555e2dfcbac1533f047021f1744ac8871150f9f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-lib": "nixpkgs-lib"
       }
     }
   },

flake.nix

@@ -2,29 +2,27 @@
   description = "NixOS modules for matrix related services";
 
   inputs = {
-    nixpkgs.url = "nixpkgs/nixos-23.11";
+    nixpkgs-lib.url = "github:nix-community/nixpkgs.lib";
+    nixpkgs.url = "nixpkgs/nixos-unstable";
   };
 
-  outputs = { self, nixpkgs }: {
+  outputs = { self, nixpkgs, nixpkgs-lib }: {
     nixosModules = {
       default = import ./module.nix;
     };
 
-    lib = import ./lib.nix { lib = nixpkgs.lib; };
+    lib = import ./lib.nix { lib = nixpkgs-lib.lib; };
 
     packages = let
-      forAllSystems = f:
-        nixpkgs.lib.genAttrs [
-          "x86_64-linux"
-          "aarch64-linux"
-          "x86_64-darwin"
-          "aarch64-darwin"
-        ] (system: f nixpkgs.legacyPackages.${system});
-    in forAllSystems (pkgs: {
-      tests = import ./tests {
-        inherit nixpkgs pkgs;
-        matrix-lib = self.lib;
+      system = "x86_64-linux";
+      pkgs = nixpkgs.legacyPackages.${system};
+    in {
+        ${system}.tests = import ./tests {
+        inherit system;
+        inherit nixpkgs;
+        inherit pkgs;
+        nixosModule = self.outputs.nixosModules.synapse;
       };
-    });
+    };
   };
 }

lib.nix

@@ -4,7 +4,7 @@ rec {
   isListenerType = type: l: lib.any (r: lib.any (n: n == type) r.names) l.resources;
   # Get the first listener that includes the given resource from worker
   firstListenerOfType = type: ls: lib.lists.findFirst (isListenerType type)
-    (throw "No listener with resource: ${type} configured")
+    (lib.throw "No listener with resource: ${type} configured")
     ls;
   # Get an attrset of the host and port from a listener 
   connectionInfo = l: {
@@ -17,64 +17,4 @@ rec {
     l = firstListenerOfType r w.settings.worker_listeners;
   in connectionInfo l;
 
-  mapWorkersToUpstreamsByType = workerInstances:
-    lib.pipe workerInstances [
-      lib.attrValues
-
-      # Index by worker type
-      (lib.foldl (acc: worker: acc // {
-        ${worker.type} = (acc.${worker.type} or [ ]) ++ [ worker ];
-      }) { })
-
-      # Subindex by resource names, listener types, and convert to upstreams
-      (lib.mapAttrs (_: workers: lib.pipe workers [
-        (lib.concatMap (worker: [ (lib.lists.head  worker.settings.worker_listeners) ]))
-        lib.flatten
-        mapListenersToUpstreamsByType
-      ]))
-    ];
-
-    mapListenersToUpstreamsByType = listenerInstances:
-      lib.pipe listenerInstances [
-        # Index by resource names
-        (lib.concatMap (listener: lib.pipe listener [
-          (listener: let
-            allResourceNames = lib.pipe listener.resources [
-              (map (resource: resource.names))
-              lib.flatten
-              lib.unique
-            ];
-          in if allResourceNames == [ ]
-            then { "empty" = listener; }
-            else lib.genAttrs allResourceNames (_: listener))
-          lib.attrsToList
-        ]))
-
-        (lib.foldl (acc: listener: acc // {
-          ${listener.name} = (acc.${listener.name} or [ ]) ++ [ listener.value ];
-        }) { })
-
-        # Index by listener type
-        (lib.mapAttrs (_:
-          (lib.foldl (acc: listener: acc // {
-            ${listener.type} = (acc.${listener.type} or [ ]) ++ [ listener ];
-          }) { })
-        ))
-
-        # Convert listeners to upstream URIs
-        (lib.mapAttrs (_:
-          (lib.mapAttrs (_: listeners:
-            lib.pipe listeners [
-              (lib.concatMap (listener:
-                if listener.path != null
-                  then [ "unix:${listener.path}" ]
-                  else (map (addr: "${addr}:${toString listener.port}") listener.bind_addresses)
-              ))
-              # NOTE: Adding ` = { }` to every upstream might seem unnecessary in isolation,
-              #       but it makes it easier to set upstreams in the nginx module.
-              (uris: lib.genAttrs uris (_: { }))
-            ]
-          ))
-        ))
-      ];
 }

pkgs/matrix-sliding-sync/default.nix

@@ -0,0 +1,37 @@
+{ lib
+, buildGoModule
+, fetchFromGitHub
+}:
+
+buildGoModule rec {
+  pname = "matrix-sliding-sync";
+  version = "0.99.11";
+
+  src = fetchFromGitHub {
+    owner = "matrix-org";
+    repo = "sliding-sync";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-Wd/nnJhKg+BDyOIz42zEScjzQRrpEq6YG9/9Tk24hgg=";
+  };
+
+  vendorHash = "sha256-0QSyYhOht1j1tWNxHQh+NUZA/W1xy7ANu+29H/gusOE=";
+
+  subPackages = [ "cmd/syncv3" ];
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X main.GitCommit=${src.rev}"
+  ];
+
+  # requires a running matrix-synapse
+  doCheck = false;
+
+  meta = with lib; {
+    description = "A sliding sync implementation of MSC3575 for matrix";
+    homepage = "https://github.com/matrix-org/sliding-sync";
+    license = with licenses; [ asl20 ];
+    maintainers = with maintainers; [ emilylange ];
+    mainProgram = "syncv3";
+  };
+}

sliding-sync/default.nix

@@ -11,7 +11,7 @@ in
 
     package = lib.mkOption {
       type = lib.types.package;
-      default = pkgs.matrix-sliding-sync;
+      default = pkgs.callPackage ../pkgs/matrix-sliding-sync { };
       description = "What package to use for the sliding-sync proxy.";
     };
 
@@ -89,7 +89,7 @@ in
       ensureDatabases = [ "matrix-sliding-sync" ];
       ensureUsers = [ rec {
         name = "matrix-sliding-sync";
-        ensureDBOwnership = true;
+        ensurePermissions."DATABASE \"${name}\"" = "ALL PRIVILEGES";
       } ];
     };
 
@@ -107,7 +107,7 @@ in
     };
 
     services.nginx.virtualHosts.${cfg.publicBaseUrl} = lib.mkIf cfg.enableNginx {
-      enableACME = lib.mkDefault true;
+      enableACME = true;
       forceSSL = true;
       locations."/" = {
         proxyPass = lib.replaceStrings [ "0.0.0.0" "::" ] [ "127.0.0.1" "::1" ] "http://${cfg.settings.SYNCV3_BINDADDR}";

synapse-module/default.nix

@@ -10,25 +10,9 @@ let
   wcfgText = "config.services.matrix-synapse-next.workers";
 
   format = pkgs.formats.yaml {};
-  matrix-synapse-common-config = format.generate "matrix-synapse-common-config.yaml" (cfg.settings // {
-    listeners = map (lib.filterAttrsRecursive (_: v: v != null)) cfg.settings.listeners;
-  });
-
-  # TODO: Align better with the upstream module
-  wrapped = cfg.package.override { 
-    inherit (cfg) plugins;
-    extras = [
-      "postgres"
-      "saml2"
-      "oidc"
-      "systemd"
-      "url-preview"
-      "sentry"
-      "jwt"
-      "redis"
-      "cache-memory"
-      "user-search"
-    ];
+  matrix-synapse-common-config = format.generate "matrix-synapse-common-config.yaml" cfg.settings;
+  pluginsEnv = cfg.package.python.buildEnv.override {
+    extraLibs = cfg.plugins;
   };
 
   inherit (lib)
@@ -49,7 +33,7 @@ in
   imports = [
     ./nginx.nix
     (import ./workers.nix {
-      inherit matrix-lib throw' format matrix-synapse-common-config wrapped;
+      inherit matrix-lib throw' format matrix-synapse-common-config pluginsEnv;
     })
   ];
 
@@ -81,15 +65,6 @@ in
       '';
     };
 
-    socketDir = mkOption {
-      type = types.path;
-      default = "/run/matrix-synapse";
-      description = ''
-        The directory where matrix-synapse by default stores the sockets of
-        all listeners that bind to UNIX sockets.
-      '';
-    };
-
     enableNginx = mkEnableOption "The synapse module managing nginx";
 
     public_baseurl = mkOption {
@@ -146,42 +121,14 @@ in
             type = types.listOf (types.submodule {
               options = {
                 port = mkOption {
-                  type = with types; nullOr types.port;
-                  default = null;
-                  description = ''
-                    The TCP port to bind to.
-
-                    ::: {.note}
-                      This option will be ignored if {option}`path` is set to a non-null value.
-                    :::
-                  '';
+                  type = types.port;
+                  description = "The TCP port to bind to";
                   example = 8448;
                 };
 
-                path = mkOption {
-                  type = with types; nullOr path;
-                  default = null;
-                  description = ''
-                    The UNIX socket to bind to.
-
-                    ::: {.note}
-                      This option will override {option}`bind_addresses` and {option}`port`
-                      if set to a non-null value.
-                    :::
-                  '';
-                  example = literalExpression ''''${${cfgText}.socketDir}/matrix-synapse.sock'';
-                };
-
                 bind_addresses = mkOption {
                   type = types.listOf types.str;
-                  default = [ ];
-                  description = ''
-                    A list of local addresses to listen on.
-
-                    ::: {.note}
-                      This option will be ignored if {option}`path` is set to a non-null value.
-                    :::
-                  '';
+                  description = "A list of local addresses to listen on";
                 };
 
                 type = mkOption {
@@ -240,14 +187,16 @@ in
             # TODO: add defaultText
             default = [
               {
-                path = "${cfg.socketDir}/matrix-synapse.sock";
+                port = 8008;
+                bind_addresses = [ "127.0.0.1" ];
                 resources = [
                   { names = [ "client" ]; compress = true; }
                   { names = [ "federation" ]; compress = false; }
                 ];
               }
               (mkIf (wcfg.instances != { }) {
-                path = "${cfg.socketDir}/matrix-synapse-replication.sock";
+                port = 9093;
+                bind_addresses = [ "127.0.0.1" ];
                 resources = [
                   {  names = [ "replication" ]; }
                 ];
@@ -389,12 +338,6 @@ in
   };
 
   config = mkIf cfg.enable {
-    assertions = [ ]
-      ++ (map (l: {
-        assertion = l.path == null -> (l.bind_addresses != [ ] && l.port != null);
-        message = "Some listeners are missing either a socket path or a bind_address + port to listen on";
-      }) cfg.settings.listeners);
-
     users.users.matrix-synapse = {
       group = "matrix-synapse";
       home = cfg.dataDir;
@@ -433,20 +376,21 @@ in
           };
         in "${cfg.package}/bin/synapse_homeserver ${flags}";
 
+        environment.PYTHONPATH =
+          lib.makeSearchPathOutput "lib" cfg.package.python.sitePackages [ pluginsEnv ];
+
         serviceConfig = {
           Type = "notify";
           User = "matrix-synapse";
           Group = "matrix-synapse";
           Slice = "system-matrix-synapse.slice";
           WorkingDirectory = cfg.dataDir;
-          StateDirectory = "matrix-synapse";
-          RuntimeDirectory = "matrix-synapse";
           ExecStart = let
             flags = lib.cli.toGNUCommandLineShell {} {
               config-path = [ matrix-synapse-common-config ] ++ cfg.extraConfigFiles;
               keys-directory = cfg.dataDir;
             };
-          in "${wrapped}/bin/synapse_homeserver ${flags}";
+          in "${cfg.package}/bin/synapse_homeserver ${flags}";
           ExecReload = "${pkgs.utillinux}/bin/kill -HUP $MAINPID";
           Restart = "on-failure";
         };

synapse-module/nginx.nix

@@ -2,10 +2,13 @@
 let
   cfg = config.services.matrix-synapse-next;
 
-  matrix-lib = (import ../lib.nix { inherit lib; });
-
-  workerUpstreams = matrix-lib.mapWorkersToUpstreamsByType cfg.workers.instances;
-  listenerUpstreams = matrix-lib.mapListenersToUpstreamsByType cfg.settings.listeners;
+  getWorkersOfType = type: lib.filterAttrs (_: w: w.type == type) cfg.workers.instances;
+  isListenerType = type: listener: lib.lists.any (r: lib.lists.any (n: n == type) r.names) listener.resources;
+  firstListenerOfType = type: worker: lib.lists.findFirst (isListenerType type) (throw "No federation endpoint on receiver") worker.settings.worker_listeners;
+  wAddressOfType = type: w: lib.lists.findFirst (_: true) (throw "No address in receiver") (firstListenerOfType type w).bind_addresses;
+  wPortOfType = type: w: (firstListenerOfType type w).port;
+  wSocketAddressOfType = type: w: "${wAddressOfType type w}:${builtins.toString (wPortOfType type w)}";
+  generateSocketAddresses = type: workers: lib.mapAttrsToList (_: w: "${wSocketAddressOfType type w}") workers;
 in
 {
   config = lib.mkIf cfg.enableNginx {
@@ -135,17 +138,24 @@ in
     '';
 
     services.nginx.upstreams.synapse_master.servers = let
-      mainListeners = builtins.intersectAttrs
-        (listenerUpstreams.client.http or { })
-        (listenerUpstreams.federation.http or { });
-    in
-      assert lib.assertMsg (mainListeners != { })
-        "No catch-all listener configured, or listener is not bound to an address";
-        mainListeners;
+      isMainListener = l: isListenerType "client" l && isListenerType "federation" l;
+      firstMainListener = lib.findFirst isMainListener
+        (throw "No catch-all listener configured") cfg.settings.listeners;
+        address = lib.findFirst (_: true) (throw "No address in main listener") firstMainListener.bind_addresses;
+        port = firstMainListener.port;
+        socketAddress = "${address}:${builtins.toString port}";
+    in {
+      "${socketAddress}" = { };
+    };
 
 
     services.nginx.upstreams.synapse_worker_federation = {
-      servers = workerUpstreams.fed-receiver.federation.http or config.services.nginx.upstreams.synapse_master.servers;
+      servers = let
+        fedReceivers = getWorkersOfType "fed-receiver";
+        socketAddresses = generateSocketAddresses "federation" fedReceivers;
+        in if fedReceivers != { } then
+          lib.genAttrs socketAddresses (_: { })
+      else config.services.nginx.upstreams.synapse_master.servers;
       extraConfig = ''
         ip_hash;
       '';
@@ -153,7 +163,12 @@ in
 
 
     services.nginx.upstreams.synapse_worker_initial_sync = {
-      servers = workerUpstreams.initial-sync.client.http or config.services.nginx.upstreams.synapse_master.servers;
+      servers = let
+        initialSyncers = getWorkersOfType "initial-sync";
+        socketAddresses = generateSocketAddresses "client" initialSyncers;
+      in if initialSyncers != { } then
+        lib.genAttrs socketAddresses (_: { })
+      else config.services.nginx.upstreams.synapse_master.servers;
       extraConfig = ''
         hash $mxid_localpart consistent;
       '';
@@ -161,7 +176,12 @@ in
 
 
     services.nginx.upstreams.synapse_worker_normal_sync = {
-      servers = workerUpstreams.normal-sync.client.http or config.services.nginx.upstreams.synapse_master.servers;
+      servers = let
+        normalSyncers = getWorkersOfType "normal-sync";
+        socketAddresses = generateSocketAddresses "client" normalSyncers;
+      in if normalSyncers != { } then
+        lib.genAttrs socketAddresses (_: { })
+      else config.services.nginx.upstreams.synapse_master.servers;
       extraConfig = ''
         hash $mxid_localpart consistent;
       '';
@@ -169,7 +189,12 @@ in
 
 
     services.nginx.upstreams.synapse_worker_user-dir = {
-      servers = workerUpstreams.user-dir.client.http or config.services.nginx.upstreams.synapse_master.servers;
+      servers = let
+        workers = getWorkersOfType "user-dir";
+        socketAddresses = generateSocketAddresses "client" workers;
+      in if workers != { } then
+        lib.genAttrs socketAddresses (_: { })
+      else config.services.nginx.upstreams.synapse_master.servers;
     };
 
     services.nginx.virtualHosts."${cfg.public_baseurl}" = {

synapse-module/workers.nix

@@ -1,6 +1,6 @@
 { matrix-synapse-common-config,
   matrix-lib,
-  wrapped,
+  pluginsEnv,
   throw',
   format
 }:
@@ -19,6 +19,7 @@
     type = types.ints.unsigned;
     description = "How many automatically configured ${workerType} workers to set up";
     default = 0;
+    example = 1;
   };
 
   genAttrs' = items: f: g: builtins.listToAttrs (map (i: lib.nameValuePair (f i) (g i)) items);
@@ -33,11 +34,25 @@ in {
           type = types.bool;
           internal = true;
           default = false;
+          description = ''
+            This is an internal flag that signals that this worker is part of the
+            workers generated by either of the following:
+
+            - federationSenders
+            - federationReceivers
+            - initialSyncers
+            - normalSyncers
+            - eventPersisters
+            - useUserDirectoryWorker
+          '';
         };
 
         index = mkOption {
           internal = true;
           type = types.ints.positive;
+          description = ''
+            This is an internal variable that indexes the worker of this type.
+          '';
         };
 
         # The custom string type here is mainly for the name to use
@@ -85,24 +100,25 @@ in {
           default = "http";
         };
 
-        port = mkOption {
-          type = with types; nullOr port;
-          default = null;
-          description = "The TCP port to bind to";
-        };
-
         path = mkOption {
-          type = with types; nullOr path;
-          default = null;
-          description = "The UNIX socket to bind to";
+          type = types.path;
+          default = instanceCfg.name;
+          description = ''
+            A path and filename for a Unix socket.
+          '';
         };
 
-        bind_addresses = mkOption {
-          type = with types; listOf str;
-          description = "A list of local addresses to listen on";
-          default = [ wcfg.defaultListenerAddress ];
-          defaultText = literalExpression "[ ${wcfgText}.defaultListenerAddress ]";
-        };
+        # port = mkOption {
+        #   type = types.port;
+        #   description = "The TCP port to bind to";
+        # };
+
+        # bind_addresses = mkOption {
+        #   type = with types; listOf str;
+        #   description = "A list of local addresses to listen on";
+        #   default = [ wcfg.defaultListenerAddress ];
+        #   defaultText = literalExpression "[ ${wcfgText}.defaultListenerAddress ]";
+        # };
 
         tls = mkOption {
           type = types.bool;
@@ -167,52 +183,6 @@ in {
       };
     };
   in {
-    mainReplicationHost = mkOption {
-      type = with types; nullOr str;
-      default = let
-        host = (matrix-lib.connectionInfo mainReplicationListener).host;
-      in
-        # To avoid connecting to 0.0.0.0 and so on
-        if builtins.elem host [ "0.0.0.0" "::" ]
-          then "127.0.0.1"
-          else host;
-      # TODO: add defaultText
-      description = "Host of the main synapse instance's replication listener";
-    };
-
-    mainReplicationPort = mkOption {
-      type = with types; nullOr port;
-      default = mainReplicationListener.port;
-      # TODO: add defaultText
-      description = "Port for the main synapse instance's replication listener";
-    };
-
-    mainReplicationPath = mkOption {
-      type = with types; nullOr path;
-      default = mainReplicationListener.path;
-      # TODO: add defaultText
-      description = "Path to the UNIX socket of the main synapse instance's replication listener";
-    };
-
-    defaultListenerAddress = mkOption {
-      type = types.str;
-      default = "127.0.0.1";
-      description = "The default listener address for the worker";
-    };
-
-    workersUsePath = mkOption {
-      type = types.bool;
-      description = "Whether to enable UNIX sockets for all automatically generated workers";
-      default = true;
-      example = false;
-    };
-
-    workerStartingPort = mkOption {
-      type = types.port;
-      description = "What port should the automatically configured workers start enumerating from";
-      default = 8083;
-    };
-
     enableMetrics = mkOption {
       type = types.bool;
       default = cfg.settings.enable_metrics;
@@ -220,12 +190,6 @@ in {
       # TODO: add description
     };
 
-    metricsStartingPort = mkOption {
-      type = types.port;
-      default = 18083;
-      # TODO: add description
-    };
-
     federationSenders = mkWorkerCountOption "federation-sender";
     federationReceivers = mkWorkerCountOption "federation-reciever";
     initialSyncers = mkWorkerCountOption "initial-syncer";
@@ -239,13 +203,14 @@ in {
       default = { };
       description = "Worker configuration";
       example = {
-        "federation_sender1" = {
+        "federation-sender-1" = {
           settings = {
-            worker_name = "federation_sender1";
+            worker_name = "federation-sender-1";
             worker_app = "synapse.app.generic_worker";
 
-            worker_replication_host = "127.0.0.1";
-            worker_replication_http_port = 9093;
+            path = "/run/matrix-synapse/federation-sender-1.sock";
+            # worker_replication_host = "127.0.0.1";
+            # worker_replication_http_port = 9093;
             worker_listeners = [ ];
           };
         };
@@ -253,135 +218,87 @@ in {
     };
   };
 
-  config = {
-    assertions = [ ]
-      ++ (lib.concatMap (worker:
-        (map (l: {
-          assertion = l.path == null -> (l.bind_addresses != [ ] && l.port != null);
-          message = "At least one worker listener is missing either a socket path or a bind_address + port to listen on";
-        }) worker.settings.worker_listeners)
-      ) (lib.attrValues wcfg.instances));
-
+  config = let 
+    genList1 = f: builtins.genList (i: f (i + 1));
+  in {
     services.matrix-synapse-next.settings = {
       federation_sender_instances =
-        lib.genList (i: "auto-fed-sender${toString (i + 1)}") wcfg.federationSenders;
+        genList1 (i: "auto-fed-sender-${toString i}") wcfg.federationSenders;
 
-      instance_map = (lib.mkIf (cfg.workers.instances != { }) ({
-        main = if wcfg.mainReplicationPath != null then {
-          path = wcfg.mainReplicationPath;
-        } else {
-          host = wcfg.mainReplicationHost;
-          port = wcfg.mainReplicationPort;
-        };
-      } // genAttrs' (lib.lists.range 1 wcfg.eventPersisters)
-        (i: "auto-event-persist${toString i}")
-        (i: let
-          wRL = matrix-lib.firstListenerOfType "replication" wcfg.instances."auto-event-persist${toString i}".settings.worker_listeners;
-        in if wRL.path != null then {
-          inherit (wRL) path;
-        } else matrix-lib.connectionInfo wRL)));
+      instance_map = lib.mkIf (cfg.workers.instances != { }) ({
+        main.path = "/run/matrix-synapse/main-replication-worker.sock";
+      } // builtins.mapAttrs (n: v: {
+        inherit (builtins.head v.settings.worker_listeners) path;
+      }) wcfg.instances);
 
       stream_writers.events =
         mkIf (wcfg.eventPersisters > 0)
-        (lib.genList (i: "auto-event-persist${toString (i + 1)}") wcfg.eventPersisters); 
+          (genList1 (i: "auto-event-persist-${toString i}") wcfg.eventPersisters); 
 
       update_user_directory_from_worker =
-        mkIf wcfg.useUserDirectoryWorker "auto-user-dir";
+        mkIf wcfg.useUserDirectoryWorker "auto-user-dir-1";
     };
 
-    services.matrix-synapse-next.workers.instances = let
-      sum = lib.foldl lib.add 0;
-      workerListenersWithMetrics = portOffset: name:
-        [(if wcfg.workersUsePath
-          then {
-            path = "${cfg.socketDir}/matrix-synapse-worker-${name}.sock";
-          }
-          else {
-            port = wcfg.workerStartingPort + portOffset - 1;
-          }
-        )]
-        ++ lib.optional wcfg.enableMetrics {
-          port = wcfg.metricsStartingPort + portOffset;
-          resources = [ { names = [ "metrics" ]; } ];
+    services.matrix-synapse-next.workers.instances =
+      let
+        workerInstances = {
+          "fed-sender" = wcfg.federationSenders;
+          "fed-receiver" = wcfg.federationReceivers;
+          "initial-sync" = wcfg.initialSyncers;
+          "normal-sync" = wcfg.normalSyncers;
+          "event-persist" = wcfg.eventPersisters;
+          "user-dir" = if wcfg.useUserDirectoryWorker then 1 else 0;
         };
-
-      makeWorkerInstances = {
-        type,
-        numberOfWorkers,
-        portOffset ? 0,
-        nameFn ? i: "auto-${type}${toString i}",
-        workerListenerFn ? i: name: workerListenersWithMetrics (portOffset + i) name
-      }: genAttrs'
-        (lib.lists.range 1 numberOfWorkers)
-        nameFn
-        (i: {
-          isAuto = true;
-          inherit type;
-          index = i;
-          settings.worker_listeners = workerListenerFn i (nameFn i);
-        });
-
-      workerInstances = {
-        "fed-sender" = wcfg.federationSenders;
-        "fed-receiver" = wcfg.federationReceivers;
-        "initial-sync" = wcfg.initialSyncers;
-        "normal-sync" = wcfg.normalSyncers;
-        "event-persist" = wcfg.eventPersisters;
-      } // (lib.optionalAttrs wcfg.useUserDirectoryWorker {
-        "user-dir" = {
-          numberOfWorkers = 1;
-          nameFn = _: "auto-user-dir";
-        };
-      });
-
-      coerceWorker = { name, value }: if builtins.isInt value then {
-        type = name;
-        numberOfWorkers = value;
-      } else { type = name; } // value;
-
-      # Like foldl, but keeps all intermediate values
-      #
-      # (b -> a -> b) -> b -> [a] -> [b]
-      scanl = f: x1: list: let
-        x2 = lib.head list;
-        x1' = f x1 x2;
-      in if list == [] then [] else [x1'] ++ (scanl f x1' (lib.tail list));
-
-      f = { portOffset, numberOfWorkers, ... }: x: x // { portOffset = portOffset + numberOfWorkers; };
-      init = { portOffset = 0; numberOfWorkers = 0; };
-    in lib.pipe workerInstances [
-      (lib.mapAttrsToList lib.nameValuePair)
-      (map coerceWorker)
-      (scanl f init)
-      (map makeWorkerInstances)
-      mkMerge
-    ];
+      in
+        lib.pipe workerInstances [
+          (lib.mapAttrsToList (type: count: { inherit type count; }))
+          (map ({ type, count }: genList1 (i: rec {
+            name = "auto-${type}-${toString i}";
+            value = {
+              inherit type;
+              isAuto = true;
+              index = i;
+              settings.worker_listeners =
+                [
+                  { path = "/run/matrix-synapse/${name}.sock"; }
+                ] ++ lib.optionals wcfg.enableMetrics [{
+                  path = "/run/matrix-synapse/${name}-metrics.sock";
+                  resources = [{ names = [ "metrics" ]; }];
+                  type = "metrics";
+                }];
+            };
+          }) count))
+          lib.flatten
+          builtins.listToAttrs
+        ];
 
     systemd.services = let
       workerList = lib.mapAttrsToList lib.nameValuePair wcfg.instances;
-      workerConfig = worker:
-        format.generate "matrix-synapse-worker-${worker.name}-config.yaml"
-        (worker.value.settings // {
-          worker_name = worker.name;
-          worker_listeners =
-            map (lib.filterAttrsRecursive (_: v: v != null)) worker.value.settings.worker_listeners;
-        });
+      workerConfig = worker: format.generate "matrix-synapse-worker-${worker.name}-config.yaml"
+        (worker.value.settings // { worker_name = worker.name; });
     in builtins.listToAttrs (lib.flip map workerList (worker: {
       name = "matrix-synapse-worker-${worker.name}";
       value = {
+        documentation = [ "https://github.com/matrix-org/synapse/blob/develop/docs/workers.md" ];
         description = "Synapse Matrix Worker";
         partOf = [ "matrix-synapse.target" ];
         wantedBy = [ "matrix-synapse.target" ];
         after = [ "matrix-synapse.service" ];
         requires = [ "matrix-synapse.service" ];
+        restartTriggers = [ matrix-synapse-common-config (workerConfig worker) ] ++ cfg.extraConfigFiles;
+        environment = {
+          PYTHONPATH = lib.makeSearchPathOutput "lib" cfg.package.python.sitePackages [
+            pluginsEnv
+          ];
+        };
         serviceConfig = {
+          Restart = "always";
           Type = "notify";
           User = "matrix-synapse";
           Group = "matrix-synapse";
           Slice = "system-matrix-synapse.slice";
           WorkingDirectory = cfg.dataDir;
-          RuntimeDirectory = "matrix-synapse";
-          StateDirectory = "matrix-synapse";
+          RuntimeDirectory = [ "matrix-synapse" ];
           ExecStartPre = pkgs.writers.writeBash "wait-for-synapse" ''
             # From https://md.darmstadt.ccc.de/synapse-at-work
             while ! systemctl is-active -q matrix-synapse.service; do
@@ -393,7 +310,7 @@ in {
               config-path = [ matrix-synapse-common-config (workerConfig worker) ] ++ cfg.extraConfigFiles;
               keys-directory = cfg.dataDir;
             };
-          in "${wrapped}/bin/synapse_worker ${flags}";
+          in "${cfg.package}/bin/synapse_worker ${flags}";
         };
       };
     }));

tests/auto-workers-config.nix

@@ -0,0 +1,18 @@
+{ pkgs, lib, ... }:
+{
+  services.matrix-synapse-next = {
+    enable = true;
+    settings.server_name = "matrix.example.com";
+
+    workers = {
+      enableMetrics = true;
+
+      federationSenders = 2;
+      federationReceivers = 2;
+      initialSyncers = 2;
+      normalSyncers = 2;
+      eventPersisters = 2;
+      useUserDirectoryWorker = true;
+    };
+  };
+}

tests/base-config.nix

@@ -0,0 +1,7 @@
+{ pkgs, lib, ... }:
+{
+  services.matrix-synapse-next = {
+    enable = true;
+    settings.server_name = "matrix.example.com";
+  };
+}

tests/default.nix

@@ -1,4 +1,16 @@
-{ nixpkgs, pkgs, matrix-lib, ... }:
-{
-  nginx-pipeline = pkgs.callPackage ./nginx-pipeline { inherit nixpkgs matrix-lib; };
+{ nixpkgs, pkgs, system ? pkgs.system, nixosModule, ... }: let
+  buildSystemWithConfig = configPath: (nixpkgs.lib.nixosSystem {
+    inherit system;
+    modules = [
+      nixosModule
+      configPath
+      {
+        boot.isContainer = true;
+      }
+    ];
+  }).config.system.build.toplevel;
+in {
+  a = pkgs.writeText "hello-world" ''a'';
+  base-config = buildSystemWithConfig ./base-config.nix;
+  auto-workers-config = buildSystemWithConfig ./auto-workers-config.nix;
 }

tests/nginx-pipeline/default.nix

@@ -1,53 +0,0 @@
-{ nixpkgs, lib, matrix-lib, writeText, ... }:
-let
-  nixosConfig = nixpkgs.lib.nixosSystem {
-    system = "x86_64-linux";
-    modules = [
-      ../../module.nix
-      {
-        system.stateVersion = "23.11";
-        boot.isContainer = true;
-        services.matrix-synapse-next = {
-          enable = true;
-          enableNginx = true;
-
-          workers = {
-            enableMetrics = true;
-
-            federationSenders = 3;
-            federationReceivers = 3;
-            initialSyncers = 1;
-            normalSyncers = 1;
-            eventPersisters = 1;
-            useUserDirectoryWorker = true;
-
-            instances.auto-fed-receiver1.settings.worker_listeners = [
-              {
-                bind_addresses = [
-                  "127.0.0.2"
-                ];
-                port = 1337;
-                resources = [
-                  { compress = false;
-                    names = [ "federation" ];
-                  }
-                ];
-              }
-            ];
-          };
-
-          settings.server_name = "example.com";
-        };
-      }
-    ];
-  };
-
-  inherit (nixosConfig.config.services.matrix-synapse-next.workers) instances;
-in
-  writeText "matrix-synapse-next-nginx-pipeline-test.txt" ''
-    ${(lib.generators.toPretty {}) instances}
-
-    ====================================================
-
-    ${(lib.generators.toPretty {}) (matrix-lib.mapWorkersToUpstreamsByType instances)}
-  ''