WIP: create tests

Co-authored-by: h7x4 <h7x4@nani.wtf>

.gitignore

@@ -0,0 +1 @@
+result

COPYING

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020, 2022-2023 Daniel Løvbrøtte Olsen and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MIGRATIONS.MD

@@ -0,0 +1,10 @@
+# Migrations
+
+This is a best effort document descibing neccecary changes you might have to do when updating
+
+## 0.5.0 UNRELEASED
+
+The module has been renamed from `synapse` to `default`
+
+
+

README.MD

@@ -9,6 +9,10 @@ With matrix.YOURDOMAIN pointing at the server:
 
     workers.federationSenders = 1;
     workers.federationReceivers = 1;
+    workers.initialSyncers = 1;
+    workers.normalSyncers = 1;
+    workers.eventPersisters = 2;
+    workers.useUserDirectoryWorker = true;
 
     enableNginx = true;
 
@@ -31,4 +35,20 @@ With matrix.YOURDOMAIN pointing at the server:
 }
 ```
 
-is ~enough to get a functional matrix-server running one federation sender and one federation receiver
+is ~enough to get a functional matrix-server running with some workers
+
+## Sliding Sync (Element X)
+
+Just add the following to your config and point `slidingsync.YOURDOMAIN` at the server
+
+```
+services.matrix-synapse-next = {
+  enableSlidingSync = true;
+};
+
+services.matrix-synapse.sliding-sync.environmentFile = "/some/file/containing/SYNCV3_SECRET=<some secret>";
+
+```
+
+If using [well-known delagation](https://matrix-org.github.io/synapse/v1.37/delegate.html) make sure `YOURDOMAIN/.well-known/matrix/client` matches
+what's in `matrix.YOURDOMAIN/.well-known/matrix/client`

flake.lock

@@ -0,0 +1,42 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1690789960,
+        "narHash": "sha256-3K+2HuyGTiJUSZNJxXXvc0qj4xFx1FHC/ItYtEa7/Xs=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "fb942492b7accdee4e6d17f5447091c65897dde4",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-unstable",
+        "type": "indirect"
+      }
+    },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1673743903,
+        "narHash": "sha256-sloY6KYyVOozJ1CkbgJPpZ99TKIjIvM+04V48C04sMQ=",
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "rev": "7555e2dfcbac1533f047021f1744ac8871150f9f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs",
+        "nixpkgs-lib": "nixpkgs-lib"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -1,9 +1,28 @@
 {
   description = "NixOS modules for matrix related services";
 
-  outputs = { self }: {
+  inputs = {
+    nixpkgs-lib.url = "github:nix-community/nixpkgs.lib";
+    nixpkgs.url = "nixpkgs/nixos-unstable";
+  };
+
+  outputs = { self, nixpkgs, nixpkgs-lib }: {
     nixosModules = {
-      synapse = import ./synapse-module;
+      default = import ./module.nix;
+    };
+
+    lib = import ./lib.nix { lib = nixpkgs-lib.lib; };
+
+    packages = let
+      system = "x86_64-linux";
+      pkgs = nixpkgs.legacyPackages.${system};
+    in {
+        ${system}.tests = import ./tests {
+        inherit system;
+        inherit nixpkgs;
+        inherit pkgs;
+        nixosModule = self.outputs.nixosModules.synapse;
+      };
     };
   };
 }

lib.nix

@@ -0,0 +1,20 @@
+{ lib }:
+rec {
+  # checks if given listener configuration has type as a resource
+  isListenerType = type: l: lib.any (r: lib.any (n: n == type) r.names) l.resources;
+  # Get the first listener that includes the given resource from worker
+  firstListenerOfType = type: ls: lib.lists.findFirst (isListenerType type)
+    (lib.throw "No listener with resource: ${type} configured")
+    ls;
+  # Get an attrset of the host and port from a listener 
+  connectionInfo = l: {
+    host = lib.head l.bind_addresses;
+    port = l.port;
+  };
+
+  # Get an attrset of the host and port from a worker given a type
+  workerConnectionResource = r: w: let
+    l = firstListenerOfType r w.settings.worker_listeners;
+  in connectionInfo l;
+
+}

module.nix

@@ -0,0 +1,8 @@
+{ ... }:
+
+{
+  imports = [
+    ./synapse-module
+    ./sliding-sync
+  ];
+}

pkgs/matrix-sliding-sync/default.nix

@@ -0,0 +1,37 @@
+{ lib
+, buildGoModule
+, fetchFromGitHub
+}:
+
+buildGoModule rec {
+  pname = "matrix-sliding-sync";
+  version = "0.99.11";
+
+  src = fetchFromGitHub {
+    owner = "matrix-org";
+    repo = "sliding-sync";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-Wd/nnJhKg+BDyOIz42zEScjzQRrpEq6YG9/9Tk24hgg=";
+  };
+
+  vendorHash = "sha256-0QSyYhOht1j1tWNxHQh+NUZA/W1xy7ANu+29H/gusOE=";
+
+  subPackages = [ "cmd/syncv3" ];
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X main.GitCommit=${src.rev}"
+  ];
+
+  # requires a running matrix-synapse
+  doCheck = false;
+
+  meta = with lib; {
+    description = "A sliding sync implementation of MSC3575 for matrix";
+    homepage = "https://github.com/matrix-org/sliding-sync";
+    license = with licenses; [ asl20 ];
+    maintainers = with maintainers; [ emilylange ];
+    mainProgram = "syncv3";
+  };
+}

sliding-sync/default.nix

@@ -0,0 +1,117 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.services.matrix-synapse.sliding-sync;
+in
+{
+  disabledModules = [ "services/matrix/matrix-sliding-sync.nix" ];
+
+  options.services.matrix-synapse.sliding-sync = {
+    enable = lib.mkEnableOption (lib.mdDoc "sliding sync");
+
+    package = lib.mkOption {
+      type = lib.types.package;
+      default = pkgs.callPackage ../pkgs/matrix-sliding-sync { };
+      description = "What package to use for the sliding-sync proxy.";
+    };
+
+    enableNginx = lib.mkEnableOption (lib.mdDoc "autogenerated nginx config");
+    publicBaseUrl = lib.mkOption {
+      type = lib.types.str;
+      description = "The domain where clients connect, only has an effect with enableNginx";
+      example = "slidingsync.matrix.org";
+    };
+
+    settings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = with lib.types; attrsOf str;
+        options = {
+          SYNCV3_SERVER = lib.mkOption {
+            type = lib.types.str;
+            description = lib.mdDoc ''
+              The destination homeserver to talk to not including `/_matrix/` e.g `https://matrix.example.org`.
+            '';
+          };
+
+          SYNCV3_DB = lib.mkOption {
+            type = lib.types.str;
+            default = "postgresql:///matrix-sliding-sync?host=/run/postgresql";
+            description = lib.mdDoc ''
+              The postgres connection string.
+              Refer to <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING>.
+            '';
+          };
+
+          SYNCV3_BINDADDR = lib.mkOption {
+            type = lib.types.str;
+            default = "127.0.0.1:8009";
+            example = "[::]:8008";
+            description = lib.mdDoc "The interface and port to listen on.";
+          };
+
+          SYNCV3_LOG_LEVEL = lib.mkOption {
+            type = lib.types.enum [ "trace" "debug" "info" "warn" "error" "fatal" ];
+            default = "info";
+            description = lib.mdDoc "The level of verbosity for messages logged.";
+          };
+        };
+      };
+      default = { };
+      description = ''
+        Freeform environment variables passed to the sliding sync proxy.
+        Refer to <https://github.com/matrix-org/sliding-sync#setup> for all supported values.
+      '';
+    };
+
+    createDatabase = lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = lib.mdDoc ''
+        Whether to enable and configure `services.postgres` to ensure that the database user `matrix-sliding-sync`
+        and the database `matrix-sliding-sync` exist.
+      '';
+    };
+
+    environmentFile = lib.mkOption {
+      type = lib.types.str;
+      description = lib.mdDoc ''
+        Environment file as defined in {manpage}`systemd.exec(5)`.
+
+        This must contain the {env}`SYNCV3_SECRET` variable which should
+        be generated with {command}`openssl rand -hex 32`.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.postgresql = lib.optionalAttrs cfg.createDatabase {
+      enable = true;
+      ensureDatabases = [ "matrix-sliding-sync" ];
+      ensureUsers = [ rec {
+        name = "matrix-sliding-sync";
+        ensurePermissions."DATABASE \"${name}\"" = "ALL PRIVILEGES";
+      } ];
+    };
+
+    systemd.services.matrix-sliding-sync = {
+      after = lib.optional cfg.createDatabase "postgresql.service";
+      wantedBy = [ "multi-user.target" ];
+      environment = cfg.settings;
+      serviceConfig = {
+        DynamicUser = true;
+        EnvironmentFile = cfg.environmentFile;
+        ExecStart = lib.getExe cfg.package;
+        StateDirectory = "matrix-sliding-sync";
+        WorkingDirectory = "%S/matrix-sliding-sync";
+      };
+    };
+
+    services.nginx.virtualHosts.${cfg.publicBaseUrl} = lib.mkIf cfg.enableNginx {
+      enableACME = true;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = lib.replaceStrings [ "0.0.0.0" "::" ] [ "127.0.0.1" "::1" ] "http://${cfg.settings.SYNCV3_BINDADDR}";
+      };
+    };
+  };
+}

synapse-module/nginx.nix

@@ -1,4 +1,4 @@
-{ lib, pkgs, config, ...}:
+{ pkgs, lib, config, ... }:
 let
   cfg = config.services.matrix-synapse-next;
 
@@ -12,224 +12,227 @@ let
 in
 {
   config = lib.mkIf cfg.enableNginx {
-  services.nginx.commonHttpConfig = ''
-    # No since argument means its initialSync
-    map $arg_since $synapse_unknown_sync {
-      default synapse_normal_sync;
-      ''' synapse_initial_sync;
-    }
+    services.nginx.commonHttpConfig = ''
+      # No since argument means its initialSync
+      map $arg_since $synapse_unknown_sync {
+        default synapse_normal_sync;
+        ''' synapse_initial_sync;
+      }
 
-    map $uri $synapse_uri_group {
-      # Sync requests
-      ~^/_matrix/client/(r0|v3)/sync$ $synapse_unknown_sync;
-      ~^/_matrix/client/(api/v1|r0|v3)/event$ synapse_normal_sync;
-      ~^/_matrix/client/(api/v1|r0|v3)/initialSync$ synapse_initial_sync;
-      ~^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ synapse_initial_sync;
+      map $uri $synapse_uri_group {
+        # Sync requests
+        ~^/_matrix/client/(r0|v3)/sync$ $synapse_unknown_sync;
+        ~^/_matrix/client/(api/v1|r0|v3)/event$ synapse_normal_sync;
+        ~^/_matrix/client/(api/v1|r0|v3)/initialSync$ synapse_initial_sync;
+        ~^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ synapse_initial_sync;
 
-      # Federation requests
-      ~^/_matrix/federation/v1/event/ synapse_federation;
-      ~^/_matrix/federation/v1/state/ synapse_federation;
-      ~^/_matrix/federation/v1/state_ids/ synapse_federation;
-      ~^/_matrix/federation/v1/backfill/ synapse_federation;
-      ~^/_matrix/federation/v1/get_missing_events/ synapse_federation;
-      ~^/_matrix/federation/v1/publicRooms synapse_federation;
-      ~^/_matrix/federation/v1/query/ synapse_federation;
-      ~^/_matrix/federation/v1/make_join/ synapse_federation;
-      ~^/_matrix/federation/v1/make_leave/ synapse_federation;
-      ~^/_matrix/federation/(v1|v2)/send_join/ synapse_federation;
-      ~^/_matrix/federation/(v1|v2)/send_leave/ synapse_federation;
-      ~^/_matrix/federation/(v1|v2)/invite/ synapse_federation;
-      ~^/_matrix/federation/v1/event_auth/ synapse_federation;
-      ~^/_matrix/federation/v1/timestamp_to_event/ synapse_federation;
-      ~^/_matrix/federation/v1/exchange_third_party_invite/ synapse_federation;
-      ~^/_matrix/federation/v1/user/devices/ synapse_federation;
-      ~^/_matrix/key/v2/query synapse_federation;
-      ~^/_matrix/federation/v1/hierarchy/ synapse_federation;
+        # Federation requests
+        ~^/_matrix/federation/v1/event/ synapse_federation;
+        ~^/_matrix/federation/v1/state/ synapse_federation;
+        ~^/_matrix/federation/v1/state_ids/ synapse_federation;
+        ~^/_matrix/federation/v1/backfill/ synapse_federation;
+        ~^/_matrix/federation/v1/get_missing_events/ synapse_federation;
+        ~^/_matrix/federation/v1/publicRooms synapse_federation;
+        ~^/_matrix/federation/v1/query/ synapse_federation;
+        ~^/_matrix/federation/v1/make_join/ synapse_federation;
+        ~^/_matrix/federation/v1/make_leave/ synapse_federation;
+        ~^/_matrix/federation/(v1|v2)/send_join/ synapse_federation;
+        ~^/_matrix/federation/(v1|v2)/send_leave/ synapse_federation;
+        ~^/_matrix/federation/(v1|v2)/invite/ synapse_federation;
+        ~^/_matrix/federation/v1/event_auth/ synapse_federation;
+        ~^/_matrix/federation/v1/timestamp_to_event/ synapse_federation;
+        ~^/_matrix/federation/v1/exchange_third_party_invite/ synapse_federation;
+        ~^/_matrix/federation/v1/user/devices/ synapse_federation;
+        ~^/_matrix/key/v2/query synapse_federation;
+        ~^/_matrix/federation/v1/hierarchy/ synapse_federation;
 
-      # Inbound federation transaction request
-      ~^/_matrix/federation/v1/send/ synapse_federation_transaction;
+        # Inbound federation transaction request
+        ~^/_matrix/federation/v1/send/ synapse_federation_transaction;
 
-      # Client API requests
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/createRoom$ synapse_client_interaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/publicRooms$ synapse_client_interaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/joined_members$ synapse_client_interaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/context/.*$ synapse_client_interaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$ synapse_client_interaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$ synapse_client_interaction;
-      ~^/_matrix/client/v1/rooms/.*/hierarchy$ synapse_client_interaction;
-      ~^/_matrix/client/(v1|unstable)/rooms/.*/relations/ synapse_client_interaction;
-      ~^/_matrix/client/v1/rooms/.*/threads$ synapse_client_interaction;
-      ~^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$ synapse_client_interaction;
-      ~^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$ synapse_client_interaction;
-      ~^/_matrix/client/(r0|v3|unstable)/account/3pid$ synapse_client_interaction;
-      ~^/_matrix/client/(r0|v3|unstable)/account/whoami$ synapse_client_interaction;
-      ~^/_matrix/client/(r0|v3|unstable)/devices$ synapse_client_interaction;
-      ~^/_matrix/client/versions$ synapse_client_interaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$ synapse_client_interaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/ synapse_client_interaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms$ synapse_client_interaction;
-      ~^/_matrix/client/v1/rooms/.*/timestamp_to_event$ synapse_client_interaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/search$ synapse_client_interaction;
+        # Client API requests
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/createRoom$ synapse_client_interaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/publicRooms$ synapse_client_interaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/joined_members$ synapse_client_interaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/context/.*$ synapse_client_interaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$ synapse_client_interaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$ synapse_client_interaction;
+        ~^/_matrix/client/v1/rooms/.*/hierarchy$ synapse_client_interaction;
+        ~^/_matrix/client/(v1|unstable)/rooms/.*/relations/ synapse_client_interaction;
+        ~^/_matrix/client/v1/rooms/.*/threads$ synapse_client_interaction;
+        ~^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$ synapse_client_interaction;
+        ~^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$ synapse_client_interaction;
+        ~^/_matrix/client/(r0|v3|unstable)/account/3pid$ synapse_client_interaction;
+        ~^/_matrix/client/(r0|v3|unstable)/account/whoami$ synapse_client_interaction;
+        ~^/_matrix/client/(r0|v3|unstable)/devices$ synapse_client_interaction;
+        ~^/_matrix/client/versions$ synapse_client_interaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$ synapse_client_interaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/ synapse_client_interaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms$ synapse_client_interaction;
+        ~^/_matrix/client/v1/rooms/.*/timestamp_to_event$ synapse_client_interaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/search$ synapse_client_interaction;
 
-      # Encryption requests
-      ~^/_matrix/client/(r0|v3|unstable)/keys/query$ synapse_client_encryption;
-      ~^/_matrix/client/(r0|v3|unstable)/keys/changes$ synapse_client_encryption;
-      ~^/_matrix/client/(r0|v3|unstable)/keys/claim$ synapse_client_encryption;
-      ~^/_matrix/client/(r0|v3|unstable)/room_keys/ synapse_client_encryption;
-      ~^/_matrix/client/(r0|v3|unstable)/keys/upload/ synapse_client_encryption;
+        # Encryption requests
+        ~^/_matrix/client/(r0|v3|unstable)/keys/query$ synapse_client_encryption;
+        ~^/_matrix/client/(r0|v3|unstable)/keys/changes$ synapse_client_encryption;
+        ~^/_matrix/client/(r0|v3|unstable)/keys/claim$ synapse_client_encryption;
+        ~^/_matrix/client/(r0|v3|unstable)/room_keys/ synapse_client_encryption;
+        ~^/_matrix/client/(r0|v3|unstable)/keys/upload/ synapse_client_encryption;
 
-      # Registration/login requests
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/login$ synapse_client_login;
-      ~^/_matrix/client/(r0|v3|unstable)/register$ synapse_client_login;
-      ~^/_matrix/client/v1/register/m.login.registration_token/validity$ synapse_client_login;
+        # Registration/login requests
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/login$ synapse_client_login;
+        ~^/_matrix/client/(r0|v3|unstable)/register$ synapse_client_login;
+        ~^/_matrix/client/v1/register/m.login.registration_token/validity$ synapse_client_login;
 
-      # Event sending requests
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact synapse_client_transaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send synapse_client_transaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state/ synapse_client_transaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$ synapse_client_transaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/join/ synapse_client_transaction;
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/profile/ synapse_client_transaction;
+        # Event sending requests
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact synapse_client_transaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send synapse_client_transaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state/ synapse_client_transaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$ synapse_client_transaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/join/ synapse_client_transaction;
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/profile/ synapse_client_transaction;
 
-      # Account data requests
-      ~^/_matrix/client/(r0|v3|unstable)/.*/tags synapse_client_data;
-      ~^/_matrix/client/(r0|v3|unstable)/.*/account_data synapse_client_data;
+        # Account data requests
+        ~^/_matrix/client/(r0|v3|unstable)/.*/tags synapse_client_data;
+        ~^/_matrix/client/(r0|v3|unstable)/.*/account_data synapse_client_data;
 
-      # Receipts requests
-      ~^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt synapse_client_interaction;
-      ~^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers synapse_client_interaction;
+        # Receipts requests
+        ~^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt synapse_client_interaction;
+        ~^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers synapse_client_interaction;
 
-      # Presence requests
-      ~^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ synapse_client_presence;
+        # Presence requests
+        ~^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ synapse_client_presence;
 
-      # User directory search requests;
-      ~^/_matrix/client/(r0|v3|unstable)/user_directory/search$ synapse_client_user-dir;
-    }
+        # User directory search requests;
+        ~^/_matrix/client/(r0|v3|unstable)/user_directory/search$ synapse_client_user-dir;
+      }
 
-    #Plugboard for url -> workers
-    map $synapse_uri_group $synapse_backend {
-      default synapse_master;
+      #Plugboard for url -> workers
+      map $synapse_uri_group $synapse_backend {
+        default synapse_master;
 
-      synapse_initial_sync synapse_worker_initial_sync;
-      synapse_normal_sync  synapse_worker_normal_sync;
+        synapse_initial_sync synapse_worker_initial_sync;
+        synapse_normal_sync  synapse_worker_normal_sync;
 
-      synapse_federation synapse_worker_federation;
-      synapse_federation_transaction synapse_worker_federation;
+        synapse_federation synapse_worker_federation;
+        synapse_federation_transaction synapse_worker_federation;
 
-      synapse_client_user-dir synapse_worker_user-dir;
-    }
+        synapse_client_user-dir synapse_worker_user-dir;
+      }
 
-    # from https://github.com/tswfi/synapse/commit/b3704b936663cc692241e978dce4ac623276b1a6
-    map $arg_access_token $accesstoken_from_urlparam {
-      # Defaults to just passing back the whole accesstoken
-      default   $arg_access_token;
-      # Try to extract username part from accesstoken URL parameter
-      "~syt_(?<username>.*?)_.*"           $username;
-    }
+      # from https://github.com/tswfi/synapse/commit/b3704b936663cc692241e978dce4ac623276b1a6
+      map $arg_access_token $accesstoken_from_urlparam {
+        # Defaults to just passing back the whole accesstoken
+        default   $arg_access_token;
+        # Try to extract username part from accesstoken URL parameter
+        "~syt_(?<username>.*?)_.*"           $username;
+      }
 
-    map $http_authorization $mxid_localpart {
-      # Defaults to just passing back the whole accesstoken
-      default                              $http_authorization;
-      # Try to extract username part from accesstoken header
-      "~Bearer syt_(?<username>.*?)_.*"    $username;
-      # if no authorization-header exist, try mapper for URL parameter "access_token"
-      ""                                   $accesstoken_from_urlparam;
-    }
-  '';
+      map $http_authorization $mxid_localpart {
+        # Defaults to just passing back the whole accesstoken
+        default                              $http_authorization;
+        # Try to extract username part from accesstoken header
+        "~Bearer syt_(?<username>.*?)_.*"    $username;
+        # if no authorization-header exist, try mapper for URL parameter "access_token"
+        ""                                   $accesstoken_from_urlparam;
+      }
+    '';
 
-  services.nginx.upstreams.synapse_master.servers = let
-    isMainListener = l: isListenerType "client" l && isListenerType "federation" l;
-    firstMainListener = lib.findFirst isMainListener
-      (throw "No cartch-all listener configured") cfg.settings.listeners;
-      address = lib.findFirst (_: true) (throw "No address in main listener") firstMainListener.bind_addresses;
-      port = firstMainListener.port;
-      socketAddress = "${address}:${builtins.toString port}";
-  in {
-    "${socketAddress}" = { };
-  };
+    services.nginx.upstreams.synapse_master.servers = let
+      isMainListener = l: isListenerType "client" l && isListenerType "federation" l;
+      firstMainListener = lib.findFirst isMainListener
+        (throw "No catch-all listener configured") cfg.settings.listeners;
+        address = lib.findFirst (_: true) (throw "No address in main listener") firstMainListener.bind_addresses;
+        port = firstMainListener.port;
+        socketAddress = "${address}:${builtins.toString port}";
+    in {
+      "${socketAddress}" = { };
+    };
 
-  
-  services.nginx.upstreams.synapse_worker_federation = {
-    servers = let
-      fedReceivers = getWorkersOfType "fed-receiver";
-      socketAddresses = generateSocketAddresses "federation" fedReceivers;
-      in if fedReceivers != { } then 
+
+    services.nginx.upstreams.synapse_worker_federation = {
+      servers = let
+        fedReceivers = getWorkersOfType "fed-receiver";
+        socketAddresses = generateSocketAddresses "federation" fedReceivers;
+        in if fedReceivers != { } then
+          lib.genAttrs socketAddresses (_: { })
+      else config.services.nginx.upstreams.synapse_master.servers;
+      extraConfig = ''
+        ip_hash;
+      '';
+    };
+
+
+    services.nginx.upstreams.synapse_worker_initial_sync = {
+      servers = let
+        initialSyncers = getWorkersOfType "initial-sync";
+        socketAddresses = generateSocketAddresses "client" initialSyncers;
+      in if initialSyncers != { } then
         lib.genAttrs socketAddresses (_: { })
-    else config.services.nginx.upstreams.synapse_master.servers;
-    extraConfig = ''
-      ip_hash;
-    '';
-  };
-
-
-  services.nginx.upstreams.synapse_worker_initial_sync = {
-    servers = let
-      initialSyncers = getWorkersOfType "initial-sync";
-      socketAddresses = generateSocketAddresses "client" initialSyncers;
-    in if initialSyncers != { } then
-      lib.genAttrs socketAddresses (_: { })
-    else config.services.nginx.upstreams.synapse_master.servers;
-    extraConfig = ''
-      hash $mxid_localpart consistent;
-    '';
-  };
-
-
-  services.nginx.upstreams.synapse_worker_normal_sync = {
-    servers = let
-      normalSyncers = getWorkersOfType "normal-sync";
-      socketAddresses = generateSocketAddresses "client" normalSyncers;
-    in if normalSyncers != { } then
-      lib.genAttrs socketAddresses (_: { })
-    else config.services.nginx.upstreams.synapse_master.servers;
-    extraConfig = ''
-      hash $mxid_localpart consistent;
-    '';
-  };
-
-
-  services.nginx.upstreams.synapse_worker_user-dir = {
-    servers = let
-      workers = getWorkersOfType "user-dir";
-      socketAddresses = generateSocketAddresses "client" workers;
-    in if workers != { } then
-      lib.genAttrs socketAddresses (_: { })
-    else config.services.nginx.upstreams.synapse_master.servers;
-  };
-
-  services.nginx.virtualHosts."${cfg.public_baseurl}" = {
-    enableACME = true;
-    forceSSL = true;
-    locations."/_matrix" = {
-      proxyPass = "http://$synapse_backend";
+      else config.services.nginx.upstreams.synapse_master.servers;
       extraConfig = ''
-        add_header X-debug-backend $synapse_backend;
-        add_header X-debug-group $synapse_uri_group;
-        client_max_body_size ${cfg.settings.max_upload_size};
-        proxy_read_timeout 10m;
+        hash $mxid_localpart consistent;
       '';
     };
-    locations."~ ^/_matrix/client/(r0|v3)/sync$" = {
-      proxyPass = "http://$synapse_backend";
+
+
+    services.nginx.upstreams.synapse_worker_normal_sync = {
+      servers = let
+        normalSyncers = getWorkersOfType "normal-sync";
+        socketAddresses = generateSocketAddresses "client" normalSyncers;
+      in if normalSyncers != { } then
+        lib.genAttrs socketAddresses (_: { })
+      else config.services.nginx.upstreams.synapse_master.servers;
       extraConfig = ''
-        proxy_read_timeout 1h;
+        hash $mxid_localpart consistent;
       '';
     };
-    locations."~ ^/_matrix/client/(api/v1|r0|v3)/initialSync$" = {
-      proxyPass = "http://synapse_worker_initial_sync";
-      extraConfig = ''
-        proxy_read_timeout 1h;
-      '';
+
+
+    services.nginx.upstreams.synapse_worker_user-dir = {
+      servers = let
+        workers = getWorkersOfType "user-dir";
+        socketAddresses = generateSocketAddresses "client" workers;
+      in if workers != { } then
+        lib.genAttrs socketAddresses (_: { })
+      else config.services.nginx.upstreams.synapse_master.servers;
     };
-    locations."~ ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$" = {
-      proxyPass = "http://synapse_worker_initial_sync";
-      extraConfig = ''
-        proxy_read_timeout 1h;
-      '';
-    };
-    locations."/_synapse/client" = {
-      proxyPass = "http://$synapse_backend";
+
+    services.nginx.virtualHosts."${cfg.public_baseurl}" = {
+      enableACME = lib.mkDefault true;
+      forceSSL = true;
+      locations."/_matrix" = {
+        proxyPass = "http://$synapse_backend";
+        extraConfig = ''
+          add_header X-debug-backend $synapse_backend;
+          add_header X-debug-group $synapse_uri_group;
+          client_max_body_size ${cfg.settings.max_upload_size};
+          proxy_read_timeout 10m;
+        '';
+      };
+      locations."~ ^/_matrix/client/(r0|v3)/sync$" = {
+        proxyPass = "http://$synapse_backend";
+        extraConfig = ''
+          proxy_read_timeout 1h;
+        '';
+      };
+      locations."~ ^/_matrix/client/(api/v1|r0|v3)/initialSync$" = {
+        proxyPass = "http://synapse_worker_initial_sync";
+        extraConfig = ''
+          proxy_read_timeout 1h;
+        '';
+      };
+      locations."~ ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$" = {
+        proxyPass = "http://synapse_worker_initial_sync";
+        extraConfig = ''
+          proxy_read_timeout 1h;
+        '';
+      };
+      locations."/_synapse/client" = {
+        proxyPass = "http://$synapse_backend";
+      };
+      locations."/.well-known/matrix" = {
+        proxyPass = "http://$synapse_backend";
+      };
     };
   };
-};
-}
+}

synapse-module/workers.nix

@@ -0,0 +1,318 @@
+{ matrix-synapse-common-config,
+  matrix-lib,
+  pluginsEnv,
+  throw',
+  format
+}:
+{ pkgs, lib, config, ... }: let
+
+  cfg = config.services.matrix-synapse-next;
+  wcfg = config.services.matrix-synapse-next.workers;
+
+  # Used to generate proper defaultTexts.
+  cfgText = "config.services.matrix-synapse-next";
+  wcfgText = "config.services.matrix-synapse-next.workers";
+
+  inherit (lib) types mkOption mkEnableOption mkIf mkMerge literalExpression;
+
+  mkWorkerCountOption = workerType: mkOption {
+    type = types.ints.unsigned;
+    description = "How many automatically configured ${workerType} workers to set up";
+    default = 0;
+    example = 1;
+  };
+
+  genAttrs' = items: f: g: builtins.listToAttrs (map (i: lib.nameValuePair (f i) (g i)) items);
+
+  mainReplicationListener = matrix-lib.firstListenerOfType "replication" cfg.settings.listeners;
+in {
+  # See https://github.com/matrix-org/synapse/blob/develop/docs/workers.md for more info
+  options.services.matrix-synapse-next.workers = let
+    workerInstanceType = types.submodule ({ config, ... }: {
+      options = {
+        isAuto = mkOption {
+          type = types.bool;
+          internal = true;
+          default = false;
+          description = ''
+            This is an internal flag that signals that this worker is part of the
+            workers generated by either of the following:
+
+            - federationSenders
+            - federationReceivers
+            - initialSyncers
+            - normalSyncers
+            - eventPersisters
+            - useUserDirectoryWorker
+          '';
+        };
+
+        index = mkOption {
+          internal = true;
+          type = types.ints.positive;
+          description = ''
+            This is an internal variable that indexes the worker of this type.
+          '';
+        };
+
+        # The custom string type here is mainly for the name to use
+        # for the metrics of custom worker types
+        type = mkOption {
+          type = types.str;
+          # TODO: add description and possibly default value?
+        };
+
+        settings = mkOption {
+          type = workerSettingsType config;
+          default = { };
+        };
+      };
+    });
+
+    workerSettingsType = instanceCfg: types.submodule {
+      freeformType = format.type;
+      
+      options = {
+        worker_app = mkOption {
+          type = types.enum [
+            "synapse.app.generic_worker"
+            "synapse.app.appservice"
+            "synapse.app.media_repository"
+            "synapse.app.user_dir"
+          ];
+          description = "The type of worker application";
+          default = "synapse.app.generic_worker";
+        };
+
+        worker_listeners = mkOption {
+          type = types.listOf (workerListenerType instanceCfg);
+          description = "Listener configuration for the worker, similar to the main synapse listener";
+          default = [ ];
+        };
+      };
+    };
+
+    workerListenerType = instanceCfg: types.submodule {
+      options = {
+        type = mkOption {
+          type = types.enum [ "http" "metrics" ];
+          description = "The type of the listener";
+          default = "http";
+        };
+
+        path = mkOption {
+          type = types.path;
+          default = instanceCfg.name;
+          description = ''
+            A path and filename for a Unix socket.
+          '';
+        };
+
+        # port = mkOption {
+        #   type = types.port;
+        #   description = "The TCP port to bind to";
+        # };
+
+        # bind_addresses = mkOption {
+        #   type = with types; listOf str;
+        #   description = "A list of local addresses to listen on";
+        #   default = [ wcfg.defaultListenerAddress ];
+        #   defaultText = literalExpression "[ ${wcfgText}.defaultListenerAddress ]";
+        # };
+
+        tls = mkOption {
+          type = types.bool;
+          description = ''
+            Whether to enable TLS for this listener.
+            Will use the TLS key/cert specified in tls_private_key_path / tls_certificate_path.
+          '';
+          default = false;
+          example = true;
+        };
+
+        x_forwarded = mkOption {
+          type = types.bool;
+          description = ''
+            Whether to use the X-Forwarded-For HTTP header as the client IP.
+
+            This option is only valid for an 'http' listener.
+            It is useful when Synapse is running behind a reverse-proxy.
+          '';
+          default = true;
+          example = false;
+        };
+
+        resources = let
+          typeToResources = t: {
+            "fed-receiver"  = [ "federation" ];
+            "fed-sender"    = [ ];
+            "initial-sync"  = [ "client" ];
+            "normal-sync"   = [ "client" ];
+            "event-persist" = [ "replication" ];
+            "user-dir"      = [ "client" ];
+          }.${t};
+        in mkOption {
+          type = types.listOf (types.submodule {
+            options = {
+              names = mkOption {
+                type = with types; listOf (enum [
+                  "client"
+                  "consent"
+                  "federation"
+                  "keys"
+                  "media"
+                  "metrics"
+                  "openid"
+                  "replication"
+                  "static"
+                  "webclient"
+                ]);
+                description = "A list of resources to host on this port";
+                default = lib.optionals instanceCfg.isAuto (typeToResources instanceCfg.type);
+                defaultText = ''
+                  If the worker is generated from other config, the resource type will
+                  be determined automatically.
+                '';
+              };
+
+              compress = mkEnableOption "HTTP compression for this resource";
+            };
+          });
+          default = [{ }];
+        };
+      };
+    };
+  in {
+    enableMetrics = mkOption {
+      type = types.bool;
+      default = cfg.settings.enable_metrics;
+      defaultText = literalExpression "${cfgText}.settings.enable_metrics";
+      # TODO: add description
+    };
+
+    federationSenders = mkWorkerCountOption "federation-sender";
+    federationReceivers = mkWorkerCountOption "federation-reciever";
+    initialSyncers = mkWorkerCountOption "initial-syncer";
+    normalSyncers = mkWorkerCountOption "sync";
+    eventPersisters = mkWorkerCountOption "event-persister";
+
+    useUserDirectoryWorker = mkEnableOption "user directory worker";
+
+    instances = mkOption {
+      type = types.attrsOf workerInstanceType;
+      default = { };
+      description = "Worker configuration";
+      example = {
+        "federation-sender-1" = {
+          settings = {
+            worker_name = "federation-sender-1";
+            worker_app = "synapse.app.generic_worker";
+
+            path = "/run/matrix-synapse/federation-sender-1.sock";
+            # worker_replication_host = "127.0.0.1";
+            # worker_replication_http_port = 9093;
+            worker_listeners = [ ];
+          };
+        };
+      };
+    };
+  };
+
+  config = let 
+    genList1 = f: builtins.genList (i: f (i + 1));
+  in {
+    services.matrix-synapse-next.settings = {
+      federation_sender_instances =
+        genList1 (i: "auto-fed-sender-${toString i}") wcfg.federationSenders;
+
+      instance_map = lib.mkIf (cfg.workers.instances != { }) ({
+        main.path = "/run/matrix-synapse/main-replication-worker.sock";
+      } // builtins.mapAttrs (n: v: {
+        inherit (builtins.head v.settings.worker_listeners) path;
+      }) wcfg.instances);
+
+      stream_writers.events =
+        mkIf (wcfg.eventPersisters > 0)
+          (genList1 (i: "auto-event-persist-${toString i}") wcfg.eventPersisters); 
+
+      update_user_directory_from_worker =
+        mkIf wcfg.useUserDirectoryWorker "auto-user-dir-1";
+    };
+
+    services.matrix-synapse-next.workers.instances =
+      let
+        workerInstances = {
+          "fed-sender" = wcfg.federationSenders;
+          "fed-receiver" = wcfg.federationReceivers;
+          "initial-sync" = wcfg.initialSyncers;
+          "normal-sync" = wcfg.normalSyncers;
+          "event-persist" = wcfg.eventPersisters;
+          "user-dir" = if wcfg.useUserDirectoryWorker then 1 else 0;
+        };
+      in
+        lib.pipe workerInstances [
+          (lib.mapAttrsToList (type: count: { inherit type count; }))
+          (map ({ type, count }: genList1 (i: rec {
+            name = "auto-${type}-${toString i}";
+            value = {
+              inherit type;
+              isAuto = true;
+              index = i;
+              settings.worker_listeners =
+                [
+                  { path = "/run/matrix-synapse/${name}.sock"; }
+                ] ++ lib.optionals wcfg.enableMetrics [{
+                  path = "/run/matrix-synapse/${name}-metrics.sock";
+                  resources = [{ names = [ "metrics" ]; }];
+                  type = "metrics";
+                }];
+            };
+          }) count))
+          lib.flatten
+          builtins.listToAttrs
+        ];
+
+    systemd.services = let
+      workerList = lib.mapAttrsToList lib.nameValuePair wcfg.instances;
+      workerConfig = worker: format.generate "matrix-synapse-worker-${worker.name}-config.yaml"
+        (worker.value.settings // { worker_name = worker.name; });
+    in builtins.listToAttrs (lib.flip map workerList (worker: {
+      name = "matrix-synapse-worker-${worker.name}";
+      value = {
+        documentation = [ "https://github.com/matrix-org/synapse/blob/develop/docs/workers.md" ];
+        description = "Synapse Matrix Worker";
+        partOf = [ "matrix-synapse.target" ];
+        wantedBy = [ "matrix-synapse.target" ];
+        after = [ "matrix-synapse.service" ];
+        requires = [ "matrix-synapse.service" ];
+        restartTriggers = [ matrix-synapse-common-config (workerConfig worker) ] ++ cfg.extraConfigFiles;
+        environment = {
+          PYTHONPATH = lib.makeSearchPathOutput "lib" cfg.package.python.sitePackages [
+            pluginsEnv
+          ];
+        };
+        serviceConfig = {
+          Restart = "always";
+          Type = "notify";
+          User = "matrix-synapse";
+          Group = "matrix-synapse";
+          Slice = "system-matrix-synapse.slice";
+          WorkingDirectory = cfg.dataDir;
+          RuntimeDirectory = [ "matrix-synapse" ];
+          ExecStartPre = pkgs.writers.writeBash "wait-for-synapse" ''
+            # From https://md.darmstadt.ccc.de/synapse-at-work
+            while ! systemctl is-active -q matrix-synapse.service; do
+                sleep 1
+            done
+          '';
+          ExecStart = let
+            flags = lib.cli.toGNUCommandLineShell {} {
+              config-path = [ matrix-synapse-common-config (workerConfig worker) ] ++ cfg.extraConfigFiles;
+              keys-directory = cfg.dataDir;
+            };
+          in "${cfg.package}/bin/synapse_worker ${flags}";
+        };
+      };
+    }));
+  };
+}

tests/auto-workers-config.nix

@@ -0,0 +1,18 @@
+{ pkgs, lib, ... }:
+{
+  services.matrix-synapse-next = {
+    enable = true;
+    settings.server_name = "matrix.example.com";
+
+    workers = {
+      enableMetrics = true;
+
+      federationSenders = 2;
+      federationReceivers = 2;
+      initialSyncers = 2;
+      normalSyncers = 2;
+      eventPersisters = 2;
+      useUserDirectoryWorker = true;
+    };
+  };
+}

tests/base-config.nix

@@ -0,0 +1,7 @@
+{ pkgs, lib, ... }:
+{
+  services.matrix-synapse-next = {
+    enable = true;
+    settings.server_name = "matrix.example.com";
+  };
+}

tests/default.nix

@@ -0,0 +1,16 @@
+{ nixpkgs, pkgs, system ? pkgs.system, nixosModule, ... }: let
+  buildSystemWithConfig = configPath: (nixpkgs.lib.nixosSystem {
+    inherit system;
+    modules = [
+      nixosModule
+      configPath
+      {
+        boot.isContainer = true;
+      }
+    ];
+  }).config.system.build.toplevel;
+in {
+  a = pkgs.writeText "hello-world" ''a'';
+  base-config = buildSystemWithConfig ./base-config.nix;
+  auto-workers-config = buildSystemWithConfig ./auto-workers-config.nix;
+}