out-of-your-element: init at 1.2

.gitignore

@@ -1 +0,0 @@
-result

README.MD

@@ -46,7 +46,7 @@ services.matrix-synapse-next = {
   enableSlidingSync = true;
 };
 
-services.matrix-synapse.sliding-sync.environmentFile = "/some/file/containing/SYNCV3_SECRET=<some secret>";
+matrix-synapse.sliding-sync.environmentFile = "/some/file/containing/SYNCV3_SECRET=<some secret>";
 
 ```
 

flake.lock

@@ -2,38 +2,22 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1690789960,
-        "narHash": "sha256-3K+2HuyGTiJUSZNJxXXvc0qj4xFx1FHC/ItYtEa7/Xs=",
-        "owner": "NixOS",
+        "lastModified": 1697239051,
+        "narHash": "sha256-TvKERJH5h470GWyTOgEKzxdBz9tjQdSGlAFDbHhAI7g=",
+        "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "fb942492b7accdee4e6d17f5447091c65897dde4",
+        "rev": "e0c8b6f719e911224906f3c059208a7f0e3441c7",
         "type": "github"
       },
       "original": {
-        "id": "nixpkgs",
-        "ref": "nixos-unstable",
-        "type": "indirect"
-      }
-    },
-    "nixpkgs-lib": {
-      "locked": {
-        "lastModified": 1673743903,
-        "narHash": "sha256-sloY6KYyVOozJ1CkbgJPpZ99TKIjIvM+04V48C04sMQ=",
-        "owner": "nix-community",
-        "repo": "nixpkgs.lib",
-        "rev": "7555e2dfcbac1533f047021f1744ac8871150f9f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nixpkgs.lib",
+        "owner": "nixos",
+        "repo": "nixpkgs",
         "type": "github"
       }
     },
     "root": {
       "inputs": {
-        "nixpkgs": "nixpkgs",
-        "nixpkgs-lib": "nixpkgs-lib"
+        "nixpkgs": "nixpkgs"
       }
     }
   },

flake.nix

@@ -2,27 +2,20 @@
   description = "NixOS modules for matrix related services";
 
   inputs = {
-    nixpkgs-lib.url = "github:nix-community/nixpkgs.lib";
-    nixpkgs.url = "nixpkgs/nixos-unstable";
+    # nixpkgs-lib.url = github:nix-community/nixpkgs.lib;
+    nixpkgs.url = github:nixos/nixpkgs;
   };
 
-  outputs = { self, nixpkgs, nixpkgs-lib }: {
+  outputs = { self, nixpkgs }:
+  let
+    systems = [ "x86_64-linux"];
+  in {
     nixosModules = {
       default = import ./module.nix;
     };
-
-    lib = import ./lib.nix { lib = nixpkgs-lib.lib; };
-
-    packages = let
-      system = "x86_64-linux";
-      pkgs = nixpkgs.legacyPackages.${system};
-    in {
-        ${system}.tests = import ./tests {
-        inherit system;
-        inherit nixpkgs;
-        inherit pkgs;
-        nixosModule = self.outputs.nixosModules.synapse;
-      };
-    };
+    lib = import ./lib.nix { lib = nixpkgs.lib; };
+    packages = nixpkgs.lib.genAttrs systems (system: {
+      out-of-your-element = nixpkgs.legacyPackages.${system}.callPackage ./pkgs/out-of-your-element {};
+    });
   };
 }

pkgs/matrix-sliding-sync/default.nix

@@ -5,16 +5,16 @@
 
 buildGoModule rec {
   pname = "matrix-sliding-sync";
-  version = "0.99.11";
+  version = "0.99.10";
 
   src = fetchFromGitHub {
     owner = "matrix-org";
     repo = "sliding-sync";
     rev = "refs/tags/v${version}";
-    hash = "sha256-Wd/nnJhKg+BDyOIz42zEScjzQRrpEq6YG9/9Tk24hgg=";
+    hash = "sha256-TjChelGVicA59q6iIIDTI7nSj8KtjpA4bfZqsfoiOQg=";
   };
 
-  vendorHash = "sha256-0QSyYhOht1j1tWNxHQh+NUZA/W1xy7ANu+29H/gusOE=";
+  vendorHash = "sha256-E3nCcw6eTKKcL55ls6n5pYlRFffsefsN0G1Hwd49uh8=";
 
   subPackages = [ "cmd/syncv3" ];
 

pkgs/out-of-your-element/default.nix

@@ -0,0 +1,26 @@
+{ lib
+, buildNpmPackage
+, fetchFromGitea
+, git
+}:
+
+buildNpmPackage rec {
+  pname = "out-of-your-element";
+  version = "1.2";
+
+  src = fetchFromGitea {
+    domain = "gitdab.com";
+    owner = "cadence";
+    repo = "out-of-your-element";
+    rev = "v1.2";
+    hash = "sha256-rlp6Eens5gV0dwLpICjKaVhxNXXeb/S7l628eXYvZaY=";
+  };
+
+  npmDepsHash = "sha256-ComQ8ua7k8zg0Dzih+MVgjnySpSlLJmLqnwxADCUv7M=";
+
+  dontNpmBuild = true;
+
+  makeCacheWritable = true;
+  npmFlags = [ "--loglevel=verbose" ]; #"--legacy-peer-deps" ];
+
+}

synapse-module/workers.nix

@@ -19,7 +19,6 @@
     type = types.ints.unsigned;
     description = "How many automatically configured ${workerType} workers to set up";
     default = 0;
-    example = 1;
   };
 
   genAttrs' = items: f: g: builtins.listToAttrs (map (i: lib.nameValuePair (f i) (g i)) items);
@@ -34,25 +33,11 @@ in {
           type = types.bool;
           internal = true;
           default = false;
-          description = ''
-            This is an internal flag that signals that this worker is part of the
-            workers generated by either of the following:
-
-            - federationSenders
-            - federationReceivers
-            - initialSyncers
-            - normalSyncers
-            - eventPersisters
-            - useUserDirectoryWorker
-          '';
         };
 
         index = mkOption {
           internal = true;
           type = types.ints.positive;
-          description = ''
-            This is an internal variable that indexes the worker of this type.
-          '';
         };
 
         # The custom string type here is mainly for the name to use
@@ -100,25 +85,17 @@ in {
           default = "http";
         };
 
-        path = mkOption {
-          type = types.path;
-          default = instanceCfg.name;
-          description = ''
-            A path and filename for a Unix socket.
-          '';
+        port = mkOption {
+          type = types.port;
+          description = "The TCP port to bind to";
         };
 
-        # port = mkOption {
-        #   type = types.port;
-        #   description = "The TCP port to bind to";
-        # };
-
-        # bind_addresses = mkOption {
-        #   type = with types; listOf str;
-        #   description = "A list of local addresses to listen on";
-        #   default = [ wcfg.defaultListenerAddress ];
-        #   defaultText = literalExpression "[ ${wcfgText}.defaultListenerAddress ]";
-        # };
+        bind_addresses = mkOption {
+          type = with types; listOf str;
+          description = "A list of local addresses to listen on";
+          default = [ wcfg.defaultListenerAddress ];
+          defaultText = literalExpression "[ ${wcfgText}.defaultListenerAddress ]";
+        };
 
         tls = mkOption {
           type = types.bool;
@@ -183,6 +160,38 @@ in {
       };
     };
   in {
+    mainReplicationHost = mkOption {
+      type = types.str;
+      default = let
+        host = (matrix-lib.connectionInfo mainReplicationListener).host;
+      in
+        # To avoid connecting to 0.0.0.0 and so on
+        if builtins.elem host [ "0.0.0.0" "::" ]
+          then "127.0.0.1"
+          else host;
+      # TODO: add defaultText
+      description = "Host of the main synapse instance's replication listener";
+    };
+
+    mainReplicationPort = mkOption {
+      type = types.port;
+      default = mainReplicationListener.port;
+      # TODO: add defaultText
+      description = "Port for the main synapse instance's replication listener";
+    };
+
+    defaultListenerAddress = mkOption {
+      type = types.str;
+      default = "127.0.0.1";
+      description = "The default listener address for the worker";
+    };
+
+    workerStartingPort = mkOption {
+      type = types.port;
+      description = "What port should the automatically configured workers start enumerating from";
+      default = 8083;
+    };
+
     enableMetrics = mkOption {
       type = types.bool;
       default = cfg.settings.enable_metrics;
@@ -190,6 +199,12 @@ in {
       # TODO: add description
     };
 
+    metricsStartingPort = mkOption {
+      type = types.port;
+      default = 18083;
+      # TODO: add description
+    };
+
     federationSenders = mkWorkerCountOption "federation-sender";
     federationReceivers = mkWorkerCountOption "federation-reciever";
     initialSyncers = mkWorkerCountOption "initial-syncer";
@@ -203,14 +218,13 @@ in {
       default = { };
       description = "Worker configuration";
       example = {
-        "federation-sender-1" = {
+        "federation_sender1" = {
           settings = {
-            worker_name = "federation-sender-1";
+            worker_name = "federation_sender1";
             worker_app = "synapse.app.generic_worker";
 
-            path = "/run/matrix-synapse/federation-sender-1.sock";
-            # worker_replication_host = "127.0.0.1";
-            # worker_replication_http_port = 9093;
+            worker_replication_host = "127.0.0.1";
+            worker_replication_http_port = 9093;
             worker_listeners = [ ];
           };
         };
@@ -218,59 +232,94 @@ in {
     };
   };
 
-  config = let 
-    genList1 = f: builtins.genList (i: f (i + 1));
-  in {
+  config = {
     services.matrix-synapse-next.settings = {
       federation_sender_instances =
-        genList1 (i: "auto-fed-sender-${toString i}") wcfg.federationSenders;
+        lib.genList (i: "auto-fed-sender${toString (i + 1)}") wcfg.federationSenders;
 
-      instance_map = lib.mkIf (cfg.workers.instances != { }) ({
-        main.path = "/run/matrix-synapse/main-replication-worker.sock";
-      } // builtins.mapAttrs (n: v: {
-        inherit (builtins.head v.settings.worker_listeners) path;
-      }) wcfg.instances);
+      instance_map = (lib.mkIf (cfg.workers.instances != { }) ({
+        main = let
+          host = lib.head mainReplicationListener.bind_addresses;
+        in {
+          host = if builtins.elem host [ "0.0.0.0" "::"] then "127.0.0.1" else host;
+          port = mainReplicationListener.port;
+        };
+      } // genAttrs' (lib.lists.range 1 wcfg.eventPersisters)
+        (i: "auto-event-persist${toString i}")
+        (i: let
+          wRL = matrix-lib.firstListenerOfType "replication" wcfg.instances."auto-event-persist${toString i}".settings.worker_listeners;
+        in matrix-lib.connectionInfo wRL)));
 
       stream_writers.events =
         mkIf (wcfg.eventPersisters > 0)
-          (genList1 (i: "auto-event-persist-${toString i}") wcfg.eventPersisters); 
+        (lib.genList (i: "auto-event-persist${toString (i + 1)}") wcfg.eventPersisters); 
 
       update_user_directory_from_worker =
-        mkIf wcfg.useUserDirectoryWorker "auto-user-dir-1";
+        mkIf wcfg.useUserDirectoryWorker "auto-user-dir";
     };
 
-    services.matrix-synapse-next.workers.instances =
-      let
-        workerInstances = {
-          "fed-sender" = wcfg.federationSenders;
-          "fed-receiver" = wcfg.federationReceivers;
-          "initial-sync" = wcfg.initialSyncers;
-          "normal-sync" = wcfg.normalSyncers;
-          "event-persist" = wcfg.eventPersisters;
-          "user-dir" = if wcfg.useUserDirectoryWorker then 1 else 0;
+    services.matrix-synapse-next.workers.instances = let
+      sum = lib.foldl lib.add 0;
+      workerListenersWithMetrics = portOffset:
+        lib.singleton ({
+          port = wcfg.workerStartingPort + portOffset - 1;
+        }) 
+        ++ lib.optional wcfg.enableMetrics {
+          port = wcfg.metricsStartingPort + portOffset;
+          resources = [ { names = [ "metrics" ]; } ];
         };
-      in
-        lib.pipe workerInstances [
-          (lib.mapAttrsToList (type: count: { inherit type count; }))
-          (map ({ type, count }: genList1 (i: rec {
-            name = "auto-${type}-${toString i}";
-            value = {
-              inherit type;
-              isAuto = true;
-              index = i;
-              settings.worker_listeners =
-                [
-                  { path = "/run/matrix-synapse/${name}.sock"; }
-                ] ++ lib.optionals wcfg.enableMetrics [{
-                  path = "/run/matrix-synapse/${name}-metrics.sock";
-                  resources = [{ names = [ "metrics" ]; }];
-                  type = "metrics";
-                }];
-            };
-          }) count))
-          lib.flatten
-          builtins.listToAttrs
-        ];
+
+      makeWorkerInstances = {
+        type,
+        numberOfWorkers,
+        portOffset ? 0,
+        nameFn ? i: "auto-${type}${toString i}",
+        workerListenerFn ? i: workerListenersWithMetrics (portOffset + i)
+      }: genAttrs'
+        (lib.lists.range 1 numberOfWorkers)
+        nameFn
+        (i: {
+          isAuto = true;
+          inherit type;
+          index = i;
+          settings.worker_listeners = workerListenerFn i;
+        });
+
+      workerInstances = {
+        "fed-sender" = wcfg.federationSenders;
+        "fed-receiver" = wcfg.federationReceivers;
+        "initial-sync" = wcfg.initialSyncers;
+        "normal-sync" = wcfg.normalSyncers;
+        "event-persist" = wcfg.eventPersisters;
+      } // (lib.optionalAttrs wcfg.useUserDirectoryWorker {
+        "user-dir" = {
+          numberOfWorkers = 1;
+          nameFn = _: "auto-user-dir";
+        };
+      });
+
+      coerceWorker = { name, value }: if builtins.isInt value then {
+        type = name;
+        numberOfWorkers = value;
+      } else { type = name; } // value;
+
+      # Like foldl, but keeps all intermediate values
+      #
+      # (b -> a -> b) -> b -> [a] -> [b]
+      scanl = f: x1: list: let
+        x2 = lib.head list;
+        x1' = f x1 x2;
+      in if list == [] then [] else [x1'] ++ (scanl f x1' (lib.tail list));
+
+      f = { portOffset, numberOfWorkers, ... }: x: x // { portOffset = portOffset + numberOfWorkers; };
+      init = { portOffset = 0; numberOfWorkers = 0; };
+    in lib.pipe workerInstances [
+      (lib.mapAttrsToList lib.nameValuePair)
+      (map coerceWorker)
+      (scanl f init)
+      (map makeWorkerInstances)
+      mkMerge
+    ];
 
     systemd.services = let
       workerList = lib.mapAttrsToList lib.nameValuePair wcfg.instances;
@@ -279,26 +328,22 @@ in {
     in builtins.listToAttrs (lib.flip map workerList (worker: {
       name = "matrix-synapse-worker-${worker.name}";
       value = {
-        documentation = [ "https://github.com/matrix-org/synapse/blob/develop/docs/workers.md" ];
         description = "Synapse Matrix Worker";
         partOf = [ "matrix-synapse.target" ];
         wantedBy = [ "matrix-synapse.target" ];
         after = [ "matrix-synapse.service" ];
         requires = [ "matrix-synapse.service" ];
-        restartTriggers = [ matrix-synapse-common-config (workerConfig worker) ] ++ cfg.extraConfigFiles;
         environment = {
           PYTHONPATH = lib.makeSearchPathOutput "lib" cfg.package.python.sitePackages [
             pluginsEnv
           ];
         };
         serviceConfig = {
-          Restart = "always";
           Type = "notify";
           User = "matrix-synapse";
           Group = "matrix-synapse";
           Slice = "system-matrix-synapse.slice";
           WorkingDirectory = cfg.dataDir;
-          RuntimeDirectory = [ "matrix-synapse" ];
           ExecStartPre = pkgs.writers.writeBash "wait-for-synapse" ''
             # From https://md.darmstadt.ccc.de/synapse-at-work
             while ! systemctl is-active -q matrix-synapse.service; do

tests/auto-workers-config.nix

@@ -1,18 +0,0 @@
-{ pkgs, lib, ... }:
-{
-  services.matrix-synapse-next = {
-    enable = true;
-    settings.server_name = "matrix.example.com";
-
-    workers = {
-      enableMetrics = true;
-
-      federationSenders = 2;
-      federationReceivers = 2;
-      initialSyncers = 2;
-      normalSyncers = 2;
-      eventPersisters = 2;
-      useUserDirectoryWorker = true;
-    };
-  };
-}

tests/base-config.nix

@@ -1,7 +0,0 @@
-{ pkgs, lib, ... }:
-{
-  services.matrix-synapse-next = {
-    enable = true;
-    settings.server_name = "matrix.example.com";
-  };
-}

tests/default.nix

@@ -1,16 +0,0 @@
-{ nixpkgs, pkgs, system ? pkgs.system, nixosModule, ... }: let
-  buildSystemWithConfig = configPath: (nixpkgs.lib.nixosSystem {
-    inherit system;
-    modules = [
-      nixosModule
-      configPath
-      {
-        boot.isContainer = true;
-      }
-    ];
-  }).config.system.build.toplevel;
-in {
-  a = pkgs.writeText "hello-world" ''a'';
-  base-config = buildSystemWithConfig ./base-config.nix;
-  auto-workers-config = buildSystemWithConfig ./auto-workers-config.nix;
-}