Support persistent Matrix sims for webhooks

src/d2m/actions/register-pk-user.js

@@ -13,13 +13,6 @@ const file = sync.require("../../matrix/file")
 /** @type {import("./register-user")} */
 const registerUser = sync.require("./register-user")
 
-/**
- * @typedef WebhookAuthor Discord API message->author. A webhook as an author.
- * @prop {string} username
- * @prop {string?} avatar
- * @prop {string} id
- */
-
 /** @returns {Promise<Ty.PkMessage>} */
 async function fetchMessage(messageID) {
 	try {
@@ -111,7 +104,7 @@ async function ensureSimJoined(pkMessage, roomID) {
 /**
  * Generate profile data based on webhook displayname and configured avatar.
  * @param {Ty.PkMessage} pkMessage
- * @param {WebhookAuthor} author
+ * @param {Ty.WebhookAuthor} author
  */
 async function memberToStateContent(pkMessage, author) {
 	// We prefer to use the member's avatar URL data since the image upload can be cached across channels,
@@ -137,7 +130,7 @@ async function memberToStateContent(pkMessage, author) {
  * 5. Compare against the previously known state content, which is helpfully stored in the database
  * 6. If the state content has changed, send it to Matrix and update it in the database for next time
  * @param {string} messageID to call API with
- * @param {WebhookAuthor} author for profile data
+ * @param {Ty.WebhookAuthor} author for profile data
  * @param {string} roomID room to join member to
  * @param {boolean} shouldActuallySync whether to actually sync updated user data or just ensure it's joined
  * @returns {Promise<string>} mxid of the updated sim

src/d2m/actions/register-webhook-user.js

@@ -0,0 +1,146 @@
+// @ts-check
+
+const assert = require("assert")
+const {reg} = require("../../matrix/read-registration")
+const Ty = require("../../types")
+
+const passthrough = require("../../passthrough")
+const {sync, db, select, from} = passthrough
+/** @type {import("../../matrix/api")} */
+const api = sync.require("../../matrix/api")
+/** @type {import("../../matrix/file")} */
+const file = sync.require("../../matrix/file")
+/** @type {import("./register-user")} */
+const registerUser = sync.require("./register-user")
+/** @type {import("../converters/user-to-mxid")} */
+const userToMxid = sync.require("../converters/user-to-mxid")
+
+/**
+ * A sim is an account that is being simulated by the bridge to copy events from the other side.
+ * @param {string} fakeUserID
+ * @param {Ty.WebhookAuthor} author
+ * @returns mxid
+ */
+async function createSim(fakeUserID, author) {
+	// Choose sim name
+	const simName = userToMxid.webhookAuthorToSimName(author)
+	const localpart = reg.ooye.namespace_prefix + simName
+	const mxid = `@${localpart}:${reg.ooye.server_name}`
+
+	// Save chosen name in the database forever
+	db.prepare("INSERT INTO sim (user_id, username, sim_name, mxid) VALUES (?, ?, ?, ?)").run(fakeUserID, author.username, simName, mxid)
+
+	// Register matrix user with that name
+	try {
+		await api.register(localpart)
+	} catch (e) {
+		// If user creation fails, manually undo the database change. Still isn't perfect, but should help.
+		// (I would prefer a transaction, but it's not safe to leave transactions open across event loop ticks.)
+		db.prepare("DELETE FROM sim WHERE user_id = ?").run(fakeUserID)
+		throw e
+	}
+	return mxid
+}
+
+/**
+ * Ensure a sim is registered for the user.
+ * If there is already a sim, use that one. If there isn't one yet, register a new sim.
+ * @param {string} fakeUserID
+ * @param {Ty.WebhookAuthor} author
+ * @returns {Promise<string>} mxid
+ */
+async function ensureSim(fakeUserID, author) {
+	let mxid = null
+	const existing = select("sim", "mxid", {user_id: fakeUserID}).pluck().get()
+	if (existing) {
+		mxid = existing
+	} else {
+		mxid = await createSim(fakeUserID, author)
+	}
+	return mxid
+}
+
+/**
+ * Ensure a sim is registered for the user and is joined to the room.
+ * @param {string} fakeUserID
+ * @param {Ty.WebhookAuthor} author
+ * @param {string} roomID
+ * @returns {Promise<string>} mxid
+ */
+async function ensureSimJoined(fakeUserID, author, roomID) {
+	// Ensure room ID is really an ID, not an alias
+	assert.ok(roomID[0] === "!")
+
+	// Ensure user
+	const mxid = await ensureSim(fakeUserID, author)
+
+	// Ensure joined
+	const existing = select("sim_member", "mxid", {room_id: roomID, mxid}).pluck().get()
+	if (!existing) {
+		try {
+			await api.inviteToRoom(roomID, mxid)
+			await api.joinRoom(roomID, mxid)
+		} catch (e) {
+			if (e.message.includes("is already in the room.")) {
+				// Sweet!
+			} else {
+				throw e
+			}
+		}
+		db.prepare("INSERT OR IGNORE INTO sim_member (room_id, mxid) VALUES (?, ?)").run(roomID, mxid)
+	}
+	return mxid
+}
+
+/**
+ * Generate profile data based on webhook displayname and configured avatar.
+ * @param {Ty.WebhookAuthor} author
+ */
+async function authorToStateContent(author) {
+	// We prefer to use the member's avatar URL data since the image upload can be cached across channels,
+	// unlike the userAvatar URL which is unique per channel, due to the webhook ID being in the URL.
+	const avatar = file.userAvatar(author)
+
+	const content = {
+		displayname: author.username,
+		membership: "join",
+	}
+	if (avatar) content.avatar_url = await file.uploadDiscordFileToMxc(avatar)
+
+	return content
+}
+
+/**
+ * Sync profile data for a sim webhook user. This function follows the following process:
+ * 1. Create and join the sim to the room if needed
+ * 2. Make an object of what the new room member state content would be, including uploading the profile picture if it hasn't been done before
+ * 3. Compare against the previously known state content, which is helpfully stored in the database
+ * 4. If the state content has changed, send it to Matrix and update it in the database for next time
+ * @param {Ty.WebhookAuthor} author for profile data
+ * @param {string} roomID room to join member to
+ * @param {boolean} shouldActuallySync whether to actually sync updated user data or just ensure it's joined
+ * @returns {Promise<string>} mxid of the updated sim
+ */
+async function syncUser(author, roomID, shouldActuallySync) {
+	const fakeUserID = userToMxid.webhookAuthorToFakeUserID(author)
+
+	// Create and join the sim to the room if needed
+	const mxid = await ensureSimJoined(fakeUserID, author, roomID)
+
+	if (shouldActuallySync) {
+		// Build current profile data
+		const content = await authorToStateContent(author)
+		const currentHash = registerUser._hashProfileContent(content, 0)
+		const existingHash = select("sim_member", "hashed_profile_content", {room_id: roomID, mxid}).safeIntegers().pluck().get()
+
+		// Only do the actual sync if the hash has changed since we last looked
+		if (existingHash !== currentHash) {
+			await api.sendState(roomID, "m.room.member", mxid, content, mxid)
+			db.prepare("UPDATE sim_member SET hashed_profile_content = ? WHERE room_id = ? AND mxid = ?").run(currentHash, roomID, mxid)
+		}
+	}
+
+	return mxid
+}
+
+module.exports.syncUser = syncUser

src/d2m/actions/send-message.js

@@ -4,7 +4,7 @@ const assert = require("assert").strict
 const DiscordTypes = require("discord-api-types/v10")
 
 const passthrough = require("../../passthrough")
-const { discord, sync, db } = passthrough
+const { discord, sync, db, select } = passthrough
 /** @type {import("../converters/message-to-event")} */
 const messageToEvent = sync.require("../converters/message-to-event")
 /** @type {import("../../matrix/api")} */
@@ -13,6 +13,8 @@ const api = sync.require("../../matrix/api")
 const registerUser = sync.require("./register-user")
 /** @type {import("./register-pk-user")} */
 const registerPkUser = sync.require("./register-pk-user")
+/** @type {import("./register-webhook-user")} */
+const registerWebhookUser = sync.require("./register-webhook-user")
 /** @type {import("../actions/create-room")} */
 const createRoom = sync.require("../actions/create-room")
 /** @type {import("../../discord/utils")} */
@@ -28,17 +30,23 @@ async function sendMessage(message, channel, guild, row) {
 	const roomID = await createRoom.ensureRoom(message.channel_id)
 
 	let senderMxid = null
-	if (!dUtils.isWebhookMessage(message)) {
+	if (dUtils.isWebhookMessage(message)) {
+		const useWebhookProfile = select("guild_space", "webhook_profile", {guild_id: guild.id}) ?? 0
+		if (row && row.speedbump_webhook_id === message.webhook_id) {
+			// Handle the PluralKit public instance
+			if (row.speedbump_id === "466378653216014359") {
+				senderMxid = await registerPkUser.syncUser(message.id, message.author, roomID, true)
+			}
+		} else if (useWebhookProfile) {
+			senderMxid = await registerWebhookUser.syncUser(message.author, roomID, true)
+		}
+	} else {
+		// not a webhook
 		if (message.author.id === discord.application.id) {
 			// no need to sync the bot's own user
 		} else {
 			senderMxid = await registerUser.syncUser(message.author, message.member, channel, guild, roomID)
 		}
-	} else if (row && row.speedbump_webhook_id === message.webhook_id) {
-		// Handle the PluralKit public instance
-		if (row.speedbump_id === "466378653216014359") {
-			senderMxid = await registerPkUser.syncUser(message.id, message.author, roomID, true)
-		}
 	}
 
 	const events = await messageToEvent.messageToEvent(message, guild, {}, {api, snow: discord.snow})

src/d2m/converters/user-to-mxid.js

@@ -2,6 +2,7 @@
 
 const assert = require("assert")
 const {reg} = require("../../matrix/read-registration")
+const Ty = require("../../types")
 
 const passthrough = require("../../passthrough")
 const {select} = passthrough
@@ -13,7 +14,7 @@ const SPECIAL_USER_MAPPINGS = new Map([
 /**
  * Downcased and stripped username. Can only include a basic set of characters.
  * https://spec.matrix.org/v1.6/appendices/#user-identifiers
- * @param {import("discord-api-types/v10").APIUser} user
+ * @param {import("discord-api-types/v10").APIUser | Ty.WebhookAuthor} user
  * @returns {string} localpart
  */
 function downcaseUsername(user) {
@@ -85,4 +86,49 @@ function userToSimName(user) {
 	throw new Error(`Ran out of suggestions when generating sim name. downcased: "${downcased}"`)
 }
 
+/**
+ * Webhooks have an ID specific to that webhook, but a single webhook can send messages with any user name.
+ * The point of this feature (gated by guild_space webhook_profile) is to create persistent Matrix accounts for individual webhook "users".
+ * This is convenient when using a bridge to a platform that does not assign persistent user IDs (e.g. IRC, Minecraft).
+ * In this case, webhook "users" are disambiguated by their username (downcased).
+ * @param {Ty.WebhookAuthor} author
+ * @returns {string}
+ */
+function webhookAuthorToFakeUserID(author) {
+	const downcased = downcaseUsername(author)
+	return `webhook_${downcased}`
+}
+
+/**
+ * @param {Ty.WebhookAuthor} author
+ * @returns {string}
+ */
+function webhookAuthorToSimName(author) {
+	if (SPECIAL_USER_MAPPINGS.has(author.id)) {
+		const error = new Error("Special users should have followed the other code path.")
+		// @ts-ignore
+		error.author = author
+		throw error
+	}
+
+	// 1. Is sim user already registered?
+	const fakeUserID = webhookAuthorToFakeUserID(author)
+	const existing = select("sim", "user_id", {user_id: fakeUserID}).pluck().get()
+	assert.equal(existing, null, "Shouldn't try to create a new name for an existing sim")
+
+	// 2. Register based on username (could be new or old format)
+	const downcased = "webhook_" + downcaseUsername(author)
+
+	// Check for conflicts with already registered sims
+	const matches = select("sim", "sim_name", {}, "WHERE sim_name LIKE ? ESCAPE '@'").pluck().all(downcased + "%")
+	// Keep generating until we get a suggestion that doesn't conflict
+	for (const suggestion of generateLocalpartAlternatives([downcased])) {
+		if (!matches.includes(suggestion)) return suggestion
+	}
+	/* c8 ignore next */
+	throw new Error(`Ran out of suggestions when generating sim name. downcased: "${downcased}"`)
+}
+
 module.exports.userToSimName = userToSimName
+module.exports.webhookAuthorToFakeUserID = webhookAuthorToFakeUserID
+module.exports.webhookAuthorToSimName = webhookAuthorToSimName

src/db/migrations/0025-add-webhook-profile-to-guild-space.sql

@@ -0,0 +1,5 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE guild_space ADD COLUMN webhook_profile INTEGER NOT NULL DEFAULT 0;
+
+COMMIT;

src/db/orm-defs.d.ts

@@ -52,6 +52,7 @@ export type Models = {
 		privacy_level: number
 		presence: 0 | 1
 		url_preview: 0 | 1
+		webhook_profile: 0 | 1
 	}
 
 	guild_active: {

src/types.d.ts

@@ -72,6 +72,13 @@ export type WebhookCreds = {
 	token: string
 }
 
+/** Discord API message->author. A webhook as an author. */
+export type WebhookAuthor = {
+	username: string
+	avatar: string | null
+	id: string
+}
+
 export type PkSystem = {
 	id: string
 	uuid: string

src/web/pug/guild.pug

@@ -124,6 +124,15 @@ block body
           | Show online statuses on Matrix
           p.s-description This might cause lag on really big Discord servers.
 
+      form.d-flex.ai-center.g16
+        #webhook-profile-loading.p8
+        - value = !!select("guild_space", "webhook_profile", {guild_id}).pluck().get()
+        input(type="hidden" name="guild_id" value=guild_id)
+        input.s-toggle-switch#webhook-profile(name="webhook_profile" type="checkbox" hx-post=rel("/api/webhook-profile") hx-indicator="#webhook-profile-loading" hx-disabled-elt="this" checked=value autocomplete="off")
+        label.s-label(for="webhook-profile")
+          | Create persistent Matrix sims for webhooks
+          p.s-description Useful when using other Discord bridges. Otherwise, not ideal, as sims will clutter the Matrix user list and will never be cleaned up.
+
   if space_id
     h2.mt48.fs-headline1 Channel setup
 

src/web/routes/guild-settings.js

@@ -74,6 +74,8 @@ as.router.post("/api/autocreate", defineToggle("autocreate", {
 
 as.router.post("/api/url-preview", defineToggle("url_preview"))
 
+as.router.post("/api/webhook-profile", defineToggle("webhook_profile"))
+
 as.router.post("/api/presence", defineToggle("presence", {
 	after() {
 		setPresence.guildPresenceSetting.update()