Backfill missed pins and pins from the past

d2m/actions/update-pins.js

@@ -1,22 +1,37 @@
 // @ts-check
 
 const passthrough = require("../../passthrough")
-const {discord, sync} = passthrough
+const {discord, sync, db} = passthrough
 /** @type {import("../converters/pins-to-list")} */
 const pinsToList = sync.require("../converters/pins-to-list")
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
 
 /**
- * @param {string} channelID
- * @param {string} roomID
+ * @template {string | null | undefined} T
+ * @param {T} timestamp
+ * @returns {T extends string ? number : null}
  */
-async function updatePins(channelID, roomID) {
-	const pins = await discord.snow.channel.getChannelPinnedMessages(channelID)
-	const eventIDs = pinsToList.pinsToList(pins)
-	await api.sendState(roomID, "m.room.pinned_events", "", {
-		pinned: eventIDs
-	})
+function convertTimestamp(timestamp) {
+	// @ts-ignore
+	return typeof timestamp === "string" ? Math.floor(new Date(timestamp).getTime() / 1000) : null
 }
 
+/**
+ * @param {string} channelID
+ * @param {string} roomID
+ * @param {number?} convertedTimestamp
+ */
+async function updatePins(channelID, roomID, convertedTimestamp) {
+	const pins = await discord.snow.channel.getChannelPinnedMessages(channelID)
+	const eventIDs = pinsToList.pinsToList(pins)
+	if (pins.length === eventIDs.length || eventIDs.length) {
+		await api.sendState(roomID, "m.room.pinned_events", "", {
+			pinned: eventIDs
+		})
+	}
+	db.prepare("UPDATE channel_room SET last_bridged_pin_timestamp = ? WHERE channel_id = ?").run(convertedTimestamp || 0, channelID)
+}
+
+module.exports.convertTimestamp = convertTimestamp
 module.exports.updatePins = updatePins

d2m/converters/pins-to-list.test.js

@@ -6,7 +6,7 @@ test("pins2list: converts known IDs, ignores unknown IDs", t => {
 	const result = pinsToList(data.pins.faked)
 	t.deepEqual(result, [
 		"$lnAF9IosAECTnlv9p2e18FG8rHn-JgYKHEHIh5qdFv4",
-		"$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuAno",
+		"$mtR8cJqM4fKno1bVsm8F4wUVqSntt2sq6jav1lyavuA",
 		"$X16nfVks1wsrhq4E9SSLiqrf2N8KD0erD0scZG7U5xg"
 	])
 })

d2m/discord-packets.js

@@ -43,6 +43,7 @@ const utils = {
 			}
 			if (listen === "full") {
 				eventDispatcher.checkMissedExpressions(message.d)
+				eventDispatcher.checkMissedPins(client, message.d)
 				eventDispatcher.checkMissedMessages(client, message.d)
 			}
 
@@ -94,6 +95,13 @@ const utils = {
 			client.channels.set(message.d.id, message.d)
 
 
+		} else if (message.t === "CHANNEL_PINS_UPDATE") {
+			const channel = client.channels.get(message.d.channel_id)
+			if (channel) {
+				channel["last_pin_timestamp"] = message.d.last_pin_timestamp
+			}
+
+
 		} else if (message.t === "GUILD_DELETE") {
 			client.guilds.delete(message.d.id)
 			const channels = client.guildChannelMap.get(message.d.id)

d2m/event-dispatcher.js

@@ -25,9 +25,15 @@ const createSpace = sync.require("./actions/create-space")
 const updatePins = sync.require("./actions/update-pins")
 /** @type {import("../matrix/api")}) */
 const api = sync.require("../matrix/api")
+/** @type {import("../discord/utils")} */
+const utils = sync.require("../discord/utils")
 /** @type {import("../discord/discord-command-handler")}) */
 const discordCommandHandler = sync.require("../discord/discord-command-handler")
 
+/** @type {any} */ // @ts-ignore bad types from semaphore
+const Semaphore = require("@chriscdn/promise-semaphore")
+const checkMissedPinsSema = new Semaphore()
+
 let lastReportedEvent = 0
 
 // Grab Discord events we care about for the bridge, check them, and pass them on
@@ -103,6 +109,14 @@ module.exports = {
 			const latestWasBridged = prepared.get(channel.last_message_id)
 			if (latestWasBridged) continue
 
+			// Permissions check
+			const member = guild.members.find(m => m.user?.id === client.user.id)
+			if (!member) return
+			if (!("permission_overwrites" in channel)) continue
+			const permissions = utils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
+			const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY
+			if ((permissions & wants) !== wants) continue // We don't have permission to look back in this channel
+
 			/** More recent messages come first. */
 			// console.log(`[check missed messages] in ${channel.id} (${guild.name} / ${channel.name}) because its last message ${channel.last_message_id} is not in the database`)
 			let messages
@@ -132,6 +146,34 @@ module.exports = {
 		}
 	},
 
+	/**
+	 * When logging back in, check if the pins on Matrix-side are up to date. If they aren't, update all pins.
+	 * Rather than query every room on Matrix-side, we cache the latest pinned message in the database and compare against that.
+	 * @param {import("./discord-client")} client
+	 * @param {DiscordTypes.GatewayGuildCreateDispatchData} guild
+	 */
+	async checkMissedPins(client, guild) {
+		if (guild.unavailable) return
+		const member = guild.members.find(m => m.user?.id === client.user.id)
+		if (!member) return
+		for (const channel of guild.channels) {
+			if (!("last_pin_timestamp" in channel) || !channel.last_pin_timestamp) continue // Only care about channels that have pins
+			if (!("permission_overwrites" in channel)) continue
+			const lastPin = updatePins.convertTimestamp(channel.last_pin_timestamp)
+
+			// Permissions check
+			const permissions = utils.getPermissions(member.roles, guild.roles, client.user.id, channel.permission_overwrites)
+			const wants = BigInt(1 << 10) | BigInt(1 << 16) // VIEW_CHANNEL + READ_MESSAGE_HISTORY
+			if ((permissions & wants) !== wants) continue // We don't have permission to look up the pins in this channel
+
+			const row = select("channel_room", ["room_id", "last_bridged_pin_timestamp"], {channel_id: channel.id}).get()
+			if (!row) continue // Only care about already bridged channels
+			if (row.last_bridged_pin_timestamp == null || lastPin > row.last_bridged_pin_timestamp) {
+				checkMissedPinsSema.request(() => updatePins.updatePins(channel.id, row.room_id, lastPin))
+			}
+		}
+	},
+
 	/**
 	 * When logging back in, check if we missed any changes to emojis or stickers. Apply the changes if so.
 	 * @param {DiscordTypes.GatewayGuildCreateDispatchData} guild
@@ -183,7 +225,8 @@ module.exports = {
 	async onChannelPinsUpdate(client, data) {
 		const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
 		if (!roomID) return // No target room to update pins in
-		await updatePins.updatePins(data.channel_id, roomID)
+		const convertedTimestamp = updatePins.convertTimestamp(data.last_pin_timestamp)
+		await updatePins.updatePins(data.channel_id, roomID, convertedTimestamp)
 	},
 
 	/**

db/migrations/0008-add-last-bridged-pin-timestamp.sql

@@ -0,0 +1,5 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE channel_room ADD COLUMN last_bridged_pin_timestamp INTEGER;
+
+COMMIT;

db/orm-defs.d.ts

@@ -6,6 +6,7 @@ export type Models = {
 		nick: string | null
 		thread_parent: string | null
 		custom_avatar: string | null
+		last_bridged_pin_timestamp: number | null
 	}
 
 	event_message: {

discord/utils.js

@@ -34,7 +34,7 @@ function getPermissions(userRoles, guildRoles, userID, channelOverwrites) {
 			// Role deny
 			overwrite => userRoles.includes(overwrite.id) && (allowed &= ~BigInt(overwrite.deny)),
 			// Role allow
-			overwrite => userRoles.includes(overwrite.id) && (allowed |= ~BigInt(overwrite.allow)),
+			overwrite => userRoles.includes(overwrite.id) && (allowed |= BigInt(overwrite.allow)),
 			// User deny
 			overwrite => overwrite.id === userID && (allowed &= ~BigInt(overwrite.deny)),
 			// User allow

discord/utils.test.js

@@ -18,6 +18,67 @@ test("discord utils: converts snowflake to timestamp", t => {
 	t.equal(utils.snowflakeToTimestampExact("86913608335773696"), 1440792219004)
 })
 
-test("discerd utils: converts timestamp to snowflake", t => {
+test("discord utils: converts timestamp to snowflake", t => {
 	t.match(utils.timestampToSnowflakeInexact(1440792219004), /^869136083357.....$/)
 })
+
+test("getPermissions: channel overwrite to allow role works", t => {
+	const guildRoles = [
+		{
+			version: 1695412489043,
+			unicode_emoji: null,
+			tags: {},
+			position: 0,
+			permissions: "559623605571137",
+			name: "@everyone",
+			mentionable: false,
+			managed: false,
+			id: "1154868424724463687",
+			icon: null,
+			hoist: false,
+			flags: 0,
+			color: 0
+		},
+		{
+			version: 1695412604262,
+			unicode_emoji: null,
+			tags: { bot_id: "466378653216014359" },
+			position: 1,
+			permissions: "536995904",
+			name: "PluralKit",
+			mentionable: false,
+			managed: true,
+			id: "1154868908336099444",
+			icon: null,
+			hoist: false,
+			flags: 0,
+			color: 0
+		},
+		{
+			version: 1698778936921,
+			unicode_emoji: null,
+			tags: {},
+			position: 1,
+			permissions: "536870912",
+			name: "web hookers",
+			mentionable: false,
+			managed: false,
+			id: "1168988246680801360",
+			icon: null,
+			hoist: false,
+			flags: 0,
+			color: 0
+		}
+	]
+	const userRoles = [ "1168988246680801360" ]
+	const userID = "684280192553844747"
+	const overwrites = [
+		{ type: 0, id: "1154868908336099444", deny: "0", allow: "1024" },
+		{ type: 0, id: "1154868424724463687", deny: "1024", allow: "0" },
+		{ type: 0, id: "1168988246680801360", deny: "0", allow: "1024" },
+		{ type: 1, id: "353373325575323648", deny: "0", allow: "1024" }
+	]
+	const permissions = utils.getPermissions(userRoles, guildRoles, userID, overwrites)
+	const want = BigInt(1 << 10 | 1 << 16)
+	t.equal((permissions & want), want)
+})

stdin.js

@@ -16,6 +16,7 @@ const api = sync.require("./matrix/api")
 const file = sync.require("./matrix/file")
 const sendEvent = sync.require("./m2d/actions/send-event")
 const eventDispatcher = sync.require("./d2m/event-dispatcher")
+const updatePins = sync.require("./d2m/actions/update-pins")
 const ks = sync.require("./matrix/kstate")
 const guildID = "112760669178241024"