From cae591e5fd63a7afe66ac93f617a7ff53e5c430c Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 15 Aug 2023 17:20:31 +1200 Subject: [PATCH] script for capturing message update events --- d2m/actions/edit-message.js | 96 +---------------------- d2m/converters/edit-to-changes.js | 96 +++++++++++++++++++++++ d2m/discord-client.js | 6 +- scripts/capture-message-update-events.js | 51 ++++++++++++ scripts/events.db | Bin 0 -> 8192 bytes 5 files changed, 154 insertions(+), 95 deletions(-) create mode 100644 d2m/converters/edit-to-changes.js create mode 100644 scripts/capture-message-update-events.js create mode 100644 scripts/events.db diff --git a/d2m/actions/edit-message.js b/d2m/actions/edit-message.js index 662e124..933267c 100644 --- a/d2m/actions/edit-message.js +++ b/d2m/actions/edit-message.js @@ -1,94 +1,5 @@ -// @ts-check - -const assert = require("assert") - -const passthrough = require("../../passthrough") -const { discord, sync, db } = passthrough -/** @type {import("../converters/message-to-event")} */ -const messageToEvent = sync.require("../converters/message-to-event") -/** @type {import("../../matrix/api")} */ -const api = sync.require("../../matrix/api") -/** @type {import("./register-user")} */ -const registerUser = sync.require("./register-user") -/** @type {import("../actions/create-room")} */ -const createRoom = sync.require("../actions/create-room") - -/** - * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message - * @param {import("discord-api-types/v10").APIGuild} guild - */ -async function editMessage(message, guild) { - // Figure out what events we will be replacing - - const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").get(message.channel_id) - const senderMxid = await registerUser.ensureSimJoined(message.author, roomID) - /** @type {{event_id: string, event_type: string, event_subtype: string?, part: number}[]} */ - const oldEventRows = db.prepare("SELECT event_id, event_type, event_subtype, part FROM event_message WHERE message_id = ?").all(message.id) - - // Figure out what we will be replacing them with - - const newEvents = await messageToEvent.messageToEvent(message, guild, api) - - // Match the new events to the old events - - /* - Rules: - + The events must have the same type. - + The events must have the same subtype. - Events will therefore be divided into three categories: - */ - /** 1. Events that are matched, and should be edited by sending another m.replace event */ - let eventsToReplace = [] - /** 2. Events that are present in the old version only, and should be blanked or redacted */ - let eventsToRedact = [] - /** 3. Events that are present in the new version only, and should be sent as new, with references back to the context */ - let eventsToSend = [] - - // For each old event... - outer: while (newEvents.length) { - const newe = newEvents[0] - // Find a new event to pair it with... - let handled = false - for (let i = 0; i < oldEventRows.length; i++) { - const olde = oldEventRows[i] - if (olde.event_type === newe.$type && olde.event_subtype === (newe.msgtype || null)) { - // Found one! - // Set up the pairing - eventsToReplace.push({ - old: olde, - new: newe - }) - // These events have been handled now, so remove them from the source arrays - newEvents.shift() - oldEventRows.splice(i, 1) - // Go all the way back to the start of the next iteration of the outer loop - continue outer - } - } - // If we got this far, we could not pair it to an existing event, so it'll have to be a new one - eventsToSend.push(newe) - newEvents.shift() - } - // Anything remaining in oldEventRows is present in the old version only and should be redacted. - eventsToRedact = oldEventRows - - // Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed! - // (Consider a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.) - // So we'll remove entries from eventsToReplace that *definitely* cannot have changed. Everything remaining *may* have changed. - eventsToReplace = eventsToReplace.filter(ev => { - // Discord does not allow files, images, attachments, or videos to be edited. - if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote") { - return false - } - // Discord does not allow stickers to be edited. - if (ev.old.event_type === "m.sticker") { - return false - } - // Anything else is fair game. - return true - }) - - // Action time! +async function editMessage() { + // Action time! // 1. Replace all the things. @@ -113,6 +24,5 @@ async function editMessage(message, guild) { } return eventIDs -} -module.exports.editMessage = editMessage +{eventsToReplace, eventsToRedact, eventsToSend} diff --git a/d2m/converters/edit-to-changes.js b/d2m/converters/edit-to-changes.js new file mode 100644 index 0000000..0dd084f --- /dev/null +++ b/d2m/converters/edit-to-changes.js @@ -0,0 +1,96 @@ +// @ts-check + +const assert = require("assert") + +const passthrough = require("../../passthrough") +const { discord, sync, db } = passthrough +/** @type {import("./message-to-event")} */ +const messageToEvent = sync.require("../converters/message-to-event") +/** @type {import("../../matrix/api")} */ +const api = sync.require("../../matrix/api") +/** @type {import("../actions/register-user")} */ +const registerUser = sync.require("./register-user") +/** @type {import("../actions/create-room")} */ +const createRoom = sync.require("../actions/create-room") + +/** + * @param {import("discord-api-types/v10").GatewayMessageCreateDispatchData} message + * IMPORTANT: This may not have all the normal fields! The API documentation doesn't provide possible types, just says it's all optional! + * Since I don't have a spec, I will have to capture some real traffic and add it as test cases... I hope they don't change anything later... + * @param {import("discord-api-types/v10").APIGuild} guild + */ +async function editToChanges(message, guild) { + // Figure out what events we will be replacing + + const roomID = db.prepare("SELECT room_id FROM channel_room WHERE channel_id = ?").get(message.channel_id) + const senderMxid = await registerUser.ensureSimJoined(message.author, roomID) + /** @type {{event_id: string, event_type: string, event_subtype: string?, part: number}[]} */ + const oldEventRows = db.prepare("SELECT event_id, event_type, event_subtype, part FROM event_message WHERE message_id = ?").all(message.id) + + // Figure out what we will be replacing them with + + const newEvents = await messageToEvent.messageToEvent(message, guild, api) + + // Match the new events to the old events + + /* + Rules: + + The events must have the same type. + + The events must have the same subtype. + Events will therefore be divided into three categories: + */ + /** 1. Events that are matched, and should be edited by sending another m.replace event */ + let eventsToReplace = [] + /** 2. Events that are present in the old version only, and should be blanked or redacted */ + let eventsToRedact = [] + /** 3. Events that are present in the new version only, and should be sent as new, with references back to the context */ + let eventsToSend = [] + + // For each old event... + outer: while (newEvents.length) { + const newe = newEvents[0] + // Find a new event to pair it with... + let handled = false + for (let i = 0; i < oldEventRows.length; i++) { + const olde = oldEventRows[i] + if (olde.event_type === newe.$type && olde.event_subtype === (newe.msgtype || null)) { + // Found one! + // Set up the pairing + eventsToReplace.push({ + old: olde, + new: newe + }) + // These events have been handled now, so remove them from the source arrays + newEvents.shift() + oldEventRows.splice(i, 1) + // Go all the way back to the start of the next iteration of the outer loop + continue outer + } + } + // If we got this far, we could not pair it to an existing event, so it'll have to be a new one + eventsToSend.push(newe) + newEvents.shift() + } + // Anything remaining in oldEventRows is present in the old version only and should be redacted. + eventsToRedact = oldEventRows + + // Now, everything in eventsToSend and eventsToRedact is a real change, but everything in eventsToReplace might not have actually changed! + // (Consider a MESSAGE_UPDATE for a text+image message - Discord does not allow the image to be changed, but the text might have been.) + // So we'll remove entries from eventsToReplace that *definitely* cannot have changed. Everything remaining *may* have changed. + eventsToReplace = eventsToReplace.filter(ev => { + // Discord does not allow files, images, attachments, or videos to be edited. + if (ev.old.event_type === "m.room.message" && ev.old.event_subtype !== "m.text" && ev.old.event_subtype !== "m.emote") { + return false + } + // Discord does not allow stickers to be edited. + if (ev.old.event_type === "m.sticker") { + return false + } + // Anything else is fair game. + return true + }) + + return {eventsToReplace, eventsToRedact, eventsToSend} +} + +module.exports.editMessage = editMessage diff --git a/d2m/discord-client.js b/d2m/discord-client.js index 91682bd..5e90d85 100644 --- a/d2m/discord-client.js +++ b/d2m/discord-client.js @@ -14,7 +14,7 @@ class DiscordClient { * @param {string} discordToken * @param {boolean} listen whether to set up the event listeners for OOYE to operate */ - constructor(discordToken, listen) { + constructor(discordToken, listen = true) { this.discordToken = discordToken this.snow = new SnowTransfer(discordToken) this.cloud = new CloudStorm(discordToken, { @@ -44,7 +44,9 @@ class DiscordClient { this.guilds = new Map() /** @type {Map>} */ this.guildChannelMap = new Map() - this.cloud.on("event", message => discordPackets.onPacket(this, message)) + if (listen) { + this.cloud.on("event", message => discordPackets.onPacket(this, message)) + } this.cloud.on("error", console.error) } } diff --git a/scripts/capture-message-update-events.js b/scripts/capture-message-update-events.js new file mode 100644 index 0000000..2ff9b49 --- /dev/null +++ b/scripts/capture-message-update-events.js @@ -0,0 +1,51 @@ +// @ts-check + +// **** +const interestingFields = ["author", "content", "edited_timestamp", "mentions", "attachments", "embeds", "type", "message_reference", "referenced_message", "sticker_items"] +// ***** + +function fieldToPresenceValue(field) { + if (field === undefined) return 0 + else if (field === null) return 1 + else if (Array.isArray(field) && field.length === 0) return 10 + else if (typeof field === "object" && Object.keys(field).length === 0) return 20 + else if (field === "") return 30 + else return 99 +} + +const sqlite = require("better-sqlite3") +const HeatSync = require("heatsync") + +const config = require("../config") +const passthrough = require("../passthrough") + +const sync = new HeatSync({watchFS: false}) + +Object.assign(passthrough, {config, sync}) + +const DiscordClient = require("../d2m/discord-client", false) + +const discord = new DiscordClient(config.discordToken, false) +passthrough.discord = discord + +;(async () => { + await discord.cloud.connect() + console.log("Discord gateway started") + + const f = event => onPacket(discord, event, () => discord.cloud.off("event", f)) + discord.cloud.on("event", f) +})() + +const events = new sqlite("scripts/events.db") +const sql = "INSERT INTO \"update\" (json, " + interestingFields.join(", ") + ") VALUES (" + "?".repeat(interestingFields.length + 1).split("").join(", ") + ")" +console.log(sql) +const prepared = events.prepare(sql) + +/** @param {DiscordClient} discord */ +function onPacket(discord, event, unsubscribe) { + if (event.t === "MESSAGE_UPDATE") { + const data = [JSON.stringify(event.d), ...interestingFields.map(f => fieldToPresenceValue(event.d[f]))] + console.log(data) + prepared.run(...data) + } +} diff --git a/scripts/events.db b/scripts/events.db new file mode 100644 index 0000000000000000000000000000000000000000..c8f5bad88858b0ddc7a3b54ec36577c94791c001 GIT binary patch literal 8192 zcmWFz^vNtqRY=P(%1ta$FlG>7U}R))P*7lCU|?lnU|?cE0A>aT1{MUDff0#~i^yeRY3_NrKF(2#i^84oS&z}8R8ld zqTuHrqTmCA3zsISZ_dQ=n1Oc{$Kzp^)<(TE8Umvs zFd71*Aut*OqaiSIL!i@ziFZ;iBLr}8a8xUSn_pH2I!Yxa#Y$Fbi8;loI!cJvpp}x5 zfswJUfrYN2X^4TTm7%4TiLsuck)?rwfwqBxm4Shhj#5EpUS4VnTsNp82yQG|MaSwW zL3!~-`8lb@N+?39WvNA#`FW{GI&+g!i%#4gpEln+rEX|a3 zlnf0mER9So3{6bU%#92U4Gb)mbd-!OjSNi;O-(E;jg1V9EzC?Uz$%T5P0S1|O$;oJ zO)V`AA^MGt%uEa{4U8z{1GP(7@2l+``mCDON|R zpeQvrvotrpI1|)@waP2a$J2gciAt67%pjaU#Gesd&p{O)ZA-}Xl zAwNwasVFfsuOPoXwMd~PKVKm=u_#AZHzU7XAtSL^A+tmwDK#}u!NfozH#4uaB(+!} zF{dasF{LsgK}W$;p(wSuAU`j~O2IilFTE(Sq%;Q{cEySck@=D!3L25A#hNjBF?k-DB?>lHNqO-_saD2jM&>4lCdQUVhK8mF#%6|Qb_yl=3Pq_! z`8hdyN;*o(`MCx8d8v7z%m>OX$r*`xd8s+^kgzv1Ftju`H!v|WGBYu@G&BRHXlPH* zs#>YEIJGD*F*jAoO35idD=|4+!6Uz@I5jsh50ocMlX5bX;}H=BHqqSN&=iz8Ow7y; zO$^P9jX=pcJtsdYF()2k6eNA7WELkEW#(q)C6?qDDOo95nj3(!NK$@@l2u7jX{wGA zB(21!q$cMVfx`uoY+w>fR!S*`CTS+-hL$EtW~oW3CMk(VsfHGbrevz zI!eerM{p8KFU`zJf%+bl^9{_*EDg;qj7$s-j7*ejlarIVxVVyAf*E-`xgn((Be)nV zPfg0m&(DVFG&eA|GynydsfB@=p^2F}D9Msq(3x2o80i^U7$D~@sugtL$be)mGgD(@ zGh=fDGb3|jP%(mtG19V@g_Wtfo|&16ktL#FMr5s+7%hd8)Z!8aP*6}Q(IqFQfP3Z8 zk^*a}qbDR#a4P92VI(9aE2Y##Qxiku6k|h!WDC>OB+Jy4#6-)KBnvasR0{(`Q&X(z kC9$9&2Ue(o1J%sJ#K^+H(9+1%)Y!tr#M}f_iW8Tr05Q;uvj6}9 literal 0 HcmV?d00001