diff --git a/scripts/estimate-size.js b/scripts/estimate-size.js new file mode 100644 index 0000000..341abc0 --- /dev/null +++ b/scripts/estimate-size.js @@ -0,0 +1,65 @@ +// @ts-check + +const pb = require("prettier-bytes") +const sqlite = require("better-sqlite3") +const HeatSync = require("heatsync") + +const {reg} = require("../src/matrix/read-registration") +const passthrough = require("../src/passthrough") + +const sync = new HeatSync({watchFS: false}) +Object.assign(passthrough, {reg, sync}) + +const DiscordClient = require("../src/d2m/discord-client") + +const discord = new DiscordClient(reg.ooye.discord_token, "no") +passthrough.discord = discord + +const db = new sqlite("ooye.db") +passthrough.db = db + +const api = require("../src/matrix/api") + +const {room: roomID} = require("minimist")(process.argv.slice(2), {string: ["room"]}) +if (!roomID) { + console.error("Usage: ./scripts/estimate-size.js --room=") + process.exit(1) +} + +const {channel_id, guild_id} = db.prepare("SELECT channel_id, guild_id FROM channel_room WHERE room_id = ?").get(roomID) + +const max = 1000 + +;(async () => { + let total = 0 + let size = 0 + let from + + while (total < max) { + const events = await api.getEvents(roomID, "b", {limit: 1000, from}) + total += events.chunk.length + from = events.end + console.log(`Fetched ${total} events so far`) + + for (const e of events.chunk) { + if (e.content?.info?.size) { + size += e.content.info.size + } + } + + if (events.chunk.length === 0 || !events.end) break + } + + console.log(`Total size of uploads: ${pb(size)}`) + + const searchResults = await discord.snow.requestHandler.request(`/guilds/${guild_id}/messages/search`, { + channel_id, + offset: "0", + limit: "1" + }, "get", "json") + + const totalAllTime = searchResults.total_results + const fractionCounted = total / totalAllTime + console.log(`That counts for ${(fractionCounted*100).toFixed(2)}% of the history on Discord (${totalAllTime.toLocaleString()} messages)`) + console.log(`The size of uploads for the whole history would be approx: ${pb(Math.floor(size/total*totalAllTime))}`) +})() diff --git a/src/matrix/api.js b/src/matrix/api.js index 70cb50b..87bbf0c 100644 --- a/src/matrix/api.js +++ b/src/matrix/api.js @@ -136,7 +136,7 @@ async function getEventForTimestamp(roomID, ts) { */ async function getEvents(roomID, dir, pagination = {}, filter) { filter = filter && JSON.stringify(filter) - /** @type {Ty.Pagination>} */ + /** @type {Ty.MessagesPagination>} */ const root = await mreq.mreq("GET", path(`/client/v3/rooms/${roomID}/messages`, null, {...pagination, dir, filter})) return root } diff --git a/src/types.d.ts b/src/types.d.ts index 6ee2eb1..a85907d 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -498,7 +498,13 @@ export type Membership = "invite" | "knock" | "join" | "leave" | "ban" export type Pagination = { chunk: T[] next_batch?: string - prev_match?: string + prev_batch?: string +} + +export type MessagesPagination = { + chunk: T[] + start: string + end?: string } export type HierarchyPagination = {