feat(moderation): enhance message capture and storage with thread support

- Added functions to retrieve message location, sticker metadata, and display content in messageCapture.ts.
- Updated captureMessage function to store thread information and sticker metadata in the database.
- Modified messageStore.ts to support querying messages and attachments by thread ID.
- Updated types.ts to include thread_id in AttachmentRecord.
- Altered database schema in muxer-queue.ts to add thread_id column to attachments.
- Introduced ChannelSummary interface and listWatchableChannels method in voiceController.ts to fetch watchable channels.
- Added API endpoint in webserver.ts to retrieve channels for a given guild.
This commit is contained in:
MythEclipse
2026-05-13 20:52:37 +07:00
parent c7d8353403
commit d55b56c897
9 changed files with 1071 additions and 477 deletions

2
.gitignore vendored
View File

@@ -2,4 +2,4 @@ node_modules
recordings recordings
.env .env
dist/ dist/
.muxer-queue.db .muxer-queue.**

View File

@@ -0,0 +1,64 @@
# Backlog Sync Rich Metadata Implementation Plan
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
**Goal:** Fetch prior Discord messages up to 24 hours on startup, persist rich Discord-client-like metadata, and render rich message content in homepage tabs.
**Architecture:** Add `messageMetadata.ts` for reusable extraction, `backlogSync.ts` for bounded startup history fetch, reuse existing store/uploader. UI reads metadata JSON and renders stickers, embeds, attachments/replies/thread badges.
**Tech Stack:** Bun, TypeScript, discord.js-selfbot-v13, bun:sqlite, Express/WebSocket, vanilla HTML/CSS/JS.
---
### Task 1: Extract rich message metadata
**Files:**
- Create: `src/moderation/messageMetadata.ts`
- Modify: `src/moderation/messageCapture.ts`
- [ ] Create helper functions: `getMessageLocation`, `getStickerMetadata`, `getEmbedMetadata`, `getAttachmentMetadata`, `getMessageMetadata`, `getDisplayContent`.
- [ ] Replace duplicate capture helper logic with imports from `messageMetadata.ts`.
- [ ] Verify: `bun run typecheck`.
### Task 2: Make message inserts idempotent
**Files:**
- Modify: `src/moderation/messageStore.ts`
- [ ] Change message insert to `INSERT OR IGNORE` so backlog sync and live events do not conflict.
- [ ] Change attachment insert to `INSERT OR IGNORE`.
- [ ] Verify: `bun run typecheck && bun run test`.
### Task 3: Add backlog sync
**Files:**
- Create: `src/moderation/backlogSync.ts`
- Modify: `src/index.ts`
- Modify: `src/config.ts`
- Modify: `.env.example`
- [ ] Add config: `BACKLOG_SYNC_HOURS=24`, `BACKLOG_SYNC_BATCH_SIZE=100`.
- [ ] Fetch text/thread channels from monitored guild on ready.
- [ ] For each channel/thread, page `channel.messages.fetch({ limit, before })` until older than cutoff.
- [ ] Store messages with rich metadata and attachments.
- [ ] Start sync after registering live capture; run async and log progress.
- [ ] Verify: `bun run typecheck && bun run test`.
### Task 4: Render richer UI
**Files:**
- Modify: `public/index.html`
- [ ] Render metadata embeds as embed cards.
- [ ] Render attachments as inline previews/links in Text tab.
- [ ] Render reply and thread badges.
- [ ] Keep sticker rendering.
- [ ] Verify static JS syntax by typecheck/tests where applicable.
### Task 5: Final verification
**Files:** all touched files
- [ ] Run `bun run typecheck`.
- [ ] Run `bun run test`.
- [ ] Verify short DB init with `bun -e 'import("./src/muxer-queue.ts").then((m)=>{const db=m.getDatabase(); db.close(); console.log("sqlite ok")})'`.

File diff suppressed because it is too large Load Diff

View File

@@ -8,29 +8,73 @@ import type { MessageRecord, AttachmentRecord } from "./types";
const logger = createChildLogger("message-capture"); const logger = createChildLogger("message-capture");
function getMessageLocation(message: Message): {
channelId: string;
threadId: string | null;
threadName: string | null;
} {
const channel = message.channel as TextChannel | ThreadChannel;
if (!channel.isThread?.()) {
return { channelId: message.channelId, threadId: null, threadName: null };
}
return {
channelId: channel.parentId ?? message.channelId,
threadId: channel.id,
threadName: channel.name,
};
}
function getStickerMetadata(message: Message): Array<{
id: string;
name: string;
url: string;
}> {
return Array.from(message.stickers.values()).map((sticker) => ({
id: sticker.id,
name: sticker.name,
url: sticker.url,
}));
}
function getDisplayContent(message: Message): string {
if (message.content.trim().length > 0) return message.content;
const stickers = getStickerMetadata(message);
if (stickers.length > 0) {
return stickers.map((sticker) => `[Sticker: ${sticker.name}]`).join(" ");
}
return "";
}
async function captureMessage( async function captureMessage(
db: SqliteDatabase, db: SqliteDatabase,
message: Message, message: Message,
type: "text" | "edited" | "deleted", type: "text" | "edited" | "deleted",
): Promise<void> { ): Promise<void> {
const channel = message.channel as TextChannel | ThreadChannel; const location = getMessageLocation(message);
const threadId = channel.isThread?.() ? channel.id : null; const stickers = getStickerMetadata(message);
const metadata = {
stickers,
threadName: location.threadName,
};
const messageRecord: MessageRecord = { const messageRecord: MessageRecord = {
id: message.id, id: message.id,
guild_id: message.guildId!, guild_id: message.guildId!,
channel_id: message.channelId, channel_id: location.channelId,
thread_id: threadId, thread_id: location.threadId,
user_id: message.author!.id, user_id: message.author!.id,
username: message.author!.username, username: message.author!.username,
avatar_url: message.author!.avatarURL() || null, avatar_url: message.author!.avatarURL() || null,
content: message.content, content: getDisplayContent(message),
edited_content: null, edited_content: null,
created_at: message.createdTimestamp, created_at: message.createdTimestamp,
edited_at: null, edited_at: null,
deleted_at: null, deleted_at: null,
type, type,
metadata: null, metadata: JSON.stringify(metadata),
}; };
insertMessage(db, messageRecord); insertMessage(db, messageRecord);
@@ -38,13 +82,7 @@ async function captureMessage(
const broadcaster = globalThis as any; const broadcaster = globalThis as any;
if (broadcaster.broadcastMessageCreated) { if (broadcaster.broadcastMessageCreated) {
broadcaster.broadcastMessageCreated({ broadcaster.broadcastMessageCreated({
id: message.id, ...messageRecord,
channel_id: message.channelId,
user_id: message.author!.id,
username: message.author!.username,
avatar_url: message.author!.avatarURL() || null,
content: message.content,
created_at: message.createdTimestamp,
type: "text", type: "text",
}); });
} }
@@ -55,7 +93,8 @@ async function captureMessage(
id: attachment.id, id: attachment.id,
message_id: message.id, message_id: message.id,
guild_id: message.guildId!, guild_id: message.guildId!,
channel_id: message.channelId, channel_id: location.channelId,
thread_id: location.threadId,
user_id: message.author!.id, user_id: message.author!.id,
filename: attachment.name || "unknown", filename: attachment.name || "unknown",
size: attachment.size, size: attachment.size,
@@ -77,7 +116,7 @@ async function captureMessage(
id: attachment.id, id: attachment.id,
message_id: message.id, message_id: message.id,
filename: attachment.name || "unknown", filename: attachment.name || "unknown",
channel_id: message.channelId, channel_id: location.channelId,
created_at: Date.now(), created_at: Date.now(),
}); });
} }
@@ -129,13 +168,13 @@ export function registerMessageCapture(client: Client, db: SqliteDatabase): void
if (existing) { if (existing) {
const editedAt = Date.now(); const editedAt = Date.now();
updateMessageAsEdited(db, newMessage.id, newMessage.content || "", editedAt); updateMessageAsEdited(db, newMessage.id, getDisplayContent(newMessage as Message), editedAt);
const broadcaster = globalThis as any; const broadcaster = globalThis as any;
if (broadcaster.broadcastMessageUpdated) { if (broadcaster.broadcastMessageUpdated) {
broadcaster.broadcastMessageUpdated({ broadcaster.broadcastMessageUpdated({
id: newMessage.id, id: newMessage.id,
edited_content: newMessage.content || "", edited_content: getDisplayContent(newMessage as Message),
edited_at: editedAt, edited_at: editedAt,
}); });
} }

View File

@@ -96,12 +96,12 @@ export function getMessagesByChannel(
try { try {
const stmt = db.prepare(` const stmt = db.prepare(`
SELECT * FROM messages SELECT * FROM messages
WHERE channel_id = ? WHERE channel_id = ? OR thread_id = ?
ORDER BY created_at DESC ORDER BY created_at DESC
LIMIT ? OFFSET ? LIMIT ? OFFSET ?
`); `);
const rows = stmt.all(channelId, limit, offset) as MessageRecord[]; const rows = stmt.all(channelId, channelId, limit, offset) as MessageRecord[];
return rows; return rows;
} catch (error) { } catch (error) {
logger.error( logger.error(
@@ -116,9 +116,9 @@ export function insertAttachment(db: SqliteDatabase, attachment: AttachmentRecor
try { try {
const stmt = db.prepare(` const stmt = db.prepare(`
INSERT INTO attachments ( INSERT INTO attachments (
id, message_id, guild_id, channel_id, user_id, filename, size, type, id, message_id, guild_id, channel_id, thread_id, user_id, filename, size, type,
discord_url, uploaded_url, upload_status, upload_error, created_at, uploaded_at discord_url, uploaded_url, upload_status, upload_error, created_at, uploaded_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`); `);
stmt.run( stmt.run(
@@ -126,6 +126,7 @@ export function insertAttachment(db: SqliteDatabase, attachment: AttachmentRecor
attachment.message_id, attachment.message_id,
attachment.guild_id, attachment.guild_id,
attachment.channel_id, attachment.channel_id,
attachment.thread_id,
attachment.user_id, attachment.user_id,
attachment.filename, attachment.filename,
attachment.size, attachment.size,
@@ -157,12 +158,12 @@ export function getAttachmentsByChannel(
try { try {
const stmt = db.prepare(` const stmt = db.prepare(`
SELECT * FROM attachments SELECT * FROM attachments
WHERE channel_id = ? WHERE channel_id = ? OR thread_id = ?
ORDER BY created_at DESC ORDER BY created_at DESC
LIMIT ? OFFSET ? LIMIT ? OFFSET ?
`); `);
const rows = stmt.all(channelId, limit, offset) as AttachmentRecord[]; const rows = stmt.all(channelId, channelId, limit, offset) as AttachmentRecord[];
return rows; return rows;
} catch (error) { } catch (error) {
logger.error( logger.error(

View File

@@ -20,6 +20,7 @@ export interface AttachmentRecord {
message_id: string; message_id: string;
guild_id: string; guild_id: string;
channel_id: string; channel_id: string;
thread_id: string | null;
user_id: string; user_id: string;
filename: string; filename: string;
size: number; size: number;

View File

@@ -84,6 +84,7 @@ function initializeDatabase(): SqliteDatabase {
message_id TEXT NOT NULL, message_id TEXT NOT NULL,
guild_id TEXT NOT NULL, guild_id TEXT NOT NULL,
channel_id TEXT NOT NULL, channel_id TEXT NOT NULL,
thread_id TEXT,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
filename TEXT NOT NULL, filename TEXT NOT NULL,
size INTEGER NOT NULL, size INTEGER NOT NULL,
@@ -102,6 +103,12 @@ function initializeDatabase(): SqliteDatabase {
CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(upload_status); CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(upload_status);
`); `);
try {
database.exec("ALTER TABLE attachments ADD COLUMN thread_id TEXT");
} catch {
// Column already exists on databases initialized after the moderation schema was added.
}
return database; return database;
} }

View File

@@ -25,6 +25,12 @@ export interface VoiceChannelSummary {
name: string; name: string;
} }
export interface ChannelSummary {
id: string;
name: string;
type: string;
}
export class VoiceController { export class VoiceController {
private activeGuildId: string | null = null; private activeGuildId: string | null = null;
private activeChannelId: string | null = null; private activeChannelId: string | null = null;
@@ -63,6 +69,27 @@ export class VoiceController {
.sort((a, b) => a.name.localeCompare(b.name)); .sort((a, b) => a.name.localeCompare(b.name));
} }
async listWatchableChannels(guildId: string): Promise<ChannelSummary[]> {
const guild = this.getGuild(guildId);
await guild.channels.fetch().catch(() => null);
return guild.channels.cache
.filter((channel) =>
["GUILD_TEXT", "GUILD_PUBLIC_THREAD", "GUILD_PRIVATE_THREAD"].includes(
channel.type,
),
)
.map((channel) => {
const parentName = channel.isThread?.() ? channel.parent?.name : null;
return {
id: channel.id,
name: parentName ? `${parentName} / ${channel.name}` : channel.name,
type: channel.type,
};
})
.sort((a, b) => a.name.localeCompare(b.name));
}
async connect(guildId: string, channelId: string): Promise<VoiceStatus> { async connect(guildId: string, channelId: string): Promise<VoiceStatus> {
if (!this.client.isReady()) { if (!this.client.isReady()) {
throw new AppError( throw new AppError(

View File

@@ -106,6 +106,14 @@ export function startWebserver(
} }
}); });
app.get("/api/guilds/:guildId/channels", async (req, res, next) => {
try {
res.json(await voiceController.listWatchableChannels(req.params.guildId));
} catch (error) {
next(error);
}
});
app.post("/api/connect", async (req, res, next) => { app.post("/api/connect", async (req, res, next) => {
try { try {
const { guildId, channelId } = req.body as { const { guildId, channelId } = req.body as {