feat(moderation): enhance message capture and storage with thread support
- Added functions to retrieve message location, sticker metadata, and display content in messageCapture.ts. - Updated captureMessage function to store thread information and sticker metadata in the database. - Modified messageStore.ts to support querying messages and attachments by thread ID. - Updated types.ts to include thread_id in AttachmentRecord. - Altered database schema in muxer-queue.ts to add thread_id column to attachments. - Introduced ChannelSummary interface and listWatchableChannels method in voiceController.ts to fetch watchable channels. - Added API endpoint in webserver.ts to retrieve channels for a given guild.
This commit is contained in:
2
.gitignore
vendored
2
.gitignore
vendored
@@ -2,4 +2,4 @@ node_modules
|
|||||||
recordings
|
recordings
|
||||||
.env
|
.env
|
||||||
dist/
|
dist/
|
||||||
.muxer-queue.db
|
.muxer-queue.**
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
# Backlog Sync Rich Metadata Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Fetch prior Discord messages up to 24 hours on startup, persist rich Discord-client-like metadata, and render rich message content in homepage tabs.
|
||||||
|
|
||||||
|
**Architecture:** Add `messageMetadata.ts` for reusable extraction, `backlogSync.ts` for bounded startup history fetch, reuse existing store/uploader. UI reads metadata JSON and renders stickers, embeds, attachments/replies/thread badges.
|
||||||
|
|
||||||
|
**Tech Stack:** Bun, TypeScript, discord.js-selfbot-v13, bun:sqlite, Express/WebSocket, vanilla HTML/CSS/JS.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 1: Extract rich message metadata
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `src/moderation/messageMetadata.ts`
|
||||||
|
- Modify: `src/moderation/messageCapture.ts`
|
||||||
|
|
||||||
|
- [ ] Create helper functions: `getMessageLocation`, `getStickerMetadata`, `getEmbedMetadata`, `getAttachmentMetadata`, `getMessageMetadata`, `getDisplayContent`.
|
||||||
|
- [ ] Replace duplicate capture helper logic with imports from `messageMetadata.ts`.
|
||||||
|
- [ ] Verify: `bun run typecheck`.
|
||||||
|
|
||||||
|
### Task 2: Make message inserts idempotent
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `src/moderation/messageStore.ts`
|
||||||
|
|
||||||
|
- [ ] Change message insert to `INSERT OR IGNORE` so backlog sync and live events do not conflict.
|
||||||
|
- [ ] Change attachment insert to `INSERT OR IGNORE`.
|
||||||
|
- [ ] Verify: `bun run typecheck && bun run test`.
|
||||||
|
|
||||||
|
### Task 3: Add backlog sync
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `src/moderation/backlogSync.ts`
|
||||||
|
- Modify: `src/index.ts`
|
||||||
|
- Modify: `src/config.ts`
|
||||||
|
- Modify: `.env.example`
|
||||||
|
|
||||||
|
- [ ] Add config: `BACKLOG_SYNC_HOURS=24`, `BACKLOG_SYNC_BATCH_SIZE=100`.
|
||||||
|
- [ ] Fetch text/thread channels from monitored guild on ready.
|
||||||
|
- [ ] For each channel/thread, page `channel.messages.fetch({ limit, before })` until older than cutoff.
|
||||||
|
- [ ] Store messages with rich metadata and attachments.
|
||||||
|
- [ ] Start sync after registering live capture; run async and log progress.
|
||||||
|
- [ ] Verify: `bun run typecheck && bun run test`.
|
||||||
|
|
||||||
|
### Task 4: Render richer UI
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `public/index.html`
|
||||||
|
|
||||||
|
- [ ] Render metadata embeds as embed cards.
|
||||||
|
- [ ] Render attachments as inline previews/links in Text tab.
|
||||||
|
- [ ] Render reply and thread badges.
|
||||||
|
- [ ] Keep sticker rendering.
|
||||||
|
- [ ] Verify static JS syntax by typecheck/tests where applicable.
|
||||||
|
|
||||||
|
### Task 5: Final verification
|
||||||
|
|
||||||
|
**Files:** all touched files
|
||||||
|
|
||||||
|
- [ ] Run `bun run typecheck`.
|
||||||
|
- [ ] Run `bun run test`.
|
||||||
|
- [ ] Verify short DB init with `bun -e 'import("./src/muxer-queue.ts").then((m)=>{const db=m.getDatabase(); db.close(); console.log("sqlite ok")})'`.
|
||||||
1353
public/index.html
1353
public/index.html
File diff suppressed because it is too large
Load Diff
@@ -8,29 +8,73 @@ import type { MessageRecord, AttachmentRecord } from "./types";
|
|||||||
|
|
||||||
const logger = createChildLogger("message-capture");
|
const logger = createChildLogger("message-capture");
|
||||||
|
|
||||||
|
function getMessageLocation(message: Message): {
|
||||||
|
channelId: string;
|
||||||
|
threadId: string | null;
|
||||||
|
threadName: string | null;
|
||||||
|
} {
|
||||||
|
const channel = message.channel as TextChannel | ThreadChannel;
|
||||||
|
if (!channel.isThread?.()) {
|
||||||
|
return { channelId: message.channelId, threadId: null, threadName: null };
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
channelId: channel.parentId ?? message.channelId,
|
||||||
|
threadId: channel.id,
|
||||||
|
threadName: channel.name,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getStickerMetadata(message: Message): Array<{
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
url: string;
|
||||||
|
}> {
|
||||||
|
return Array.from(message.stickers.values()).map((sticker) => ({
|
||||||
|
id: sticker.id,
|
||||||
|
name: sticker.name,
|
||||||
|
url: sticker.url,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
function getDisplayContent(message: Message): string {
|
||||||
|
if (message.content.trim().length > 0) return message.content;
|
||||||
|
|
||||||
|
const stickers = getStickerMetadata(message);
|
||||||
|
if (stickers.length > 0) {
|
||||||
|
return stickers.map((sticker) => `[Sticker: ${sticker.name}]`).join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
async function captureMessage(
|
async function captureMessage(
|
||||||
db: SqliteDatabase,
|
db: SqliteDatabase,
|
||||||
message: Message,
|
message: Message,
|
||||||
type: "text" | "edited" | "deleted",
|
type: "text" | "edited" | "deleted",
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const channel = message.channel as TextChannel | ThreadChannel;
|
const location = getMessageLocation(message);
|
||||||
const threadId = channel.isThread?.() ? channel.id : null;
|
const stickers = getStickerMetadata(message);
|
||||||
|
const metadata = {
|
||||||
|
stickers,
|
||||||
|
threadName: location.threadName,
|
||||||
|
};
|
||||||
|
|
||||||
const messageRecord: MessageRecord = {
|
const messageRecord: MessageRecord = {
|
||||||
id: message.id,
|
id: message.id,
|
||||||
guild_id: message.guildId!,
|
guild_id: message.guildId!,
|
||||||
channel_id: message.channelId,
|
channel_id: location.channelId,
|
||||||
thread_id: threadId,
|
thread_id: location.threadId,
|
||||||
user_id: message.author!.id,
|
user_id: message.author!.id,
|
||||||
username: message.author!.username,
|
username: message.author!.username,
|
||||||
avatar_url: message.author!.avatarURL() || null,
|
avatar_url: message.author!.avatarURL() || null,
|
||||||
content: message.content,
|
content: getDisplayContent(message),
|
||||||
edited_content: null,
|
edited_content: null,
|
||||||
created_at: message.createdTimestamp,
|
created_at: message.createdTimestamp,
|
||||||
edited_at: null,
|
edited_at: null,
|
||||||
deleted_at: null,
|
deleted_at: null,
|
||||||
type,
|
type,
|
||||||
metadata: null,
|
metadata: JSON.stringify(metadata),
|
||||||
};
|
};
|
||||||
|
|
||||||
insertMessage(db, messageRecord);
|
insertMessage(db, messageRecord);
|
||||||
@@ -38,13 +82,7 @@ async function captureMessage(
|
|||||||
const broadcaster = globalThis as any;
|
const broadcaster = globalThis as any;
|
||||||
if (broadcaster.broadcastMessageCreated) {
|
if (broadcaster.broadcastMessageCreated) {
|
||||||
broadcaster.broadcastMessageCreated({
|
broadcaster.broadcastMessageCreated({
|
||||||
id: message.id,
|
...messageRecord,
|
||||||
channel_id: message.channelId,
|
|
||||||
user_id: message.author!.id,
|
|
||||||
username: message.author!.username,
|
|
||||||
avatar_url: message.author!.avatarURL() || null,
|
|
||||||
content: message.content,
|
|
||||||
created_at: message.createdTimestamp,
|
|
||||||
type: "text",
|
type: "text",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -55,7 +93,8 @@ async function captureMessage(
|
|||||||
id: attachment.id,
|
id: attachment.id,
|
||||||
message_id: message.id,
|
message_id: message.id,
|
||||||
guild_id: message.guildId!,
|
guild_id: message.guildId!,
|
||||||
channel_id: message.channelId,
|
channel_id: location.channelId,
|
||||||
|
thread_id: location.threadId,
|
||||||
user_id: message.author!.id,
|
user_id: message.author!.id,
|
||||||
filename: attachment.name || "unknown",
|
filename: attachment.name || "unknown",
|
||||||
size: attachment.size,
|
size: attachment.size,
|
||||||
@@ -77,7 +116,7 @@ async function captureMessage(
|
|||||||
id: attachment.id,
|
id: attachment.id,
|
||||||
message_id: message.id,
|
message_id: message.id,
|
||||||
filename: attachment.name || "unknown",
|
filename: attachment.name || "unknown",
|
||||||
channel_id: message.channelId,
|
channel_id: location.channelId,
|
||||||
created_at: Date.now(),
|
created_at: Date.now(),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -129,13 +168,13 @@ export function registerMessageCapture(client: Client, db: SqliteDatabase): void
|
|||||||
|
|
||||||
if (existing) {
|
if (existing) {
|
||||||
const editedAt = Date.now();
|
const editedAt = Date.now();
|
||||||
updateMessageAsEdited(db, newMessage.id, newMessage.content || "", editedAt);
|
updateMessageAsEdited(db, newMessage.id, getDisplayContent(newMessage as Message), editedAt);
|
||||||
|
|
||||||
const broadcaster = globalThis as any;
|
const broadcaster = globalThis as any;
|
||||||
if (broadcaster.broadcastMessageUpdated) {
|
if (broadcaster.broadcastMessageUpdated) {
|
||||||
broadcaster.broadcastMessageUpdated({
|
broadcaster.broadcastMessageUpdated({
|
||||||
id: newMessage.id,
|
id: newMessage.id,
|
||||||
edited_content: newMessage.content || "",
|
edited_content: getDisplayContent(newMessage as Message),
|
||||||
edited_at: editedAt,
|
edited_at: editedAt,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -96,12 +96,12 @@ export function getMessagesByChannel(
|
|||||||
try {
|
try {
|
||||||
const stmt = db.prepare(`
|
const stmt = db.prepare(`
|
||||||
SELECT * FROM messages
|
SELECT * FROM messages
|
||||||
WHERE channel_id = ?
|
WHERE channel_id = ? OR thread_id = ?
|
||||||
ORDER BY created_at DESC
|
ORDER BY created_at DESC
|
||||||
LIMIT ? OFFSET ?
|
LIMIT ? OFFSET ?
|
||||||
`);
|
`);
|
||||||
|
|
||||||
const rows = stmt.all(channelId, limit, offset) as MessageRecord[];
|
const rows = stmt.all(channelId, channelId, limit, offset) as MessageRecord[];
|
||||||
return rows;
|
return rows;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(
|
logger.error(
|
||||||
@@ -116,9 +116,9 @@ export function insertAttachment(db: SqliteDatabase, attachment: AttachmentRecor
|
|||||||
try {
|
try {
|
||||||
const stmt = db.prepare(`
|
const stmt = db.prepare(`
|
||||||
INSERT INTO attachments (
|
INSERT INTO attachments (
|
||||||
id, message_id, guild_id, channel_id, user_id, filename, size, type,
|
id, message_id, guild_id, channel_id, thread_id, user_id, filename, size, type,
|
||||||
discord_url, uploaded_url, upload_status, upload_error, created_at, uploaded_at
|
discord_url, uploaded_url, upload_status, upload_error, created_at, uploaded_at
|
||||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
`);
|
`);
|
||||||
|
|
||||||
stmt.run(
|
stmt.run(
|
||||||
@@ -126,6 +126,7 @@ export function insertAttachment(db: SqliteDatabase, attachment: AttachmentRecor
|
|||||||
attachment.message_id,
|
attachment.message_id,
|
||||||
attachment.guild_id,
|
attachment.guild_id,
|
||||||
attachment.channel_id,
|
attachment.channel_id,
|
||||||
|
attachment.thread_id,
|
||||||
attachment.user_id,
|
attachment.user_id,
|
||||||
attachment.filename,
|
attachment.filename,
|
||||||
attachment.size,
|
attachment.size,
|
||||||
@@ -157,12 +158,12 @@ export function getAttachmentsByChannel(
|
|||||||
try {
|
try {
|
||||||
const stmt = db.prepare(`
|
const stmt = db.prepare(`
|
||||||
SELECT * FROM attachments
|
SELECT * FROM attachments
|
||||||
WHERE channel_id = ?
|
WHERE channel_id = ? OR thread_id = ?
|
||||||
ORDER BY created_at DESC
|
ORDER BY created_at DESC
|
||||||
LIMIT ? OFFSET ?
|
LIMIT ? OFFSET ?
|
||||||
`);
|
`);
|
||||||
|
|
||||||
const rows = stmt.all(channelId, limit, offset) as AttachmentRecord[];
|
const rows = stmt.all(channelId, channelId, limit, offset) as AttachmentRecord[];
|
||||||
return rows;
|
return rows;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(
|
logger.error(
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ export interface AttachmentRecord {
|
|||||||
message_id: string;
|
message_id: string;
|
||||||
guild_id: string;
|
guild_id: string;
|
||||||
channel_id: string;
|
channel_id: string;
|
||||||
|
thread_id: string | null;
|
||||||
user_id: string;
|
user_id: string;
|
||||||
filename: string;
|
filename: string;
|
||||||
size: number;
|
size: number;
|
||||||
|
|||||||
@@ -84,6 +84,7 @@ function initializeDatabase(): SqliteDatabase {
|
|||||||
message_id TEXT NOT NULL,
|
message_id TEXT NOT NULL,
|
||||||
guild_id TEXT NOT NULL,
|
guild_id TEXT NOT NULL,
|
||||||
channel_id TEXT NOT NULL,
|
channel_id TEXT NOT NULL,
|
||||||
|
thread_id TEXT,
|
||||||
user_id TEXT NOT NULL,
|
user_id TEXT NOT NULL,
|
||||||
filename TEXT NOT NULL,
|
filename TEXT NOT NULL,
|
||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
@@ -102,6 +103,12 @@ function initializeDatabase(): SqliteDatabase {
|
|||||||
CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(upload_status);
|
CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(upload_status);
|
||||||
`);
|
`);
|
||||||
|
|
||||||
|
try {
|
||||||
|
database.exec("ALTER TABLE attachments ADD COLUMN thread_id TEXT");
|
||||||
|
} catch {
|
||||||
|
// Column already exists on databases initialized after the moderation schema was added.
|
||||||
|
}
|
||||||
|
|
||||||
return database;
|
return database;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,12 @@ export interface VoiceChannelSummary {
|
|||||||
name: string;
|
name: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ChannelSummary {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
}
|
||||||
|
|
||||||
export class VoiceController {
|
export class VoiceController {
|
||||||
private activeGuildId: string | null = null;
|
private activeGuildId: string | null = null;
|
||||||
private activeChannelId: string | null = null;
|
private activeChannelId: string | null = null;
|
||||||
@@ -63,6 +69,27 @@ export class VoiceController {
|
|||||||
.sort((a, b) => a.name.localeCompare(b.name));
|
.sort((a, b) => a.name.localeCompare(b.name));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async listWatchableChannels(guildId: string): Promise<ChannelSummary[]> {
|
||||||
|
const guild = this.getGuild(guildId);
|
||||||
|
await guild.channels.fetch().catch(() => null);
|
||||||
|
|
||||||
|
return guild.channels.cache
|
||||||
|
.filter((channel) =>
|
||||||
|
["GUILD_TEXT", "GUILD_PUBLIC_THREAD", "GUILD_PRIVATE_THREAD"].includes(
|
||||||
|
channel.type,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.map((channel) => {
|
||||||
|
const parentName = channel.isThread?.() ? channel.parent?.name : null;
|
||||||
|
return {
|
||||||
|
id: channel.id,
|
||||||
|
name: parentName ? `${parentName} / ${channel.name}` : channel.name,
|
||||||
|
type: channel.type,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.sort((a, b) => a.name.localeCompare(b.name));
|
||||||
|
}
|
||||||
|
|
||||||
async connect(guildId: string, channelId: string): Promise<VoiceStatus> {
|
async connect(guildId: string, channelId: string): Promise<VoiceStatus> {
|
||||||
if (!this.client.isReady()) {
|
if (!this.client.isReady()) {
|
||||||
throw new AppError(
|
throw new AppError(
|
||||||
|
|||||||
@@ -106,6 +106,14 @@ export function startWebserver(
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
app.get("/api/guilds/:guildId/channels", async (req, res, next) => {
|
||||||
|
try {
|
||||||
|
res.json(await voiceController.listWatchableChannels(req.params.guildId));
|
||||||
|
} catch (error) {
|
||||||
|
next(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
app.post("/api/connect", async (req, res, next) => {
|
app.post("/api/connect", async (req, res, next) => {
|
||||||
try {
|
try {
|
||||||
const { guildId, channelId } = req.body as {
|
const { guildId, channelId } = req.body as {
|
||||||
|
|||||||
Reference in New Issue
Block a user