feat: implement age restriction handling in message analysis and metadata

This commit is contained in:
MythEclipse
2026-05-30 21:12:25 +07:00
parent c9e79c8c7c
commit 10f44138bc
3 changed files with 136 additions and 11 deletions
+80 -5
View File
@@ -9,6 +9,7 @@ import { retryWithBackoff } from "../retry.js";
import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js"; import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js";
import { buildConversationContext } from "./conversationContext.js"; import { buildConversationContext } from "./conversationContext.js";
import { runModerationAnalysis } from "./llmModerationClient.js"; import { runModerationAnalysis } from "./llmModerationClient.js";
import { isAgeRestrictedMetadata } from "./messageMetadata.js";
import { import {
getAttachmentsForMessages, getAttachmentsForMessages,
getConversationContextBefore, getConversationContextBefore,
@@ -17,6 +18,7 @@ import {
getMessageById, getMessageById,
getPendingConversationKeys, getPendingConversationKeys,
getPendingMessagesByConversation, getPendingMessagesByConversation,
updateMessageAIAnalysis,
updateMessagesAIAnalysisBulk, updateMessagesAIAnalysisBulk,
} from "./messageStore.js"; } from "./messageStore.js";
import type { import type {
@@ -56,6 +58,59 @@ function scheduleAutoDelete(row: MessageRecord): void {
setImmediate(run); setImmediate(run);
} }
function isAgeRestrictedMessage(message: MessageRecord): boolean {
return isAgeRestrictedMetadata(message.metadata);
}
function buildAgeRestrictedSkipResult(): {
status: "clean";
flags: string | null;
score: number;
analysis: string;
categories: string[];
severity: "none";
confidence: number;
recommendedAction: "none";
analyzedAt: number;
error: null;
} {
return {
status: "clean",
flags: JSON.stringify(["age_restricted"]),
score: 0,
analysis: "Skipped moderation for age-restricted content.",
categories: ["age_restricted"],
severity: "none",
confidence: 1,
recommendedAction: "none",
analyzedAt: Date.now(),
error: null,
};
}
async function skipAgeRestrictedMessages(
messages: MessageRecord[],
): Promise<MessageRecord[]> {
const ageRestrictedMessages = messages.filter(isAgeRestrictedMessage);
if (ageRestrictedMessages.length === 0) {
return messages;
}
const skippedRows = await updateMessagesAIAnalysisBulk(
ageRestrictedMessages.map((message) => ({
messageId: message.id,
result: buildAgeRestrictedSkipResult(),
})),
);
for (const row of skippedRows) {
getModerationBroadcaster()?.messageAnalyzed(row);
}
const skippedIds = new Set(ageRestrictedMessages.map((message) => message.id));
return messages.filter((message) => !skippedIds.has(message.id));
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Batch pipeline state // Batch pipeline state
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -672,13 +727,16 @@ function scheduleConversationAnalysis(conversationKey: string): void {
conversationKey, conversationKey,
config.AI_ANALYSIS_MAX_BATCH_SIZE, config.AI_ANALYSIS_MAX_BATCH_SIZE,
) )
.then((messages) => { .then(async (messages) => {
if (messages.length === 0) return; if (messages.length === 0) return;
const processableMessages = await skipAgeRestrictedMessages(messages);
if (processableMessages.length === 0) return;
// FIX #6: trim to token budget before sending to LLM. // FIX #6: trim to token budget before sending to LLM.
// 50 tokens overhead accounts for JSON structure + id/username fields. // 50 tokens overhead accounts for JSON structure + id/username fields.
let trimmed = pickBatchWithinBudget( let trimmed = pickBatchWithinBudget(
messages, processableMessages,
config.AI_ANALYSIS_MAX_TARGET_TOKENS, config.AI_ANALYSIS_MAX_TARGET_TOKENS,
50, 50,
); );
@@ -687,12 +745,12 @@ function scheduleConversationAnalysis(conversationKey: string): void {
// pickBatchWithinBudget returns [] — which would leave them permanently // pickBatchWithinBudget returns [] — which would leave them permanently
// stuck as `pending`. Fall back to the first message alone so at // stuck as `pending`. Fall back to the first message alone so at
// least one makes progress; the rest will be processed in later ticks. // least one makes progress; the rest will be processed in later ticks.
if (trimmed.length === 0 && messages.length > 0) { if (trimmed.length === 0 && processableMessages.length > 0) {
trimmed = messages.slice(0, 1); trimmed = processableMessages.slice(0, 1);
logger.warn( logger.warn(
{ {
conversationKey, conversationKey,
messageId: messages[0]?.id, messageId: processableMessages[0]?.id,
tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS, tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS,
}, },
"All messages exceed token budget — processing first message alone to avoid stuck-pending deadlock", "All messages exceed token budget — processing first message alone to avoid stuck-pending deadlock",
@@ -731,6 +789,19 @@ export async function queueMessageAnalysis(messageId: string): Promise<void> {
logger.warn({ messageId }, "Message not found for analysis queue"); logger.warn({ messageId }, "Message not found for analysis queue");
return; return;
} }
if (isAgeRestrictedMessage(message)) {
const updated = await updateMessageAIAnalysis(
message.id,
buildAgeRestrictedSkipResult(),
);
if (updated) {
getModerationBroadcaster()?.messageAnalyzed(updated);
}
logger.info({ messageId }, "Skipped AI analysis for age-restricted message");
return;
}
queueConversationAnalysis(getConversationKey(message)); queueConversationAnalysis(getConversationKey(message));
} catch (error) { } catch (error) {
logger.error( logger.error(
@@ -830,6 +901,10 @@ export function startPendingAIAnalysisWorker(client?: Client): void {
key, key,
config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT, config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT,
) )
.then(async (msgs) => {
const processableMessages = await skipAgeRestrictedMessages(msgs);
return processableMessages;
})
.then((msgs) => { .then((msgs) => {
if (msgs.length > 0) { if (msgs.length > 0) {
enqueueIndividualFallbacks(msgs); enqueueIndividualFallbacks(msgs);
+19 -6
View File
@@ -1,5 +1,6 @@
import { executeAll, executeGet } from "../database/drizzle.js"; import { executeAll, executeGet } from "../database/drizzle.js";
import { createChildLogger } from "../logger.js"; import { createChildLogger } from "../logger.js";
import { config } from "../config.js";
import type { MessageRecord } from "./types.js"; import type { MessageRecord } from "./types.js";
const logger = createChildLogger("analytics-store"); const logger = createChildLogger("analytics-store");
@@ -106,10 +107,16 @@ export async function getHourlyStats(input: {
try { try {
const since = Date.now() - hours * 3600_000; const since = Date.now() - hours * 3600_000;
const sqliteRows = await executeAll( const isPg = config.DATABASE_TYPE === "postgres";
const hourExpr = isPg
? `to_char(to_timestamp((created_at / 3600000) * 3600), 'YYYY-MM-DD HH24:MI:SS') as hour`
: `datetime((created_at / 3600000) * 3600, 'unixepoch') as hour`;
const rows = await executeAll(
` `
SELECT SELECT
datetime((created_at / 3600000) * 3600, 'unixepoch') as hour, ${hourExpr},
count(*) as count, count(*) as count,
count(case when ai_status = 'clean' then 1 end) as clean, count(case when ai_status = 'clean' then 1 end) as clean,
count(case when ai_status = 'warn' then 1 end) as warned, count(case when ai_status = 'warn' then 1 end) as warned,
@@ -141,7 +148,7 @@ export async function getHourlyStats(input: {
buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 }); buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 });
} }
for (const row of sqliteRows) { for (const row of rows) {
const d = new Date(row.hour.replace(" ", "T") + "Z"); const d = new Date(row.hour.replace(" ", "T") + "Z");
const key = d.toISOString().slice(0, 13) + ":00:00Z"; const key = d.toISOString().slice(0, 13) + ":00:00Z";
const bucket = buckets.get(key); const bucket = buckets.get(key);
@@ -344,7 +351,7 @@ export async function getUserLeaderboard(input: {
AND created_at >= ? AND created_at >= ?
AND deleted_at IS NULL AND deleted_at IS NULL
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""} ${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
GROUP BY user_id GROUP BY user_id, username, avatar_url
ORDER BY message_count DESC ORDER BY message_count DESC
LIMIT ? LIMIT ?
`, `,
@@ -379,6 +386,12 @@ export async function getModerationStats(input: {
try { try {
const since = Date.now() - hours * 3600_000; const since = Date.now() - hours * 3600_000;
const isPg = config.DATABASE_TYPE === "postgres";
const avgScoreExpr = isPg
? `round(avg(ai_moderation_score)::numeric, 2)`
: `round(avg(ai_moderation_score), 2)`;
const row = await executeGet( const row = await executeGet(
` `
SELECT SELECT
@@ -388,7 +401,7 @@ export async function getModerationStats(input: {
count(case when ai_status = 'flagged' then 1 end) as flagged, count(case when ai_status = 'flagged' then 1 end) as flagged,
count(case when ai_status = 'error' then 1 end) as error, count(case when ai_status = 'error' then 1 end) as error,
count(case when ai_status = 'pending' or ai_status IS NULL then 1 end) as pending, count(case when ai_status = 'pending' or ai_status IS NULL then 1 end) as pending,
round(avg(ai_moderation_score), 2) as average_score ${avgScoreExpr} as average_score
FROM messages FROM messages
WHERE guild_id = ? WHERE guild_id = ?
AND created_at >= ? AND created_at >= ?
@@ -501,7 +514,7 @@ export async function getTopViolators(input: {
AND created_at >= ? AND created_at >= ?
AND deleted_at IS NULL AND deleted_at IS NULL
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""} ${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
GROUP BY user_id GROUP BY user_id, username, avatar_url
HAVING flagged_count > 0 OR warned_count > 0 HAVING flagged_count > 0 OR warned_count > 0
ORDER BY (flagged_count * 3 + warned_count) DESC ORDER BY (flagged_count * 3 + warned_count) DESC
LIMIT ? LIMIT ?
+37
View File
@@ -9,6 +9,9 @@ export interface MessageLocation {
threadId: string | null; threadId: string | null;
threadName: string | null; threadName: string | null;
channelName: string | null; channelName: string | null;
nsfw?: boolean;
nsfwLevel?: string | null;
ageRestricted?: boolean;
} }
export interface StickerEvidence { export interface StickerEvidence {
@@ -74,12 +77,25 @@ export interface RichMessageMetadata {
export function getMessageLocation(message: Message): MessageLocation { export function getMessageLocation(message: Message): MessageLocation {
const channel = message.channel as TextChannel | ThreadChannel; const channel = message.channel as TextChannel | ThreadChannel;
const safetyChannel = channel as TextChannel & {
nsfw?: boolean;
nsfwLevel?: string | null;
};
if (!channel.isThread?.()) { if (!channel.isThread?.()) {
return { return {
channelId: message.channelId, channelId: message.channelId,
threadId: null, threadId: null,
threadName: null, threadName: null,
channelName: "name" in channel ? channel.name : null, channelName: "name" in channel ? channel.name : null,
nsfw: typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined,
nsfwLevel:
typeof safetyChannel.nsfwLevel === "string"
? safetyChannel.nsfwLevel
: null,
ageRestricted:
typeof safetyChannel.nsfw === "boolean"
? safetyChannel.nsfw
: undefined,
}; };
} }
@@ -88,6 +104,13 @@ export function getMessageLocation(message: Message): MessageLocation {
threadId: channel.id, threadId: channel.id,
threadName: channel.name, threadName: channel.name,
channelName: channel.parent?.name ?? null, channelName: channel.parent?.name ?? null,
nsfw: typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined,
nsfwLevel:
typeof safetyChannel.nsfwLevel === "string"
? safetyChannel.nsfwLevel
: null,
ageRestricted:
typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined,
}; };
} }
@@ -200,6 +223,20 @@ export function parseRichMessageMetadata(
} }
} }
export function isAgeRestrictedMetadata(
metadata: string | null | undefined,
): boolean {
const parsed = parseRichMessageMetadata(metadata);
if (!parsed) return false;
const nsfwLevel = parsed.channel.nsfwLevel?.toUpperCase();
return Boolean(
parsed.channel.nsfw ||
parsed.channel.ageRestricted ||
nsfwLevel === "AGE_RESTRICTED",
);
}
export function extractMessageMediaEvidence( export function extractMessageMediaEvidence(
metadata: string | null | undefined, metadata: string | null | undefined,
): MessageMediaEvidence { ): MessageMediaEvidence {