feat: implement age restriction handling in message analysis and metadata
This commit is contained in:
@@ -9,6 +9,7 @@ import { retryWithBackoff } from "../retry.js";
|
|||||||
import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js";
|
import { attemptAutoDeleteFlaggedMessage } from "./autoDeleteManager.js";
|
||||||
import { buildConversationContext } from "./conversationContext.js";
|
import { buildConversationContext } from "./conversationContext.js";
|
||||||
import { runModerationAnalysis } from "./llmModerationClient.js";
|
import { runModerationAnalysis } from "./llmModerationClient.js";
|
||||||
|
import { isAgeRestrictedMetadata } from "./messageMetadata.js";
|
||||||
import {
|
import {
|
||||||
getAttachmentsForMessages,
|
getAttachmentsForMessages,
|
||||||
getConversationContextBefore,
|
getConversationContextBefore,
|
||||||
@@ -17,6 +18,7 @@ import {
|
|||||||
getMessageById,
|
getMessageById,
|
||||||
getPendingConversationKeys,
|
getPendingConversationKeys,
|
||||||
getPendingMessagesByConversation,
|
getPendingMessagesByConversation,
|
||||||
|
updateMessageAIAnalysis,
|
||||||
updateMessagesAIAnalysisBulk,
|
updateMessagesAIAnalysisBulk,
|
||||||
} from "./messageStore.js";
|
} from "./messageStore.js";
|
||||||
import type {
|
import type {
|
||||||
@@ -56,6 +58,59 @@ function scheduleAutoDelete(row: MessageRecord): void {
|
|||||||
setImmediate(run);
|
setImmediate(run);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isAgeRestrictedMessage(message: MessageRecord): boolean {
|
||||||
|
return isAgeRestrictedMetadata(message.metadata);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAgeRestrictedSkipResult(): {
|
||||||
|
status: "clean";
|
||||||
|
flags: string | null;
|
||||||
|
score: number;
|
||||||
|
analysis: string;
|
||||||
|
categories: string[];
|
||||||
|
severity: "none";
|
||||||
|
confidence: number;
|
||||||
|
recommendedAction: "none";
|
||||||
|
analyzedAt: number;
|
||||||
|
error: null;
|
||||||
|
} {
|
||||||
|
return {
|
||||||
|
status: "clean",
|
||||||
|
flags: JSON.stringify(["age_restricted"]),
|
||||||
|
score: 0,
|
||||||
|
analysis: "Skipped moderation for age-restricted content.",
|
||||||
|
categories: ["age_restricted"],
|
||||||
|
severity: "none",
|
||||||
|
confidence: 1,
|
||||||
|
recommendedAction: "none",
|
||||||
|
analyzedAt: Date.now(),
|
||||||
|
error: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function skipAgeRestrictedMessages(
|
||||||
|
messages: MessageRecord[],
|
||||||
|
): Promise<MessageRecord[]> {
|
||||||
|
const ageRestrictedMessages = messages.filter(isAgeRestrictedMessage);
|
||||||
|
if (ageRestrictedMessages.length === 0) {
|
||||||
|
return messages;
|
||||||
|
}
|
||||||
|
|
||||||
|
const skippedRows = await updateMessagesAIAnalysisBulk(
|
||||||
|
ageRestrictedMessages.map((message) => ({
|
||||||
|
messageId: message.id,
|
||||||
|
result: buildAgeRestrictedSkipResult(),
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const row of skippedRows) {
|
||||||
|
getModerationBroadcaster()?.messageAnalyzed(row);
|
||||||
|
}
|
||||||
|
|
||||||
|
const skippedIds = new Set(ageRestrictedMessages.map((message) => message.id));
|
||||||
|
return messages.filter((message) => !skippedIds.has(message.id));
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Batch pipeline state
|
// Batch pipeline state
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -672,13 +727,16 @@ function scheduleConversationAnalysis(conversationKey: string): void {
|
|||||||
conversationKey,
|
conversationKey,
|
||||||
config.AI_ANALYSIS_MAX_BATCH_SIZE,
|
config.AI_ANALYSIS_MAX_BATCH_SIZE,
|
||||||
)
|
)
|
||||||
.then((messages) => {
|
.then(async (messages) => {
|
||||||
if (messages.length === 0) return;
|
if (messages.length === 0) return;
|
||||||
|
|
||||||
|
const processableMessages = await skipAgeRestrictedMessages(messages);
|
||||||
|
if (processableMessages.length === 0) return;
|
||||||
|
|
||||||
// FIX #6: trim to token budget before sending to LLM.
|
// FIX #6: trim to token budget before sending to LLM.
|
||||||
// 50 tokens overhead accounts for JSON structure + id/username fields.
|
// 50 tokens overhead accounts for JSON structure + id/username fields.
|
||||||
let trimmed = pickBatchWithinBudget(
|
let trimmed = pickBatchWithinBudget(
|
||||||
messages,
|
processableMessages,
|
||||||
config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
||||||
50,
|
50,
|
||||||
);
|
);
|
||||||
@@ -687,12 +745,12 @@ function scheduleConversationAnalysis(conversationKey: string): void {
|
|||||||
// pickBatchWithinBudget returns [] — which would leave them permanently
|
// pickBatchWithinBudget returns [] — which would leave them permanently
|
||||||
// stuck as `pending`. Fall back to the first message alone so at
|
// stuck as `pending`. Fall back to the first message alone so at
|
||||||
// least one makes progress; the rest will be processed in later ticks.
|
// least one makes progress; the rest will be processed in later ticks.
|
||||||
if (trimmed.length === 0 && messages.length > 0) {
|
if (trimmed.length === 0 && processableMessages.length > 0) {
|
||||||
trimmed = messages.slice(0, 1);
|
trimmed = processableMessages.slice(0, 1);
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{
|
{
|
||||||
conversationKey,
|
conversationKey,
|
||||||
messageId: messages[0]?.id,
|
messageId: processableMessages[0]?.id,
|
||||||
tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
||||||
},
|
},
|
||||||
"All messages exceed token budget — processing first message alone to avoid stuck-pending deadlock",
|
"All messages exceed token budget — processing first message alone to avoid stuck-pending deadlock",
|
||||||
@@ -731,6 +789,19 @@ export async function queueMessageAnalysis(messageId: string): Promise<void> {
|
|||||||
logger.warn({ messageId }, "Message not found for analysis queue");
|
logger.warn({ messageId }, "Message not found for analysis queue");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isAgeRestrictedMessage(message)) {
|
||||||
|
const updated = await updateMessageAIAnalysis(
|
||||||
|
message.id,
|
||||||
|
buildAgeRestrictedSkipResult(),
|
||||||
|
);
|
||||||
|
if (updated) {
|
||||||
|
getModerationBroadcaster()?.messageAnalyzed(updated);
|
||||||
|
}
|
||||||
|
logger.info({ messageId }, "Skipped AI analysis for age-restricted message");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
queueConversationAnalysis(getConversationKey(message));
|
queueConversationAnalysis(getConversationKey(message));
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(
|
logger.error(
|
||||||
@@ -830,6 +901,10 @@ export function startPendingAIAnalysisWorker(client?: Client): void {
|
|||||||
key,
|
key,
|
||||||
config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT,
|
config.AI_ANALYSIS_INDIVIDUAL_MAX_CONCURRENT,
|
||||||
)
|
)
|
||||||
|
.then(async (msgs) => {
|
||||||
|
const processableMessages = await skipAgeRestrictedMessages(msgs);
|
||||||
|
return processableMessages;
|
||||||
|
})
|
||||||
.then((msgs) => {
|
.then((msgs) => {
|
||||||
if (msgs.length > 0) {
|
if (msgs.length > 0) {
|
||||||
enqueueIndividualFallbacks(msgs);
|
enqueueIndividualFallbacks(msgs);
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { executeAll, executeGet } from "../database/drizzle.js";
|
import { executeAll, executeGet } from "../database/drizzle.js";
|
||||||
import { createChildLogger } from "../logger.js";
|
import { createChildLogger } from "../logger.js";
|
||||||
|
import { config } from "../config.js";
|
||||||
import type { MessageRecord } from "./types.js";
|
import type { MessageRecord } from "./types.js";
|
||||||
|
|
||||||
const logger = createChildLogger("analytics-store");
|
const logger = createChildLogger("analytics-store");
|
||||||
@@ -106,10 +107,16 @@ export async function getHourlyStats(input: {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const since = Date.now() - hours * 3600_000;
|
const since = Date.now() - hours * 3600_000;
|
||||||
const sqliteRows = await executeAll(
|
const isPg = config.DATABASE_TYPE === "postgres";
|
||||||
|
|
||||||
|
const hourExpr = isPg
|
||||||
|
? `to_char(to_timestamp((created_at / 3600000) * 3600), 'YYYY-MM-DD HH24:MI:SS') as hour`
|
||||||
|
: `datetime((created_at / 3600000) * 3600, 'unixepoch') as hour`;
|
||||||
|
|
||||||
|
const rows = await executeAll(
|
||||||
`
|
`
|
||||||
SELECT
|
SELECT
|
||||||
datetime((created_at / 3600000) * 3600, 'unixepoch') as hour,
|
${hourExpr},
|
||||||
count(*) as count,
|
count(*) as count,
|
||||||
count(case when ai_status = 'clean' then 1 end) as clean,
|
count(case when ai_status = 'clean' then 1 end) as clean,
|
||||||
count(case when ai_status = 'warn' then 1 end) as warned,
|
count(case when ai_status = 'warn' then 1 end) as warned,
|
||||||
@@ -141,7 +148,7 @@ export async function getHourlyStats(input: {
|
|||||||
buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 });
|
buckets.set(key, { count: 0, clean: 0, warned: 0, flagged: 0, error: 0 });
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const row of sqliteRows) {
|
for (const row of rows) {
|
||||||
const d = new Date(row.hour.replace(" ", "T") + "Z");
|
const d = new Date(row.hour.replace(" ", "T") + "Z");
|
||||||
const key = d.toISOString().slice(0, 13) + ":00:00Z";
|
const key = d.toISOString().slice(0, 13) + ":00:00Z";
|
||||||
const bucket = buckets.get(key);
|
const bucket = buckets.get(key);
|
||||||
@@ -344,7 +351,7 @@ export async function getUserLeaderboard(input: {
|
|||||||
AND created_at >= ?
|
AND created_at >= ?
|
||||||
AND deleted_at IS NULL
|
AND deleted_at IS NULL
|
||||||
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||||
GROUP BY user_id
|
GROUP BY user_id, username, avatar_url
|
||||||
ORDER BY message_count DESC
|
ORDER BY message_count DESC
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
`,
|
`,
|
||||||
@@ -379,6 +386,12 @@ export async function getModerationStats(input: {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const since = Date.now() - hours * 3600_000;
|
const since = Date.now() - hours * 3600_000;
|
||||||
|
const isPg = config.DATABASE_TYPE === "postgres";
|
||||||
|
|
||||||
|
const avgScoreExpr = isPg
|
||||||
|
? `round(avg(ai_moderation_score)::numeric, 2)`
|
||||||
|
: `round(avg(ai_moderation_score), 2)`;
|
||||||
|
|
||||||
const row = await executeGet(
|
const row = await executeGet(
|
||||||
`
|
`
|
||||||
SELECT
|
SELECT
|
||||||
@@ -388,7 +401,7 @@ export async function getModerationStats(input: {
|
|||||||
count(case when ai_status = 'flagged' then 1 end) as flagged,
|
count(case when ai_status = 'flagged' then 1 end) as flagged,
|
||||||
count(case when ai_status = 'error' then 1 end) as error,
|
count(case when ai_status = 'error' then 1 end) as error,
|
||||||
count(case when ai_status = 'pending' or ai_status IS NULL then 1 end) as pending,
|
count(case when ai_status = 'pending' or ai_status IS NULL then 1 end) as pending,
|
||||||
round(avg(ai_moderation_score), 2) as average_score
|
${avgScoreExpr} as average_score
|
||||||
FROM messages
|
FROM messages
|
||||||
WHERE guild_id = ?
|
WHERE guild_id = ?
|
||||||
AND created_at >= ?
|
AND created_at >= ?
|
||||||
@@ -501,7 +514,7 @@ export async function getTopViolators(input: {
|
|||||||
AND created_at >= ?
|
AND created_at >= ?
|
||||||
AND deleted_at IS NULL
|
AND deleted_at IS NULL
|
||||||
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
${channelId ? `AND (channel_id = ? OR thread_id = ?)` : ""}
|
||||||
GROUP BY user_id
|
GROUP BY user_id, username, avatar_url
|
||||||
HAVING flagged_count > 0 OR warned_count > 0
|
HAVING flagged_count > 0 OR warned_count > 0
|
||||||
ORDER BY (flagged_count * 3 + warned_count) DESC
|
ORDER BY (flagged_count * 3 + warned_count) DESC
|
||||||
LIMIT ?
|
LIMIT ?
|
||||||
|
|||||||
@@ -9,6 +9,9 @@ export interface MessageLocation {
|
|||||||
threadId: string | null;
|
threadId: string | null;
|
||||||
threadName: string | null;
|
threadName: string | null;
|
||||||
channelName: string | null;
|
channelName: string | null;
|
||||||
|
nsfw?: boolean;
|
||||||
|
nsfwLevel?: string | null;
|
||||||
|
ageRestricted?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StickerEvidence {
|
export interface StickerEvidence {
|
||||||
@@ -74,12 +77,25 @@ export interface RichMessageMetadata {
|
|||||||
|
|
||||||
export function getMessageLocation(message: Message): MessageLocation {
|
export function getMessageLocation(message: Message): MessageLocation {
|
||||||
const channel = message.channel as TextChannel | ThreadChannel;
|
const channel = message.channel as TextChannel | ThreadChannel;
|
||||||
|
const safetyChannel = channel as TextChannel & {
|
||||||
|
nsfw?: boolean;
|
||||||
|
nsfwLevel?: string | null;
|
||||||
|
};
|
||||||
if (!channel.isThread?.()) {
|
if (!channel.isThread?.()) {
|
||||||
return {
|
return {
|
||||||
channelId: message.channelId,
|
channelId: message.channelId,
|
||||||
threadId: null,
|
threadId: null,
|
||||||
threadName: null,
|
threadName: null,
|
||||||
channelName: "name" in channel ? channel.name : null,
|
channelName: "name" in channel ? channel.name : null,
|
||||||
|
nsfw: typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined,
|
||||||
|
nsfwLevel:
|
||||||
|
typeof safetyChannel.nsfwLevel === "string"
|
||||||
|
? safetyChannel.nsfwLevel
|
||||||
|
: null,
|
||||||
|
ageRestricted:
|
||||||
|
typeof safetyChannel.nsfw === "boolean"
|
||||||
|
? safetyChannel.nsfw
|
||||||
|
: undefined,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,6 +104,13 @@ export function getMessageLocation(message: Message): MessageLocation {
|
|||||||
threadId: channel.id,
|
threadId: channel.id,
|
||||||
threadName: channel.name,
|
threadName: channel.name,
|
||||||
channelName: channel.parent?.name ?? null,
|
channelName: channel.parent?.name ?? null,
|
||||||
|
nsfw: typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined,
|
||||||
|
nsfwLevel:
|
||||||
|
typeof safetyChannel.nsfwLevel === "string"
|
||||||
|
? safetyChannel.nsfwLevel
|
||||||
|
: null,
|
||||||
|
ageRestricted:
|
||||||
|
typeof safetyChannel.nsfw === "boolean" ? safetyChannel.nsfw : undefined,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -200,6 +223,20 @@ export function parseRichMessageMetadata(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isAgeRestrictedMetadata(
|
||||||
|
metadata: string | null | undefined,
|
||||||
|
): boolean {
|
||||||
|
const parsed = parseRichMessageMetadata(metadata);
|
||||||
|
if (!parsed) return false;
|
||||||
|
|
||||||
|
const nsfwLevel = parsed.channel.nsfwLevel?.toUpperCase();
|
||||||
|
return Boolean(
|
||||||
|
parsed.channel.nsfw ||
|
||||||
|
parsed.channel.ageRestricted ||
|
||||||
|
nsfwLevel === "AGE_RESTRICTED",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function extractMessageMediaEvidence(
|
export function extractMessageMediaEvidence(
|
||||||
metadata: string | null | undefined,
|
metadata: string | null | undefined,
|
||||||
): MessageMediaEvidence {
|
): MessageMediaEvidence {
|
||||||
|
|||||||
Reference in New Issue
Block a user