import { createChildLogger } from "@/shared/logger/index"; import { config } from "../../shared/config/index.js"; import type { AnalysisResult, MessageRecord, } from "../message-capture/types.js"; const logger = createChildLogger("auto-delete-eligibility"); /** Parse a config value that may be a JSON array string or a comma-separated list. */ export function parseStringList(value?: string | null): string[] { if (!value) return []; try { const parsed = JSON.parse(value) as unknown; return Array.isArray(parsed) ? parsed.filter((item): item is string => typeof item === "string") : []; } catch { return value .split(",") .map((item) => item.trim()) .filter(Boolean); } } /** Derive severity from legacy messages that lack structured AI fields. */ export function deriveSeverity(msg: MessageRecord): string { if (msg.ai_severity) return msg.ai_severity; const score = msg.ai_confidence ?? msg.ai_moderation_score ?? 0; if (msg.ai_status === "flagged") return score >= 0.9 ? "critical" : score >= 0.7 ? "high" : "medium"; if (msg.ai_status === "warn") return score >= 0.6 ? "medium" : "low"; return "none"; } /** Derive recommended action from legacy messages that lack structured AI fields. */ export function deriveRecommendedAction(msg: MessageRecord): string { const severity = deriveSeverity(msg); // Flagged at high/critical severity is ALWAYS delete — the stored // recommended_action from the LLM is conservative (often "review") and // must not override severity for severe violations. if ( msg.ai_status === "flagged" && (severity === "critical" || severity === "high") ) { return "delete"; } if (msg.ai_recommended_action) return msg.ai_recommended_action; if ( msg.ai_status === "flagged" && (severity === "critical" || severity === "high" || severity === "medium") ) return "delete"; if (msg.ai_status === "flagged") return "review"; if (msg.ai_status === "warn") return "warn"; return "none"; } /** Parse the flag list from a structured result or the stored column. */ export function parseModerationFlags( message: MessageRecord, analysisResult?: AnalysisResult, ): string[] { const flags = analysisResult?.flags ?? null; if (flags && flags.length > 0) return flags; const stored = message.ai_moderation_flags; if (!stored) return []; try { const parsed = JSON.parse(stored) as unknown; return Array.isArray(parsed) ? parsed.filter((f): f is string => typeof f === "string") : []; } catch { return []; } } /** * True when the ONLY violation is the member's server nickname — the message * content itself is clean. Such messages must NOT be auto-deleted; the * correct enforcement is resetting the nickname to the default username. * Any other flag (sara, harassment, vulgar_language, ...) keeps the normal * delete path. * * Two detection paths (belt-and-suspenders): * 1. Exact flags — the ONLY flag is `offensive_username`. * 2. Analysis attribution — the flag set is exactly the username-attributable * set (`offensive_username`, `sara`, `conflict_instigation`) AND the * analysis text explicitly attributes the violation SOLELY to the * username while stating the message content itself is clean/ordinary. * This catches the false-positive case where the LLM wrongly applies a * content-level zero-tolerance flag (e.g. `sara`/`conflict_instigation` * for a username like "matikanetanyahu") instead of `offensive_username`. */ const USERNAME_ATTRIBUTABLE_FLAGS = new Set([ "offensive_username", "sara", "conflict_instigation", ]); /** Analysis text that clearly states the message content itself is clean. */ const CONTENT_CLEAN_PATTERN = /(?:isi pesan\s*(?:hanya|bersih|tidak (?:melanggar|ada)|membahas|berisi|bukan)|pesan\s*(?:bersih|tidak (?:melanggar|ada))|tidak ada (?:diskusi|konten|indikasi|pelanggaran))/i; /** Analysis text that clearly attributes the violation to the username. */ const USERNAME_ATTRIBUTION_PATTERN = /(?:username|nickname|nama pengguna)[\s\S]{0,40}?(?:mengandung|memiliki|berisi|melanggar|ofensif|mengecam|menyerang)/i; export function isNicknameOnlyViolation( message: MessageRecord, analysisResult?: AnalysisResult, ): boolean { const flags = parseModerationFlags(message, analysisResult); if (flags.length === 0) return false; // Path 1: exact offensive_username-only. if (flags.every((f) => f === "offensive_username")) return true; // Path 2: username-attributable flag set + analysis text corroborates // that the violation is username-only with clean content. const onlyUsernameAttributable = flags.every((f) => USERNAME_ATTRIBUTABLE_FLAGS.has(f), ); if (!onlyUsernameAttributable) return false; const analysis = (analysisResult?.analysis ?? typeof message.ai_analysis === "string") ? message.ai_analysis : ""; if (!analysis) return false; return ( USERNAME_ATTRIBUTION_PATTERN.test(analysis) && CONTENT_CLEAN_PATTERN.test(analysis) ); } /** * Check whether a message qualifies for auto-deletion. * Uses the structured `analysisResult` fields when provided, falling back * to legacy message-level AI fields otherwise. */ export function isEligibleForAutoDelete( message: MessageRecord, analysisResult?: AnalysisResult, ): boolean { // If analysisResult is provided, use its status field; otherwise use message.ai_status const status = analysisResult?.status ?? message.ai_status; if (status !== "flagged" && status !== "warn") { logger.debug( { messageId: message.id, status }, "Message not eligible for auto-delete: status is not flagged or warn", ); return false; } // Confidence check const confidence = analysisResult?.confidence ?? message.ai_confidence ?? message.ai_moderation_score ?? 0; if (confidence < config.AUTO_DELETE_MIN_CONFIDENCE) { logger.debug( { messageId: message.id, confidence, threshold: config.AUTO_DELETE_MIN_CONFIDENCE, }, "Message not eligible for auto-delete: confidence below threshold", ); return false; } // Severity check const severity = analysisResult?.severity ?? deriveSeverity(message); const allowedSeverities = parseStringList( config.AUTO_DELETE_ALLOWED_SEVERITIES, ); if (allowedSeverities.length > 0 && !allowedSeverities.includes(severity)) { logger.debug( { messageId: message.id, severity, allowed: allowedSeverities }, "Message not eligible for auto-delete: severity not in allowed list", ); return false; } // Recommended action check. // CRITICAL: the LLM's `recommended_action` is CONSERVATIVE — for a flagged // message at high/critical severity it frequently emits "review" (it sees a // screenshot/context ambiguity and hedges) even when the violation itself is // severe. Trusting that value lets serious violations (harassment, SARA, // threats) slip through undeleted. So: flagged + high/critical severity is // ALWAYS eligible regardless of the LLM's recommended action. The action // check only gates warn/flagged-medium (where a review is legitimate). if ( status === "flagged" && (severity === "high" || severity === "critical") ) { logger.debug( { messageId: message.id, status, severity }, "Message eligible for auto-delete: flagged with high/critical severity", ); } else { const recommendedAction = analysisResult?.recommendedAction ?? deriveRecommendedAction(message); if ( recommendedAction !== "delete" && recommendedAction !== "escalate" && recommendedAction !== "warn" ) { logger.debug( { messageId: message.id, recommendedAction }, "Message not eligible for auto-delete: recommended action is not delete/escalate/warn", ); return false; } } // Categories check const allowedCategories = parseStringList( config.AUTO_DELETE_ALLOWED_CATEGORIES, ); if (allowedCategories.length > 0) { const messageCategories = analysisResult?.categories ?? parseStringList(message.ai_categories ?? message.ai_moderation_flags); const hasAllowedCategory = messageCategories.some((cat) => allowedCategories.includes(cat), ); if (!hasAllowedCategory) { logger.debug( { messageId: message.id, categories: messageCategories, allowed: allowedCategories, }, "Message not eligible for auto-delete: no allowed categories match", ); return false; } } // Excluded channels check const excludedChannels = parseStringList( config.AUTO_DELETE_EXCLUDED_CHANNEL_IDS, ); if (excludedChannels.length > 0) { const channelId = message.thread_id ?? message.channel_id; if (excludedChannels.includes(channelId)) { logger.debug( { messageId: message.id, channelId }, "Message not eligible for auto-delete: channel excluded", ); return false; } } // Excluded users check const excludedUsers = parseStringList(config.AUTO_DELETE_EXCLUDED_USER_IDS); if (excludedUsers.length > 0 && excludedUsers.includes(message.user_id)) { logger.debug( { messageId: message.id, userId: message.user_id }, "Message not eligible for auto-delete: user excluded", ); return false; } return true; }