Merge branch worktree-neo-surveillance-redesign into main — Neo Surveillance redesign
Full frontend redesign with glassmorphic dark theme, floating top nav, Live2D mascot, split-pane messages, and Ops Center dashboard. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com
This commit is contained in:
Developer
co-authored by
Claude Opus 4.8 (1M context) <noreply@anthropic.com
Claude Opus 4.8 (1M context) <noreply@anthropic.com
parent
102b3bac1f
commit
3f4fa42098
@@ -0,0 +1,500 @@
|
||||
/**
|
||||
* ai-analysis-worker.ts
|
||||
*
|
||||
* Two-pass AI moderation analysis worker (Piscina-compatible).
|
||||
*
|
||||
* ## Pipeline
|
||||
*
|
||||
* Message → Layer 1 (fast classifier / heuristic)
|
||||
* │
|
||||
* ├─ clear match → final result (no LLM call)
|
||||
* └─ ambiguous → Layer 2 (LLM evaluator)
|
||||
*
|
||||
* Layer 1 runs synchronously in-memory. Layer 2 calls the LLM API via
|
||||
* the existing moderation pipeline (moderationOrchestrator).
|
||||
*
|
||||
* This file replaces the old `aiAnalysisWorker.ts` (archived) with a
|
||||
* simpler, unified worker that combines both layers.
|
||||
*/
|
||||
|
||||
import { createChildLogger } from "@bete/shared/logger";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { initializeDatabase } from "../../shared/database/drizzle.js";
|
||||
import { messageStore } from "../message-capture/messageStore.js";
|
||||
import type { MessageRecord } from "../message-capture/types.js";
|
||||
import { buildConversationContext } from "./conversationContext.js";
|
||||
import { classifyMessage } from "./fastClassifier.js";
|
||||
import type { Layer1Result } from "./fastClassifier.js";
|
||||
import { runModerationAnalysis } from "./moderationOrchestrator.js";
|
||||
|
||||
const logger = createChildLogger("ai-analysis-worker");
|
||||
|
||||
let dbInitialized = false;
|
||||
let dbInitPromise: Promise<unknown> | null = null;
|
||||
|
||||
async function ensureDb(): Promise<void> {
|
||||
if (dbInitialized) return;
|
||||
if (!dbInitPromise) {
|
||||
dbInitPromise = initializeDatabase().then(() => {
|
||||
dbInitialized = true;
|
||||
});
|
||||
}
|
||||
await dbInitPromise;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface AnalysisInput {
|
||||
batch: MessageBatch;
|
||||
config: WorkerConfig;
|
||||
}
|
||||
|
||||
export interface AnalysisResult {
|
||||
messageId: string;
|
||||
status: "clean" | "warn" | "flagged" | "error";
|
||||
flags: string[];
|
||||
categories: string[];
|
||||
severity: "none" | "low" | "medium" | "high" | "critical";
|
||||
confidence: number;
|
||||
recommendedAction:
|
||||
| "none"
|
||||
| "monitor"
|
||||
| "warn"
|
||||
| "review"
|
||||
| "delete"
|
||||
| "escalate";
|
||||
toxicityScore: number;
|
||||
harmScore: number;
|
||||
jailbreakScore: number;
|
||||
safetyScore: number;
|
||||
explanation: string;
|
||||
correctedFlags?: string[];
|
||||
}
|
||||
|
||||
export interface WorkerConfig {
|
||||
aiLlmApiKey: string;
|
||||
aiLlmBaseUrl: string;
|
||||
aiLlmModel: string;
|
||||
aiLlmTimeoutMs: number;
|
||||
}
|
||||
|
||||
export interface MessageBatch {
|
||||
conversationKey: string;
|
||||
messages: MessageRecord[];
|
||||
contextMessages: string[];
|
||||
}
|
||||
|
||||
// Worker job types (Piscina entry point)
|
||||
type WorkerJob =
|
||||
| { type: "batch"; conversationKey: string; messages: MessageRecord[] }
|
||||
| {
|
||||
type: "individual";
|
||||
message: MessageRecord;
|
||||
skipNormalAnalysis: boolean;
|
||||
};
|
||||
|
||||
type BatchOkResponse = {
|
||||
ok: true;
|
||||
conversationKey: string;
|
||||
rows: MessageRecord[];
|
||||
};
|
||||
type BatchErrorResponse = {
|
||||
ok: false;
|
||||
conversationKey: string;
|
||||
rows: MessageRecord[];
|
||||
error: string;
|
||||
};
|
||||
type IndividualOkResponse = { ok: true; results: AnalysisResult[] };
|
||||
type IndividualErrorResponse = {
|
||||
ok: false;
|
||||
results: AnalysisResult[];
|
||||
error: string;
|
||||
};
|
||||
|
||||
type WorkerResponse =
|
||||
| BatchOkResponse
|
||||
| BatchErrorResponse
|
||||
| IndividualOkResponse
|
||||
| IndividualErrorResponse;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Default export — Piscina worker entry point
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export default async function workerRouter(
|
||||
job: WorkerJob,
|
||||
): Promise<WorkerResponse> {
|
||||
if (!config.AI_LLM_API_KEY) {
|
||||
const errorMsg =
|
||||
"AI_LLM_API_KEY is missing from environment. Worker cannot process moderation requests without credentials.";
|
||||
logger.error({ error: errorMsg }, "AI_LLM_API_KEY is missing from environment");
|
||||
|
||||
if (job.type === "batch") {
|
||||
return {
|
||||
ok: false,
|
||||
conversationKey: job.conversationKey,
|
||||
rows: [],
|
||||
error: errorMsg,
|
||||
};
|
||||
}
|
||||
return { ok: false, results: [], error: errorMsg };
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureDb();
|
||||
} catch (dbError) {
|
||||
const msg = dbError instanceof Error ? dbError.message : String(dbError);
|
||||
if (job.type === "batch") {
|
||||
return {
|
||||
ok: false,
|
||||
conversationKey: job.conversationKey,
|
||||
rows: [],
|
||||
error: `Database init failed: ${msg}`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
results: [],
|
||||
error: `Database init failed: ${msg}`,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
if (job.type === "batch") {
|
||||
return await processBatch(job);
|
||||
}
|
||||
return await processIndividual(job);
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
const errorStack = error instanceof Error ? error.stack : undefined;
|
||||
logger.error(
|
||||
{ type: job.type, error: errorMessage, stack: errorStack },
|
||||
"Worker job failed",
|
||||
);
|
||||
if (job.type === "batch") {
|
||||
return {
|
||||
ok: false,
|
||||
conversationKey: job.conversationKey,
|
||||
rows: [],
|
||||
error: errorMessage,
|
||||
};
|
||||
}
|
||||
return { ok: false, results: [], error: errorMessage };
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Two-pass pipeline
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Runs the two-pass pipeline on a single message:
|
||||
* 1. Layer 1 — fast heuristic classifier
|
||||
* 2. Layer 2 (if cascade) — LLM-based evaluator
|
||||
*
|
||||
* Returns the combined AnalysisResult.
|
||||
*/
|
||||
async function runTwoPassPipeline(
|
||||
message: MessageRecord,
|
||||
contextText: string,
|
||||
attachments: Awaited<ReturnType<typeof messageStore.getAttachmentsForMessages>>,
|
||||
): Promise<AnalysisResult> {
|
||||
// ── Layer 1: Fast classifier ──────────────────────────────────────────
|
||||
const layer1Result: Layer1Result = classifyMessage(message);
|
||||
|
||||
logger.debug(
|
||||
{
|
||||
messageId: message.id,
|
||||
layer1Flags: layer1Result.flags,
|
||||
cascade: layer1Result.cascadeToLayer2,
|
||||
},
|
||||
"Layer 1 classification complete",
|
||||
);
|
||||
|
||||
if (!layer1Result.cascadeToLayer2) {
|
||||
// Layer 1 result is final — no LLM call needed
|
||||
return buildResultFromLayer1(message.id, layer1Result);
|
||||
}
|
||||
|
||||
// ── Layer 2: LLM-based evaluation ─────────────────────────────────────
|
||||
try {
|
||||
const moderationResult = await runModerationAnalysis({
|
||||
targets: [message],
|
||||
contextText,
|
||||
attachments,
|
||||
});
|
||||
|
||||
if (moderationResult.results.length === 0) {
|
||||
return buildFallbackResult(message.id, "No LLM result returned");
|
||||
}
|
||||
|
||||
const llmResult = moderationResult.results[0];
|
||||
return mergeLayers(layer1Result, llmResult);
|
||||
} catch (error) {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||
logger.warn(
|
||||
{ messageId: message.id, error: errorMsg },
|
||||
"Layer 2 (LLM) analysis failed — falling back to Layer 1 result",
|
||||
);
|
||||
// Fallback to Layer 1 with reduced confidence
|
||||
const fallback = buildResultFromLayer1(message.id, layer1Result);
|
||||
fallback.confidence = Math.min(fallback.confidence, 0.4);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result builders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function buildResultFromLayer1(
|
||||
messageId: string,
|
||||
layer1: Layer1Result,
|
||||
): AnalysisResult {
|
||||
const status = layer1.severity === "none" ? "clean" as const : "flagged" as const;
|
||||
const recommendedAction = mapSeverityToAction(layer1.severity);
|
||||
|
||||
return {
|
||||
messageId,
|
||||
status,
|
||||
flags: layer1.flags,
|
||||
categories: layer1.flags,
|
||||
severity: layer1.severity === "high" ? "high" as const : layer1.severity === "medium" ? "medium" as const : "low" as const,
|
||||
confidence: layer1.confidence,
|
||||
recommendedAction,
|
||||
toxicityScore: layer1.toxicityScore,
|
||||
harmScore: layer1.harmScore,
|
||||
jailbreakScore: 0,
|
||||
safetyScore: 0,
|
||||
explanation: layer1.explanation,
|
||||
};
|
||||
}
|
||||
|
||||
function buildFallbackResult(
|
||||
messageId: string,
|
||||
reason: string,
|
||||
): AnalysisResult {
|
||||
return {
|
||||
messageId,
|
||||
status: "error",
|
||||
flags: ["analysis_incomplete"],
|
||||
categories: ["analysis_incomplete"],
|
||||
severity: "none",
|
||||
confidence: 0,
|
||||
recommendedAction: "review",
|
||||
toxicityScore: 0,
|
||||
harmScore: 0,
|
||||
jailbreakScore: 0,
|
||||
safetyScore: 0,
|
||||
explanation: reason,
|
||||
};
|
||||
}
|
||||
|
||||
function mergeLayers(
|
||||
layer1: Layer1Result,
|
||||
llmResult: AnalysisResult,
|
||||
): AnalysisResult {
|
||||
// Merge flags from both layers (deduplicate)
|
||||
const flagSet = new Set<string>([...layer1.flags, ...(llmResult.flags || [])]);
|
||||
|
||||
// Take the max severity
|
||||
const severityOrder = ["none", "low", "medium", "high", "critical"] as const;
|
||||
const l1Idx = severityOrder.indexOf(layer1.severity);
|
||||
const l2Idx = severityOrder.indexOf(
|
||||
(llmResult.severity ?? "none") as (typeof severityOrder)[number],
|
||||
);
|
||||
const finalSeverity = severityOrder[Math.max(l1Idx, l2Idx)];
|
||||
|
||||
// Combined confidence: weighted average favoring LLM when available
|
||||
const combinedConfidence =
|
||||
0.3 * layer1.confidence + 0.7 * (llmResult.confidence ?? 0.5);
|
||||
|
||||
// Combine scores (take max per dimension)
|
||||
const toxicityScore = Math.max(
|
||||
layer1.toxicityScore,
|
||||
llmResult.toxicityScore ?? 0,
|
||||
);
|
||||
const harmScore = Math.max(layer1.harmScore, llmResult.harmScore ?? 0);
|
||||
|
||||
return {
|
||||
messageId: llmResult.messageId,
|
||||
status: llmResult.status === "error" ? "error" as const : llmResult.status ?? "clean" as const,
|
||||
flags: Array.from(flagSet),
|
||||
categories: [
|
||||
...new Set([
|
||||
...layer1.flags,
|
||||
...(llmResult.categories ?? []),
|
||||
]),
|
||||
],
|
||||
severity: finalSeverity,
|
||||
confidence: Math.min(combinedConfidence, 1),
|
||||
recommendedAction: llmResult.recommendedAction ?? mapSeverityToAction(finalSeverity),
|
||||
toxicityScore,
|
||||
harmScore,
|
||||
jailbreakScore: llmResult.jailbreakScore ?? 0,
|
||||
safetyScore: llmResult.safetyScore ?? 0,
|
||||
explanation: llmResult.explanation ?? layer1.explanation,
|
||||
};
|
||||
}
|
||||
|
||||
function mapSeverityToAction(
|
||||
severity: "none" | "low" | "medium" | "high" | "critical",
|
||||
): AnalysisResult["recommendedAction"] {
|
||||
switch (severity) {
|
||||
case "none":
|
||||
return "none";
|
||||
case "low":
|
||||
return "monitor";
|
||||
case "medium":
|
||||
return "review";
|
||||
case "high":
|
||||
return "delete";
|
||||
case "critical":
|
||||
return "escalate";
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Batch handler
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function processBatch(job: {
|
||||
type: "batch";
|
||||
conversationKey: string;
|
||||
messages: MessageRecord[];
|
||||
}): Promise<BatchOkResponse | BatchErrorResponse> {
|
||||
const { conversationKey, messages } = job;
|
||||
const firstMessage = messages[0];
|
||||
if (!firstMessage) return { ok: true, conversationKey, rows: [] };
|
||||
|
||||
// Fetch context
|
||||
const contextBefore = await messageStore.getConversationContextBefore({
|
||||
channelId: firstMessage.channel_id,
|
||||
threadId: firstMessage.thread_id,
|
||||
beforeCreatedAt: firstMessage.created_at,
|
||||
limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT,
|
||||
});
|
||||
|
||||
const contextLines = buildConversationContext({
|
||||
contextBefore,
|
||||
targets: messages,
|
||||
maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS,
|
||||
});
|
||||
const contextText = contextLines.join("\n");
|
||||
|
||||
// Fetch attachments
|
||||
const targetIds = messages.map((m) => m.id);
|
||||
const contextIds = contextBefore.map((m) => m.id);
|
||||
const allMessageIds = [...targetIds, ...contextIds];
|
||||
const attachments =
|
||||
await messageStore.getAttachmentsForMessages(allMessageIds);
|
||||
|
||||
// Run two-pass pipeline for each message
|
||||
const analysisResults = await Promise.all(
|
||||
messages.map(async (msg) => {
|
||||
try {
|
||||
return await runTwoPassPipeline(msg, contextText, attachments);
|
||||
} catch (error) {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||
logger.error(
|
||||
{ messageId: msg.id, error: errorMsg },
|
||||
"Two-pass pipeline failed for message",
|
||||
);
|
||||
return buildFallbackResult(msg.id, errorMsg);
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
// Save results to DB
|
||||
const updates = analysisResults.map((result) => ({
|
||||
messageId: result.messageId,
|
||||
result: {
|
||||
status: result.status,
|
||||
flags: JSON.stringify(result.flags),
|
||||
score: result.toxicityScore,
|
||||
analysis: result.explanation,
|
||||
categories: result.categories,
|
||||
severity: result.severity,
|
||||
confidence: result.confidence,
|
||||
recommendedAction: result.recommendedAction,
|
||||
analyzedAt: Date.now(),
|
||||
error: result.status === "error" ? result.explanation : null,
|
||||
},
|
||||
}));
|
||||
|
||||
let allRows: MessageRecord[] = [];
|
||||
if (updates.length > 0) {
|
||||
allRows = await messageStore.updateMessagesAIAnalysisBulk(updates);
|
||||
}
|
||||
|
||||
logger.info(
|
||||
{
|
||||
total: messages.length,
|
||||
saved: allRows.length,
|
||||
conversationKey,
|
||||
},
|
||||
"Two-pass batch analysis complete",
|
||||
);
|
||||
|
||||
return { ok: true, conversationKey, rows: allRows };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Individual fallback handler
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function processIndividual(job: {
|
||||
type: "individual";
|
||||
message: MessageRecord;
|
||||
skipNormalAnalysis: boolean;
|
||||
}): Promise<IndividualOkResponse | IndividualErrorResponse> {
|
||||
const { message, skipNormalAnalysis } = job;
|
||||
|
||||
if (skipNormalAnalysis) {
|
||||
// Use Layer 1 only (fast)
|
||||
const layer1Result = classifyMessage(message);
|
||||
|
||||
if (!layer1Result.cascadeToLayer2) {
|
||||
// Layer 1 is sufficient
|
||||
return {
|
||||
ok: true,
|
||||
results: [buildResultFromLayer1(message.id, layer1Result)],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Full analysis (context + two-pass)
|
||||
const contextBefore = await messageStore.getConversationContextBefore({
|
||||
channelId: message.channel_id,
|
||||
threadId: message.thread_id,
|
||||
beforeCreatedAt: message.created_at,
|
||||
limit: config.AI_ANALYSIS_CONTEXT_MESSAGE_LIMIT,
|
||||
});
|
||||
|
||||
const contextLines = buildConversationContext({
|
||||
contextBefore,
|
||||
targets: [message],
|
||||
maxTokens: config.AI_ANALYSIS_MAX_CONTEXT_TOKENS,
|
||||
});
|
||||
const contextText = contextLines.join("\n");
|
||||
|
||||
const contextIds = contextBefore.map((m) => m.id);
|
||||
const attachments = await messageStore.getAttachmentsForMessages([
|
||||
message.id,
|
||||
...contextIds,
|
||||
]);
|
||||
|
||||
try {
|
||||
const result = await runTwoPassPipeline(message, contextText, attachments);
|
||||
return { ok: true, results: [result] };
|
||||
} catch (error) {
|
||||
const errorMsg = error instanceof Error ? error.message : String(error);
|
||||
logger.error(
|
||||
{ messageId: message.id, error: errorMsg },
|
||||
"Individual two-pass analysis failed",
|
||||
);
|
||||
return { ok: true, results: [buildFallbackResult(message.id, errorMsg)] };
|
||||
}
|
||||
}
|
||||
@@ -11,9 +11,9 @@ import type { MessageRecord } from "../message-capture/types.js";
|
||||
|
||||
function getAnalysisWorkerUrl(): URL {
|
||||
const candidates = [
|
||||
new URL("./aiAnalysisWorker.js", import.meta.url),
|
||||
new URL("../aiAnalysisWorker.js", import.meta.url),
|
||||
new URL("./aiAnalysisWorker.ts", import.meta.url),
|
||||
new URL("./ai-analysis-worker.js", import.meta.url),
|
||||
new URL("../ai-analysis-worker.js", import.meta.url),
|
||||
new URL("./ai-analysis-worker.ts", import.meta.url),
|
||||
];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
|
||||
@@ -0,0 +1,399 @@
|
||||
/**
|
||||
* fastClassifier.ts
|
||||
*
|
||||
* Layer 1 — Synchronous heuristic classifier for the two-pass moderation pipeline.
|
||||
* Runs BEFORE any LLM call. Catches obvious spam, NSFW patterns, repeated characters,
|
||||
* and other low-hanging fruit with zero network cost.
|
||||
*
|
||||
* When a strong heuristic match is found, `cascadeToLayer2` is `false` and the
|
||||
* result is used as the final verdict. Otherwise the message proceeds to the LLM.
|
||||
*/
|
||||
|
||||
import type { MessageRecord } from "../message-capture/types.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface Layer1Result {
|
||||
flags: string[];
|
||||
severity: "none" | "low" | "medium" | "high";
|
||||
toxicityScore: number;
|
||||
harmScore: number;
|
||||
cascadeToLayer2: boolean;
|
||||
confidence: number;
|
||||
explanation: string;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pattern definitions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface Pattern {
|
||||
name: string;
|
||||
test: (content: string, mentions: number) => boolean;
|
||||
severity: "low" | "medium" | "high";
|
||||
score: number; // contribution to toxicity/harm score
|
||||
category: "toxicity" | "harm" | "spam" | "safety";
|
||||
}
|
||||
|
||||
// ── Zalgo / zero-width detection ─────────────────────────────────────────
|
||||
|
||||
const ZALGO_RE =
|
||||
/[̀-ͯ҃-҉ؐ-ًؚ-ٰٟۖ-ۜ۟-ۤۧ-۪ۨ-ܑۭܰ-݊ަ-ްँ-ः़ा-्॑-॔ॢ-ॣঁ-ঃ়া-ৄে-ৈো-্ৗৢ-ৣ৾ਁ-ਃ਼ਾ-ੂੇ-ੈੋ-੍ੑੰ-ੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢ-ૣૺ-૿ଁ-ଃ଼ା-ୄେ-ୈୋ-୍ୖ-ୗୢ-ୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕ-ౖౢ-ౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕ-ೖೢ-ೣഀ-ഃ഻-഼ാ-ൄെ-ൈൊ-്ൗൢ-ൣඁ-ඃ්ා-ුූෘ-ෟෲ-ෳัิ-ฺ็-๎ັິ-ູົ-ຼ່-ໍ༘-༹༙༵༷༾-༿ཱ-྄྆-྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒ-ᝓᝲ-ᝳ឴-៓៝᠋-᠍ᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪾ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮁᮢ-ᮥᮨ-ᮩ᮫-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳷-᳹᷀-᷿-
- -⃐-⃰⳯-⵿⳱ⷠ-〪ⷿ-゙〯-゚꙯-꙲ꙴ-꙽ꚞ-ꚟ꛰-꛱ꠂ꠆ꠋꠣ-ꠧ꠬ꢀ-ꢁꢴ-ꣅ꣠-꣱ꣿ-꤉ꤦ-꤭ꥇ-꥓ꥠ-ꥼꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷ-ꪸꪾ-꪿꫁ꫫ-ꫯꫵ-꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯-]||͏|ᅟᅠ឴឵ - --ㅤﾠ-\U000e0001\U000e0020-\U000e007f/;
|
||||
|
||||
const ZERO_WIDTH_RE = /[-]/;
|
||||
|
||||
// ── URL / invite / phone / email / crypto patterns ─────────────────────
|
||||
|
||||
const URL_RE = /https?:\/\/[^\s"]+/gi;
|
||||
const INVITE_RE = /(?:discord\.(?:gg|com\/invite)|dsc\.gg)\/[a-zA-Z0-9_-]+/gi;
|
||||
const INVITE_CODE_RE = /(?:^|\s)([a-zA-Z0-9_-]{6,12})(?:\s|$)/g;
|
||||
const PHONE_RE =
|
||||
/(?:\+?\d{1,3}[-.\s]?)?\(?\d{2,4}\)?[-.\s]?\d{3,4}[-.\s]?\d{3,4}/g;
|
||||
const EMAIL_RE = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/gi;
|
||||
const CRYPTO_RE =
|
||||
/(?:0x[a-fA-F0-9]{40}|bc1[a-z0-9]{39,59}|1[a-km-zA-HJ-NP-Z1-9]{25,34}|3[a-km-zA-HJ-NP-Z1-9]{25,34})/g;
|
||||
const IP_RE = /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g;
|
||||
|
||||
// ── Spam / low-quality patterns ─────────────────────────────────────────
|
||||
|
||||
const REPEATED_CHAR_RE = /(.)\1{8,}/; // 9+ repeated chars
|
||||
const REPEATED_WORD_RE = /\b(\w{3,})\b\s*\b\1\b\s*\b\1\b/; // same word 3x
|
||||
const EXCESSIVE_CAPS_RE = /[A-Z]{6,}/;
|
||||
const EXCESSIVE_EMOJI_RE =
|
||||
/[\u{1F300}-\u{1F9FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]/gu;
|
||||
const BASE64_RE =
|
||||
/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
|
||||
const PHISHING_RE =
|
||||
/(?:free\s*(?:nitro|gift|prime|steam|vbucks?)|click\s*(?:here|this)\s*(?:to|for)\s*(?:claim|win|verify)|login\s*:\s*\w+\s*password\s*:\s*\w+)/gi;
|
||||
|
||||
// ── Toxicity patterns ───────────────────────────────────────────────────
|
||||
|
||||
const HARASSMENT_RE =
|
||||
/\b(?:fuck|shit|asshole|bitch|dickhead|cunt|motherfucker|bastard|piss\s*off|screw\s*you|go\s*(?:to\s*)?hell|kys|kill\s*(?:yourself| urself))\b/i;
|
||||
const HATE_SPEECH_RE =
|
||||
/\b(?:nazi|white\s*supremacy|heil|racial\s*purity|race\s*war)\b/i;
|
||||
|
||||
// ── Harm patterns ───────────────────────────────────────────────────────
|
||||
|
||||
const SELF_HARM_RE =
|
||||
/\b(?:kill\s*(?:myself|me)|end\s*(?:my|the)\s*(?:life|own)|suicide|want\s*(?:to\s*)?die|cut\s*(?:myself|my\s*wrists)|harm\s*myself)\b/i;
|
||||
const VIOLENCE_RE =
|
||||
/\b(?:shoot|stab|bomb|massacre|terrorist|behead|torture|murder)\b/i;
|
||||
|
||||
// ── Safety patterns ─────────────────────────────────────────────────────
|
||||
|
||||
const PERSONAL_INFO_RE =
|
||||
/\b(?:\d{3}-\d{2}-\d{4}|(?:\d{3}\s?){2}\d{4})\b/; // SSN / IDs
|
||||
const SEXTORTION_RE =
|
||||
/\b(?:nudes?\s*(?:pic|photo|video|send|trade)|cp\s*(?:content|link|loli|shotacon)|underage|minor\s*(?:girl|boy|content))\b/i;
|
||||
const GROOMING_RE =
|
||||
/\b(?:how\s*old\s*are\s*you|are\s*you\s*(?:alone|home\s*alone)|dm\s*me\s*(?:baby|honey|sweetie|cutie)|send\s*(?:nudes|pics))\b/i;
|
||||
|
||||
// ── Mass-mention patterns ───────────────────────────────────────────────
|
||||
|
||||
const EVERYONE_MENTION = /@everyone/g;
|
||||
const HERE_MENTION = /@here/g;
|
||||
const ROLE_MENTION = /<@&(\d+)>/g;
|
||||
|
||||
// ── Pattern registry (ordered roughly by specificity) ───────────────────
|
||||
|
||||
const PATTERNS: Pattern[] = [
|
||||
// ── High severity ─────────────────────────────────────────────────
|
||||
{
|
||||
name: "self_harm",
|
||||
test: (c) => SELF_HARM_RE.test(c),
|
||||
severity: "high",
|
||||
score: 0.9,
|
||||
category: "harm",
|
||||
},
|
||||
{
|
||||
name: "violence_threat",
|
||||
test: (c) => VIOLENCE_RE.test(c),
|
||||
severity: "high",
|
||||
score: 0.85,
|
||||
category: "harm",
|
||||
},
|
||||
{
|
||||
name: "sextortion",
|
||||
test: (c) => SEXTORTION_RE.test(c),
|
||||
severity: "high",
|
||||
score: 0.95,
|
||||
category: "safety",
|
||||
},
|
||||
{
|
||||
name: "grooming",
|
||||
test: (c) => GROOMING_RE.test(c),
|
||||
severity: "high",
|
||||
score: 0.9,
|
||||
category: "safety",
|
||||
},
|
||||
{
|
||||
name: "hate_speech",
|
||||
test: (c) => HATE_SPEECH_RE.test(c),
|
||||
severity: "high",
|
||||
score: 0.85,
|
||||
category: "toxicity",
|
||||
},
|
||||
{
|
||||
name: "harassment",
|
||||
test: (c) => HARASSMENT_RE.test(c),
|
||||
severity: "medium",
|
||||
score: 0.6,
|
||||
category: "toxicity",
|
||||
},
|
||||
{
|
||||
name: "phishing",
|
||||
test: (c) => PHISHING_RE.test(c),
|
||||
severity: "high",
|
||||
score: 0.85,
|
||||
category: "harm",
|
||||
},
|
||||
// ── Spam / low quality ────────────────────────────────────────────
|
||||
{
|
||||
name: "mass_everyone_mention",
|
||||
test: (_c, mentions) => mentions >= 3,
|
||||
severity: "medium",
|
||||
score: 0.65,
|
||||
category: "spam",
|
||||
},
|
||||
{
|
||||
name: "excessive_caps",
|
||||
test: (c) => {
|
||||
const caps = (c.match(EXCESSIVE_CAPS_RE) || []).join("");
|
||||
return caps.length > 0 && caps.length / Math.max(c.length, 1) > 0.5;
|
||||
},
|
||||
severity: "low",
|
||||
score: 0.3,
|
||||
category: "spam",
|
||||
},
|
||||
{
|
||||
name: "repeated_characters",
|
||||
test: (c) => REPEATED_CHAR_RE.test(c),
|
||||
severity: "low",
|
||||
score: 0.25,
|
||||
category: "spam",
|
||||
},
|
||||
{
|
||||
name: "repeated_words",
|
||||
test: (c) => REPEATED_WORD_RE.test(c),
|
||||
severity: "low",
|
||||
score: 0.25,
|
||||
category: "spam",
|
||||
},
|
||||
{
|
||||
name: "zalgo_text",
|
||||
test: (c) => ZALGO_RE.test(c) || ZERO_WIDTH_RE.test(c),
|
||||
severity: "medium",
|
||||
score: 0.6,
|
||||
category: "spam",
|
||||
},
|
||||
{
|
||||
name: "excessive_emojis",
|
||||
test: (c) => {
|
||||
const emojiCount = (c.match(EXCESSIVE_EMOJI_RE) || []).length;
|
||||
const textLen = c.replace(EXCESSIVE_EMOJI_RE, "").trim().length;
|
||||
return emojiCount >= 5 && textLen < emojiCount;
|
||||
},
|
||||
severity: "low",
|
||||
score: 0.2,
|
||||
category: "spam",
|
||||
},
|
||||
{
|
||||
name: "base64_gibberish",
|
||||
test: (c) => c.length >= 20 && BASE64_RE.test(c.trim()),
|
||||
severity: "low",
|
||||
score: 0.2,
|
||||
category: "spam",
|
||||
},
|
||||
// ── Medium severity ───────────────────────────────────────────────
|
||||
{
|
||||
name: "personal_info",
|
||||
test: (c) => PERSONAL_INFO_RE.test(c),
|
||||
severity: "medium",
|
||||
score: 0.6,
|
||||
category: "safety",
|
||||
},
|
||||
{
|
||||
name: "discord_invite",
|
||||
test: (c) => INVITE_RE.test(c),
|
||||
severity: "low",
|
||||
score: 0.2,
|
||||
category: "spam",
|
||||
},
|
||||
{
|
||||
name: "url_only",
|
||||
test: (c) => {
|
||||
const urls = c.match(URL_RE);
|
||||
if (!urls) return false;
|
||||
const textWithoutUrls = c.replace(URL_RE, "").trim();
|
||||
return urls.length >= 3 && textWithoutUrls.length === 0;
|
||||
},
|
||||
severity: "low",
|
||||
score: 0.25,
|
||||
category: "spam",
|
||||
},
|
||||
{
|
||||
name: "phone_number",
|
||||
test: (c) => PHONE_RE.test(c),
|
||||
severity: "medium",
|
||||
score: 0.5,
|
||||
category: "safety",
|
||||
},
|
||||
{
|
||||
name: "email_address",
|
||||
test: (c) => EMAIL_RE.test(c),
|
||||
severity: "low",
|
||||
score: 0.3,
|
||||
category: "safety",
|
||||
},
|
||||
{
|
||||
name: "crypto_address",
|
||||
test: (c) => CRYPTO_RE.test(c),
|
||||
severity: "medium",
|
||||
score: 0.5,
|
||||
category: "spam",
|
||||
},
|
||||
{
|
||||
name: "ip_address_sharing",
|
||||
test: (c) => IP_RE.test(c),
|
||||
severity: "low",
|
||||
score: 0.3,
|
||||
category: "safety",
|
||||
},
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pattern matcher — count mentions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function countMentions(content: string): number {
|
||||
let count = 0;
|
||||
const everyoneMatches = content.match(EVERYONE_MENTION);
|
||||
if (everyoneMatches) count += everyoneMatches.length;
|
||||
const hereMatches = content.match(HERE_MENTION);
|
||||
if (hereMatches) count += hereMatches.length;
|
||||
const roleMatches = content.match(ROLE_MENTION);
|
||||
if (roleMatches) count += roleMatches.length;
|
||||
return count;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main classifier
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Runs Layer 1 heuristic classification on a message.
|
||||
* Returns a `Layer1Result` with matched flags and a decision on
|
||||
* whether to cascade to Layer 2 (LLM).
|
||||
*/
|
||||
export function classifyMessage(message: MessageRecord): Layer1Result {
|
||||
const content = message.edited_content ?? message.content;
|
||||
if (!content || content.trim().length === 0) {
|
||||
return {
|
||||
flags: [],
|
||||
severity: "none",
|
||||
toxicityScore: 0,
|
||||
harmScore: 0,
|
||||
cascadeToLayer2: true, // empty content still needs metadata check
|
||||
confidence: 0,
|
||||
explanation: "No text content to classify",
|
||||
};
|
||||
}
|
||||
|
||||
const mentions = countMentions(content);
|
||||
const matchedFlags: string[] = [];
|
||||
const severityWeights: Record<string, number> = {
|
||||
low: 1,
|
||||
medium: 2,
|
||||
high: 3,
|
||||
};
|
||||
|
||||
let maxSeverityWeight = 0;
|
||||
let totalToxicityScore = 0;
|
||||
let totalHarmScore = 0;
|
||||
let totalSafetyScore = 0;
|
||||
let totalSpamScore = 0;
|
||||
const matchedPatternDetails: string[] = [];
|
||||
|
||||
for (const pattern of PATTERNS) {
|
||||
if (pattern.test(content, mentions)) {
|
||||
matchedFlags.push(pattern.name);
|
||||
matchedPatternDetails.push(pattern.name);
|
||||
|
||||
const weight = severityWeights[pattern.severity] || 1;
|
||||
maxSeverityWeight = Math.max(maxSeverityWeight, weight);
|
||||
|
||||
switch (pattern.category) {
|
||||
case "toxicity":
|
||||
totalToxicityScore += pattern.score;
|
||||
break;
|
||||
case "harm":
|
||||
totalHarmScore += pattern.score;
|
||||
break;
|
||||
case "safety":
|
||||
totalSafetyScore += pattern.score;
|
||||
break;
|
||||
case "spam":
|
||||
totalSpamScore += pattern.score;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Determine final severity ───────────────────────────────────────
|
||||
|
||||
let finalSeverity: "none" | "low" | "medium" | "high" = "none";
|
||||
if (maxSeverityWeight >= 3) finalSeverity = "high";
|
||||
else if (maxSeverityWeight >= 2) finalSeverity = "medium";
|
||||
else if (maxSeverityWeight >= 1) finalSeverity = "low";
|
||||
|
||||
// ── Determine if we should cascade ─────────────────────────────────
|
||||
// Cascade to Layer 2 whenever:
|
||||
// 1. No high-severity match was found, OR
|
||||
// 2. Only spam/low-quality patterns matched (need LLM for nuance)
|
||||
// Do NOT cascade when a clear high-severity harm/safety/toxicity match
|
||||
// was found — the heuristic is sufficient.
|
||||
|
||||
const hasHighSeverityPattern = matchedFlags.some((f) => {
|
||||
const p = PATTERNS.find((p) => p.name === f);
|
||||
return p && p.severity === "high";
|
||||
});
|
||||
const hasOnlyLowSeveritySpam = matchedFlags.every((f) => {
|
||||
const p = PATTERNS.find((p) => p.name === f);
|
||||
return p && p.category === "spam" && p.severity !== "high";
|
||||
});
|
||||
|
||||
// Cascade if no matches, only spam, or low/medium severity toxicity/safety
|
||||
const cascadeToLayer2 =
|
||||
matchedFlags.length === 0 ||
|
||||
hasOnlyLowSeveritySpam ||
|
||||
(finalSeverity !== "high" && hasHighSeverityPattern === false);
|
||||
|
||||
// ── Compute final scores (clamped 0-1) ─────────────────────────────
|
||||
|
||||
const toxicityScore = Math.min(totalToxicityScore, 1);
|
||||
const harmScore = Math.min(totalHarmScore, 1);
|
||||
|
||||
// ── Confidence ─────────────────────────────────────────────────────
|
||||
|
||||
const confidence = cascadeToLayer2
|
||||
? 0.4 + 0.1 * matchedFlags.length // low confidence when punting to LLM
|
||||
: 0.6 + 0.4 * (1 - matchedFlags.length / PATTERNS.length);
|
||||
|
||||
const explanation =
|
||||
matchedFlags.length > 0
|
||||
? `Layer 1 matched: ${matchedPatternDetails.join(", ")}`
|
||||
: "No heuristic patterns matched";
|
||||
|
||||
return {
|
||||
flags: matchedFlags,
|
||||
severity: finalSeverity,
|
||||
toxicityScore,
|
||||
harmScore,
|
||||
cascadeToLayer2,
|
||||
confidence: Math.min(confidence, 0.99),
|
||||
explanation,
|
||||
};
|
||||
}
|
||||
@@ -2,3 +2,8 @@ export { startPendingAIAnalysisWorker } from "./aiAnalyzer.js";
|
||||
export { runModerationAnalysis } from "./moderationOrchestrator.js";
|
||||
export { buildSystemPrompt } from "./moderationPrompt.js";
|
||||
export { runSimpleTextFallback } from "./simpleFallback.js";
|
||||
|
||||
// ── New two-pass pipeline exports ──────────────────────────────────────────
|
||||
export { classifyMessage } from "./fastClassifier.js";
|
||||
export type { Layer1Result } from "./fastClassifier.js";
|
||||
export type { AnalysisInput, AnalysisResult, WorkerConfig, MessageBatch } from "./ai-analysis-worker.js";
|
||||
Reference in New Issue
Block a user