MythEclipse/GMW
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(gateway): implement separate Piscina pools for text and media analysis to optimize processing

Browse Source
This commit is contained in:
asepharyana
2026-08-31 22:59:27 +07:00
parent 3ce594d9b5
commit 12cc956329
9 changed files with 272 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/discord-gateway/src/modules/ai-moderation/batchProcessor.ts
+80 -39
View File
@@ -5,7 +5,7 @@ import { messageStore } from "../message-capture/messageStore.js";
import type { MessageRecord } from "../message-capture/types.js";
import { pickBatchWithinBudget as pickBatchWithinBudgetPure } from "./batchBudget.js";
import { partitionBatchOutcome } from "./batchOutcomeClassifier.js";
import { workerPool } from "./circuitBreaker.js";
import { mediaWorkerPool, textWorkerPool } from "./circuitBreaker.js";
import { estimateTokens } from "./conversationContext.js";
import {
conversationErrorCooldown,
@@ -14,6 +14,7 @@ import {
resetConversationBatchFailures,
} from "./conversationState.js";
import { enqueueIndividualFallbacks } from "./individualFallbackProcessor.js";
import { hasMediaContent } from "./mediaAnalysisClient.js";
import {
broadcastAnalysisCompleted,
LAST_ERROR,
@@ -121,29 +122,31 @@ export async function skipAgeRestrictedMessages(
// Batch pipeline
// ---------------------------------------------------------------------------
export async function processBatch(
/**
* Runs one worker job (either the text-only or the media sub-batch of a
* conversation) end-to-end: dispatch → broadcast/save → fallback routing.
* Returns whether the *caller* should schedule the next debounce pass for
* this conversation (mirrors the old single-job semantics, now evaluated
* per queue).
*
* Broadcasting happens here, inside each queue's own call — NOT after
* waiting on the other queue. That's the actual fix for "text menunggu
* image": previously one mixed conversation batch made ONE worker call
* with both text and media targets, and `runModerationAnalysis` only
* resolves (so results only get saved/broadcast) once BOTH finish — so a
* fast text verdict sat unused until the slow vision/image verdict was
* ready too. Splitting into two independent jobs means the text queue
* saves+broadcasts its rows the moment IT finishes, regardless of how long
* the media queue takes.
*/
async function runQueueBatch(
pool: typeof textWorkerPool,
conversationKey: string,
messages: MessageRecord[],
processingStartedAt: number,
): Promise<void> {
if (messages.length === 0) {
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
return;
}
const cooldownUntil = conversationErrorCooldown.get(conversationKey) ?? 0;
if (Date.now() < cooldownUntil) {
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
return;
}
): Promise<boolean> {
activeRequests++;
let shouldScheduleNext = false;
try {
const result = (await workerPool.run({
const result = (await pool.run({
type: "batch",
conversationKey,
messages,
@@ -191,7 +194,7 @@ export async function processBatch(
},
"Batch analysis failed, will retry after cooldown",
);
return;
return false;
}
// Batch succeeded -- partition per-message outcome (2026-08-25).
@@ -281,20 +284,13 @@ export async function processBatch(
conversationErrorCooldown.set(conversationKey, newCooldown);
}
// Release the processing lock immediately so the cooldown timer controls retry
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
// Do NOT schedule next -- let the cooldown gate it
shouldScheduleNext = false;
return false;
}
if (apiFailedMessages.length === 0) {
resetConversationBatchFailures(conversationKey);
conversationErrorCooldown.delete(conversationKey);
}
shouldScheduleNext = true;
resetConversationBatchFailures(conversationKey);
conversationErrorCooldown.delete(conversationKey);
return true;
} catch (error) {
recordConversationBatchFailure(conversationKey);
@@ -326,18 +322,63 @@ export async function processBatch(
},
"Analysis worker failed, will retry after cooldown",
);
return false;
} finally {
activeRequests--;
}
}
export async function processBatch(
conversationKey: string,
messages: MessageRecord[],
processingStartedAt: number,
): Promise<void> {
if (messages.length === 0) {
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
if (shouldScheduleNext) {
setImmediate(() => {
// Dynamic import to avoid circular dependency at module scope
import("./batchScheduler.js").then((m) =>
m.scheduleConversationAnalysis(conversationKey),
);
});
return;
}
const cooldownUntil = conversationErrorCooldown.get(conversationKey) ?? 0;
if (Date.now() < cooldownUntil) {
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
return;
}
// Split the batch itself — not just route it — so text and media never
// share one worker call. A conversation batch commonly mixes plain-text
// messages with an image/sticker from someone else; without this split,
// ALL of it (including the plain-text messages) would ride along on the
// media job and wait for vision analysis to finish. Each sub-batch is now
// dispatched to its own pool AND handled independently below, so the text
// queue's results land as soon as text analysis completes, full stop.
const textMessages = messages.filter((m) => !hasMediaContent(m));
const mediaMessages = messages.filter((m) => hasMediaContent(m));
const jobs: Promise<boolean>[] = [];
if (textMessages.length > 0) {
jobs.push(runQueueBatch(textWorkerPool, conversationKey, textMessages));
}
if (mediaMessages.length > 0) {
jobs.push(runQueueBatch(mediaWorkerPool, conversationKey, mediaMessages));
}
const outcomes = await Promise.allSettled(jobs);
const shouldScheduleNext = outcomes.every(
(o) => o.status === "fulfilled" && o.value,
);
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
conversationProcessing.delete(conversationKey);
}
if (shouldScheduleNext) {
setImmediate(() => {
// Dynamic import to avoid circular dependency at module scope
import("./batchScheduler.js").then((m) =>
m.scheduleConversationAnalysis(conversationKey),
);
});
}
}
services/discord-gateway/src/modules/ai-moderation/circuitBreaker.ts
+32 -3
View File
@@ -6,7 +6,16 @@ import { config } from "../../shared/config/config.js";
import type { MessageRecord } from "../message-capture/types.js";
// ---------------------------------------------------------------------------
// Piscina worker pool (shared by batch + individual pipelines)
// Piscina worker pools (2026-08-31: split text vs media)
//
// Both the batch and individual-fallback pipelines used to share ONE pool.
// A conversation batch containing images/stickers/embeds routes through
// runMediaBatch (download + vision + LLM — tens of seconds per batch), and
// Piscina hands each worker thread exactly one task at a time. With a small
// fixed thread count, a handful of slow media batches could occupy every
// thread and leave fast text-only batches for OTHER conversations queued
// behind them for the whole media duration. Text and media now get their
// own dedicated pools so a media backlog can never starve text analysis.
// ---------------------------------------------------------------------------
function getAnalysisWorkerUrl(): URL {
@@ -25,12 +34,32 @@ function getAnalysisWorkerUrl(): URL {
return candidates[2];
}
export const workerPool = new Piscina({
filename: fileURLToPath(getAnalysisWorkerUrl()),
const analysisWorkerFilename = fileURLToPath(getAnalysisWorkerUrl());
/** Dedicated pool for text-only batch/individual analysis jobs. */
export const textWorkerPool = new Piscina({
filename: analysisWorkerFilename,
execArgv: process.execArgv,
maxThreads: config.PISCINA_MAX_THREADS ?? availableParallelism(),
});
/**
* Dedicated pool for jobs whose batch contains at least one message with
* media (attachments/stickers/embeds). Kept separate so slow image/vision
* analysis never blocks the text pool above.
*/
export const mediaWorkerPool = new Piscina({
filename: analysisWorkerFilename,
execArgv: process.execArgv,
maxThreads: config.PISCINA_MEDIA_MAX_THREADS ?? 2,
});
/**
* @deprecated Use `textWorkerPool` or `mediaWorkerPool` directly. Kept as an
* alias to the text pool only for anything not yet migrated.
*/
export const workerPool = textWorkerPool;
/**
* Gets the conversation key for a message (thread_id or channel_id).
*/
services/discord-gateway/src/modules/ai-moderation/individualFallbackProcessor.ts
+12 -3
View File
@@ -6,9 +6,14 @@ import type {
AnalysisResult,
MessageRecord,
} from "../message-capture/types.js";
import { getConversationKey, workerPool } from "./circuitBreaker.js";
import {
getConversationKey,
mediaWorkerPool,
textWorkerPool,
} from "./circuitBreaker.js";
import { fireAlert } from "./conversationState.js";
import { classifyIndividualWorkerResult } from "./fallbackResultClassifier.js";
import { hasMediaContent } from "./mediaAnalysisClient.js";
import {
broadcastAnalysisCompleted,
LAST_ERROR,
@@ -73,8 +78,12 @@ async function processIndividualFallback(
let exhaustedOnIncomplete = false;
try {
// Run the LLM-heavy work in the worker thread
const workerResult = (await workerPool.run({
// Run the LLM-heavy work in the worker thread — same text/media pool
// split as the batch pipeline (see circuitBreaker.ts), so a single
// media message falling back to individual analysis can't queue behind
// (or block) text-only fallbacks, and vice versa.
const pool = hasMediaContent(message) ? mediaWorkerPool : textWorkerPool;
const workerResult = (await pool.run({
type: "individual",
message,
skipNormalAnalysis: false,