256 lines
8.9 KiB
TypeScript
256 lines
8.9 KiB
TypeScript
import { createChildLogger } from "@/shared/logger/index";
|
|
import { config } from "../../shared/config/index.js";
|
|
import { messageStore } from "../message-capture/messageStore.js";
|
|
import type { MessageRecord } from "../message-capture/types.js";
|
|
import { type AnalysisLane, splitMessagesByLane } from "./analysisLanes.js";
|
|
import { computeBudgetOverflowMessages } from "./batchBudget.js";
|
|
import {
|
|
pickBatchWithinBudget,
|
|
processBatch,
|
|
skipAgeRestrictedMessages,
|
|
skipAnalysisUserMessages,
|
|
} from "./batchProcessor.js";
|
|
import {
|
|
clearConversationProcessing,
|
|
conversationConsecutiveErrors,
|
|
conversationDebounceTimers,
|
|
conversationErrorCooldown,
|
|
getConversationProcessingStartedAt,
|
|
isConversationProcessingLocked,
|
|
MAX_CONSECUTIVE_ERRORS,
|
|
setConversationProcessing,
|
|
} from "./conversationState.js";
|
|
|
|
const logger = createChildLogger("batch-scheduler");
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scheduling
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Timer key namespaced by lane so one conversation can hold a text timer AND
|
|
* a media timer independently.
|
|
*/
|
|
function timerKey(conversationKey: string, lane: AnalysisLane): string {
|
|
return `${conversationKey}::${lane}`;
|
|
}
|
|
|
|
/**
|
|
* Schedules a debounced analysis run for a conversation.
|
|
*
|
|
* `lane` optional:
|
|
* - With a lane: takes that lane's processing lock; if the SAME lane is
|
|
* already processing, skip. The OTHER lane's lock does not block this one —
|
|
* text and media of one conversation never block each other.
|
|
* - Without a lane (recovery worker / whole-conversation): takes BOTH lane
|
|
* locks (each lock independently) and dispatches both lanes concurrently.
|
|
* Each lane releases its own lock when its worker job finishes.
|
|
*
|
|
* The async work inside setTimeout is wrapped in an explicit .catch() so
|
|
* DB errors don't produce unhandled promise rejections. Uses a unified
|
|
* single-timer path: always clear-and-reset one timer per conversation+lane
|
|
* regardless of whether a cooldown is active.
|
|
*/
|
|
export function scheduleConversationAnalysis(
|
|
conversationKey: string,
|
|
lane?: AnalysisLane,
|
|
): void {
|
|
const lanesToSchedule: AnalysisLane[] = lane ? [lane] : ["text", "media"];
|
|
|
|
const convoCooldown = conversationErrorCooldown.get(conversationKey) ?? 0;
|
|
const convoErrors = conversationConsecutiveErrors.get(conversationKey) ?? 0;
|
|
|
|
// Hard-block: circuit breaker threshold reached AND cooldown still active.
|
|
if (convoErrors >= MAX_CONSECUTIVE_ERRORS && Date.now() < convoCooldown) {
|
|
return;
|
|
}
|
|
|
|
// Unified delay: honour the cooldown window if active, otherwise use the
|
|
// normal debounce interval. Always clear-and-reset so only ONE timer is
|
|
// ever pending per conversation+lane regardless of call source.
|
|
const now = Date.now();
|
|
const delayMs =
|
|
convoCooldown > now
|
|
? convoCooldown - now + 500
|
|
: config.AI_ANALYSIS_DEBOUNCE_MS;
|
|
|
|
for (const targetLane of lanesToSchedule) {
|
|
if (isConversationProcessingLocked(conversationKey, targetLane)) {
|
|
continue;
|
|
}
|
|
scheduleLaneTimer(conversationKey, targetLane, delayMs);
|
|
}
|
|
}
|
|
|
|
function scheduleLaneTimer(
|
|
conversationKey: string,
|
|
lane: AnalysisLane,
|
|
delayMs: number,
|
|
): void {
|
|
const tKey = timerKey(conversationKey, lane);
|
|
const existingTimer = conversationDebounceTimers.get(tKey);
|
|
if (existingTimer) {
|
|
clearTimeout(existingTimer);
|
|
}
|
|
|
|
const timer = setTimeout(() => {
|
|
conversationDebounceTimers.delete(tKey);
|
|
|
|
if (isConversationProcessingLocked(conversationKey, lane)) {
|
|
logger.warn(
|
|
{ conversationKey, lane, tKey },
|
|
"scheduleLaneTimer: lane locked, skipping dispatch",
|
|
);
|
|
return;
|
|
}
|
|
const processingStartedAt = Date.now();
|
|
setConversationProcessing(conversationKey, lane, processingStartedAt);
|
|
logger.debug(
|
|
{ conversationKey, lane, processingStartedAt },
|
|
"scheduleLaneTimer: lock acquired, dispatching batch fetch",
|
|
);
|
|
|
|
messageStore
|
|
.getPendingMessagesByConversation(
|
|
conversationKey,
|
|
config.AI_ANALYSIS_MAX_BATCH_SIZE,
|
|
)
|
|
.then(async (messages: MessageRecord[]) => {
|
|
// Filter to THIS lane only. The DB fetch is lane-agnostic (a
|
|
// conversation key can have both text and media pending); each lane
|
|
// picks its own subset so text and media never share a worker job.
|
|
const { [lane]: laneMessages } = splitMessagesByLane(messages);
|
|
if (laneMessages.length === 0) {
|
|
// No work for this lane — the other lane (if scheduled) owns the
|
|
// rest. Clear this lane's lock so the debounce can re-arm.
|
|
releaseLaneSlot(conversationKey, lane, processingStartedAt);
|
|
return;
|
|
}
|
|
|
|
const processableMessages = await skipAnalysisUserMessages(
|
|
await skipAgeRestrictedMessages(laneMessages),
|
|
);
|
|
if (processableMessages.length === 0) {
|
|
releaseLaneSlot(conversationKey, lane, processingStartedAt);
|
|
return;
|
|
}
|
|
|
|
let trimmed = pickBatchWithinBudget(
|
|
processableMessages,
|
|
config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
|
50,
|
|
);
|
|
|
|
// If every message individually exceeds the token budget,
|
|
// fall back to the first message alone to avoid stuck-pending deadlock.
|
|
if (trimmed.length === 0 && processableMessages.length > 0) {
|
|
trimmed = processableMessages.slice(0, 1);
|
|
logger.warn(
|
|
{
|
|
conversationKey,
|
|
lane,
|
|
messageId: processableMessages[0]?.id,
|
|
tokenBudget: config.AI_ANALYSIS_MAX_TARGET_TOKENS,
|
|
},
|
|
"All messages exceed token budget -- processing first message alone to avoid stuck-pending deadlock",
|
|
);
|
|
}
|
|
|
|
// Un-claim messages that did NOT make it into the trimmed batch.
|
|
// getPendingMessagesByConversation() flips EVERY fetched pending row
|
|
// to `processing`; pickBatchWithinBudget() may then stop early on the
|
|
// token budget, leaving the tail rows stuck in `processing` forever
|
|
// (recovery only reverts rows older than 120s, and these keep getting
|
|
// re-claimed each wave). Return them to `pending` so the next wave
|
|
// picks them up instead of leaking processing slots.
|
|
const unclaimed = computeBudgetOverflowMessages(
|
|
processableMessages,
|
|
trimmed,
|
|
);
|
|
if (unclaimed.length > 0) {
|
|
const unclaimedRows = await messageStore
|
|
.updateMessagesAIAnalysisBulk(
|
|
unclaimed.map((msg) => ({
|
|
messageId: msg.id,
|
|
result: {
|
|
status: "pending",
|
|
flags: null,
|
|
score: null,
|
|
analysis: null,
|
|
categories: null,
|
|
severity: null,
|
|
confidence: null,
|
|
recommendedAction: null,
|
|
analyzedAt: null,
|
|
error: null,
|
|
},
|
|
})),
|
|
)
|
|
.catch((err: unknown) => {
|
|
logger.error(
|
|
{
|
|
conversationKey,
|
|
lane,
|
|
count: unclaimed.length,
|
|
error: err instanceof Error ? err.message : String(err),
|
|
},
|
|
"Failed to un-claim budget-overflow messages back to pending",
|
|
);
|
|
return null;
|
|
});
|
|
if (unclaimedRows) {
|
|
logger.debug(
|
|
{
|
|
conversationKey,
|
|
lane,
|
|
unclaimedCount: unclaimed.length,
|
|
},
|
|
"Returned budget-overflow messages to pending for next wave",
|
|
);
|
|
}
|
|
}
|
|
|
|
// processBatch releases THIS lane's lock the moment its worker job
|
|
// finishes and re-schedules the same lane — independent of the other
|
|
// lane's (possibly much slower) media batch.
|
|
return processBatch(
|
|
conversationKey,
|
|
lane,
|
|
trimmed,
|
|
processingStartedAt,
|
|
);
|
|
})
|
|
.catch((err: unknown) => {
|
|
releaseLaneSlot(conversationKey, lane, processingStartedAt);
|
|
logger.error(
|
|
{
|
|
conversationKey,
|
|
lane,
|
|
error: err instanceof Error ? err.message : String(err),
|
|
},
|
|
"Failed to fetch or dispatch pending messages for scheduled analysis",
|
|
);
|
|
});
|
|
}, delayMs);
|
|
|
|
conversationDebounceTimers.set(tKey, timer);
|
|
}
|
|
|
|
/**
|
|
* Clears the processing lock for a lane, but ONLY if this timer still owns it
|
|
* (processingStartedAt matches). Guards against clearing a newer slot that was
|
|
* taken after this timer's window expired.
|
|
*/
|
|
function releaseLaneSlot(
|
|
conversationKey: string,
|
|
lane: AnalysisLane,
|
|
processingStartedAt: number,
|
|
): void {
|
|
if (
|
|
getConversationProcessingStartedAt(conversationKey, lane) ===
|
|
processingStartedAt
|
|
) {
|
|
clearConversationProcessing(conversationKey, lane);
|
|
}
|
|
}
|