MythEclipse/GMW
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ai-moderation): never cache vision outputs that claim 'no image seen'

Browse Source 
Root cause (3rd layer after 50371bd + 4f4c435): a vision model run
(2026-08-10) returned 'Maaf, saya tidak melihat gambar apapun yang terlampir...'
and that text was cached as a VALID vision_llm result (image + phash keys,
24h/7d TTL). Every subsequent analysis of the same image (same hash/phash)
hit the poisoned cache, so image analysis looked broken forever even though
9router responded fine — the moderation LLM wrote 'lampiran yang gagal
terbaca' from a cache hit.

Also: mimo via 9router streams reasoning in delta.reasoning +
delta.reasoning_details[].text (content:"") — extractChunkText only read
delta.reasoning_content, so those runs aggregated empty → 'Vision API null
response' (observed 08:54/09:07/09:38).

Fixes:
- llmClient.extractChunkText: fall back to delta.reasoning and
  reasoning_details[].text (mimo), on top of reasoning_content (gemma).
- visionAnalyzer: isNoImageSeenText() detects 'no image' style outputs;
  such results are NEVER cached, and poisoned entries are purged when hit
  (LRU/DB/phash) so re-analysis actually re-runs vision.
- Tests: reasoning/reasoning_details extraction + isNoImageSeenText
  (Indonesian + English, no false positives on real descriptions).
This commit is contained in:
asepharyana
2026-08-11 09:55:43 +07:00
parent 4f4c43555f
commit c18431bdbf
4 changed files with 178 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/discord-gateway/src/modules/ai-moderation/llmClient.ts
+23 -4
View File
@@ -56,7 +56,16 @@ export async function withLlmConcurrency<T>(fn: () => Promise<T>): Promise<T> {
*/
type LLMResponseChunk = {
choices?: Array<{
delta?: { content?: string | null; reasoning_content?: string | null };
delta?: {
content?: string | null;
reasoning_content?: string | null;
reasoning?: string | null;
reasoning_details?: Array<{
type?: string;
text?: string;
index?: number;
}> | null;
};
message?: { content?: string | null };
finish_reason?: string | null;
text?: string;
@@ -69,18 +78,28 @@ type LLMResponseChunk = {
/**
* Extract the textual payload from a single streaming chunk. Prefers
* `delta.content`; falls back to `delta.reasoning_content` (DeepSeek-style /
* Cloudflare gemma stream ALL output there with content:"") so reasoning-only
* models still produce usable aggregated text. Exported for unit tests.
* `delta.content`; falls back to reasoning fields so reasoning-only models
* still produce usable aggregated text. Providers differ in the field name:
* - DeepSeek-style / Cloudflare gemma → `delta.reasoning_content`
* - mimo (via 9router) streams reasoning in `delta.reasoning` +
* `delta.reasoning_details[].text` (content:"") — without these fallbacks
* vision aggregation came back empty ("Vision API null response").
* Exported for unit tests.
*/
export function extractChunkText(
chunk: LLMResponseChunk | null | undefined,
): string {
if (!chunk) return "";
const choice = chunk.choices?.[0];
const reasoningDetails = choice?.delta?.reasoning_details
?.map((d) => d.text ?? "")
.filter(Boolean)
.join("");
return (
choice?.delta?.content ||
choice?.delta?.reasoning_content ||
choice?.delta?.reasoning ||
reasoningDetails ||
choice?.message?.content ||
choice?.text ||
chunk?.message?.content ||
services/discord-gateway/src/modules/ai-moderation/visionAnalyzer.ts
+68 -9
View File
@@ -29,6 +29,35 @@ import {
upsertCachedMediaByPhash,
visionLruCache,
} from "./mediaCache.js";
/**
* Detect vision outputs where the model claims it saw no image at all
* ("Maaf, saya tidak melihat gambar apapun...", "Tidak ada gambar yang
* terlampir...", "I cannot see any image..."). Such text is NOT a valid
* analysis — caching it poisons the image cache for 24h (image/phash keys),
* so every re-analysis of the same image returns the "no image" text and the
* moderation LLM writes "lampiran gagal terbaca". These outputs must be
* treated as failures: never cached, and ignored when read back from cache.
*/
export function isNoImageSeenText(text: string | null | undefined): boolean {
if (!text) return false;
const lower = text.toLowerCase();
return (
/tidak (?:melihat|ada|terlihat) (?:gambar|foto|image)/i.test(lower) ||
/tidak (?:ada )?(?:gambar|foto|image) (?:apapun|yang terlampir)/i.test(
lower,
) ||
/gambar apapun/i.test(lower) ||
/tanpa (?:input )?(?:visual|gambar|image)/i.test(lower) ||
/\bno image (?:provided|attached|detected|found|was provided)?/i.test(
lower,
) ||
/(?:cannot|can't) see (?:any |an |the )?image/i.test(lower) ||
/i (?:do not|don't) (?:see|detect) (?:any |an |the )?image/i.test(lower) ||
/there (?:is|are) no image/i.test(lower)
);
}
import {
buildMediaCandidates,
downloadAndExtractFrame,
@@ -119,18 +148,32 @@ export const analyzeSingleMediaImage = async (
// Layer 0: LRU
const lruCached = visionLruCache.get(cacheKey);
if (lruCached) {
if (lruCached && !isNoImageSeenText(lruCached)) {
log.debug({ cacheKey }, "Vision LRU cache HIT (in-memory)");
return `[Media analysis for message ${messageId}] ${image.sourceLabel}: ${lruCached}`;
}
if (lruCached) {
// Poisoned entry ("I see no image") — drop it and re-analyze.
log.warn({ cacheKey }, "Vision LRU cache HIT was no-image-seen — dropping");
visionLruCache.delete(cacheKey);
}
// Layer 1: DB
const cached = await getCachedMediaAnalysis(cacheKey);
if (cached) {
if (cached && !isNoImageSeenText(cached)) {
visionLruCache.set(cacheKey, cached);
log.debug({ cacheKey }, "Media analysis cache HIT (DB → LRU)");
return `[Media analysis for message ${messageId}] ${image.sourceLabel}: ${cached}`;
}
if (cached) {
// Poisoned DB entry — purge it so later messages re-analyze.
log.warn(
{ cacheKey },
"Media analysis cache HIT was no-image-seen — purging",
);
await deleteCachedMediaAnalysis(cacheKey).catch(() => {});
visionLruCache.delete(cacheKey);
}
// In-flight dedupe
const existing = inFlightVisionCalls.get(cacheKey);
@@ -173,7 +216,7 @@ export const analyzeSingleMediaImage = async (
phash = await computeImagePhash(imgBuffer);
if (phash) {
const phashCached = await getCachedMediaByPhash(phash);
if (phashCached) {
if (phashCached && !isNoImageSeenText(phashCached)) {
visionLruCache.set(cacheKey, phashCached);
await upsertCachedMediaAnalysis(
cacheKey,
@@ -183,6 +226,12 @@ export const analyzeSingleMediaImage = async (
).catch(() => {});
return phashCached;
}
if (phashCached) {
log.warn(
{ phash, cacheKey },
"phash cache HIT was no-image-seen — ignoring",
);
}
}
}
} catch {
@@ -195,7 +244,7 @@ export const analyzeSingleMediaImage = async (
for (let attempt = 0; attempt < 3; attempt++) {
try {
const content = await llmVision(promptText, image.image_url);
if (content) {
if (content && !isNoImageSeenText(content)) {
await upsertCachedMediaAnalysis(
cacheKey,
content,
@@ -213,7 +262,17 @@ export const analyzeSingleMediaImage = async (
}
return content;
}
log.warn({ messageId }, "Vision API null response");
if (content) {
// Model claims it saw no image — same as a null response: NOT a
// valid analysis, and caching it would poison the key for every
// re-analysis of the same image (phash TTL is 7 days).
log.warn(
{ messageId, cacheKey },
"Vision returned no-image-seen text — not caching",
);
} else {
log.warn({ messageId }, "Vision API null response");
}
break;
} catch (err) {
lastError = err instanceof Error ? err : new Error(String(err));
@@ -240,6 +299,7 @@ export const analyzeSingleMediaImage = async (
"Vision failed after 3 attempts",
);
await deleteCachedMediaAnalysis(cacheKey).catch(() => {});
visionLruCache.delete(cacheKey);
return FAILED_ANALYSIS_PREFIX;
})();
@@ -378,10 +438,9 @@ export async function prepareMediaMessage(
// Profile is emitted ONCE per batch in a <user_profiles> map (see
// mediaBatchProcessor); here we only reference it to avoid repeating the
// full summary on every message of the same user.
const profileRef =
profile && profile.profile_summary?.trim()
? buildUserProfileRef(target.user_id)
: "";
const profileRef = profile?.profile_summary?.trim()
? buildUserProfileRef(target.user_id)
: "";
// Rich reputation — same shape as the text path: attrs + optional
// <user_history> with the last flagged messages for repeat offenders.