fix(ai-moderation): read delta.reasoning_content in stream aggregation — image vision never returned text
Root cause: 9router combo 'multimodal' routes to cloudflare-ai/@cf/google/
gemma-4-26b-a4b-it which streams ALL output in delta.reasoning_content
(content:"") and finishes with 'length' at max_tokens. llmClient only read
delta.content, so llmVision returned empty → every image moderation fell back
to text-only analysis ('Meskipun analisis gambar gagal' in every ai_analysis).
Fix: extractChunkText() prefers delta.content then falls back to
delta.reasoning_content (also handles message/text/response fields), with
unit tests for the exact 9router chunk shape. Verified live against a real
DB image: oc/mimo-v2.5-free (new first model in the multimodal combo) returns
a proper description in delta.content.
This commit is contained in:
@@ -56,7 +56,7 @@ export async function withLlmConcurrency<T>(fn: () => Promise<T>): Promise<T> {
|
|||||||
*/
|
*/
|
||||||
type LLMResponseChunk = {
|
type LLMResponseChunk = {
|
||||||
choices?: Array<{
|
choices?: Array<{
|
||||||
delta?: { content?: string | null };
|
delta?: { content?: string | null; reasoning_content?: string | null };
|
||||||
message?: { content?: string | null };
|
message?: { content?: string | null };
|
||||||
finish_reason?: string | null;
|
finish_reason?: string | null;
|
||||||
text?: string;
|
text?: string;
|
||||||
@@ -67,6 +67,29 @@ type LLMResponseChunk = {
|
|||||||
finish_reason?: string;
|
finish_reason?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract the textual payload from a single streaming chunk. Prefers
|
||||||
|
* `delta.content`; falls back to `delta.reasoning_content` (DeepSeek-style /
|
||||||
|
* Cloudflare gemma stream ALL output there with content:"") so reasoning-only
|
||||||
|
* models still produce usable aggregated text. Exported for unit tests.
|
||||||
|
*/
|
||||||
|
export function extractChunkText(
|
||||||
|
chunk: LLMResponseChunk | null | undefined,
|
||||||
|
): string {
|
||||||
|
if (!chunk) return "";
|
||||||
|
const choice = chunk.choices?.[0];
|
||||||
|
return (
|
||||||
|
choice?.delta?.content ||
|
||||||
|
choice?.delta?.reasoning_content ||
|
||||||
|
choice?.message?.content ||
|
||||||
|
choice?.text ||
|
||||||
|
chunk?.message?.content ||
|
||||||
|
chunk?.response ||
|
||||||
|
chunk?.content ||
|
||||||
|
""
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Lazy singleton — created on first use so that config is always resolved.
|
// Lazy singleton — created on first use so that config is always resolved.
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -167,15 +190,7 @@ export async function llmChat(
|
|||||||
let finishReason = "stop";
|
let finishReason = "stop";
|
||||||
for await (const chunk of response as unknown as AsyncIterable<LLMResponseChunk>) {
|
for await (const chunk of response as unknown as AsyncIterable<LLMResponseChunk>) {
|
||||||
const choice = chunk?.choices?.[0];
|
const choice = chunk?.choices?.[0];
|
||||||
const textChunk =
|
content += extractChunkText(chunk);
|
||||||
choice?.delta?.content ||
|
|
||||||
choice?.message?.content ||
|
|
||||||
choice?.text ||
|
|
||||||
chunk?.message?.content ||
|
|
||||||
chunk?.response ||
|
|
||||||
chunk?.content ||
|
|
||||||
"";
|
|
||||||
content += textChunk;
|
|
||||||
const fr = choice?.finish_reason || chunk?.finish_reason;
|
const fr = choice?.finish_reason || chunk?.finish_reason;
|
||||||
if (fr) finishReason = fr;
|
if (fr) finishReason = fr;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
|
// llmClient chunk extraction — reasoning_content fallback (pure, no network)
|
||||||
|
// ═══════════════════════════════════════════════════════════════════════════
|
||||||
|
// Regression: 9router "multimodal" combo routed to cloudflare gemma-4-26b
|
||||||
|
// which streams ALL output in delta.reasoning_content with content:"" — the
|
||||||
|
// old extractor returned empty text → llmVision reported "Vision API null
|
||||||
|
// response" → every image moderation batch fell back to text-only analysis
|
||||||
|
// (LLM kept writing "Meskipun analisis gambar gagal").
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { extractChunkText } from "../src/modules/ai-moderation/llmClient.js";
|
||||||
|
|
||||||
|
describe("extractChunkText — streaming chunk text extraction", () => {
|
||||||
|
it("reads delta.content (standard OpenAI streaming)", () => {
|
||||||
|
expect(
|
||||||
|
extractChunkText({
|
||||||
|
choices: [{ delta: { content: "halo" }, finish_reason: null }],
|
||||||
|
}),
|
||||||
|
).toBe("halo");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("falls back to delta.reasoning_content when content is empty — reasoning-only models (cloudflare gemma)", () => {
|
||||||
|
// Exact shape seen from 9router → cloudflare-ai/@cf/google/gemma-4-26b:
|
||||||
|
// {"choices":[{"delta":{"content":"","reasoning_content":"Task","role":"assistant"},"finish_reason":null,...}]}
|
||||||
|
expect(
|
||||||
|
extractChunkText({
|
||||||
|
choices: [
|
||||||
|
{
|
||||||
|
delta: { content: "", reasoning_content: "Task" },
|
||||||
|
finish_reason: null,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
).toBe("Task");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("prefers content over reasoning when both present (deepseek-style final answer)", () => {
|
||||||
|
expect(
|
||||||
|
extractChunkText({
|
||||||
|
choices: [
|
||||||
|
{
|
||||||
|
delta: { content: "jawaban akhir", reasoning_content: "pikiran" },
|
||||||
|
finish_reason: null,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
).toBe("jawaban akhir");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("handles Anthropic-style message.content", () => {
|
||||||
|
expect(extractChunkText({ message: { content: "via message" } })).toBe(
|
||||||
|
"via message",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("handles top-level content / response fields (local LLM proxies)", () => {
|
||||||
|
expect(extractChunkText({ content: "top-level" })).toBe("top-level");
|
||||||
|
expect(extractChunkText({ response: "via response" })).toBe("via response");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns empty string for null/undefined/empty chunks", () => {
|
||||||
|
expect(extractChunkText(null)).toBe("");
|
||||||
|
expect(extractChunkText(undefined)).toBe("");
|
||||||
|
expect(extractChunkText({})).toBe("");
|
||||||
|
expect(
|
||||||
|
extractChunkText({
|
||||||
|
choices: [{ delta: { content: "", reasoning_content: null } }],
|
||||||
|
}),
|
||||||
|
).toBe("");
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user