feat(vision): route multimodal analysis to dedicated NVIDIA direct endpoint
- config: add AI_LLM_VISION_BASE_URL + AI_LLM_VISION_API_KEY (separate from text router) - llmClient: llmVision() now calls dedicated vision endpoint when configured (axios POST to integrate.api.nvidia.com, model nvidia/nemotron-3-nano-omni-30b-a3b-reasoning, reasoning_budget 16384, non-stream), falls back to router combo otherwise - keeps text/moderation on omniroute, vision on NVIDIA direct
This commit is contained in:
@@ -283,6 +283,15 @@ export async function llmChat(
|
|||||||
* Convenience for vision (image/sticker/emoji) analysis.
|
* Convenience for vision (image/sticker/emoji) analysis.
|
||||||
* Returns the raw completion content (trimmed) or null.
|
* Returns the raw completion content (trimmed) or null.
|
||||||
*
|
*
|
||||||
|
* When AI_LLM_VISION_BASE_URL + AI_LLM_VISION_API_KEY are configured, vision is
|
||||||
|
* sent DIRECTLY to a dedicated multimodal endpoint (e.g. NVIDIA direct API),
|
||||||
|
* separate from the text/moderation router (omniroute/9router). This keeps
|
||||||
|
* image analysis on a vision-capable model while text moderation stays on the
|
||||||
|
* router's text combo.
|
||||||
|
*
|
||||||
|
* Otherwise it falls back to the shared AI_LLM_BASE_URL (router combo) using
|
||||||
|
* AI_LLM_VISION_MODEL.
|
||||||
|
*
|
||||||
* NOTE: retries are disabled here on purpose — visionAnalyzer.ts already
|
* NOTE: retries are disabled here on purpose — visionAnalyzer.ts already
|
||||||
* wraps this call in its own 3-attempt loop with exponential backoff.
|
* wraps this call in its own 3-attempt loop with exponential backoff.
|
||||||
* A second retry layer would multiply worst-case API calls (3×3=9/image).
|
* A second retry layer would multiply worst-case API calls (3×3=9/image).
|
||||||
@@ -291,6 +300,12 @@ export async function llmVision(
|
|||||||
promptText: string,
|
promptText: string,
|
||||||
imageUrl: { url: string },
|
imageUrl: { url: string },
|
||||||
): Promise<string | null> {
|
): Promise<string | null> {
|
||||||
|
// ── Dedicated vision endpoint (NVIDIA direct, etc.) ──────────────────────
|
||||||
|
if (config.AI_LLM_VISION_BASE_URL && config.AI_LLM_VISION_API_KEY) {
|
||||||
|
return llmVisionDirect(promptText, imageUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Fallback: shared router combo ─────────────────────────────────────────
|
||||||
const completion = await llmChat({
|
const completion = await llmChat({
|
||||||
messages: [
|
messages: [
|
||||||
{
|
{
|
||||||
@@ -312,3 +327,64 @@ export async function llmVision(
|
|||||||
if (!completion) return null;
|
if (!completion) return null;
|
||||||
return completion.choices[0]?.message?.content?.trim() ?? null;
|
return completion.choices[0]?.message?.content?.trim() ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
import axios from "axios";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Direct vision call to a dedicated multimodal endpoint (NVIDIA integrate API).
|
||||||
|
* Model is fixed to the vision-capable one configured via AI_LLM_VISION_MODEL
|
||||||
|
* (default nvidia/nemotron-3-nano-omni-30b-a3b-reasoning). Uses reasoning_budget
|
||||||
|
* + non-streaming (axios JSON) — NVIDIA direct does not need the SSE streaming
|
||||||
|
* gymnastics the composite routers require.
|
||||||
|
*/
|
||||||
|
async function llmVisionDirect(
|
||||||
|
promptText: string,
|
||||||
|
imageUrl: { url: string },
|
||||||
|
): Promise<string | null> {
|
||||||
|
const visionModel = config.AI_LLM_VISION_MODEL || "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning";
|
||||||
|
try {
|
||||||
|
const response = await axios.post(
|
||||||
|
`${config.AI_LLM_VISION_BASE_URL}/chat/completions`,
|
||||||
|
{
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
{ type: "text", text: promptText },
|
||||||
|
{ type: "image_url", image_url: imageUrl },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
model: visionModel,
|
||||||
|
max_tokens: 65536,
|
||||||
|
reasoning_budget: 16384,
|
||||||
|
stream: false,
|
||||||
|
temperature: 0.6,
|
||||||
|
top_p: 0.95,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
headers: {
|
||||||
|
Authorization: `Bearer ${config.AI_LLM_VISION_API_KEY}`,
|
||||||
|
Accept: "application/json",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
timeout: 120_000,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const content: string | undefined =
|
||||||
|
response.data?.choices?.[0]?.message?.content;
|
||||||
|
if (!content) return null;
|
||||||
|
return content.trim();
|
||||||
|
} catch (err: any) {
|
||||||
|
const status = err?.response?.status ?? "n/a";
|
||||||
|
const detail = err?.response?.data
|
||||||
|
? JSON.stringify(err.response.data).slice(0, 300)
|
||||||
|
: err?.message;
|
||||||
|
log.error(
|
||||||
|
{ status, detail, model: visionModel },
|
||||||
|
"Direct vision API call failed",
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -137,7 +137,13 @@ export const configSchema = z
|
|||||||
.url()
|
.url()
|
||||||
.default("https://9router.asepharyana.my.id/v1"),
|
.default("https://9router.asepharyana.my.id/v1"),
|
||||||
AI_LLM_MODEL: z.string().default("text"),
|
AI_LLM_MODEL: z.string().default("text"),
|
||||||
AI_LLM_VISION_MODEL: z.string().optional(),
|
AI_LLM_VISION_MODEL: z.string().default("multimodal"),
|
||||||
|
// Vision can be routed to a dedicated endpoint (e.g. NVIDIA direct) that is
|
||||||
|
// separate from the text/moderation router. When both are set, llmVision()
|
||||||
|
// calls the dedicated vision endpoint directly; otherwise it falls back to
|
||||||
|
// the shared AI_LLM_BASE_URL with AI_LLM_VISION_MODEL.
|
||||||
|
AI_LLM_VISION_BASE_URL: z.string().url().optional(),
|
||||||
|
AI_LLM_VISION_API_KEY: z.string().optional(),
|
||||||
AI_LLM_EMBEDDING_MODEL: z.string().optional(),
|
AI_LLM_EMBEDDING_MODEL: z.string().optional(),
|
||||||
AI_LLM_EMBEDDING_MIN_SIMILARITY: z.coerce
|
AI_LLM_EMBEDDING_MIN_SIMILARITY: z.coerce
|
||||||
.number()
|
.number()
|
||||||
|
|||||||
Reference in New Issue
Block a user