diff --git a/services/discord-gateway/src/modules/ai-moderation/llmClient.ts b/services/discord-gateway/src/modules/ai-moderation/llmClient.ts index 1c6a749..3eb064e 100644 --- a/services/discord-gateway/src/modules/ai-moderation/llmClient.ts +++ b/services/discord-gateway/src/modules/ai-moderation/llmClient.ts @@ -147,6 +147,11 @@ export interface LlmCallOpts { top_p?: number; /** Force JSON output via response_format: { type: "json_object" }. */ jsonResponse?: { type: "json_object" }; + /** + * Disable LLM chain-of-thought (reasoning/thinking) for faster analysis. + * Defaults to config.AI_LLM_DISABLE_THINKING when omitted. + */ + disableThinking?: boolean; /** Extra retries beyond DEFAULT_RETRIES (default 2). */ retries?: number; /** Whether to use streaming (if true, will consume stream and return aggregated result) */ @@ -155,6 +160,64 @@ export interface LlmCallOpts { signal?: AbortSignal; } +/** + * Build the request params for an LLM chat completion. Pulled out of `llmChat` + * so the thinking-disable injection can be unit-tested without network access. + * + * Optional params (temperature/top_p/max_tokens) are only attached when + * explicitly provided, to maximise compatibility with various providers/local + * APIs. When `disableThinking` is set, we inject the common provider params + * used to switch OFF chain-of-thought reasoning. OpenAI-compatible routers + * ignore the variants their backend does not understand, so sending the + * OpenAI (`reasoning_effort`), OpenRouter (`reasoning.enabled`) and + * vLLM/Qwen/litellm (`chat_template_kwargs.enable_thinking`) forms together + * covers the popular reasoning backends behind a proxy. + */ +export function buildLlmParams( + opts: LlmCallOpts, + disableThinking: boolean, +): OpenAI.Chat.Completions.ChatCompletionCreateParams { + const { + messages, + model = config.AI_LLM_MODEL, + max_tokens, + temperature, + top_p, + jsonResponse, + stream, + } = opts; + + const params = { + model, + messages, + ...(stream !== undefined ? { stream } : {}), + } as OpenAI.Chat.Completions.ChatCompletionCreateParams; + + if (temperature !== undefined) params.temperature = temperature; + if (top_p !== undefined) params.top_p = top_p; + if (max_tokens !== undefined) params.max_tokens = max_tokens; + if (jsonResponse) params.response_format = jsonResponse; + + if (disableThinking) { + Object.assign( + params, + { + // OpenAI o-series + reasoning_effort: "none", + // OpenRouter + reasoning: { enabled: false }, + // vLLM / Qwen / litellm + chat_template_kwargs: { enable_thinking: false }, + // Anthropic / Claude-format (9router exposes thinkingFormat + // "claude-adaptive" / "claude-budget" on its reasoning models) + thinking: { type: "disabled" }, + } as Record, + ); + } + + return params; +} + /** * Call the LLM with sensible defaults: concurrency cap, retry, model, tokens. * @@ -167,33 +230,12 @@ export async function llmChat( const client = getClient(); if (!client) return null; - const { - messages, - model = config.AI_LLM_MODEL, - max_tokens, - temperature, - top_p, - jsonResponse, - retries = DEFAULT_RETRIES, - stream, - signal, - } = opts; + const { retries = DEFAULT_RETRIES, signal } = opts; + const disableThinking = + opts.disableThinking ?? config.AI_LLM_DISABLE_THINKING; - const params = { - model, - messages, - ...(stream !== undefined ? { stream } : {}), - } as OpenAI.Chat.Completions.ChatCompletionCreateParams; - - // Attach optional parameters only if explicitly provided to maintain - // maximum compatibility with various LLM providers and local APIs. - if (temperature !== undefined) params.temperature = temperature; - if (top_p !== undefined) params.top_p = top_p; - if (max_tokens !== undefined) params.max_tokens = max_tokens; - - if (jsonResponse) { - params.response_format = jsonResponse; - } + const params = buildLlmParams(opts, disableThinking); + const model = params.model; return retryWithBackoff( async () => { @@ -359,7 +401,7 @@ async function llmVisionDirect( ], model: visionModel, max_tokens: 65536, - reasoning_budget: 16384, + reasoning_budget: config.AI_LLM_DISABLE_THINKING ? 0 : 16384, stream: false, temperature: 0.6, top_p: 0.95, diff --git a/services/discord-gateway/src/shared/config/index.ts b/services/discord-gateway/src/shared/config/index.ts index 8dd055c..19c1bac 100644 --- a/services/discord-gateway/src/shared/config/index.ts +++ b/services/discord-gateway/src/shared/config/index.ts @@ -151,6 +151,13 @@ export const configSchema = z // the shared AI_LLM_BASE_URL with AI_LLM_VISION_MODEL. AI_LLM_VISION_BASE_URL: z.string().url().optional(), AI_LLM_VISION_API_KEY: z.string().optional(), + AI_LLM_DISABLE_THINKING: z + .string() + .default("true") + .transform((v) => v === "true") + .describe( + "Disable LLM chain-of-thought (reasoning/thinking) to speed up AI analysis. Set false to restore thinking.", + ), AI_LLM_EMBEDDING_MODEL: z.string().optional(), AI_LLM_EMBEDDING_MIN_SIMILARITY: z.coerce .number() diff --git a/services/discord-gateway/tests/llmThinkingDisable.test.ts b/services/discord-gateway/tests/llmThinkingDisable.test.ts new file mode 100644 index 0000000..671fe20 --- /dev/null +++ b/services/discord-gateway/tests/llmThinkingDisable.test.ts @@ -0,0 +1,53 @@ +// ═══════════════════════════════════════════════════════════════════════════ +// llmClient buildLlmParams — disable-thinking injection (pure, no network) +// ═══════════════════════════════════════════════════════════════════════════ +import { describe, expect, it } from "vitest"; +import { buildLlmParams } from "../src/modules/ai-moderation/llmClient.js"; + +const baseOpts = { + messages: [{ role: "user" as const, content: "halo" }], +}; + +describe("buildLlmParams — disable-thinking injection", () => { + it("injects no thinking-disabling params when disableThinking is false", () => { + const params = buildLlmParams(baseOpts, false); + expect((params as Record).reasoning_effort).toBeUndefined(); + expect((params as Record).reasoning).toBeUndefined(); + expect( + (params as Record).chat_template_kwargs, + ).toBeUndefined(); + }); + + it("injects all provider variants when disableThinking is true", () => { + const params = buildLlmParams(baseOpts, true) as Record; + expect(params.reasoning_effort).toBe("none"); + expect(params.reasoning).toEqual({ enabled: false }); + expect(params.chat_template_kwargs).toEqual({ enable_thinking: false }); + expect(params.thinking).toEqual({ type: "disabled" }); + }); + + it("keeps caller-supplied max_tokens / jsonResponse / stream intact", () => { + const params = buildLlmParams( + { + ...baseOpts, + max_tokens: 16384, + stream: true, + jsonResponse: { type: "json_object" }, + }, + true, + ); + expect(params.max_tokens).toBe(16384); + expect(params.stream).toBe(true); + expect(params.response_format).toEqual({ type: "json_object" }); + // thinking-disabled params still present + expect( + (params as Record).chat_template_kwargs, + ).toEqual({ enable_thinking: false }); + }); + + it("falls back to config default model when none supplied", () => { + // config.AI_LLM_MODEL defaults to "text" + const params = buildLlmParams(baseOpts, false); + expect(params.model).toBe("text"); + }); +});