refactor(ai-moderation): enhance llmClient flexibility and idempotency
Update the LLM client to support streaming responses and improve database migration idempotency. - Add `stream` support to `llmChat` with dynamic chunk parsing for compatibility across different LLM providers. - Remove hardcoded default temperature and top_p to allow for more flexible parameter passing. - Update database migrations to use `IF NOT EXISTS` for tables and indexes to prevent errors during re-runs.
This commit is contained in:
@@ -1,11 +1,11 @@
|
|||||||
CREATE TABLE "channel_cultures" (
|
CREATE TABLE IF NOT EXISTS "channel_cultures" (
|
||||||
"channel_id" text PRIMARY KEY NOT NULL,
|
"channel_id" text PRIMARY KEY NOT NULL,
|
||||||
"guild_id" text NOT NULL,
|
"guild_id" text NOT NULL,
|
||||||
"culture_summary" text NOT NULL,
|
"culture_summary" text NOT NULL,
|
||||||
"last_analyzed_at" bigint NOT NULL
|
"last_analyzed_at" bigint NOT NULL
|
||||||
);
|
);
|
||||||
--> statement-breakpoint
|
--> statement-breakpoint
|
||||||
CREATE TABLE "user_reputations" (
|
CREATE TABLE IF NOT EXISTS "user_reputations" (
|
||||||
"user_id" text PRIMARY KEY NOT NULL,
|
"user_id" text PRIMARY KEY NOT NULL,
|
||||||
"guild_id" text NOT NULL,
|
"guild_id" text NOT NULL,
|
||||||
"trust_score" integer DEFAULT 50 NOT NULL,
|
"trust_score" integer DEFAULT 50 NOT NULL,
|
||||||
@@ -16,6 +16,6 @@ CREATE TABLE "user_reputations" (
|
|||||||
"updated_at" bigint NOT NULL
|
"updated_at" bigint NOT NULL
|
||||||
);
|
);
|
||||||
--> statement-breakpoint
|
--> statement-breakpoint
|
||||||
CREATE INDEX "idx_channel_cultures_guild_id" ON "channel_cultures" USING btree ("guild_id");--> statement-breakpoint
|
CREATE INDEX IF NOT EXISTS "idx_channel_cultures_guild_id" ON "channel_cultures" USING btree ("guild_id");--> statement-breakpoint
|
||||||
CREATE INDEX "idx_user_reputations_guild_id" ON "user_reputations" USING btree ("guild_id");--> statement-breakpoint
|
CREATE INDEX IF NOT EXISTS "idx_user_reputations_guild_id" ON "user_reputations" USING btree ("guild_id");--> statement-breakpoint
|
||||||
CREATE INDEX "idx_user_reputations_trust_score" ON "user_reputations" USING btree ("trust_score");
|
CREATE INDEX IF NOT EXISTS "idx_user_reputations_trust_score" ON "user_reputations" USING btree ("trust_score");
|
||||||
@@ -33,12 +33,6 @@ function getClient(): OpenAI | null {
|
|||||||
return openaiClient;
|
return openaiClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Shared defaults
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
const DEFAULT_TEMPERATURE = 0.2;
|
|
||||||
const DEFAULT_TOP_P = 0.95;
|
|
||||||
const DEFAULT_RETRIES = 2;
|
const DEFAULT_RETRIES = 2;
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -60,6 +54,8 @@ export interface LlmCallOpts {
|
|||||||
jsonResponse?: { type: "json_object" };
|
jsonResponse?: { type: "json_object" };
|
||||||
/** Extra retries beyond DEFAULT_RETRIES (default 2). */
|
/** Extra retries beyond DEFAULT_RETRIES (default 2). */
|
||||||
retries?: number;
|
retries?: number;
|
||||||
|
/** Whether to use streaming (if true, will consume stream and return aggregated result) */
|
||||||
|
stream?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -77,33 +73,74 @@ export async function llmChat(
|
|||||||
const {
|
const {
|
||||||
messages,
|
messages,
|
||||||
model = config.AI_LLM_MODEL,
|
model = config.AI_LLM_MODEL,
|
||||||
max_tokens = 8192,
|
max_tokens,
|
||||||
temperature = DEFAULT_TEMPERATURE,
|
|
||||||
top_p = DEFAULT_TOP_P,
|
|
||||||
jsonResponse,
|
|
||||||
retries = DEFAULT_RETRIES,
|
|
||||||
} = opts;
|
|
||||||
|
|
||||||
const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming =
|
|
||||||
{
|
|
||||||
model,
|
|
||||||
messages,
|
|
||||||
temperature,
|
temperature,
|
||||||
top_p,
|
top_p,
|
||||||
max_tokens,
|
jsonResponse,
|
||||||
stream: false,
|
retries = DEFAULT_RETRIES,
|
||||||
|
stream = false,
|
||||||
|
} = opts;
|
||||||
|
|
||||||
|
const params: any = {
|
||||||
|
model,
|
||||||
|
messages,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Attach optional parameters only if explicitly provided to maintain
|
||||||
|
// maximum compatibility with various LLM providers and local APIs.
|
||||||
|
if (stream !== undefined) params.stream = stream;
|
||||||
|
if (temperature !== undefined) params.temperature = temperature;
|
||||||
|
if (top_p !== undefined) params.top_p = top_p;
|
||||||
|
if (max_tokens !== undefined) params.max_tokens = max_tokens;
|
||||||
|
|
||||||
if (jsonResponse) {
|
if (jsonResponse) {
|
||||||
(params as unknown as Record<string, unknown>).response_format =
|
params.response_format = jsonResponse;
|
||||||
jsonResponse;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return retryWithBackoff(
|
return retryWithBackoff(
|
||||||
async () => {
|
async () => {
|
||||||
return withLlmConcurrency(async () =>
|
return withLlmConcurrency(async () => {
|
||||||
client.chat.completions.create(params),
|
const response = await client.chat.completions.create(params);
|
||||||
);
|
if (stream) {
|
||||||
|
let content = "";
|
||||||
|
let finishReason = "stop";
|
||||||
|
for await (const chunk of response as any) {
|
||||||
|
const choice = chunk?.choices?.[0];
|
||||||
|
|
||||||
|
// Dynamic parsing to support multiple providers (OpenAI, Ollama, Groq, Anthropic via proxy, etc.)
|
||||||
|
const textChunk =
|
||||||
|
choice?.delta?.content ||
|
||||||
|
choice?.message?.content ||
|
||||||
|
choice?.text ||
|
||||||
|
chunk?.message?.content ||
|
||||||
|
chunk?.response ||
|
||||||
|
chunk?.content ||
|
||||||
|
"";
|
||||||
|
|
||||||
|
content += textChunk;
|
||||||
|
|
||||||
|
const fr = choice?.finish_reason || chunk?.finish_reason;
|
||||||
|
if (fr) {
|
||||||
|
finishReason = fr;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
id: 'stream-aggregated',
|
||||||
|
choices: [
|
||||||
|
{
|
||||||
|
message: { role: 'assistant', content, refusal: null },
|
||||||
|
finish_reason: finishReason,
|
||||||
|
index: 0,
|
||||||
|
logprobs: null,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
created: Math.floor(Date.now() / 1000),
|
||||||
|
model: model,
|
||||||
|
object: 'chat.completion',
|
||||||
|
} as OpenAI.Chat.Completions.ChatCompletion;
|
||||||
|
}
|
||||||
|
return response as OpenAI.Chat.Completions.ChatCompletion;
|
||||||
|
});
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
retries,
|
retries,
|
||||||
|
|||||||
Reference in New Issue
Block a user