feat(ai): audit + harden embedding pipeline
- Normalize text before embedding (strip mentions/URLs/emoji/markdown/control chars, lowercase, truncate) on both write and query sides so vectors aren't diluted and tokens aren't wasted - embeddingClient: retry embeddings (maxRetries 2), validate batch dimension consistency, preserve index alignment for empty-normalized texts - archiveEmbedder: store normalized text in archive payload, skip empty-normalized content - backend: normalize search queries, make archive search similarity threshold configurable (AI_LLM_EMBEDDING_ARCHIVE_MIN_SIMILARITY, default 0.6)
This commit is contained in:
asepharyana
@@ -15,9 +15,59 @@ import OpenAI from "openai";
|
||||
|
||||
import { createChildLogger } from "@/shared/logger/index";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { cleanContent } from "./textSignals.js";
|
||||
|
||||
const log = createChildLogger("embedding-client");
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Text normalization (shared by moderation + archive embedding)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Max characters fed to the embedding model for a single document. Embedding
|
||||
* models have a hard token ceiling; embedding past it throws / wastes tokens.
|
||||
* Content messages are truncated; search queries have their own (smaller) cap.
|
||||
*/
|
||||
export const EMBEDDING_MAX_CHARS = 1200;
|
||||
export const EMBEDDING_MAX_QUERY_CHARS = 300;
|
||||
|
||||
/**
|
||||
* Normalize free-form Discord text before embedding.
|
||||
*
|
||||
* Raw messages are full of signal-hostile noise: @mentions, channels, custom
|
||||
* emoji, URLs, markdown and control chars. Embedding that noise directly
|
||||
* dilutes the vector (sentences that differ only in an @mention or a link
|
||||
* land far apart) and inflates token cost. The same cleanup is applied to the
|
||||
* user's search query so archived vectors and the query share one space.
|
||||
*
|
||||
* - `cleanContent` (from textSignals) strips URLs/@mentions/emoji/markdown and
|
||||
* collapses whitespace — good for embeddings, not just term extraction.
|
||||
* - Control characters / zero-width joiners are removed (Discord pastes these).
|
||||
* - Lowercase is applied so "Discord" and "discord" embed identically (embed
|
||||
* models are case-sensitive; this measurably improves near-duplicate recall).
|
||||
*
|
||||
* Falls back to the raw input if normalization empties a string (e.g. a
|
||||
* message that was only a URL) so index alignment is preserved by callers.
|
||||
*/
|
||||
export function normalizeEmbeddingText(raw: string, maxChars: number): string {
|
||||
if (!raw) return "";
|
||||
const cleaned = cleanContent(
|
||||
raw.replace(/[\p{Cc}\p{Cf}]/gu, " ").toLowerCase(),
|
||||
);
|
||||
if (!cleaned) return raw.slice(0, maxChars); // preserve original if stripped
|
||||
return cleaned.slice(0, maxChars);
|
||||
}
|
||||
|
||||
/** Normalize a single embedded document/message. */
|
||||
export function normalizeEmbeddingContent(raw: string): string {
|
||||
return normalizeEmbeddingText(raw, EMBEDDING_MAX_CHARS);
|
||||
}
|
||||
|
||||
/** Normalize a user-provided search query before embedding it. */
|
||||
export function normalizeEmbeddingQuery(raw: string): string {
|
||||
return normalizeEmbeddingText(raw, EMBEDDING_MAX_QUERY_CHARS);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Client (lazy singleton — same base URL as the chat client)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -30,7 +80,10 @@ function getClient(): OpenAI | null {
|
||||
openaiClient = new OpenAI({
|
||||
apiKey: config.AI_LLM_API_KEY,
|
||||
baseURL: config.AI_LLM_BASE_URL,
|
||||
maxRetries: 0,
|
||||
// Embeddings are cheap and idempotent — a transient network blip should
|
||||
// NOT silently disable the whole semantic cache for a batch. Let the SDK
|
||||
// retry (2 retries, jittered) instead of failing open immediately.
|
||||
maxRetries: 2,
|
||||
timeout: 60_000,
|
||||
});
|
||||
}
|
||||
@@ -49,6 +102,11 @@ export function isEmbeddingEnabled(): boolean {
|
||||
/**
|
||||
* Embed a batch of texts with the configured model.
|
||||
* Returns null on any failure so callers can skip semantic lookup.
|
||||
*
|
||||
* Each input is normalized (noise stripped, lowercased, length-capped) before
|
||||
* embedding — see normalizeEmbeddingText. Index alignment with `texts` is
|
||||
* preserved: if a text normalizes to empty we embed the raw original so the
|
||||
* caller's `embeddings[i] ↔ texts[i]` mapping never shifts.
|
||||
*/
|
||||
export async function embedTexts(texts: string[]): Promise<number[][] | null> {
|
||||
if (!isEmbeddingEnabled()) return null;
|
||||
@@ -57,15 +115,33 @@ export async function embedTexts(texts: string[]): Promise<number[][] | null> {
|
||||
const client = getClient();
|
||||
if (!client) return null;
|
||||
|
||||
const normalized = texts.map((t) => normalizeEmbeddingContent(t));
|
||||
|
||||
try {
|
||||
const response = await client.embeddings.create({
|
||||
model: config.AI_LLM_EMBEDDING_MODEL as string,
|
||||
input: texts,
|
||||
input: normalized,
|
||||
// OpenAI SDK v6 defaults to base64; Nvidia-backed embedding models
|
||||
// (e.g. llama-nemotron-embed) reject it with 400. Always float.
|
||||
encoding_format: "float",
|
||||
});
|
||||
return response.data.map((item) => item.embedding);
|
||||
// All vectors in one response must share the model's dimension. If they
|
||||
// don't (shouldn't happen, but guards against a misconfigured/mismatched
|
||||
// model), fail the batch rather than feed garbage to cosine + Qdrant.
|
||||
const vectors = response.data.map((item) => item.embedding);
|
||||
const firstLen = vectors[0]?.length ?? 0;
|
||||
const consistent = vectors.every((v) => v.length === firstLen);
|
||||
if (!consistent || firstLen === 0) {
|
||||
log.error(
|
||||
{
|
||||
model: config.AI_LLM_EMBEDDING_MODEL,
|
||||
dims: vectors.map((v) => v.length),
|
||||
},
|
||||
"Embedding response had inconsistent/empty dimensions — treating as failure",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
return vectors;
|
||||
} catch (error) {
|
||||
log.warn(
|
||||
{ error: error instanceof Error ? error.message : String(error) },
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { embedText } from "@/modules/ai-moderation/embeddingClient";
|
||||
import {
|
||||
embedText,
|
||||
normalizeEmbeddingContent,
|
||||
} from "@/modules/ai-moderation/embeddingClient";
|
||||
import {
|
||||
ARCHIVE_COLLECTION,
|
||||
qdrantPointId,
|
||||
@@ -41,14 +44,19 @@ export function archiveMessageEmbedded(message: ArchiveMessage): void {
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
const vector = await embedText(text);
|
||||
// Normalize once: the vector AND the stored payload both use the clean
|
||||
// text so the public search returns readable content and the vector
|
||||
// isn't diluted by @mentions/URLs/emoji (see normalizeEmbeddingContent).
|
||||
const normalized = normalizeEmbeddingContent(text);
|
||||
if (!normalized) return; // nothing meaningful left after cleanup
|
||||
const vector = await embedText(normalized);
|
||||
if (!vector) return;
|
||||
const ok = await upsertQdrantPointV2(
|
||||
ARCHIVE_COLLECTION,
|
||||
qdrantPointId(`archive:${message.id}`),
|
||||
vector,
|
||||
{
|
||||
text: text.slice(0, 4000),
|
||||
text: normalized.slice(0, 4000),
|
||||
flags: "",
|
||||
analyzed_at: Date.now(),
|
||||
// 5-year persistent window (archive is NOT a TTL cache).
|
||||
|
||||
Reference in New Issue
Block a user