refactor(ai): remove semantic embedding cache + Qdrant vector store
Hapus seluruh fitur embedding/Qdrant (tidak dipakai lagi): - gateway: drop embeddingClient.ts, qdrantClient.ts, archiveEmbedder.ts dan tes qdrantEnsure.test.ts; moderationOrchestrator kembali ke exact-hash cache -> LLM (tanpa phase-2 semantic lookup); textCacheStore kehilangan findSimilarTextModeration / parseQdrantVerdict / isSemanticBandAccepted / upsertBareKeyToQdrant; cache-prune hanya menyapu Postgres. - backend: drop embed.ts + qdrant.ts, endpoint messages.semanticSearch dan schema/type terkait; kolom embedding dilepas dari schema text_analysis_cache. - frontend: hapus toggle EXACT/SEMANTIC, hook useSemanticSearch, API client + tipe SemanticSearchResult. - config: buang AI_LLM_EMBEDDING_* dan QDRANT_* (env + .env.example). - docs: ARCHITECTURE.md / AGENTS.md / README.md / diagram arsitektur disesuaikan (LLM caller - vision, cache = exact-hash saja). Verifikasi: tsc 0 (backend, gateway, frontend); bun test 135 pass + 37 pass, 0 fail; biome 0 error.
This commit is contained in:
@@ -1,11 +1,9 @@
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Semantic two-band acceptance + global exact-cache reuse guard
|
||||
// Global exact-cache reuse guard
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Design (2026-08-24): cache hits may be served MORE aggressively for
|
||||
// verdicts that cannot trigger enforcement actions, and NEVER more
|
||||
// aggressively for actionable ones. Two layers enforce this:
|
||||
// - isSemanticBandAccepted: similarity thresholds differ by verdict class
|
||||
// (clean band 0.92 default vs strict actionable band 0.97 default).
|
||||
// aggressively for actionable ones:
|
||||
// - isGloballyReusableCleanVerdict: context-free (cross-channel) reuse of
|
||||
// the legacy bare key only for clean / flagless / action=none verdicts
|
||||
// with high confidence and bounded age.
|
||||
@@ -31,64 +29,6 @@ function makeVerdict(
|
||||
};
|
||||
}
|
||||
|
||||
describe("isSemanticBandAccepted", () => {
|
||||
it("accepts a non-actionable clean verdict at the loose clean band", () => {
|
||||
// Default AI_LLM_EMBEDDING_MIN_SIMILARITY_CLEAN = 0.92.
|
||||
expect(isBandAccept(makeVerdict(), 0.93)).toBe(true);
|
||||
});
|
||||
|
||||
it("accepts a clean verdict exactly at the clean band boundary", () => {
|
||||
expect(isBandAccept(makeVerdict({ confidence: 0.99 }), 0.92)).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects a clean verdict below the clean band", () => {
|
||||
expect(isBandAccept(makeVerdict(), 0.91)).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects an actionable flagged verdict between the bands", () => {
|
||||
// 0.93 >= clean band BUT < strict band → must NOT be served.
|
||||
expect(
|
||||
isBandAccept(
|
||||
makeVerdict({ status: "flagged", flags: ["hate_speech"] }),
|
||||
0.93,
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts a flagged verdict at the strict band", () => {
|
||||
expect(
|
||||
isBandAccept(
|
||||
makeVerdict({ status: "flagged", flags: ["hate_speech"] }),
|
||||
0.98,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects a warn verdict below the strict band", () => {
|
||||
expect(
|
||||
isBandAccept(
|
||||
makeVerdict({ status: "warn", recommendedAction: "warn" }),
|
||||
0.96,
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("treats a clean verdict WITH flags as actionable (strict band)", () => {
|
||||
expect(isBandAccept(makeVerdict({ flags: ["borderline"] }), 0.93)).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("treats a clean verdict with a non-none action as actionable", () => {
|
||||
expect(
|
||||
isBandAccept(makeVerdict({ recommendedAction: "review" }), 0.93),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// Import indirection so the describe block reads cleanly.
|
||||
import { isSemanticBandAccepted as isBandAccept } from "../src/modules/ai-moderation/textCacheStore.js";
|
||||
|
||||
describe("isGloballyReusableCleanVerdict", () => {
|
||||
it("accepts a fresh, confident, flagless clean verdict", () => {
|
||||
const v = makeVerdict({ confidence: 0.9 });
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
// The qdrant client reads config at import time — we only need the
|
||||
// reset-on-failure behavior, so mock global fetch and import fresh.
|
||||
import {
|
||||
ensureQdrantCollection,
|
||||
resetQdrantCollectionCache,
|
||||
} from "../src/modules/ai-moderation/qdrantClient.js";
|
||||
|
||||
function jsonResponse(body: unknown, ok = true, status = 200): Response {
|
||||
return {
|
||||
ok,
|
||||
status,
|
||||
text: async () => JSON.stringify(body),
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
describe("ensureQdrantCollection retry-on-failure", () => {
|
||||
beforeEach(() => {
|
||||
resetQdrantCollectionCache();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetQdrantCollectionCache();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("returns false on failure and does NOT get stuck — retries on next call", async () => {
|
||||
// First call: GET collection fails hard (not a 404) → ensure rejects.
|
||||
const fetchMock = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockRejectedValueOnce(new Error("network down"))
|
||||
// Second call: GET returns a matching-size collection → success.
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({
|
||||
result: { config: { params: { vectors: { size: 3072 } } } },
|
||||
}),
|
||||
);
|
||||
|
||||
const first = await ensureQdrantCollection(3072);
|
||||
expect(first).toBe(false);
|
||||
|
||||
// Without the fix, the second call returns the memoised rejected promise
|
||||
// and fetch is never called again. With the fix, it retries.
|
||||
const second = await ensureQdrantCollection(3072);
|
||||
expect(second).toBe(true);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("recreates collection when vector size changes (DELETE + PUT)", async () => {
|
||||
const fetchMock = vi
|
||||
.spyOn(globalThis, "fetch")
|
||||
.mockResolvedValueOnce(
|
||||
jsonResponse({
|
||||
result: { config: { params: { vectors: { size: 2048 } } } },
|
||||
}),
|
||||
) // GET old size
|
||||
.mockResolvedValueOnce(jsonResponse({ result: true })) // DELETE
|
||||
.mockResolvedValueOnce(jsonResponse({ result: true })); // PUT
|
||||
|
||||
const ok = await ensureQdrantCollection(3072);
|
||||
expect(ok).toBe(true);
|
||||
|
||||
const methods = fetchMock.mock.calls.map(
|
||||
(c) => (c[1] as RequestInit).method,
|
||||
);
|
||||
expect(methods).toEqual(["GET", "DELETE", "PUT"]);
|
||||
});
|
||||
|
||||
it("is idempotent when the collection already matches", async () => {
|
||||
const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(
|
||||
jsonResponse({
|
||||
result: { config: { params: { vectors: { size: 3072 } } } },
|
||||
}),
|
||||
);
|
||||
|
||||
const ok = await ensureQdrantCollection(3072);
|
||||
expect(ok).toBe(true);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -6,10 +6,10 @@
|
||||
// ["conflict_instigation"]) fell into the legacy `flags.length === 0 ?
|
||||
// clean : flagged` branch and was read back as FLAGGED. Downstream this
|
||||
// broke auto-delete eligibility gating and mislabelled warnings on the
|
||||
// dashboard. parseQdrantVerdict had the same narrowing (warn → clean).
|
||||
// dashboard.
|
||||
//
|
||||
// Fix: normalizeStoredStatus() accepts the full clean/warn/flagged union in
|
||||
// BOTH readers; unknown/legacy values still derive from flags.
|
||||
// Fix: normalizeStoredStatus() accepts the full clean/warn/flagged union;
|
||||
// unknown/legacy values still derive from flags.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { normalizeStoredStatus } from "../src/modules/ai-moderation/textCacheStore.js";
|
||||
|
||||
|
||||
Reference in New Issue
Block a user