refactor(ai): remove semantic embedding cache + Qdrant vector store

Hapus seluruh fitur embedding/Qdrant (tidak dipakai lagi):

- gateway: drop embeddingClient.ts, qdrantClient.ts, archiveEmbedder.ts
  dan tes qdrantEnsure.test.ts; moderationOrchestrator kembali ke
  exact-hash cache -> LLM (tanpa phase-2 semantic lookup); textCacheStore
  kehilangan findSimilarTextModeration / parseQdrantVerdict /
  isSemanticBandAccepted / upsertBareKeyToQdrant; cache-prune hanya
  menyapu Postgres.
- backend: drop embed.ts + qdrant.ts, endpoint messages.semanticSearch
  dan schema/type terkait; kolom embedding dilepas dari schema
  text_analysis_cache.
- frontend: hapus toggle EXACT/SEMANTIC, hook useSemanticSearch,
  API client + tipe SemanticSearchResult.
- config: buang AI_LLM_EMBEDDING_* dan QDRANT_* (env + .env.example).
- docs: ARCHITECTURE.md / AGENTS.md / README.md / diagram arsitektur
  disesuaikan (LLM caller - vision, cache = exact-hash saja).

Verifikasi: tsc 0 (backend, gateway, frontend); bun test 135 pass +
37 pass, 0 fail; biome 0 error.
This commit is contained in:
mytheclipsebotreview
2026-09-25 01:38:51 +07:00
parent fdc7f01c26
commit 2b6ec59286
30 changed files with 41 additions and 1987 deletions
@@ -1,11 +1,9 @@
// ═══════════════════════════════════════════════════════════════════════════
// Semantic two-band acceptance + global exact-cache reuse guard
// Global exact-cache reuse guard
// ═══════════════════════════════════════════════════════════════════════════
// Design (2026-08-24): cache hits may be served MORE aggressively for
// verdicts that cannot trigger enforcement actions, and NEVER more
// aggressively for actionable ones. Two layers enforce this:
// - isSemanticBandAccepted: similarity thresholds differ by verdict class
// (clean band 0.92 default vs strict actionable band 0.97 default).
// aggressively for actionable ones:
// - isGloballyReusableCleanVerdict: context-free (cross-channel) reuse of
// the legacy bare key only for clean / flagless / action=none verdicts
// with high confidence and bounded age.
@@ -31,64 +29,6 @@ function makeVerdict(
};
}
describe("isSemanticBandAccepted", () => {
it("accepts a non-actionable clean verdict at the loose clean band", () => {
// Default AI_LLM_EMBEDDING_MIN_SIMILARITY_CLEAN = 0.92.
expect(isBandAccept(makeVerdict(), 0.93)).toBe(true);
});
it("accepts a clean verdict exactly at the clean band boundary", () => {
expect(isBandAccept(makeVerdict({ confidence: 0.99 }), 0.92)).toBe(true);
});
it("rejects a clean verdict below the clean band", () => {
expect(isBandAccept(makeVerdict(), 0.91)).toBe(false);
});
it("rejects an actionable flagged verdict between the bands", () => {
// 0.93 >= clean band BUT < strict band → must NOT be served.
expect(
isBandAccept(
makeVerdict({ status: "flagged", flags: ["hate_speech"] }),
0.93,
),
).toBe(false);
});
it("accepts a flagged verdict at the strict band", () => {
expect(
isBandAccept(
makeVerdict({ status: "flagged", flags: ["hate_speech"] }),
0.98,
),
).toBe(true);
});
it("rejects a warn verdict below the strict band", () => {
expect(
isBandAccept(
makeVerdict({ status: "warn", recommendedAction: "warn" }),
0.96,
),
).toBe(false);
});
it("treats a clean verdict WITH flags as actionable (strict band)", () => {
expect(isBandAccept(makeVerdict({ flags: ["borderline"] }), 0.93)).toBe(
false,
);
});
it("treats a clean verdict with a non-none action as actionable", () => {
expect(
isBandAccept(makeVerdict({ recommendedAction: "review" }), 0.93),
).toBe(false);
});
});
// Import indirection so the describe block reads cleanly.
import { isSemanticBandAccepted as isBandAccept } from "../src/modules/ai-moderation/textCacheStore.js";
describe("isGloballyReusableCleanVerdict", () => {
it("accepts a fresh, confident, flagless clean verdict", () => {
const v = makeVerdict({ confidence: 0.9 });
@@ -1,82 +0,0 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
// The qdrant client reads config at import time — we only need the
// reset-on-failure behavior, so mock global fetch and import fresh.
import {
ensureQdrantCollection,
resetQdrantCollectionCache,
} from "../src/modules/ai-moderation/qdrantClient.js";
function jsonResponse(body: unknown, ok = true, status = 200): Response {
return {
ok,
status,
text: async () => JSON.stringify(body),
} as unknown as Response;
}
describe("ensureQdrantCollection retry-on-failure", () => {
beforeEach(() => {
resetQdrantCollectionCache();
vi.restoreAllMocks();
});
afterEach(() => {
resetQdrantCollectionCache();
vi.restoreAllMocks();
});
it("returns false on failure and does NOT get stuck — retries on next call", async () => {
// First call: GET collection fails hard (not a 404) → ensure rejects.
const fetchMock = vi
.spyOn(globalThis, "fetch")
.mockRejectedValueOnce(new Error("network down"))
// Second call: GET returns a matching-size collection → success.
.mockResolvedValueOnce(
jsonResponse({
result: { config: { params: { vectors: { size: 3072 } } } },
}),
);
const first = await ensureQdrantCollection(3072);
expect(first).toBe(false);
// Without the fix, the second call returns the memoised rejected promise
// and fetch is never called again. With the fix, it retries.
const second = await ensureQdrantCollection(3072);
expect(second).toBe(true);
expect(fetchMock).toHaveBeenCalledTimes(2);
});
it("recreates collection when vector size changes (DELETE + PUT)", async () => {
const fetchMock = vi
.spyOn(globalThis, "fetch")
.mockResolvedValueOnce(
jsonResponse({
result: { config: { params: { vectors: { size: 2048 } } } },
}),
) // GET old size
.mockResolvedValueOnce(jsonResponse({ result: true })) // DELETE
.mockResolvedValueOnce(jsonResponse({ result: true })); // PUT
const ok = await ensureQdrantCollection(3072);
expect(ok).toBe(true);
const methods = fetchMock.mock.calls.map(
(c) => (c[1] as RequestInit).method,
);
expect(methods).toEqual(["GET", "DELETE", "PUT"]);
});
it("is idempotent when the collection already matches", async () => {
const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValueOnce(
jsonResponse({
result: { config: { params: { vectors: { size: 3072 } } } },
}),
);
const ok = await ensureQdrantCollection(3072);
expect(ok).toBe(true);
expect(fetchMock).toHaveBeenCalledTimes(1);
});
});
@@ -6,10 +6,10 @@
// ["conflict_instigation"]) fell into the legacy `flags.length === 0 ?
// clean : flagged` branch and was read back as FLAGGED. Downstream this
// broke auto-delete eligibility gating and mislabelled warnings on the
// dashboard. parseQdrantVerdict had the same narrowing (warn → clean).
// dashboard.
//
// Fix: normalizeStoredStatus() accepts the full clean/warn/flagged union in
// BOTH readers; unknown/legacy values still derive from flags.
// Fix: normalizeStoredStatus() accepts the full clean/warn/flagged union;
// unknown/legacy values still derive from flags.
import { describe, expect, it } from "vitest";
import { normalizeStoredStatus } from "../src/modules/ai-moderation/textCacheStore.js";