feat(mcpedia): Phase 2 — semantic + hybrid search, tRPC/Hono API

- @mcpedia/embeddings: OpenRouter provider (9router /v1, encoding_format float),
  chunkText + embedChunks; EMBED_DIM=2048
- document_chunks table (real[] embedding) — pgvector NOT available on shared
  imrnes Postgres, so cosine is computed in-app (KB-scale fine); pgvector deferred
- indexer: chunk + embed + upsert per document
- @mcpedia/search: semanticSearch (cosine) + hybridSearch (FTS+cosine RRF)
- apps/api: Hono + tRPC v11 (6 procedures), serve on :4020
- MCP: semantic_search + hybrid_search tools (6 total)
- web: keyword/hybrid toggle; .env.example + README + PHASES updated
This commit is contained in:
asepharyana
2026-08-19 19:00:29 +07:00
parent ec3a2867d4
commit 9397303f01
32 changed files with 1120 additions and 42 deletions
+15 -2
View File
@@ -2,12 +2,13 @@ import { db } from "@mcpedia/db";
import { documents } from "@mcpedia/db/schema";
import { parseFile } from "@mcpedia/parser";
import { CONTENT_ROOT } from "@mcpedia/config";
import { listContentFiles } from "@mcpedia/core";
import { listContentFiles, indexChunks } from "@mcpedia/core";
import { join } from "node:path";
async function main() {
const files = listContentFiles();
let indexed = 0;
let chunked = 0;
for (const rel of files) {
const abs = join(CONTENT_ROOT, rel);
const { meta, body } = parseFile(abs, rel);
@@ -47,8 +48,20 @@ async function main() {
});
indexed++;
console.log(` indexed ${rel}`);
// Phase 2: chunk + embed for semantic search.
try {
const n = await indexChunks(meta.slug, body);
chunked += n;
console.log(` embedded ${n} chunks`);
} catch (err) {
console.error(
` embed FAILED for ${meta.slug}: ${err instanceof Error ? err.message : err}`,
);
// Don't abort the whole index over one doc's embedding failure.
}
}
console.log(`indexed ${indexed} documents`);
console.log(`indexed ${indexed} documents, ${chunked} chunks embedded`);
}
main()
+1
View File
@@ -7,6 +7,7 @@
"@mcpedia/config": "workspace:*",
"@mcpedia/core": "workspace:*",
"@mcpedia/db": "workspace:*",
"@mcpedia/embeddings": "workspace:*",
"@mcpedia/parser": "workspace:*",
"@mcpedia/search": "workspace:*",
"drizzle-orm": "^0.38.0",