feat(mcpedia): Phase 2 — semantic + hybrid search, tRPC/Hono API
- @mcpedia/embeddings: OpenRouter provider (9router /v1, encoding_format float), chunkText + embedChunks; EMBED_DIM=2048 - document_chunks table (real[] embedding) — pgvector NOT available on shared imrnes Postgres, so cosine is computed in-app (KB-scale fine); pgvector deferred - indexer: chunk + embed + upsert per document - @mcpedia/search: semanticSearch (cosine) + hybridSearch (FTS+cosine RRF) - apps/api: Hono + tRPC v11 (6 procedures), serve on :4020 - MCP: semantic_search + hybrid_search tools (6 total) - web: keyword/hybrid toggle; .env.example + README + PHASES updated
This commit is contained in:
+15
-2
@@ -2,12 +2,13 @@ import { db } from "@mcpedia/db";
|
||||
import { documents } from "@mcpedia/db/schema";
|
||||
import { parseFile } from "@mcpedia/parser";
|
||||
import { CONTENT_ROOT } from "@mcpedia/config";
|
||||
import { listContentFiles } from "@mcpedia/core";
|
||||
import { listContentFiles, indexChunks } from "@mcpedia/core";
|
||||
import { join } from "node:path";
|
||||
|
||||
async function main() {
|
||||
const files = listContentFiles();
|
||||
let indexed = 0;
|
||||
let chunked = 0;
|
||||
for (const rel of files) {
|
||||
const abs = join(CONTENT_ROOT, rel);
|
||||
const { meta, body } = parseFile(abs, rel);
|
||||
@@ -47,8 +48,20 @@ async function main() {
|
||||
});
|
||||
indexed++;
|
||||
console.log(` indexed ${rel}`);
|
||||
|
||||
// Phase 2: chunk + embed for semantic search.
|
||||
try {
|
||||
const n = await indexChunks(meta.slug, body);
|
||||
chunked += n;
|
||||
console.log(` embedded ${n} chunks`);
|
||||
} catch (err) {
|
||||
console.error(
|
||||
` embed FAILED for ${meta.slug}: ${err instanceof Error ? err.message : err}`,
|
||||
);
|
||||
// Don't abort the whole index over one doc's embedding failure.
|
||||
}
|
||||
}
|
||||
console.log(`indexed ${indexed} documents`);
|
||||
console.log(`indexed ${indexed} documents, ${chunked} chunks embedded`);
|
||||
}
|
||||
|
||||
main()
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
"@mcpedia/config": "workspace:*",
|
||||
"@mcpedia/core": "workspace:*",
|
||||
"@mcpedia/db": "workspace:*",
|
||||
"@mcpedia/embeddings": "workspace:*",
|
||||
"@mcpedia/parser": "workspace:*",
|
||||
"@mcpedia/search": "workspace:*",
|
||||
"drizzle-orm": "^0.38.0",
|
||||
|
||||
Reference in New Issue
Block a user