feat(mcpedia): Phase 3 — async indexing (BullMQ), git-sync webhook, revisions, MCP Resources

- packages/queue: ioredis singleton + BullMQ Queue/Worker (prefix mcpedia:
  on shared imrnes Redis :6379); apps/worker runs startWorker()
- @mcpedia/core: indexContentFile/runFullIndex (single indexing entry point
  shared by script/worker/hook) + revision.service (list/get/restore)
- document_revisions table (migration 0002) — snapshots only on body change
- apps/api: POST /hooks/reindex + /hooks/index webhooks; tRPC revisions,
  getRevision, restoreRevision, jobStatus, queueStatus
- apps/mcp: register MCP Resources mcpedia://docs{/,+slug/chunks/revisions}
  ({+slug} RFC6570 reserved expansion for slugs containing /)
- apps/mcp zod pinned to ^4 to match MCP SDK 1.30 compiled types
  (resolves registerTool TS2589/ShapeOutput skew)
- scripts/enqueue.ts one-shot job enqueue helper; indexer refactored to runFullIndex
- PHASES.md/README/.env.example/docs updated

apps/api/package.json

@@ -13,6 +13,8 @@
     "@hono/node-server": "^1.13.0",
     "@mcpedia/config": "workspace:*",
     "@mcpedia/core": "workspace:*",
+    "@mcpedia/db": "workspace:*",
+    "@mcpedia/queue": "workspace:*",
     "@trpc/server": "^11.0.0",
     "hono": "^4.6.0",
     "zod": "^3.23.8"

apps/api/src/index.ts

@@ -4,12 +4,31 @@ import { fetchRequestHandler } from "@trpc/server/adapters/fetch";
 import { db } from "@mcpedia/db";
 import { appRouter } from "./router";
 import type { Context } from "./trpc";
+import { enqueueIndexDoc, enqueueFullIndex } from "@mcpedia/queue";
 
 const app = new Hono();
 
 // Health check.
 app.get("/health", (c) => c.json({ ok: true }));
 
+// --- Phase 3: Git synchronization hook ---
+// POST /hooks/reindex        -> enqueue a full-corpus reindex (git push webhook)
+// POST /hooks/index?slug=... -> enqueue a single document reindex
+// Returns the created job id(s). The worker processes them asynchronously.
+app.post("/hooks/reindex", async (c) => {
+  const job = await enqueueFullIndex("git-push");
+  return c.json({ ok: true, jobId: job.id, kind: "full" });
+});
+
+app.post("/hooks/index", async (c) => {
+  const slug = c.req.query("slug");
+  if (!slug) return c.json({ ok: false, error: "slug query param required" }, 400);
+  // slug is the relative path without extension, e.g. docs/websocket/contract
+  const relPath = slug.endsWith(".md") || slug.endsWith(".mdx") ? slug : `${slug}.md`;
+  const job = await enqueueIndexDoc(relPath, "git-push");
+  return c.json({ ok: true, jobId: job.id, kind: "doc", relPath });
+});
+
 // Mount tRPC at /trpc/*. The fetch adapter is the canonical Bun/Hono adapter.
 app.all("/trpc/*", (c) =>
   fetchRequestHandler({

apps/api/src/router.ts

@@ -7,7 +7,12 @@ import {
   keywordSearch,
   listDocuments,
   semanticSearch,
+  listRevisions,
+  getRevision,
+  restoreRevision,
 } from "@mcpedia/core";
+import { getQueue, INDEX_QUEUE } from "@mcpedia/queue";
+import { getConnection, BULLMQ_PREFIX } from "@mcpedia/queue/client";
 
 export const appRouter = router({
   search: publicProcedure
@@ -33,6 +38,58 @@ export const appRouter = router({
   related: publicProcedure
     .input(z.object({ slug: z.string(), limit: z.number().int().min(1).max(20).default(5) }))
     .query(async ({ input }) => getRelated(input.slug, input.limit)),
+
+  // --- Phase 3: revisions ---
+  revisions: publicProcedure
+    .input(z.object({ slug: z.string(), limit: z.number().int().min(1).max(50).default(20) }))
+    .query(async ({ input }) => listRevisions(input.slug, input.limit)),
+
+  getRevision: publicProcedure
+    .input(z.object({ id: z.string() }))
+    .query(async ({ input }) => getRevision(input.id)),
+
+  restoreRevision: publicProcedure
+    .input(z.object({ id: z.string() }))
+    .mutation(async ({ input }) => restoreRevision(input.id)),
+
+  // --- Phase 3: async job status ---
+  jobStatus: publicProcedure
+    .input(z.object({ id: z.string() }))
+    .query(async ({ input }) => {
+      const queue = getQueue();
+      const job = await queue.getJob(input.id);
+      if (!job) return { exists: false };
+      const state = await job.getState();
+      const failedReason = job.failedReason;
+      const returnvalue = job.returnvalue;
+      const progress = job.progress;
+      return {
+        exists: true,
+        id: job.id,
+        name: job.name,
+        state,
+        progress,
+        failedReason,
+        returnvalue,
+        attemptsMade: job.attemptsMade,
+      };
+    }),
+
+  queueStatus: publicProcedure.query(async () => {
+    const queue = getQueue();
+    const [waiting, active, completed, failed, delayed] = await Promise.all([
+      queue.getWaitingCount(),
+      queue.getActiveCount(),
+      queue.getCompletedCount(),
+      queue.getFailedCount(),
+      queue.getDelayedCount(),
+    ]);
+    return {
+      queue: INDEX_QUEUE,
+      prefix: BULLMQ_PREFIX,
+      counts: { waiting, active, completed, failed, delayed },
+    };
+  }),
 });
 
 export type AppRouter = typeof appRouter;

apps/mcp/package.json

@@ -17,7 +17,7 @@
     "@mcpedia/core": "workspace:*",
     "@mcpedia/search": "workspace:*",
     "@modelcontextprotocol/sdk": "^1.29.0",
-    "zod": "^3.23.8"
+    "zod": "^4.0.0"
   },
   "devDependencies": {
     "@types/node": "^20",

apps/mcp/src/index.ts

@@ -1,7 +1,19 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { z } from "zod";
-import { listDocuments, getDocument, getRelated, semanticSearch, hybridSearch, keywordSearch } from "@mcpedia/core";
+import {
+  listDocuments,
+  getDocument,
+  getRelated,
+  semanticSearch,
+  hybridSearch,
+  keywordSearch,
+  listRevisions,
+  readContentFile,
+} from "@mcpedia/core";
+import { CONTENT_ROOT } from "@mcpedia/config";
+import { join } from "node:path";
 
 export function createMcpServer(): McpServer {
   const server = new McpServer({
@@ -119,6 +131,116 @@ export function createMcpServer(): McpServer {
     },
   );
 
+  // --- Phase 3: MCP Resources (read-only knowledge base surfaced via URIs) ---
+  // mcpedia://docs                       -> list all published documents
+  // mcpedia://docs/{slug}                -> full markdown body (from disk)
+  // mcpedia://docs/{slug}/chunks         -> chunked preview (semantic slices)
+  // mcpedia://docs/{slug}/revisions      -> revision history summary
+  server.registerResource(
+    "mcpedia-docs-list",
+    "mcpedia://docs",
+    {
+      title: "MCPedia document index",
+      description: "List of all published documents in the knowledge base.",
+      mimeType: "application/json",
+    },
+    async (uri) => {
+      const docs = await listDocuments();
+      return {
+        contents: [
+          {
+            uri: uri.href,
+            mimeType: "application/json",
+            text: JSON.stringify(docs, null, 2),
+          },
+        ],
+      };
+    },
+  );
+
+  server.registerResource(
+    "mcpedia-doc-chunks",
+    new ResourceTemplate("mcpedia://docs/{+slug}/chunks", { list: undefined }),
+    {
+      title: "MCPedia document chunks",
+      description: "Preview of the embedded semantic chunks for a document.",
+      mimeType: "application/json",
+    },
+    async (uri, vars) => {
+      const slug = String(vars.slug);
+      const doc = await getDocument(slug);
+      if (!doc) throw new Error(`Document not found: ${slug}`);
+      // Chunk the body the same way the indexer does (size 1000 / overlap 150)
+      // so the resource mirrors what semantic search actually sees.
+      const { chunkText } = await import("@mcpedia/embeddings");
+      const chunks = chunkText(doc.body, { size: 1000, overlap: 150 });
+      return {
+        contents: [
+          {
+            uri: uri.href,
+            mimeType: "application/json",
+            text: JSON.stringify(
+              chunks.map((c, i) => ({ index: i, length: c.length, preview: c.slice(0, 200) })),
+              null,
+              2,
+            ),
+          },
+        ],
+      };
+    },
+  );
+
+  server.registerResource(
+    "mcpedia-doc-revisions",
+    new ResourceTemplate("mcpedia://docs/{+slug}/revisions", { list: undefined }),
+    {
+      title: "MCPedia document revisions",
+      description: "Revision history summary for a document.",
+      mimeType: "application/json",
+    },
+    async (uri, vars) => {
+      const slug = String(vars.slug);
+      const revs = await listRevisions(slug, 20);
+      return {
+        contents: [
+          {
+            uri: uri.href,
+            mimeType: "application/json",
+            text: JSON.stringify(revs, null, 2),
+          },
+        ],
+      };
+    },
+  );
+
+  // Registered LAST: the bare {+slug} template is greedy and would otherwise
+  // swallow /chunks and /revisions URIs. Specific templates must match first.
+  server.registerResource(
+    "mcpedia-doc",
+    new ResourceTemplate("mcpedia://docs/{+slug}", { list: undefined }),
+    {
+      title: "MCPedia document",
+      description: "Full markdown body of a single document, read from disk (source of truth).",
+      mimeType: "text/markdown",
+    },
+    async (uri, vars) => {
+      const slug = String(vars.slug);
+      const doc = await getDocument(slug);
+      if (!doc) {
+        throw new Error(`Document not found: ${slug}`);
+      }
+      return {
+        contents: [
+          {
+            uri: uri.href,
+            mimeType: "text/markdown",
+            text: doc.body,
+          },
+        ],
+      };
+    },
+  );
+
   return server;
 }
 

apps/mcp/src/smoke.test.ts

@@ -92,6 +92,35 @@ async function main() {
   }
   console.log(`hybrid_search => ${hybHits.length} docs, top: ${hybHits[0].doc.slug}`);
 
+  // 8) resources: list
+  const resList = await client.listResources();
+  const resNames = resList.resources.map((r: any) => r.name).sort();
+  console.log("resources:", resNames.join(", "));
+  if (!resNames.includes("mcpedia-docs-list")) {
+    throw new Error("expected mcpedia-docs-list resource");
+  }
+
+  // 9) resource: read the docs list (must not throw, returns JSON content)
+  const readList = await client.readResource({ uri: "mcpedia://docs" });
+  const listText = (readList.contents as any)[0].text;
+  if (!listText.includes("docs/websocket/contract")) {
+    throw new Error("mcpedia://docs did not list the websocket contract doc");
+  }
+  console.log("readResource(mcpedia://docs) => ok");
+
+  // 10) resource: read a single doc body + revisions
+  const readDoc = await client.readResource({ uri: "mcpedia://docs/docs/websocket/contract" });
+  const docText = (readDoc.contents as any)[0].text;
+  if (!docText.includes("WebSocket Contract")) {
+    throw new Error("mcpedia://docs/{slug} returned unexpected body");
+  }
+  console.log("readResource(mcpedia://docs/docs/websocket/contract) => ok");
+
+  const readRev = await client.readResource({
+    uri: "mcpedia://docs/docs/websocket/contract/revisions",
+  });
+  console.log("readResource(.../revisions) => ok");
+
   await client.close();
   await server.close();
   console.log("\nSMOKE OK");

apps/worker/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@mcpedia/worker",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "start": "bun run src/index.ts",
+    "lint": "tsc --noEmit",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@mcpedia/config": "workspace:*",
+    "@mcpedia/core": "workspace:*",
+    "@mcpedia/db": "workspace:*",
+    "@mcpedia/queue": "workspace:*"
+  },
+  "devDependencies": {
+    "typescript": "^5.6.0"
+  }
+}

apps/worker/src/index.ts

@@ -0,0 +1,12 @@
+import { startWorker } from "@mcpedia/queue/worker";
+
+// Keep the process alive: the worker listens on the BullMQ queue until a
+// SIGINT/SIGTERM closes it (handled inside startWorker).
+const worker = await startWorker();
+
+// Heartbeat so the supervisor/operator can see liveness without scraping logs.
+const heartbeat = setInterval(() => {
+  console.log(`[worker] alive, ${worker.name} queue="${worker.name}"`);
+}, 30_000);
+
+worker.on("closed", () => clearInterval(heartbeat));

apps/worker/tsconfig.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "paths": {
+      "@mcpedia/db": ["../../packages/db/src/index.ts"],
+      "@mcpedia/db/schema": ["../../packages/db/src/schema.ts"],
+      "@mcpedia/*": ["../../packages/*"]
+    }
+  },
+  "include": ["src/**/*.ts"]
+}