feat(mcpedia): Phase 3 — async indexing (BullMQ), git-sync webhook, revisions, MCP Resources

- packages/queue: ioredis singleton + BullMQ Queue/Worker (prefix mcpedia:
  on shared imrnes Redis :6379); apps/worker runs startWorker()
- @mcpedia/core: indexContentFile/runFullIndex (single indexing entry point
  shared by script/worker/hook) + revision.service (list/get/restore)
- document_revisions table (migration 0002) — snapshots only on body change
- apps/api: POST /hooks/reindex + /hooks/index webhooks; tRPC revisions,
  getRevision, restoreRevision, jobStatus, queueStatus
- apps/mcp: register MCP Resources mcpedia://docs{/,+slug/chunks/revisions}
  ({+slug} RFC6570 reserved expansion for slugs containing /)
- apps/mcp zod pinned to ^4 to match MCP SDK 1.30 compiled types
  (resolves registerTool TS2589/ShapeOutput skew)
- scripts/enqueue.ts one-shot job enqueue helper; indexer refactored to runFullIndex
- PHASES.md/README/.env.example/docs updated

packages/queue/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@mcpedia/queue",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts",
+    "./client": "./src/client.ts",
+    "./queue": "./src/queue.ts",
+    "./worker": "./src/worker.ts"
+  },
+  "dependencies": {
+    "@mcpedia/config": "workspace:*",
+    "@mcpedia/core": "workspace:*",
+    "@mcpedia/db": "workspace:*",
+    "bullmq": "^6.1.2",
+    "ioredis": "^6.0.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.6.0"
+  }
+}

packages/queue/src/client.ts

@@ -0,0 +1,34 @@
+import { REDIS_URL, REDIS_PASSWORD, QUEUE_PREFIX } from "@mcpedia/config";
+import IORedis, { type RedisOptions } from "ioredis";
+
+/**
+ * Shared ioredis connection for BullMQ. BullMQ requires an ioredis instance and
+ * internally duplicates it for blocking commands, so we keep the option objects
+ * explicit (maxRetriesPerRequest: null is REQUIRED for the blocking
+ * connection — a finite retry count causes "Connection in key mode" errors).
+ */
+function buildOptions(): RedisOptions {
+  const opts: RedisOptions = {
+    maxRetriesPerRequest: null,
+    lazyConnect: true,
+    enableOfflineQueue: true,
+  };
+  if (REDIS_PASSWORD) opts.password = REDIS_PASSWORD;
+  return opts;
+}
+
+let _connection: IORedis | null = null;
+
+/** Lazily-created singleton ioredis connection. */
+export function getConnection(): IORedis {
+  if (!_connection) {
+    _connection = new IORedis(REDIS_URL, buildOptions());
+    _connection.on("error", (err) => {
+      // Log but don't crash the process on transient Redis errors.
+      console.error("[queue] redis error:", err.message);
+    });
+  }
+  return _connection;
+}
+
+export const BULLMQ_PREFIX = QUEUE_PREFIX;

packages/queue/src/index.ts

@@ -0,0 +1,9 @@
+export { getConnection, BULLMQ_PREFIX } from "./client";
+export {
+  getQueue,
+  enqueueIndexDoc,
+  enqueueFullIndex,
+  INDEX_QUEUE,
+} from "./queue";
+export type { IndexDocJobData, IndexAllJobData, JobType } from "./queue";
+export { createWorker, startWorker } from "./worker";

packages/queue/src/queue.ts

@@ -0,0 +1,66 @@
+import { Queue, type Job } from "bullmq";
+import { getConnection, BULLMQ_PREFIX } from "./client";
+
+export const INDEX_QUEUE = "mcpedia-index";
+
+/** Lazily-created singleton BullMQ queue. */
+let _queue: Queue | null = null;
+
+export function getQueue(): Queue {
+  if (!_queue) {
+    _queue = new Queue(INDEX_QUEUE, {
+      connection: getConnection(),
+      prefix: BULLMQ_PREFIX,
+    });
+  }
+  return _queue;
+}
+
+export interface IndexDocJobData {
+  relPath: string;
+  reason: string;
+}
+
+export interface IndexAllJobData {
+  reason: string;
+}
+
+export type JobType = "index-doc" | "index-all";
+
+/**
+ * Enqueue a single-document reindex job. Keyed by slug so repeated edits
+ * collapse into one pending job (BullMQ dedup by jobId within the window).
+ */
+export async function enqueueIndexDoc(
+  relPath: string,
+  reason = "index",
+): Promise<Job<IndexDocJobData>> {
+  const slug = relPath.replace(/\.mdx?$/, "");
+  return getQueue().add(
+    "index-doc",
+    { relPath, reason },
+    {
+      jobId: `doc__${slug}`,
+      removeOnComplete: 1000,
+      removeOnFail: 5000,
+      attempts: 3,
+      backoff: { type: "exponential", delay: 2000 },
+    },
+  );
+}
+
+/** Enqueue a full-corpus reindex (used by the git-sync hook). */
+export async function enqueueFullIndex(
+  reason = "reindex",
+): Promise<Job<IndexAllJobData>> {
+  return getQueue().add(
+    "index-all",
+    { reason },
+    {
+      jobId: `full__${Date.now()}`,
+      removeOnComplete: 100,
+      removeOnFail: 1000,
+      attempts: 1,
+    },
+  );
+}

packages/queue/src/worker.ts

@@ -0,0 +1,63 @@
+import { Worker, type Job } from "bullmq";
+import { getConnection, BULLMQ_PREFIX } from "./client";
+import { INDEX_QUEUE } from "./queue";
+import { indexContentFile, runFullIndex } from "@mcpedia/core";
+
+export function createWorker(): Worker {
+  const worker = new Worker(
+    INDEX_QUEUE,
+    async (job: Job) => {
+      switch (job.name) {
+        case "index-doc": {
+          const { relPath, reason } = job.data as {
+            relPath: string;
+            reason: string;
+          };
+          await job.log(`indexing ${relPath}`);
+          const r = await indexContentFile(relPath, reason);
+          return r;
+        }
+        case "index-all": {
+          const { reason } = job.data as { reason: string };
+          await job.log(`full index (${reason})`);
+          return await runFullIndex(reason);
+        }
+        default:
+          throw new Error(`unknown job type: ${job.name}`);
+      }
+    },
+    {
+      connection: getConnection(),
+      prefix: BULLMQ_PREFIX,
+      concurrency: 4,
+    },
+  );
+
+  worker.on("completed", (job) => {
+    console.log(`[worker] completed ${job.name} (${job.id})`);
+  });
+  worker.on("failed", (job, err) => {
+    console.error(`[worker] failed ${job?.name} (${job?.id}): ${err.message}`);
+  });
+  worker.on("error", (err) => {
+    console.error(`[worker] error:`, err.message);
+  });
+
+  return worker;
+}
+
+/** Start the worker and wire graceful shutdown. */
+export async function startWorker(): Promise<Worker> {
+  const worker = createWorker();
+  console.log("[worker] indexing worker started");
+
+  const shutdown = async (sig: string) => {
+    console.log(`[worker] ${sig} received, closing...`);
+    await worker.close();
+    process.exit(0);
+  };
+  process.on("SIGINT", () => void shutdown("SIGINT"));
+  process.on("SIGTERM", () => void shutdown("SIGTERM"));
+
+  return worker;
+}