diff --git a/apps/api/src/router.ts b/apps/api/src/router.ts index 72ea3f9..ee60c7a 100644 --- a/apps/api/src/router.ts +++ b/apps/api/src/router.ts @@ -83,6 +83,7 @@ export const appRouter = router({ status: z.enum(["published", "draft"]).optional(), author: z.string().optional(), tags: z.array(z.string()).optional(), + extraFields: z.record(z.string(), z.unknown()).optional(), }), ) .mutation(async ({ input }) => createDocument(input)), @@ -98,6 +99,7 @@ export const appRouter = router({ status: z.enum(["published", "draft"]).optional(), tags: z.array(z.string()).optional(), author: z.string().optional(), + extraFields: z.record(z.string(), z.unknown()).optional(), }), ) .mutation(async ({ input }) => { diff --git a/apps/mcp-client/src/client.ts b/apps/mcp-client/src/client.ts index 6cc82cf..d694e33 100644 --- a/apps/mcp-client/src/client.ts +++ b/apps/mcp-client/src/client.ts @@ -130,6 +130,7 @@ export class McpediaClient { status?: "published" | "draft"; author?: string; tags?: string[]; + extraFields?: Record; }): Promise { const result = await this.callTool("create_document", input); return JSON.parse(extractText(result) || "{}"); diff --git a/apps/mcp/src/index.ts b/apps/mcp/src/index.ts index 024b089..a1edd2e 100644 --- a/apps/mcp/src/index.ts +++ b/apps/mcp/src/index.ts @@ -213,9 +213,10 @@ export function createMcpServer(authSecret?: string): McpServer { status: z.enum(["published", "draft"]).optional(), author: z.string().optional(), tags: z.array(z.string()).optional(), + extraFields: z.record(z.string(), z.unknown()).optional().describe("Dynamic custom metadata key-value pairs"), }), }, - async ({ slug, title, section, body, type, status, author, tags }) => { + async ({ slug, title, section, body, type, status, author, tags, extraFields }) => { requireMcpAuth(authSecret); const doc = await createDocument({ slug, @@ -226,6 +227,7 @@ export function createMcpServer(authSecret?: string): McpServer { status, author, tags, + extraFields, }); return { content: [{ type: "text", text: JSON.stringify({ ok: true, slug: doc.slug }) }], @@ -246,9 +248,10 @@ export function createMcpServer(authSecret?: string): McpServer { status: z.enum(["published", "draft"]).optional(), tags: z.array(z.string()).optional(), author: z.string().optional(), + extraFields: z.record(z.string(), z.unknown()).optional().describe("Dynamic custom metadata key-value pairs"), }), }, - async ({ slug, title, body, type, status, tags, author }) => { + async ({ slug, title, body, type, status, tags, author, extraFields }) => { requireMcpAuth(authSecret); const doc = await updateDocument(slug, { title, @@ -257,6 +260,7 @@ export function createMcpServer(authSecret?: string): McpServer { status, tags, author, + extraFields, }); return { content: [{ type: "text", text: JSON.stringify({ ok: true, slug: doc.slug }) }], diff --git a/apps/web/app/api/docs/[...slug]/route.ts b/apps/web/app/api/docs/[...slug]/route.ts index 9410d2a..057fabd 100644 --- a/apps/web/app/api/docs/[...slug]/route.ts +++ b/apps/web/app/api/docs/[...slug]/route.ts @@ -22,6 +22,26 @@ const STANDARD_FIELDS = new Set([ "author", "tags", "createdAt", "updatedAt", "id", "path", "slug", ]); +function parseCustomValue(value: unknown): unknown { + if (typeof value === "string") { + const trimmed = value.trim(); + if (trimmed === "true") return true; + if (trimmed === "false") return false; + if (/^-?\d+(\.\d+)?$/.test(trimmed) && !(trimmed.startsWith("0") && trimmed.length > 1 && !trimmed.startsWith("0."))) { + const num = Number(trimmed); + if (!isNaN(num)) return num; + } + if ((trimmed.startsWith("[") && trimmed.endsWith("]")) || (trimmed.startsWith("{") && trimmed.endsWith("}"))) { + try { + return JSON.parse(trimmed); + } catch { + return value; + } + } + } + return value; +} + function splitPayload(body: Record): { standard: Record; extraFields: Record; @@ -29,10 +49,12 @@ function splitPayload(body: Record): { const standard: Record = {}; const extraFields: Record = {}; for (const [key, value] of Object.entries(body)) { - if (STANDARD_FIELDS.has(key)) { + if (key === "extraFields" && typeof value === "object" && value !== null) { + Object.assign(extraFields, value); + } else if (STANDARD_FIELDS.has(key)) { standard[key] = value; } else if (value !== undefined && value !== null) { - extraFields[key] = value; + extraFields[key] = parseCustomValue(value); } } return { standard, extraFields }; diff --git a/apps/web/app/api/docs/route.ts b/apps/web/app/api/docs/route.ts index d30c08e..f968088 100644 --- a/apps/web/app/api/docs/route.ts +++ b/apps/web/app/api/docs/route.ts @@ -30,6 +30,26 @@ const STANDARD_FIELDS = new Set([ "author", "tags", "createdAt", "updatedAt", "id", "path", ]); +function parseCustomValue(value: unknown): unknown { + if (typeof value === "string") { + const trimmed = value.trim(); + if (trimmed === "true") return true; + if (trimmed === "false") return false; + if (/^-?\d+(\.\d+)?$/.test(trimmed) && !(trimmed.startsWith("0") && trimmed.length > 1 && !trimmed.startsWith("0."))) { + const num = Number(trimmed); + if (!isNaN(num)) return num; + } + if ((trimmed.startsWith("[") && trimmed.endsWith("]")) || (trimmed.startsWith("{") && trimmed.endsWith("}"))) { + try { + return JSON.parse(trimmed); + } catch { + return value; + } + } + } + return value; +} + /** * Separate a flat payload into standard CRUD fields + extraFields (custom metadata). * The DocForm sends all fields flat — any key not in STANDARD_FIELDS becomes an @@ -42,10 +62,12 @@ function splitPayload(body: Record): { const standard: Record = {}; const extraFields: Record = {}; for (const [key, value] of Object.entries(body)) { - if (STANDARD_FIELDS.has(key)) { + if (key === "extraFields" && typeof value === "object" && value !== null) { + Object.assign(extraFields, value); + } else if (STANDARD_FIELDS.has(key)) { standard[key] = value; - } else { - extraFields[key] = value; + } else if (value !== undefined && value !== null) { + extraFields[key] = parseCustomValue(value); } } return { standard, extraFields }; diff --git a/apps/web/app/api/search/route.ts b/apps/web/app/api/search/route.ts new file mode 100644 index 0000000..91f388b --- /dev/null +++ b/apps/web/app/api/search/route.ts @@ -0,0 +1,71 @@ +import { NextRequest, NextResponse } from "next/server"; +import { keywordSearch, hybridSearch, semanticSearch } from "@mcpedia/core"; + +// GET /api/search?q=...&mode=hybrid|keyword|semantic +export async function GET(req: NextRequest) { + const { searchParams } = new URL(req.url); + const q = searchParams.get("q") ?? ""; + const mode = searchParams.get("mode") ?? "hybrid"; + const limit = Math.min(Math.max(Number(searchParams.get("limit") ?? 20), 1), 50); + + if (!q.trim()) { + return NextResponse.json({ results: [] }); + } + + try { + if (mode === "keyword") { + const hits = await keywordSearch(q, limit); + return NextResponse.json({ + results: hits.map((h) => ({ + slug: h.doc.slug, + title: h.doc.title, + section: h.doc.section, + score: h.rank, + snippet: h.snippet, + })), + }); + } + + if (mode === "semantic") { + const hits = await semanticSearch(q, limit); + return NextResponse.json({ + results: hits.map((h) => ({ + slug: h.slug, + title: h.slug.split("/").pop() ?? h.slug, + section: h.slug.split("/")[0] ?? "docs", + score: h.score, + snippet: h.content.slice(0, 160), + })), + }); + } + + // Default: hybrid search + const hits = await hybridSearch(q, limit); + return NextResponse.json({ + results: hits.map((h) => ({ + slug: h.doc.slug, + title: h.doc.title, + section: h.doc.section, + score: h.rank, + snippet: h.snippet, + })), + }); + } catch (err) { + console.error("Search API error:", err); + // Fall back to keyword search if embedding provider fails + try { + const hits = await keywordSearch(q, limit); + return NextResponse.json({ + results: hits.map((h) => ({ + slug: h.doc.slug, + title: h.doc.title, + section: h.doc.section, + score: h.rank, + snippet: h.snippet, + })), + }); + } catch { + return NextResponse.json({ results: [], error: "Search failed" }, { status: 500 }); + } + } +} diff --git a/apps/web/app/search/page.tsx b/apps/web/app/search/page.tsx index ecf058e..1314ed6 100644 --- a/apps/web/app/search/page.tsx +++ b/apps/web/app/search/page.tsx @@ -1,7 +1,8 @@ "use client"; -import { useState } from "react"; +import { useState, useEffect, useCallback, Suspense } from "react"; import Link from "next/link"; +import { useSearchParams } from "next/navigation"; interface DocHit { slug: string; @@ -11,19 +12,30 @@ interface DocHit { snippet: string; } -export default function SearchPage() { - const [query, setQuery] = useState(""); +type SearchMode = "hybrid" | "keyword" | "semantic"; + +function SearchContent() { + const searchParams = useSearchParams(); + const initialQ = searchParams.get("q") ?? ""; + const initialMode = (searchParams.get("mode") as SearchMode) ?? "hybrid"; + + const [query, setQuery] = useState(initialQ); + const [mode, setMode] = useState( + ["hybrid", "keyword", "semantic"].includes(initialMode) ? initialMode : "hybrid", + ); const [results, setResults] = useState([]); const [loading, setLoading] = useState(false); - async function handleSearch(q: string) { + const handleSearch = useCallback(async (q: string, searchMode: SearchMode) => { if (!q.trim()) { setResults([]); return; } setLoading(true); try { - const res = await fetch(`/api/search?q=${encodeURIComponent(q)}`); + const res = await fetch( + `/api/search?q=${encodeURIComponent(q)}&mode=${searchMode}`, + ); const data = await res.json(); setResults(data.results || []); } catch { @@ -31,51 +43,98 @@ export default function SearchPage() { } finally { setLoading(false); } - } + }, []); + + useEffect(() => { + if (initialQ) { + handleSearch(initialQ, mode); + } + }, [initialQ, handleSearch, mode]); function handleChange(e: React.ChangeEvent) { const val = e.target.value; setQuery(val); - handleSearch(val); + handleSearch(val, mode); + } + + function handleModeChange(newMode: SearchMode) { + setMode(newMode); + if (query.trim()) { + handleSearch(query, newMode); + } } return ( -
+

Search

-
+ +
+ + {/* Mode Selector */} +
+ Mode: + {( + [ + { id: "hybrid", label: "Hybrid (FTS + Semantic)" }, + { id: "keyword", label: "Keyword (FTS)" }, + { id: "semantic", label: "Semantic (Embeddings)" }, + ] as const + ).map((m) => ( + + ))} +
{loading &&

Searching...

} {!loading && results.length === 0 && query && ( -

No results for "{query}".

+

No results found for "{query}".

)} {!loading && results.length > 0 && ( -
    +
      {results.map((r) => ( -
    • +
    • {r.title} - - {r.section} · {Math.round(r.score * 100)}% match - +
      + {r.section} + · + {r.slug} + · + Score: {r.score.toFixed(3)} +
      {r.snippet && ( -

      - {r.snippet} -

      +

      )}

      • ))} @@ -84,3 +143,18 @@ export default function SearchPage() {
); } + +export default function SearchPage() { + return ( + +

Search

+

Loading search...

+
+ } + > + + + ); +} diff --git a/packages/core/src/revision.service.ts b/packages/core/src/revision.service.ts index f76b8c3..ea461a7 100644 --- a/packages/core/src/revision.service.ts +++ b/packages/core/src/revision.service.ts @@ -3,6 +3,8 @@ import { documents, documentRevisions, documentChunks } from "@mcpedia/db/schema import { reindexChunks } from "./index.service"; import { eq, desc, and, sql } from "drizzle-orm"; import { toMeta } from "./row-map"; +import { CONTENT_ROOT } from "@mcpedia/config"; +import { join } from "node:path"; import type { DocumentMeta } from "@mcpedia/types"; export interface RevisionSummary { @@ -77,7 +79,8 @@ export async function getRevision( } /** - * Restore a revision: write its body+metadata back into the live `documents` row. + * Restore a revision: write its body+metadata back into the live `documents` row + * and update the on-disk markdown file. * * @param id revision UUID * @param opts optional seam for testing — override the chunk-rebuild step so @@ -110,6 +113,11 @@ export async function restoreRevision( tags?: string[]; }; + const [docRow] = await db + .select({ path: documents.path }) + .from(documents) + .where(eq(documents.id, rev.documentId)); + await db .update(documents) .set({ @@ -124,6 +132,34 @@ export async function restoreRevision( }) .where(eq(documents.id, rev.documentId)); + // Sync back to disk (source of truth for file-based reads) + if (docRow?.path) { + const absPath = join(CONTENT_ROOT, docRow.path); + try { + const { stringifyFile } = await import("@mcpedia/parser"); + stringifyFile( + absPath, + docRow.path, + { + id: rev.slug, + slug: rev.slug, + title: rev.title, + type: (m.type as any) ?? "documentation", + section: (m.section as any) ?? "docs", + status: (m.status as any) ?? "published", + author: m.author ?? "", + tags: m.tags ?? [], + path: docRow.path, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + rev.body, + ); + } catch (err) { + console.error(`restoreRevision: failed to write file to disk for ${rev.slug}:`, err); + } + } + // Rebuild semantic chunks + embeddings from the restored body so semantic // and hybrid search stay consistent (otherwise document_chunks would hold // the NEW body's chunks while documents.body holds the OLD/restore body). diff --git a/packages/parser/src/index.ts b/packages/parser/src/index.ts index dec49df..1d64c0f 100644 --- a/packages/parser/src/index.ts +++ b/packages/parser/src/index.ts @@ -60,11 +60,10 @@ export function parseFile(absPath: string, relPath: string): ParsedFile { // Extract any non-standard frontmatter keys as dynamic extra fields. // These are stored in DB as JSONB + rendered as dynamic badges in the UI. - const extraFields: Record = {}; + const extraFields: Record = {}; for (const [k, v] of Object.entries(data)) { if (!STANDARD_FRONTMATTER_KEYS.has(k) && v !== undefined && v !== null) { - // Serialize non-string values (numbers, booleans) to string - extraFields[k] = typeof v === "string" ? v : JSON.stringify(v); + extraFields[k] = v; } } diff --git a/packages/parser/src/parse.test.ts b/packages/parser/src/parse.test.ts index 2f0cb9f..6dcc82f 100644 --- a/packages/parser/src/parse.test.ts +++ b/packages/parser/src/parse.test.ts @@ -79,3 +79,56 @@ test("parseFile: body excludes frontmatter delimiter", () => { expect(body).not.toContain("---"); expect(body).toContain("# Real body"); }); + +test("parseFile: extracts dynamic extra fields", () => { + const { meta } = writeDoc( + "writeups/ctf/chal.md", + [ + "---", + 'title: CTF Challenge', + 'event: DEF CON 2024', + 'points: 100', + 'solved: true', + "---", + "# Solved", + ].join("\n"), + ); + expect(meta.extraFields?.event).toBe("DEF CON 2024"); + expect(meta.extraFields?.points).toBe(100); + expect(meta.extraFields?.solved).toBe(true); +}); + +test("stringifyFile: writes file that round-trips via parseFile", async () => { + const { stringifyFile } = await import("../src/index"); + const p = join(tmp, "docs/roundtrip.md"); + const meta = { + id: "docs/roundtrip", + slug: "docs/roundtrip", + title: "Roundtrip Test", + type: "documentation" as const, + section: "docs" as const, + status: "published" as const, + author: "tester", + tags: ["a", "b"], + path: "docs/roundtrip.md", + createdAt: "2026-08-21T00:00:00.000Z", + updatedAt: "2026-08-21T00:00:00.000Z", + extraFields: { + event: "DEF CON", + difficulty: "medium", + points: 500, + }, + }; + const body = "# Content\n\nParagraph content."; + stringifyFile(p, "docs/roundtrip.md", meta, body); + + const parsed = parseFile(p, "docs/roundtrip.md"); + expect(parsed.meta.title).toBe("Roundtrip Test"); + expect(parsed.meta.author).toBe("tester"); + expect(parsed.meta.tags).toEqual(["a", "b"]); + expect(parsed.meta.extraFields?.event).toBe("DEF CON"); + expect(parsed.meta.extraFields?.difficulty).toBe("medium"); + expect(parsed.meta.extraFields?.points).toBe(500); + expect(parsed.body.trim()).toBe(body); +}); + diff --git a/packages/search/src/index.ts b/packages/search/src/index.ts index 496d235..1a7998c 100644 --- a/packages/search/src/index.ts +++ b/packages/search/src/index.ts @@ -32,7 +32,7 @@ const VALID_TYPES: DocType[] = ["documentation", "writeup", "research", "note"]; /** Map a Drizzle row (text columns, Date timestamps) into the strict types. */ function toMeta(row: DocumentRow): DocumentMeta { - const extra = row.extraFields as Record | null; + const extra = (row.extraFields ?? {}) as Record; return { id: row.id, slug: row.slug, @@ -47,8 +47,9 @@ function toMeta(row: DocumentRow): DocumentMeta { path: row.path, createdAt: row.createdAt.toISOString(), updatedAt: row.updatedAt.toISOString(), + extraFields: extra, // Spread dynamic extra fields (CTF: event, challenge, category, difficulty, points, etc.) - ...(extra ?? {}), + ...extra, }; }