MythEclipse/mcpedia
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete search REST route, restore disk sync, and extraFields support across API/MCP

Browse Source
This commit is contained in:
asepharyana
2026-08-21 11:03:04 +07:00
parent 989ee9c464
commit a67a8c6469
11 changed files with 319 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/router.ts
+2
View File
@@ -83,6 +83,7 @@ export const appRouter = router({
status: z.enum(["published", "draft"]).optional(),
author: z.string().optional(),
tags: z.array(z.string()).optional(),
extraFields: z.record(z.string(), z.unknown()).optional(),
}),
)
.mutation(async ({ input }) => createDocument(input)),
@@ -98,6 +99,7 @@ export const appRouter = router({
status: z.enum(["published", "draft"]).optional(),
tags: z.array(z.string()).optional(),
author: z.string().optional(),
extraFields: z.record(z.string(), z.unknown()).optional(),
}),
)
.mutation(async ({ input }) => {
apps/mcp-client/src/client.ts
+1
View File
@@ -130,6 +130,7 @@ export class McpediaClient {
status?: "published" | "draft";
author?: string;
tags?: string[];
extraFields?: Record<string, unknown>;
}): Promise<any> {
const result = await this.callTool("create_document", input);
return JSON.parse(extractText(result) || "{}");
apps/mcp/src/index.ts
+6 -2
View File
@@ -213,9 +213,10 @@ export function createMcpServer(authSecret?: string): McpServer {
status: z.enum(["published", "draft"]).optional(),
author: z.string().optional(),
tags: z.array(z.string()).optional(),
extraFields: z.record(z.string(), z.unknown()).optional().describe("Dynamic custom metadata key-value pairs"),
}),
},
async ({ slug, title, section, body, type, status, author, tags }) => {
async ({ slug, title, section, body, type, status, author, tags, extraFields }) => {
requireMcpAuth(authSecret);
const doc = await createDocument({
slug,
@@ -226,6 +227,7 @@ export function createMcpServer(authSecret?: string): McpServer {
status,
author,
tags,
extraFields,
});
return {
content: [{ type: "text", text: JSON.stringify({ ok: true, slug: doc.slug }) }],
@@ -246,9 +248,10 @@ export function createMcpServer(authSecret?: string): McpServer {
status: z.enum(["published", "draft"]).optional(),
tags: z.array(z.string()).optional(),
author: z.string().optional(),
extraFields: z.record(z.string(), z.unknown()).optional().describe("Dynamic custom metadata key-value pairs"),
}),
},
async ({ slug, title, body, type, status, tags, author }) => {
async ({ slug, title, body, type, status, tags, author, extraFields }) => {
requireMcpAuth(authSecret);
const doc = await updateDocument(slug, {
title,
@@ -257,6 +260,7 @@ export function createMcpServer(authSecret?: string): McpServer {
status,
tags,
author,
extraFields,
});
return {
content: [{ type: "text", text: JSON.stringify({ ok: true, slug: doc.slug }) }],
apps/web/app/api/docs/[...slug]/route.ts
+24 -2
View File
@@ -22,6 +22,26 @@ const STANDARD_FIELDS = new Set([
"author", "tags", "createdAt", "updatedAt", "id", "path", "slug",
]);
function parseCustomValue(value: unknown): unknown {
if (typeof value === "string") {
const trimmed = value.trim();
if (trimmed === "true") return true;
if (trimmed === "false") return false;
if (/^-?\d+(\.\d+)?$/.test(trimmed) && !(trimmed.startsWith("0") && trimmed.length > 1 && !trimmed.startsWith("0."))) {
const num = Number(trimmed);
if (!isNaN(num)) return num;
}
if ((trimmed.startsWith("[") && trimmed.endsWith("]")) || (trimmed.startsWith("{") && trimmed.endsWith("}"))) {
try {
return JSON.parse(trimmed);
} catch {
return value;
}
}
}
return value;
}
function splitPayload(body: Record<string, unknown>): {
standard: Record<string, unknown>;
extraFields: Record<string, unknown>;
@@ -29,10 +49,12 @@ function splitPayload(body: Record<string, unknown>): {
const standard: Record<string, unknown> = {};
const extraFields: Record<string, unknown> = {};
for (const [key, value] of Object.entries(body)) {
if (STANDARD_FIELDS.has(key)) {
if (key === "extraFields" && typeof value === "object" && value !== null) {
Object.assign(extraFields, value);
} else if (STANDARD_FIELDS.has(key)) {
standard[key] = value;
} else if (value !== undefined && value !== null) {
extraFields[key] = value;
extraFields[key] = parseCustomValue(value);
}
}
return { standard, extraFields };
apps/web/app/api/docs/route.ts
+25 -3
View File
@@ -30,6 +30,26 @@ const STANDARD_FIELDS = new Set([
"author", "tags", "createdAt", "updatedAt", "id", "path",
]);
function parseCustomValue(value: unknown): unknown {
if (typeof value === "string") {
const trimmed = value.trim();
if (trimmed === "true") return true;
if (trimmed === "false") return false;
if (/^-?\d+(\.\d+)?$/.test(trimmed) && !(trimmed.startsWith("0") && trimmed.length > 1 && !trimmed.startsWith("0."))) {
const num = Number(trimmed);
if (!isNaN(num)) return num;
}
if ((trimmed.startsWith("[") && trimmed.endsWith("]")) || (trimmed.startsWith("{") && trimmed.endsWith("}"))) {
try {
return JSON.parse(trimmed);
} catch {
return value;
}
}
}
return value;
}
/**
* Separate a flat payload into standard CRUD fields + extraFields (custom metadata).
* The DocForm sends all fields flat — any key not in STANDARD_FIELDS becomes an
@@ -42,10 +62,12 @@ function splitPayload(body: Record<string, unknown>): {
const standard: Record<string, unknown> = {};
const extraFields: Record<string, unknown> = {};
for (const [key, value] of Object.entries(body)) {
if (STANDARD_FIELDS.has(key)) {
if (key === "extraFields" && typeof value === "object" && value !== null) {
Object.assign(extraFields, value);
} else if (STANDARD_FIELDS.has(key)) {
standard[key] = value;
} else {
extraFields[key] = value;
} else if (value !== undefined && value !== null) {
extraFields[key] = parseCustomValue(value);
}
}
return { standard, extraFields };
apps/web/app/api/search/route.ts
+71
View File
@@ -0,0 +1,71 @@
import { NextRequest, NextResponse } from "next/server";
import { keywordSearch, hybridSearch, semanticSearch } from "@mcpedia/core";
// GET /api/search?q=...&mode=hybrid|keyword|semantic
export async function GET(req: NextRequest) {
const { searchParams } = new URL(req.url);
const q = searchParams.get("q") ?? "";
const mode = searchParams.get("mode") ?? "hybrid";
const limit = Math.min(Math.max(Number(searchParams.get("limit") ?? 20), 1), 50);
if (!q.trim()) {
return NextResponse.json({ results: [] });
}
try {
if (mode === "keyword") {
const hits = await keywordSearch(q, limit);
return NextResponse.json({
results: hits.map((h) => ({
slug: h.doc.slug,
title: h.doc.title,
section: h.doc.section,
score: h.rank,
snippet: h.snippet,
})),
});
}
if (mode === "semantic") {
const hits = await semanticSearch(q, limit);
return NextResponse.json({
results: hits.map((h) => ({
slug: h.slug,
title: h.slug.split("/").pop() ?? h.slug,
section: h.slug.split("/")[0] ?? "docs",
score: h.score,
snippet: h.content.slice(0, 160),
})),
});
}
// Default: hybrid search
const hits = await hybridSearch(q, limit);
return NextResponse.json({
results: hits.map((h) => ({
slug: h.doc.slug,
title: h.doc.title,
section: h.doc.section,
score: h.rank,
snippet: h.snippet,
})),
});
} catch (err) {
console.error("Search API error:", err);
// Fall back to keyword search if embedding provider fails
try {
const hits = await keywordSearch(q, limit);
return NextResponse.json({
results: hits.map((h) => ({
slug: h.doc.slug,
title: h.doc.title,
section: h.doc.section,
score: h.rank,
snippet: h.snippet,
})),
});
} catch {
return NextResponse.json({ results: [], error: "Search failed" }, { status: 500 });
}
}
}
apps/web/app/search/page.tsx
+95 -21
View File
@@ -1,7 +1,8 @@
"use client";
import { useState } from "react";
import { useState, useEffect, useCallback, Suspense } from "react";
import Link from "next/link";
import { useSearchParams } from "next/navigation";
interface DocHit {
slug: string;
@@ -11,19 +12,30 @@ interface DocHit {
snippet: string;
}
export default function SearchPage() {
const [query, setQuery] = useState("");
type SearchMode = "hybrid" | "keyword" | "semantic";
function SearchContent() {
const searchParams = useSearchParams();
const initialQ = searchParams.get("q") ?? "";
const initialMode = (searchParams.get("mode") as SearchMode) ?? "hybrid";
const [query, setQuery] = useState(initialQ);
const [mode, setMode] = useState<SearchMode>(
["hybrid", "keyword", "semantic"].includes(initialMode) ? initialMode : "hybrid",
);
const [results, setResults] = useState<DocHit[]>([]);
const [loading, setLoading] = useState(false);
async function handleSearch(q: string) {
const handleSearch = useCallback(async (q: string, searchMode: SearchMode) => {
if (!q.trim()) {
setResults([]);
return;
}
setLoading(true);
try {
const res = await fetch(`/api/search?q=${encodeURIComponent(q)}`);
const res = await fetch(
`/api/search?q=${encodeURIComponent(q)}&mode=${searchMode}`,
);
const data = await res.json();
setResults(data.results || []);
} catch {
@@ -31,51 +43,98 @@ export default function SearchPage() {
} finally {
setLoading(false);
}
}
}, []);
useEffect(() => {
if (initialQ) {
handleSearch(initialQ, mode);
}
}, [initialQ, handleSearch, mode]);
function handleChange(e: React.ChangeEvent<HTMLInputElement>) {
const val = e.target.value;
setQuery(val);
handleSearch(val);
handleSearch(val, mode);
}
function handleModeChange(newMode: SearchMode) {
setMode(newMode);
if (query.trim()) {
handleSearch(query, newMode);
}
}
return (
<div>
<div className="max-w-3xl">
<h1 className="text-2xl font-light text-[#f7f8f8] mb-6">Search</h1>
<div className="mb-6">
<div className="space-y-4 mb-8">
<input
type="search"
value={query}
onChange={handleChange}
placeholder="Search documents..."
className="w-full px-4 py-2 bg-[#0f1011] border border-[#23252a] rounded text-[#f7f8f8] placeholder-[#62666d] focus:outline-none focus:border-[#5e6ad2] focus:ring-1 focus:ring-[#5e6ad2]"
placeholder="Search documents by keyword or semantic meaning..."
className="w-full px-4 py-2.5 bg-[#0f1011] border border-[#23252a] rounded-lg text-[#f7f8f8] placeholder-[#62666d] focus:outline-none focus:border-[#5e6ad2] focus:ring-1 focus:ring-[#5e6ad2]"
autoFocus
/>
{/* Mode Selector */}
<div className="flex items-center gap-2 text-xs">
<span className="text-[#62666d]">Mode:</span>
{(
[
{ id: "hybrid", label: "Hybrid (FTS + Semantic)" },
{ id: "keyword", label: "Keyword (FTS)" },
{ id: "semantic", label: "Semantic (Embeddings)" },
] as const
).map((m) => (
<button
key={m.id}
type="button"
onClick={() => handleModeChange(m.id)}
className={`px-2.5 py-1 rounded border transition-colors ${
mode === m.id
? "bg-[#5e6ad2]/15 border-[#5e6ad2] text-[#7170ff]"
: "bg-[#0f1011] border-[#1f2022] text-[#8a8f98] hover:text-[#d0d6e0]"
}`}
>
{m.label}
</button>
))}
</div>
</div>
{loading && <p className="text-[#8a8f98]">Searching...</p>}
{!loading && results.length === 0 && query && (
<p className="text-[#8a8f98]">No results for "{query}".</p>
<p className="text-[#8a8f98]">No results found for "{query}".</p>
)}
{!loading && results.length > 0 && (
<ul className="space-y-3">
<ul className="space-y-4">
{results.map((r) => (
<li key={r.slug} className="border-b border-[#1f2022] pb-3">
<li
key={r.slug}
className="bg-[#0f1011] border border-[#1f2022] rounded-lg p-4 hover:border-[#5e6ad2]/40 transition-colors"
>
<Link
href={`/${r.slug}`}
className="text-[#f7f8f8] hover:text-[#7170ff] font-medium transition-colors"
className="text-[#f7f8f8] hover:text-[#7170ff] font-medium transition-colors block mb-1"
>
{r.title}
</Link>
<span className="block text-xs text-[#62666d]">
{r.section} · {Math.round(r.score * 100)}% match
</span>
<div className="flex items-center gap-2 text-xs text-[#62666d] mb-2">
<span className="uppercase text-[#7170ff]">{r.section}</span>
<span>·</span>
<span>{r.slug}</span>
<span>·</span>
<span>Score: {r.score.toFixed(3)}</span>
</div>
{r.snippet && (
<p className="mt-1 text-sm text-[#8a8f98] line-clamp-2">
{r.snippet}
</p>
<p
className="text-sm text-[#8a8f98] leading-relaxed [&>mark]:bg-[#5e6ad2]/30 [&>mark]:text-[#f7f8f8] [&>mark]:px-1 [&>mark]:rounded"
dangerouslySetInnerHTML={{ __html: r.snippet }}
/>
)}
</li>
))}
@@ -84,3 +143,18 @@ export default function SearchPage() {
</div>
);
}
export default function SearchPage() {
return (
<Suspense
fallback={
<div className="max-w-3xl">
<h1 className="text-2xl font-light text-[#f7f8f8] mb-6">Search</h1>
<p className="text-[#8a8f98]">Loading search...</p>
</div>
}
>
<SearchContent />
</Suspense>
);
}
packages/core/src/revision.service.ts
+37 -1
View File
@@ -3,6 +3,8 @@ import { documents, documentRevisions, documentChunks } from "@mcpedia/db/schema
import { reindexChunks } from "./index.service";
import { eq, desc, and, sql } from "drizzle-orm";
import { toMeta } from "./row-map";
import { CONTENT_ROOT } from "@mcpedia/config";
import { join } from "node:path";
import type { DocumentMeta } from "@mcpedia/types";
export interface RevisionSummary {
@@ -77,7 +79,8 @@ export async function getRevision(
}
/**
* Restore a revision: write its body+metadata back into the live `documents` row.
* Restore a revision: write its body+metadata back into the live `documents` row
* and update the on-disk markdown file.
*
* @param id revision UUID
* @param opts optional seam for testing — override the chunk-rebuild step so
@@ -110,6 +113,11 @@ export async function restoreRevision(
tags?: string[];
};
const [docRow] = await db
.select({ path: documents.path })
.from(documents)
.where(eq(documents.id, rev.documentId));
await db
.update(documents)
.set({
@@ -124,6 +132,34 @@ export async function restoreRevision(
})
.where(eq(documents.id, rev.documentId));
// Sync back to disk (source of truth for file-based reads)
if (docRow?.path) {
const absPath = join(CONTENT_ROOT, docRow.path);
try {
const { stringifyFile } = await import("@mcpedia/parser");
stringifyFile(
absPath,
docRow.path,
{
id: rev.slug,
slug: rev.slug,
title: rev.title,
type: (m.type as any) ?? "documentation",
section: (m.section as any) ?? "docs",
status: (m.status as any) ?? "published",
author: m.author ?? "",
tags: m.tags ?? [],
path: docRow.path,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
},
rev.body,
);
} catch (err) {
console.error(`restoreRevision: failed to write file to disk for ${rev.slug}:`, err);
}
}
// Rebuild semantic chunks + embeddings from the restored body so semantic
// and hybrid search stay consistent (otherwise document_chunks would hold
// the NEW body's chunks while documents.body holds the OLD/restore body).
packages/parser/src/index.ts
+2 -3
View File
@@ -60,11 +60,10 @@ export function parseFile(absPath: string, relPath: string): ParsedFile {
// Extract any non-standard frontmatter keys as dynamic extra fields.
// These are stored in DB as JSONB + rendered as dynamic badges in the UI.
const extraFields: Record<string, string> = {};
const extraFields: Record<string, unknown> = {};
for (const [k, v] of Object.entries(data)) {
if (!STANDARD_FRONTMATTER_KEYS.has(k) && v !== undefined && v !== null) {
// Serialize non-string values (numbers, booleans) to string
extraFields[k] = typeof v === "string" ? v : JSON.stringify(v);
extraFields[k] = v;
}
}
packages/parser/src/parse.test.ts
+53
View File
@@ -79,3 +79,56 @@ test("parseFile: body excludes frontmatter delimiter", () => {
expect(body).not.toContain("---");
expect(body).toContain("# Real body");
});
test("parseFile: extracts dynamic extra fields", () => {
const { meta } = writeDoc(
"writeups/ctf/chal.md",
[
"---",
'title: CTF Challenge',
'event: DEF CON 2024',
'points: 100',
'solved: true',
"---",
"# Solved",
].join("\n"),
);
expect(meta.extraFields?.event).toBe("DEF CON 2024");
expect(meta.extraFields?.points).toBe(100);
expect(meta.extraFields?.solved).toBe(true);
});
test("stringifyFile: writes file that round-trips via parseFile", async () => {
const { stringifyFile } = await import("../src/index");
const p = join(tmp, "docs/roundtrip.md");
const meta = {
id: "docs/roundtrip",
slug: "docs/roundtrip",
title: "Roundtrip Test",
type: "documentation" as const,
section: "docs" as const,
status: "published" as const,
author: "tester",
tags: ["a", "b"],
path: "docs/roundtrip.md",
createdAt: "2026-08-21T00:00:00.000Z",
updatedAt: "2026-08-21T00:00:00.000Z",
extraFields: {
event: "DEF CON",
difficulty: "medium",
points: 500,
},
};
const body = "# Content\n\nParagraph content.";
stringifyFile(p, "docs/roundtrip.md", meta, body);
const parsed = parseFile(p, "docs/roundtrip.md");
expect(parsed.meta.title).toBe("Roundtrip Test");
expect(parsed.meta.author).toBe("tester");
expect(parsed.meta.tags).toEqual(["a", "b"]);
expect(parsed.meta.extraFields?.event).toBe("DEF CON");
expect(parsed.meta.extraFields?.difficulty).toBe("medium");
expect(parsed.meta.extraFields?.points).toBe(500);
expect(parsed.body.trim()).toBe(body);
});
packages/search/src/index.ts
+3 -2
View File
@@ -32,7 +32,7 @@ const VALID_TYPES: DocType[] = ["documentation", "writeup", "research", "note"];
/** Map a Drizzle row (text columns, Date timestamps) into the strict types. */
function toMeta(row: DocumentRow): DocumentMeta {
const extra = row.extraFields as Record<string, unknown> | null;
const extra = (row.extraFields ?? {}) as Record<string, unknown>;
return {
id: row.id,
slug: row.slug,
@@ -47,8 +47,9 @@ function toMeta(row: DocumentRow): DocumentMeta {
path: row.path,
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
extraFields: extra,
// Spread dynamic extra fields (CTF: event, challenge, category, difficulty, points, etc.)
...(extra ?? {}),
...extra,
};
}