diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d232ad8..72b2a96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,9 +10,16 @@ concurrency: group: ci-${{ github.ref }} cancel-in-progress: true +env: + VPS_HOST: ${{ secrets.SSH_DEPLOY_HOST }} + VPS_USER: ${{ secrets.SSH_DEPLOY_USER }} + +permissions: + contents: read + jobs: - build: - name: typecheck + build + test: + name: typecheck + tests runs-on: ubuntu-latest steps: - name: Checkout @@ -29,84 +36,80 @@ jobs: - name: Typecheck run: bun run typecheck - - name: Build web - working-directory: apps/web - run: bun run build - - # Upload the built .next directory (hidden dir → include-hidden-files). - - name: Upload web build artifact - uses: actions/upload-artifact@v4 - with: - name: mcpedia-web-build - path: apps/web/.next - include-hidden-files: true - if-no-files-found: error - - # Smoke: requires Postgres + Redis — skipped in CI without secrets. - - name: MCP smoke test - if: env.DATABASE_URL != '' - working-directory: apps/mcp - env: - DATABASE_URL: ${{ secrets.DATABASE_URL }} - REDIS_URL: ${{ secrets.REDIS_URL }} - run: bun run smoke - - # Unit tests — bun:test mocks, no external services. - name: Test run: bun run test - deploy: - name: deploy to VPS - needs: build + build-and-deploy: + name: build + deploy (Nix) + needs: test if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + service: [web, api, mcp, worker] + steps: - name: Checkout uses: actions/checkout@v5 - - - name: Download build artifact - uses: actions/download-artifact@v4 with: - name: mcpedia-web-build - path: apps/web/.next + fetch-depth: 0 - # Tar .next, then SCP the tarball to VPS (single file = reliable transfer). - - name: Tar .next + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@v16 + with: + determinate: false + extra-conf: | + sandbox = false + accept-flake-config = true + + - name: Cache Nix + uses: DeterminateSystems/magic-nix-cache-action@v14 + with: + use-flakehub: false + + - name: Build ${{ matrix.service }} + id: build run: | - # Exclude cache to reduce tarball size (cache is ephemeral) - tar -C apps/web --exclude='.next/cache' -czf mcpedia-web-next.tar.gz .next - ls -lh mcpedia-web-next.tar.gz + nix build .#${{ matrix.service }} --impure --option sandbox false --print-build-logs + STORE_PATH=$(readlink result) + echo "store-path=$STORE_PATH" >> "$GITHUB_OUTPUT" + echo "Build OK ${{ matrix.service }}: $STORE_PATH" - - name: Copy tarball to VPS - uses: appleboy/scp-action@v1 - with: - host: ${{ secrets.SSH_DEPLOY_HOST }} - port: ${{ secrets.SSH_DEPLOY_PORT }} - username: ${{ secrets.SSH_DEPLOY_USER }} - key: ${{ secrets.SSH_DEPLOY_KEY }} - source: "mcpedia-web-next.tar.gz" - target: "/tmp/" + - name: Setup SSH key + env: + SSH_KEY: ${{ secrets.SSH_DEPLOY_KEY }} + run: | + mkdir -p ~/.ssh + echo "$SSH_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + sed -i 's/\r$//' ~/.ssh/id_ed25519 + ssh-keygen -y -f ~/.ssh/id_ed25519 >/dev/null 2>&1 || { echo "SSH key invalid"; exit 1; } + ssh-keyscan -H "$VPS_HOST" >> ~/.ssh/known_hosts 2>/dev/null - - name: Unpack and restart on VPS - uses: appleboy/ssh-action@v1 - with: - host: ${{ secrets.SSH_DEPLOY_HOST }} - port: ${{ secrets.SSH_DEPLOY_PORT }} - username: ${{ secrets.SSH_DEPLOY_USER }} - key: ${{ secrets.SSH_DEPLOY_KEY }} - envs: SSH_DEPLOY_HOST - script: | - set -e - cd /home/code/mcpedia + - name: Copy to VPS & deploy + env: + SSH_KEY: ${{ secrets.SSH_DEPLOY_KEY }} + run: | + STORE_PATH="${{ steps.build.outputs.store-path }}" + SERVICE="${{ matrix.service }}" + PROFILE="mcpedia-${SERVICE}" - git pull origin main - /home/code/.bun/bin/bun install --frozen-lockfile - /home/code/.bun/bin/bun run scripts/indexer.ts + # Copy Nix closure to VPS + nix copy --to "ssh://***@$VPS_HOST" "$STORE_PATH" - rm -rf apps/web/.next - tar -xzf /tmp/mcpedia-web-next.tar.gz -C apps/web/ - rm -f /tmp/mcpedia-web-next.tar.gz + # Deploy: set profile + restart + ssh "$VPS_USER@$VPS_HOST" \ + "sudo /nix/var/nix/profiles/default/bin/nix-env \ + --profile /nix/var/nix/profiles/$PROFILE \ + --set '$STORE_PATH' && \ + sudo systemctl restart $PROFILE && \ + sleep 3 && \ + sudo systemctl status $PROFILE --no-pager --no-legend | head -5" - sudo systemctl restart mcpedia-web mcpedia-api mcpedia-mcp mcpedia-worker - sleep 3 - systemctl --no-pager status mcpedia-web mcpedia-api mcpedia-mcp mcpedia-worker --no-legend + - name: Verify service + run: | + SERVICE="${{ matrix.service }}" + PORT_MAP="web:4016 api:4017 mcp:4021 worker:4018" + PORT=$(echo "$PORT_MAP" | grep "$SERVICE" | awk -F: '{print $2}') + ssh "$VPS_USER@$VPS_HOST" "curl -s -o /dev/null -w '%{http_code}' http://localhost:$PORT/health 2>/dev/null || systemctl is-active $PROFILE" diff --git a/.gitignore b/.gitignore index e4fe07d..f200acb 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,9 @@ next-env.d.ts # monorepo / turbo .turbo/ dist/ + +# MCPedia content — DB is source of truth via MCP API (create_document/update_document) +# Filesystem .md files are auto-generated backups, not tracked in git. +# To edit content: use the MCP create_document/update_document tools (requires x-webhook-secret) +content/**/*.md +!content/**/.gitkeep diff --git a/AGENTS.md b/AGENTS.md index badf09a..fccab55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,12 +10,15 @@ This block is written and re-added by `next dev` — verify at `node_modules/nex # MCPedia Agent Instructions -MCPedia is a content-first knowledge base where Markdown files under `content/` serve as the Git-tracked source of truth, indexed into PostgreSQL and accessed via Web UI, tRPC API, and Model Context Protocol (MCP). +MCPedia is a content-first knowledge base where documents are stored in **PostgreSQL** as the source of truth (with auto-generated .md backups on disk), indexed into embeddings + search vectors, and accessed via Web UI, tRPC API, and Model Context Protocol (MCP). + +- **DB-first**: Documents are created/updated/deleted via the MCP `create_document`/`update_document`/`delete_document` tools (require `x-webhook-secret`). Filesystem `.md` files in `content/` are auto-generated backups (gitignored). +- **Git-sync deploy**: CI builds `.next` on GitHub Actions → tarball → SCP to VPS → unpack. VPS never builds. Content changes flow via MCP API to the DB directly. ## Architecture Principles 1. **Single Core Layer (`@mcpedia/core`)**: All business logic (document CRUD, indexing, search, revisions, path classification) resides in `packages/core`. Never access the database directly from `apps/web`, `apps/mcp`, or `apps/api`. -2. **Content as Source of Truth**: Markdown files in `content/` with YAML frontmatter are the primary data store. The database stores metadata, search vectors, embeddings, and revision history. +2. **DB as Source of Truth**: PostgreSQL stores document body + metadata. Filesystem `.md` files in `content/` are auto-generated backups (gitignored). Use the MCP `create_document`/`update_document` tools to write content. 3. **Multi-Modal Search**: Keyword search (Postgres FTS `tsvector` + GIN), Semantic search (cosine similarity over chunked embeddings), and Hybrid search (Reciprocal Rank Fusion / RRF) are unified in `@mcpedia/search`. 4. **Mutations & Security**: State-changing operations (document creation/updates/deletions, reindexing, revision restoration) require authentication (`x-webhook-secret` header or session cookie). diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..7890fe1 --- /dev/null +++ b/flake.nix @@ -0,0 +1,115 @@ +{ + description = "MCPedia — Nix-native build for web, api, mcp, worker services"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { inherit system; }; + + # ─── Web: Next.js ─────────────────────────────────────────────── + web = pkgs.stdenvNoCC.mkDerivation { + pname = "mcpedia-web"; + version = "1.0.0"; + src = ./.; + nativeBuildInputs = [ pkgs.bun pkgs.makeBinaryWrapper ]; + buildPhase = '' + export HOME=$TMPDIR + bun install --frozen-lockfile 2>&1 + cd apps/web + bun run build + ''; + installPhase = '' + mkdir -p $out/share/mcpedia-web + cp -rL apps/web/.next $out/share/mcpedia-web/.next + cp -rL apps/web/node_modules $out/share/mcpedia-web/node_modules + cp apps/web/package.json $out/share/mcpedia-web/ + makeBinaryWrapper ${pkgs.bun}/bin/bun $out/bin/mcpedia-web \ + --add-flags "node_modules/.bin/next start" \ + --chdir $out/share/mcpedia-web + ''; + }; + + # ─── API: Hono ────────────────────────────────────────────────── + api = pkgs.stdenvNoCC.mkDerivation { + pname = "mcpedia-api"; + version = "1.0.0"; + src = ./.; + nativeBuildInputs = [ pkgs.bun pkgs.makeBinaryWrapper ]; + buildPhase = '' + export HOME=$TMPDIR + bun install --frozen-lockfile 2>&1 + cd apps/api + bun build src/index.ts --outdir dist --external @mcpedia/* + ''; + installPhase = '' + mkdir -p $out/share/mcpedia-api + cp -rL apps/api/dist $out/share/mcpedia-api/dist + cp -rL apps/api/node_modules $out/share/mcpedia-api/node_modules + cp apps/api/package.json $out/share/mcpedia-api/ + makeBinaryWrapper ${pkgs.bun}/bin/bun $out/bin/mcpedia-api \ + --add-flags "run dist/index.js" \ + --chdir $out/share/mcpedia-api + ''; + }; + + # ─── MCP: Streamable HTTP ─────────────────────────────────────── + mcp = pkgs.stdenvNoCC.mkDerivation { + pname = "mcpedia-mcp"; + version = "1.0.0"; + src = ./.; + nativeBuildInputs = [ pkgs.bun pkgs.makeBinaryWrapper ]; + buildPhase = '' + export HOME=$TMPDIR + bun install --frozen-lockfile 2>&1 + cd apps/mcp + bun build src/http.ts --outdir dist --external @mcpedia/* --external @modelcontextprotocol/* + ''; + installPhase = '' + mkdir -p $out/share/mcpedia-mcp + cp -rL apps/mcp/dist $out/share/mcpedia-mcp/dist + cp -rL apps/mcp/node_modules $out/share/mcpedia-mcp/node_modules + cp apps/mcp/package.json $out/share/mcpedia-mcp/ + makeBinaryWrapper ${pkgs.bun}/bin/bun $out/bin/mcpedia-mcp \ + --add-flags "run dist/http.js" \ + --chdir $out/share/mcpedia-mcp + ''; + }; + + # ─── Worker: BullMQ ───────────────────────────────────────────── + worker = pkgs.stdenvNoCC.mkDerivation { + pname = "mcpedia-worker"; + version = "1.0.0"; + src = ./.; + nativeBuildInputs = [ pkgs.bun pkgs.makeBinaryWrapper ]; + buildPhase = '' + export HOME=$TMPDIR + bun install --frozen-lockfile 2>&1 + cd apps/worker + bun build src/index.ts --outdir dist --external @mcpedia/* + ''; + installPhase = '' + mkdir -p $out/share/mcpedia-worker + cp -rL apps/worker/dist $out/share/mcpedia-worker/dist + cp -rL apps/worker/node_modules $out/share/mcpedia-worker/node_modules + cp apps/worker/package.json $out/share/mcpedia-worker/ + makeBinaryWrapper ${pkgs.bun}/bin/bun $out/bin/mcpedia-worker \ + --add-flags "run dist/index.js" \ + --chdir $out/share/mcpedia-worker + ''; + }; + + in { + packages = { + web = web; + api = api; + mcp = mcp; + worker = worker; + default = web; # `nix build` builds web by default + }; + }); +}