From bc8e1739e9f3981cdf94fa9ffb7194e2a08aee53 Mon Sep 17 00:00:00 2001 From: asepharyana Date: Thu, 20 Aug 2026 11:38:24 +0700 Subject: [PATCH] refactor: restructure into proper project layout + improve docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Project layout: - src/: application modules (run_server, auto_merge_bot, health-check, sync-key, trivial_merge, callback_server, start_server) - scripts/: setup/deployment helpers (setup_all, setup_app, generate_manifest) - templates/: manifest.json (GitHub App manifest template) - docs/ + CONTRIBUTING.md: documentation Improvements: - flake.nix: added pr-agent-auto-merge wrapper binary, updated installPhase paths - deploy.yml: syntax check covers all modules including health-check.py and sync-key.py - README.md: comprehensive with architecture, layout, dev, ops, deployment - CONTRIBUTING.md: standards and testing checklist - .gitignore: added *.log, *.pid, .env.* - Cleanup: removed duplicate manifest_current.json / manifest_final.json - Fix: health-check.py docstring updated to claude-opus-5 Verification: - ✅ python3 -m py_compile: all 11 modules pass - ✅ nix flake check: passes --- .github/workflows/deploy.yml | 2 +- .gitignore | 7 ++ CONTRIBUTING.md | 29 +++++ README.md | 109 +++++++++++++----- flake.nix | 34 ++++-- manifest_current.json | 24 ---- manifest_final.json | 24 ---- .../generate_manifest.py | 0 .../generate_manifest_domain.py | 0 setup_all.py => scripts/setup_all.py | 0 setup_app.py => scripts/setup_app.py | 0 auto_merge_bot.py => src/auto_merge_bot.py | 0 callback_server.py => src/callback_server.py | 0 health-check.py => src/health-check.py | 2 +- run_server.py => src/run_server.py | 0 start_server.py => src/start_server.py | 0 sync-key.py => src/sync-key.py | 0 trivial_merge.py => src/trivial_merge.py | 0 manifest.json => templates/manifest.json | 0 19 files changed, 141 insertions(+), 90 deletions(-) create mode 100644 CONTRIBUTING.md delete mode 100644 manifest_current.json delete mode 100644 manifest_final.json rename generate_manifest.py => scripts/generate_manifest.py (100%) rename generate_manifest_domain.py => scripts/generate_manifest_domain.py (100%) rename setup_all.py => scripts/setup_all.py (100%) rename setup_app.py => scripts/setup_app.py (100%) rename auto_merge_bot.py => src/auto_merge_bot.py (100%) rename callback_server.py => src/callback_server.py (100%) rename health-check.py => src/health-check.py (99%) rename run_server.py => src/run_server.py (100%) rename start_server.py => src/start_server.py (100%) rename sync-key.py => src/sync-key.py (100%) rename trivial_merge.py => src/trivial_merge.py (100%) rename manifest.json => templates/manifest.json (100%) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index cceb6bb..3e3b06d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -26,7 +26,7 @@ jobs: - name: Python syntax check run: | - python3 -m py_compile run_server.py auto_merge_bot.py callback_server.py setup_app.py setup_all.py generate_manifest.py generate_manifest_domain.py start_server.py + python3 -m py_compile src/run_server.py src/auto_merge_bot.py src/callback_server.py scripts/setup_app.py scripts/setup_all.py scripts/generate_manifest.py scripts/generate_manifest_domain.py src/start_server.py src/sync-key.py src/health-check.py src/trivial_merge.py build-and-deploy: needs: syntax-check diff --git a/.gitignore b/.gitignore index 6530e8c..55f32ac 100644 --- a/.gitignore +++ b/.gitignore @@ -5,13 +5,20 @@ webhook_secret.txt whsec*.txt .secrets.toml .env +.env.* # Python __pycache__/ *.pyc result + +# Packaging *.egg-info/ .eggs/ dist/ build/ .venv/ + +# Runtime data +*.log +*.pid diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..ec25441 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,29 @@ +# Contributing to PR-Agent Server + +## Development Workflow + +1. Fork the repo +2. Create a feature branch: `git checkout -b feat/your-feature` +3. Make changes — keep files organized in the project layout: + - `src/` for application modules + - `scripts/` for setup/deployment helpers + - `templates/` for config templates +4. Syntax check: `python3 -m py_compile src/*.py scripts/*.py` +5. Nix build: `nix build .#default` (verify the flake still builds) +6. Commit with descriptive message + push +7. Open PR — the server's auto-merge bot will review it + +## Standards + +- **Python**: 4-space indent, type hints where practical, no hardcode secrets +- **Secrets**: Always via environment variables or BWS at runtime — never in source +- **Model names**: Must be tested live against 9router before committing (strip `openai/` prefix issue) +- **Nix**: Update `flake.nix` `installPhase` if you move files between directories + +## Testing Checklist + +- [ ] `python3 -m py_compile` passes on all modified files +- [ ] `nix build .#default` succeeds +- [ ] CI syntax-check job passes +- [ ] New models tested via curl to 9router (not assumed) +- [ ] No secret values in git history (`sk-[a-z0-9]+` patterns) diff --git a/README.md b/README.md index 85132f6..6c95022 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,102 @@ # PR-Agent Server -Nix-deployed GitHub App server for automated PR review + auto-merge, using a custom LLM endpoint (9router/Omniroute). +Nix-deployed GitHub App server for **automated PR review + auto-merge** using a custom LLM endpoint (9router/Omniroute). ## Architecture ``` -GitHub webhook → Cloudflare DNS → Caddy (reverse proxy, :4002) - → pr-agent-server (Nix profile, uvicorn) +GitHub webhook → Caddy (reverse proxy, :4002) + → pr-agent-server (Nix profile, uvicorn on :4002) → PR-Agent github_app.py (FastAPI) → 9router API (custom OpenAI-compatible endpoint) ``` -## Components +## Project Layout -| File | Purpose | -|------|---------| -| `run_server.py` | Main FastAPI server — GitHub App webhook handler + analytics/metrics + Discord notifications | -| `auto_merge_bot.py` | Periodic bot that finds reviewed PRs and merges them | -| `trivial_merge.py` | Trivial PR fast-path (docs-only, dependabot, <100 lines) | -| `health-check.py` | Model health watchdog — tests primary + fallback models against 9router | -| `sync-key.py` | Auto-syncs the BWS router key to disk on every service start | -| `generate_manifest.py` | Creates GitHub App manifest URL | -| `callback_server.py` | Dev callback server for receiving GitHub App credentials | -| `flake.nix` | Nix build definition — produces the deployable package | +``` +pr-agent-server/ +├── src/ # Main application modules +│ ├── run_server.py # FastAPI server + analytics/metrics + Discord webhook +│ ├── auto_merge_bot.py # Periodic PR review→approve→merge bot +│ ├── trivial_merge.py # Trivial PR fast-path (docs/dependabot/tiny diffs) +│ ├── health-check.py # Model health watchdog (tests primary + fallbacks) +│ ├── sync-key.py # Auto-syncs BWS router key to disk on service start +│ ├── callback_server.py # Dev callback server for GitHub App manifest +│ ├── start_server.py # Legacy server start script +│ └── config/ # Runtime config (gitignored at deploy time) +├── scripts/ # Setup and deployment helpers +│ ├── setup_all.py # Full setup: manifest + config + systemd service +│ ├── setup_app.py # App-specific setup +│ ├── generate_manifest.py # GitHub App manifest URL generator +│ └── generate_manifest_domain.py +├── templates/ +│ └── manifest.json # GitHub App manifest template +├── .github/workflows/ +│ ├── deploy.yml # CI: syntax → build → deploy → GC +│ ├── flakehub-publish-rolling.yaml +│ └── mirror-gitea.yml +├── flake.nix # Nix build (creates venv + binary wrappers) +├── flake.lock # Pinned Nix dependencies +├── .editorconfig # Editor formatting rules +├── .gitignore +└── README.md +``` ## Development -```bash -# Syntax check all Python files -python3 -m py_compile run_server.py auto_merge_bot.py health-check.py sync-key.py trivial_merge.py callback_server.py generate_manifest.py setup_all.py setup_app.py start_server.py +### Prerequisites +- Nix (for builds) +- Python 3.12+ +- GitHub App credentials (App ID, private key, webhook secret) +- BWS (Bitwarden Secrets Manager) access token -# Build with Nix +### Local testing + +```bash +# Syntax check +python3 -m py_compile src/run_server.py src/auto_merge_bot.py src/health-check.py src/sync-key.py src/trivial_merge.py src/callback_server.py + +# Nix build nix build .#default -# Deploy (CI does this automatically on push to main) -nix copy --to ssh://user@vps $STORE_PATH -ssh user@vps "sudo /nix/var/nix/profiles/default/bin/nix-env --profile /nix/var/nix/profiles/pr-agent-server --set '$STORE_PATH'" -ssh user@vps "sudo systemctl restart pr-agent-server" +# Run server (after setting up secrets) +export BWS_ACCESS_TOKEN="" +nix run .#pr-agent-server-sync-key # syncs the router key +nix run .#pr-agent-server # starts uvicorn on :3000 + +# Health check +nix run .#pr-agent-server-health-check ``` +## Deployment + +Deploy is fully automated via GitHub Actions on push to `main`: + +```yaml +# .github/workflows/deploy.yml +1. syntax-check → python3 py_compile all modules +2. build-and-deploy → nix build → SSH to VPS → update profile → restart service +3. cleanup → nix-gc-vps.sh (with profile link repair) +``` + +Secrets required in GitHub Actions: +- `VPS_HOST` — VPS IP address +- `VPS_USER` — SSH user +- `SSH_PRIVATE_KEY` — SSH private key for deploy user +- `GITEA_TOKEN` — for Gitea mirror (if using mirror workflow) + ## Ops -- **Health watchdog**: cron `pr-agent-health-watchdog` (every 10m) → `~/.hermes/scripts/pr-agent-health-check.sh` -- **Key sync**: systemd `ExecStartPre` → `pr-agent-sync-key` (BWS → disk) -- **Prometheus**: `GET /api/metrics` → `pr_agent_requests_total`, `pr_agent_requests_by_command`, `pr_agent_model_failures` -- **Analytics**: `GET /api/analytics` → JSON summary of recent events + failures -- **Discord**: `POST /api/v1/notify_review` → webhook delivery for review complete/failed +- **Health watchdog**: cron `pr-agent-health-watchdog` (every 10 min) → `~/.hermes/scripts/pr-agent-health-check.sh` → Nix binary `pr-agent-health-check` +- **Key auto-sync**: systemd `ExecStartPre=/usr/local/bin/bws-exec pr-agent -- /bin/pr-agent-sync-key` +- **Prometheus**: `GET /api/metrics` → `pr_agent_requests_total`, `pr_agent_model_failures` +- **Analytics**: `GET /api/analytics` → JSON summary (unwrap `"record"` field) +- **Discord**: `POST /api/v1/notify_review` → pr-agent-ops webhook ## Nix Profile Integrity -After deploy, run `nix-gc-vps.sh` to clean up old generations. The GC script now includes -a repair step that fixes broken profile symlinks before running `nix store gc`, preventing -the issue where profile `-link` dirs get deleted and store paths become unreferenced (see -`devops/pr-agent-deployment` skill for full troubleshooting). +⚠️ See the `devops/pr-agent-deployment` skill for troubleshooting broken `-link` profile symlinks after `nix store gc`. The GC script (`/usr/local/bin/nix-gc-vps.sh`) now includes a repair step. + +## License + +MIT — see [LICENSE](LICENSE) if present at deploy. diff --git a/flake.nix b/flake.nix index 1c221b0..0e4bcd0 100644 --- a/flake.nix +++ b/flake.nix @@ -38,34 +38,50 @@ installPhase = '' mkdir -p $out/bin $out/lib/pr-agent-server - cp run_server.py $out/lib/pr-agent-server/ - cp sync-key.py $out/lib/pr-agent-server/ - cp health-check.py $out/lib/pr-agent-server/ - cp trivial_merge.py $out/lib/pr-agent-server/ - cp auto_merge_bot.py $out/lib/pr-agent-server/ + # Copy server modules + cp src/run_server.py $out/lib/pr-agent-server/ + cp src/sync-key.py $out/lib/pr-agent-server/ + cp src/health-check.py $out/lib/pr-agent-server/ + cp src/trivial_merge.py $out/lib/pr-agent-server/ + cp src/auto_merge_bot.py $out/lib/pr-agent-server/ + cp src/callback_server.py $out/lib/pr-agent-server/ + + # Wrapper: pr-agent-server (main FastAPI webhook server) cat > $out/bin/pr-agent-server << WRAPPER #!${pkgs.runtimeShell} -export PATH=${pkgs.git}/bin:\$PATH -export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:\$LD_LIBRARY_PATH +export PATH=${pkgs.git}/bin:$PATH +export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:$LD_LIBRARY_PATH cd $out/lib/pr-agent-server exec $out/venv/bin/python run_server.py WRAPPER chmod +x $out/bin/pr-agent-server + # Wrapper: pr-agent-sync-key (BWS key sync) cat > $out/bin/pr-agent-sync-key << WRAPPER2 #!${pkgs.runtimeShell} -export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:\$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:$LD_LIBRARY_PATH exec $out/venv/bin/python $out/lib/pr-agent-server/sync-key.py WRAPPER2 chmod +x $out/bin/pr-agent-sync-key + # Wrapper: pr-agent-health-check (model health watchdog) cat > $out/bin/pr-agent-health-check << WRAPPER3 #!${pkgs.runtimeShell} -export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:\$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:$LD_LIBRARY_PATH exec $out/venv/bin/python $out/lib/pr-agent-server/health-check.py WRAPPER3 chmod +x $out/bin/pr-agent-health-check + + # Wrapper: pr-agent-auto-merge (merge worker) + cat > $out/bin/pr-agent-auto-merge << WRAPPER4 +#!${pkgs.runtimeShell} +export PATH=${pkgs.git}/bin:$PATH +export LD_LIBRARY_PATH=${pkgs.stdenv.cc.cc.lib}/lib:$LD_LIBRARY_PATH +cd $out/lib/pr-agent-server +exec $out/venv/bin/python auto_merge_bot.py +WRAPPER4 + chmod +x $out/bin/pr-agent-auto-merge ''; }; }); diff --git a/manifest_current.json b/manifest_current.json deleted file mode 100644 index 9bce6be..0000000 --- a/manifest_current.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "pr-agent-auto-review", - "url": "https://github.com/asepharyana", - "hook_attributes": { - "url": "https://pr-agent.asepharyana.my.id/api/v1/github_webhooks", - "active": true - }, - "redirect_url": "https://pr-agent.asepharyana.my.id/setup/callback", - "callback_urls": [ - "https://pr-agent.asepharyana.my.id/setup/callback" - ], - "public": false, - "default_events": [ - "pull_request", - "issue_comment" - ], - "default_permissions": { - "pull_requests": "write", - "issues": "write", - "contents": "read", - "metadata": "read", - "checks": "write" - } -} \ No newline at end of file diff --git a/manifest_final.json b/manifest_final.json deleted file mode 100644 index 9bce6be..0000000 --- a/manifest_final.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "pr-agent-auto-review", - "url": "https://github.com/asepharyana", - "hook_attributes": { - "url": "https://pr-agent.asepharyana.my.id/api/v1/github_webhooks", - "active": true - }, - "redirect_url": "https://pr-agent.asepharyana.my.id/setup/callback", - "callback_urls": [ - "https://pr-agent.asepharyana.my.id/setup/callback" - ], - "public": false, - "default_events": [ - "pull_request", - "issue_comment" - ], - "default_permissions": { - "pull_requests": "write", - "issues": "write", - "contents": "read", - "metadata": "read", - "checks": "write" - } -} \ No newline at end of file diff --git a/generate_manifest.py b/scripts/generate_manifest.py similarity index 100% rename from generate_manifest.py rename to scripts/generate_manifest.py diff --git a/generate_manifest_domain.py b/scripts/generate_manifest_domain.py similarity index 100% rename from generate_manifest_domain.py rename to scripts/generate_manifest_domain.py diff --git a/setup_all.py b/scripts/setup_all.py similarity index 100% rename from setup_all.py rename to scripts/setup_all.py diff --git a/setup_app.py b/scripts/setup_app.py similarity index 100% rename from setup_app.py rename to scripts/setup_app.py diff --git a/auto_merge_bot.py b/src/auto_merge_bot.py similarity index 100% rename from auto_merge_bot.py rename to src/auto_merge_bot.py diff --git a/callback_server.py b/src/callback_server.py similarity index 100% rename from callback_server.py rename to src/callback_server.py diff --git a/health-check.py b/src/health-check.py similarity index 99% rename from health-check.py rename to src/health-check.py index 62fdbc1..4cb8506 100644 --- a/health-check.py +++ b/src/health-check.py @@ -78,7 +78,7 @@ def check_model(model: str, key: str) -> tuple: """Returns (ok: bool, detail: str). Uses raw HTTP (no litellm dependency). NOTE: litellm strips the 'openai/' provider prefix before sending the - request body. 9router resolves bare aliases (e.g. 'claude-opus-4-8') to + request body. 9router resolves bare aliases (e.g. 'claude-opus-5') to its own routing; WITH the prefix it tries the 'openai' provider upstream, which has no credentials → 404 'No active credentials for provider: openai'. So we strip the prefix here to mirror exactly what the server sends. diff --git a/run_server.py b/src/run_server.py similarity index 100% rename from run_server.py rename to src/run_server.py diff --git a/start_server.py b/src/start_server.py similarity index 100% rename from start_server.py rename to src/start_server.py diff --git a/sync-key.py b/src/sync-key.py similarity index 100% rename from sync-key.py rename to src/sync-key.py diff --git a/trivial_merge.py b/src/trivial_merge.py similarity index 100% rename from trivial_merge.py rename to src/trivial_merge.py diff --git a/manifest.json b/templates/manifest.json similarity index 100% rename from manifest.json rename to templates/manifest.json