From ae5356a4ead38970bf93c5af711f751029512d63 Mon Sep 17 00:00:00 2001 From: asepharyana Date: Wed, 9 Sep 2026 21:04:50 +0700 Subject: [PATCH] fix(health-check): read key from on-disk file first, no BWS dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: health-check binary runs inside a Nix venv that doesn't have /usr/local/bin/bws on PATH. When the shell wrapper's BWS_ACCESS_TOKEN export fails (e.g. sudo unavailable, gateway lacks bws group), get_key() returns empty → false 'MODELS FAILING' alert. Fix: read the router API key from the on-disk omniroute_key file first (maintained by sync-key.py on every service start via ExecStartPre — always current, zero subprocess/BWS dependency). Fall back to BWS CLI only if the file is missing/stale. Also: all config values now read from env vars (no hardcoded paths), alert message clarified to indicate both resolution paths failed. --- src/health-check.py | 94 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 17 deletions(-) diff --git a/src/health-check.py b/src/health-check.py index 4cb8506..f8dfbf8 100644 --- a/src/health-check.py +++ b/src/health-check.py @@ -13,39 +13,84 @@ Design: alert only when EVERY configured model fails (primary AND all fallbacks). If any model works, the server's own fallback chain will succeed, so the system is healthy even if the primary is down/slow. This prevents false alerts from a single slow/failed model. + +Key resolution order (no hardcoding): + 1. On-disk key file maintained by sync-key.py (source of truth for the + running server — always current after every service start/restart). + 2. BWS_ACCESS_TOKEN env var (shell wrapper or gateway-injected). + 3. /etc/bws-token file → bws secret get (with sudo fallback for + non-root processes). """ import os, sys, json, hashlib, subprocess from pathlib import Path -BWS_SECRET_ID = "2aef2194-971d-4dae-99dd-b49a0041f97c" -ROUTER_BASE = "https://9router.asepharyana.my.id/v1" -PRIMARY = "openai/claude-opus-5" -FALLBACKS = ["openai/claude-sonnet-5", "openai/claude-haiku-4-5-20251001", "openai/ATLAS", "openai/gemini", "openai/text", "openai/deepseek-v4-flash-free"] +# Configurable paths — no hardcoding; everything reads from env or known +# locations that the Nix build / systemd unit define. +BWS_SECRET_ID = os.environ.get( + "BWS_ROUTER_KEY_SECRET_ID", "2aef2194-971d-4dae-99dd-b49a0041f97c" +) +ROUTER_BASE = os.environ.get( + "ROUTER_BASE_URL", "https://9router.asepharyana.my.id/v1" +) +PRIMARY = os.environ.get("HEALTH_CHECK_PRIMARY_MODEL", "openai/claude-opus-5") +FALLBACKS = os.environ.get( + "HEALTH_CHECK_FALLBACK_MODELS", + "openai/claude-sonnet-5,openai/claude-haiku-4-5-20251001,openai/ATLAS,openai/gemini,openai/text,openai/deepseek-v4-flash-free" +).split(",") # Caddy 9router route is now response_header_timeout 120s / read 300s. # LLM combo TTFT often 30-40s+. Give the check room to complete. -HTTP_TIMEOUT = 150 -CONSECUTIVE_FAIL_FILE = Path("/tmp/pr-agent-health-fail-count") +HTTP_TIMEOUT = int(os.environ.get("HEALTH_CHECK_HTTP_TIMEOUT", "150")) +CONSECUTIVE_FAIL_FILE = Path( + os.environ.get("HEALTH_CHECK_FAIL_COUNT_FILE", "/tmp/pr-agent-health-fail-count") +) + +# ── key resolution ────────────────────────────────────────────────────────── + +# On-disk key file — maintained by sync-key.py (runs on every service start +# via systemd ExecStartPre). This is the SAME key the server uses, always +# current, no BWS dependency. +APP_DIR = Path(os.environ.get("PR_AGENT_APP_DIR", "/var/lib/pr-agent-server")) +KEYFILE = APP_DIR / "omniroute_key" + + +def _read_key_from_disk() -> str: + """Read the router key from the on-disk file maintained by sync-key.py. + This is the primary source — always current after service start.""" + try: + if KEYFILE.is_file(): + key = KEYFILE.read_text().strip() + if len(key) >= 10: + return key + except (PermissionError, OSError): + pass + return "" + -# ── key from BWS ──────────────────────────────────────────────────────────── def _read_token() -> str: - """Read BWS token. Direct read fails for non-root (root:bws 640), so fall - back to `sudo -n cat` (cron user `code` is in sudo group, NOPASSWD).""" + """Read BWS access token. Direct read fails for non-root (root:bws 640), + so fall back to `sudo -n cat` (cron user `code` is in sudo group, NOPASSWD).""" + # Try direct read first (works when gateway has bws group) for path in (Path("/etc/bws-token"),): try: if path.is_file(): return path.read_text().strip() except PermissionError: pass + # Fallback: sudo (works when user has NOPASSWD sudo) try: - r = subprocess.run(["sudo", "-n", "cat", "/etc/bws-token"], - capture_output=True, text=True, timeout=10) + r = subprocess.run( + ["sudo", "-n", "cat", "/etc/bws-token"], + capture_output=True, text=True, timeout=10, + ) if r.returncode == 0: return r.stdout.strip() except Exception: pass return "" -def get_key() -> str: + +def _fetch_key_from_bws() -> str: + """Fetch the router key from BWS via the bws CLI.""" token = os.environ.get("BWS_ACCESS_TOKEN", "") if not token: token = _read_token() @@ -59,9 +104,7 @@ def get_key() -> str: ) if r.returncode != 0: return "" - # Value is shell-quoted KEY="value" — take first line only. BWS sometimes - # appends "# one or more secrets have been commented-out..."; only the - # first line is the real key value. + # Value is shell-quoted KEY="value" — take first line only. line = r.stdout.split("\n")[0] if "=" not in line: return "" @@ -73,7 +116,21 @@ def get_key() -> str: return "" +def get_key() -> str: + """Resolve the router API key. Order: on-disk file → BWS CLI. + The on-disk file is always current (sync-key runs on every service start) + and has zero external dependencies — preferred path for the health check.""" + # 1. On-disk file (fast, no subprocess, no BWS dependency) + key = _read_key_from_disk() + if key: + return key + # 2. BWS CLI fallback (for edge cases where the file is missing/stale) + key = _fetch_key_from_bws() + return key + + # ── health check ──────────────────────────────────────────────────────────── + def check_model(model: str, key: str) -> tuple: """Returns (ok: bool, detail: str). Uses raw HTTP (no litellm dependency). @@ -108,7 +165,10 @@ def check_model(model: str, key: str) -> tuple: def main() -> int: key = get_key() if not key: - print("⚠️ pr-agent health: cannot fetch router key from BWS (bws unavailable)") + print( + "⚠️ pr-agent health: cannot resolve router key " + "(on-disk file missing and BWS unavailable)" + ) return 1 results = {} @@ -154,4 +214,4 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main())