feat(proxy-fitness): pool fitness registry + Smart rotation + pool-scoped retry

Pool/IP fitness registry (globalThis-backed, provider::model scopes, 5-min
cooldown, provider::* wildcard) fed by pool-scoped failures: freebuff
limited-IP / model-locked gates and opencode free per-IP limits. chatCore
retries a failed pool via another pool without locking the account; new
Smart rotation strategy (per-connection + no-auth providers) skips unfit
pools. Proxy Fitness dashboard page: active-block table with provider/IP
filters, per-record Clear and provider-scoped Clear All; egress column via
pool geo enrichment. Unit tests for the registry.

open-sse/executors/freebuff.js

@@ -8,6 +8,7 @@ import {
   DEFAULT_RETRY_CONFIG,
   resolveRetryEntry,
 } from "../config/runtimeConfig.js";
+import { markPoolUnfit, clearPoolUnfit } from "../services/proxyPoolFitness.js";
 
 /**
  * Freebuff Executor — OpenAI-compatible chat completions on
@@ -87,8 +88,110 @@ const FREE_ROOT_AGENT_BY_MODEL = {
 // don't share one session row). Re-claims are driven by the cache expiring or
 // by a 428 from chat — no early re-claim, so we never POST /session while our
 // own row is still active (which could come back as a spurious model_locked).
-const sessionCache = new Map(); // `${token}::${model}` -> { instanceId, expiresAt }
-const inflight = new Map();     // dedupe concurrent claims for the same key
+// All state lives on globalThis so Next dev (Turbopack) bundles share ONE copy.
+const FB_STATE_KEY = "__9routerFreebuffState__";
+const fbState = (globalThis[FB_STATE_KEY] ??= {
+  sessionCache: new Map(),      // `${token}::${model}` -> { instanceId, expiresAt }
+  inflight: new Map(),          // dedupe concurrent claims for the same key
+  modelLockCooldowns: new Map(), // `${token}::${model}` -> expiresAt (ms)
+  poolLimitCooldowns: new Map(), // `${proxyKey}::${model}` -> expiresAt (ms)
+});
+const sessionCache = fbState.sessionCache;
+const inflight = fbState.inflight;
+const modelLockCooldowns = fbState.modelLockCooldowns;
+const poolLimitCooldowns = fbState.poolLimitCooldowns;
+
+const MODEL_LOCK_COOLDOWN_MS = 10 * 60 * 1000; // session bound to another model (~1h) — re-check every 10 min
+const POOL_LIMITED_COOLDOWN_MS = 5 * 60 * 1000; // IP tier refuses this model — try a different pool/relay
+
+// Cooldown maps need pruning: expired entries are cleared on write (sweep) and
+// on read, so long-running servers don't accumulate one entry per (account,model)
+// / (proxy,model) forever.
+function setCooldown(map, key, until) {
+  const now = Date.now();
+  for (const [k, v] of map) {
+    if (v <= now) map.delete(k);
+  }
+  map.set(key, until);
+}
+
+function getCooldown(map, key) {
+  const until = map.get(key);
+  if (until == null) return null;
+  if (until <= Date.now()) {
+    map.delete(key);
+    return null;
+  }
+  return until;
+}
+
+function proxyKeyOf(proxyOptions) {
+  return proxyOptions?.vercelRelayUrl || proxyOptions?.connectionProxyUrl || "direct";
+}
+
+function sessionGateFromText(text) {
+  let parsed = {};
+  try { parsed = JSON.parse(String(text || "")); } catch { parsed = {}; }
+  return classifySessionGate(parsed.error || parsed.error_type || "", parsed.message || "", parsed.currentModel || null);
+}
+
+// Parse a 409/428/410 body into { kind, currentModel }. `msg` may be a whole
+// error string containing a JSON tail (requestSession errors embed the body).
+function sessionGateFromError(error) {
+  const msg = String(error?.message || "");
+  const start = msg.indexOf("{");
+  if (start < 0) return null;
+  try {
+    const parsed = JSON.parse(msg.slice(start));
+    return classifySessionGate(parsed.error || "", parsed.message || "", parsed.currentModel || null);
+  } catch {
+    return null;
+  }
+}
+
+function classifySessionGate(code, message, currentModel) {
+  if (code === "session_superseded") return { kind: "superseded" };
+  if (code === "model_locked") return { kind: "model_locked", currentModel };
+  // session_model_mismatch with the limited-tier message is an IP-tier refusal;
+  // without it (or unknown) treat it as a model lock so we don't reclaim in a loop.
+  if (code === "session_model_mismatch") {
+    return /limited/i.test(String(message || ""))
+      ? { kind: "limited_ip" }
+      : { kind: "model_locked", currentModel };
+  }
+  return { kind: "stale" }; // 428/410/unknown → reclaim
+}
+
+// Applies cooldowns and throws for non-reclaimable gates. Never returns for them.
+function throwSessionGateError(gate, { token, model, proxyKey, poolId, log }) {
+  if (gate.kind === "model_locked") {
+    const until = Date.now() + MODEL_LOCK_COOLDOWN_MS;
+    setCooldown(modelLockCooldowns, `${token}::${model}`, until);
+    const label = gate.currentModel ? `"${gate.currentModel}"` : "another model";
+    const err = new Error(
+      `Freebuff session is locked to ${label} — it cannot serve ${model}. End the session on freebuff.com or wait for it to expire (~1h).`,
+    );
+    err.status = 409;
+    err.resetsAtMs = until;
+    log?.warn?.("AUTH", `Freebuff model_locked (session=${label}, requested=${model}) — model cooldown ${MODEL_LOCK_COOLDOWN_MS / 60000}min`);
+    throw err;
+  }
+  if (gate.kind === "limited_ip") {
+    const until = Date.now() + POOL_LIMITED_COOLDOWN_MS;
+    setCooldown(poolLimitCooldowns, `${proxyKey}::${model}`, until);
+    const scope = `freebuff::${model}`;
+    if (poolId) markPoolUnfit(poolId, scope, until, "limited_ip");
+    // Pool-scoped, not account-scoped: the caller retries via another pool
+    // instead of locking the account (resetsAtMs intentionally absent).
+    const err = new Error(
+      `Freebuff limited-mode IP rejected ${model} — this IP only allows DeepSeek V4 Flash / MiMo 2.5. Use a full-access proxy or a different model.`,
+    );
+    err.status = 409;
+    err.poolScoped = { poolId, scope, reason: "limited_ip" };
+    log?.warn?.("AUTH", `Freebuff limited-IP refused ${model} (proxy=${proxyKey.slice(0, 40)}…) — cooldown ${POOL_LIMITED_COOLDOWN_MS / 60000}min`);
+    throw err;
+  }
+}
 
 function sessionOrigin() {
   return new URL(PROVIDERS.freebuff.baseUrl).origin; // https://www.codebuff.com
@@ -185,7 +288,11 @@ async function requestSession(token, model, proxyOptions) {
 
 async function ensureSession(token, model, proxyOptions, force = false) {
   const key = sessionCacheKey(token, model);
+  // Lazy prune: drop stale rows so the cache never accumulates expired entries.
   const cached = sessionCache.get(key);
+  if (cached && cached.expiresAt <= Date.now()) {
+    sessionCache.delete(key);
+  }
   if (!force && cached && cached.expiresAt > Date.now()) {
     return { instanceId: cached.instanceId, status: "active" };
   }
@@ -262,6 +369,32 @@ export function resetSessionCache() {
   inflight.clear();
 }
 
+// Periodic sweeper: drop stale session rows + expired cooldowns so long-running
+// servers never accumulate state for accounts/models no longer in use.
+// Returns how many entries were removed.
+export function pruneSessionState(now = Date.now()) {
+  let removed = 0;
+  for (const [key, entry] of sessionCache) {
+    if (entry?.expiresAt && entry.expiresAt <= now) {
+      sessionCache.delete(key);
+      removed += 1;
+    }
+  }
+  for (const [key, until] of modelLockCooldowns) {
+    if (until <= now) {
+      modelLockCooldowns.delete(key);
+      removed += 1;
+    }
+  }
+  for (const [key, until] of poolLimitCooldowns) {
+    if (until <= now) {
+      poolLimitCooldowns.delete(key);
+      removed += 1;
+    }
+  }
+  return removed;
+}
+
 export class FreebuffExecutor extends BaseExecutor {
   constructor() {
     super("freebuff", PROVIDERS.freebuff);
@@ -282,6 +415,12 @@ export class FreebuffExecutor extends BaseExecutor {
       cost_mode: "free",
     };
     body.provider = { allow_fallbacks: false };
+    // Freebuff agents (base2-free-*) own reasoning: the backend applies the
+    // agent's reasoningOptions.effort server-side, so a client-sent
+    // reasoning_effort / reasoning.effort collides with that default →
+    // 400 "both provided with conflicting values". Mirror the CLI: send none.
+    delete body.reasoning_effort;
+    delete body.reasoning;
     // Free-tier gate: first system message must open with the CLI marker.
     return injectFreebuffMarker(body);
   }
@@ -292,10 +431,32 @@ export class FreebuffExecutor extends BaseExecutor {
       throw new Error("Freebuff requires a connected Freebuff login (no access token found)");
     }
 
+    // Fail fast while a known-dead (account,model) / (proxy,model) pair is in
+    // cooldown — no session claim, no run registration, no upstream spam.
+    const proxyKey = proxyKeyOf(proxyOptions);
+    const poolId = proxyOptions?.proxyPoolId || null;
+    const scope = `freebuff::${model}`;
+    const lockUntil = getCooldown(modelLockCooldowns, `${token}::${model}`);
+    if (lockUntil) {
+      const err = new Error(`Freebuff session locked to another model — retry after ${new Date(lockUntil).toLocaleTimeString()}`);
+      err.status = 409;
+      err.resetsAtMs = lockUntil;
+      throw err;
+    }
+    const poolUntil = getCooldown(poolLimitCooldowns, `${proxyKey}::${model}`);
+    if (poolUntil) {
+      const err = new Error(`Freebuff limited-mode IP rejected ${model} — retry with a full-access proxy after ${new Date(poolUntil).toLocaleTimeString()}`);
+      err.status = 409;
+      err.poolScoped = { poolId, scope, reason: "limited_ip" };
+      throw err;
+    }
+
     let session;
     try {
       session = await ensureSession(token, model, proxyOptions);
     } catch (error) {
+      const gate = sessionGateFromError(error);
+      if (gate) throwSessionGateError(gate, { token, model, proxyKey, poolId, log });
       log?.error?.("AUTH", `Freebuff session failed: ${error.message}`);
       throw error;
     }
@@ -396,9 +557,17 @@ export class FreebuffExecutor extends BaseExecutor {
       //   409 session_superseded — another instance took over the session
       //   409 session_model_mismatch — session bound to a different model
       //   410 session_expired    — the active session's expires_at passed
-      // In every case: abandon the run, force a fresh session claim + a fresh
-      // run, then retry exactly once.
+      // model_locked / limited-tier mismatches are NOT reclaimable — the server
+      // keeps refusing until the session expires or the IP tier changes, so we
+      // set a cooldown and fail fast instead of force re-claiming in a loop.
       if (SESSION_STALE_CODES.has(response.status)) {
+        const text = await response.text().catch(() => "");
+        const gate = sessionGateFromText(text);
+        if (gate.kind === "model_locked" || gate.kind === "limited_ip") {
+          markFinished("cancelled");
+          throwSessionGateError(gate, { token, model, proxyKey, poolId, log });
+        }
+
         log?.debug?.("AUTH", `Freebuff ${response.status} session gate — re-claiming session`);
         markFinished("cancelled");
         try {
@@ -406,21 +575,34 @@ export class FreebuffExecutor extends BaseExecutor {
           runId = await startRun(token, model, proxyOptions);
           activeRunId = runId;
         } catch (error) {
+          const gate2 = sessionGateFromError(error);
+          if (gate2) throwSessionGateError(gate2, { token, model, proxyKey, poolId, log });
           log?.error?.("AUTH", `Freebuff session re-claim failed: ${error.message}`);
           throw error;
         }
         ({ response, transformedBody } = await doChat());
 
         if (SESSION_STALE_CODES.has(response.status)) {
-          const text = await response.text().catch(() => "");
+          const text2 = await response.text().catch(() => "");
+          const gate3 = sessionGateFromText(text2);
+          if (gate3.kind === "model_locked" || gate3.kind === "limited_ip") {
+            throwSessionGateError(gate3, { token, model, proxyKey, poolId, log });
+          }
           const err = new Error(
-            `Freebuff session gate refused (${response.status}) — another freebuff instance may be holding the session. ${text.slice(0, 160)}`,
+            `Freebuff session gate refused (${response.status}) — another freebuff instance may be holding the session. ${text2.slice(0, 160)}`,
           );
           err.status = response.status;
           throw err;
         }
       }
 
+      // A successful chat means the pair is healthy again — lift any cooldowns.
+      if (response.ok) {
+        modelLockCooldowns.delete(`${token}::${model}`);
+        poolLimitCooldowns.delete(`${proxyKey}::${model}`);
+        if (poolId) clearPoolUnfit(poolId, scope);
+      }
+
       // The authToken has no refresh path — when it dies, the user re-logs in.
       // Drop the cached session for this token so a re-login starts clean.
       if (response.status === 401) {

open-sse/executors/opencode.js

@@ -5,6 +5,12 @@ import { injectReasoningContent } from "../utils/reasoningContentInjector.js";
 // Models that use /zen/v1/messages (claude format)
 const MESSAGES_MODELS = new Set();
 
+// OpenCode free tier is limited per egress IP — a 429/403 with a limit-ish
+// body means the POOL's IP is exhausted, not the account. Declare it
+// pool-scoped so chatCore marks the pool unfit, retries via another pool, and
+// it shows up (clearable) on the Proxy Fitness page.
+const IP_LIMIT_BODY = /limit|rate|quota|exhausted|capacity|too many|retry/i;
+
 export class OpenCodeExecutor extends BaseExecutor {
   constructor() {
     super("opencode", PROVIDERS.opencode);
@@ -29,4 +35,17 @@ export class OpenCodeExecutor extends BaseExecutor {
       "Accept": "text/event-stream"
     };
   }
+
+  parseError(response, bodyText) {
+    const status = response?.status || 0;
+    const text = String(bodyText || "");
+    if ((status === 429 || status === 403) && IP_LIMIT_BODY.test(text)) {
+      return {
+        status,
+        message: text.slice(0, 300) || `OpenCode free limit (${status})`,
+        poolScoped: { reason: "ip-limit" },
+      };
+    }
+    return null; // fall through to default parsing
+  }
 }

open-sse/handlers/chatCore.js

@@ -29,6 +29,12 @@ import { getCapabilitiesForModel } from "../providers/capabilities.js";
 import { stripUnsupportedModalities } from "../translator/concerns/modality.js";
 import { prefetchRemoteImages } from "../translator/concerns/prefetch.js";
 import { resolveSessionId } from "../utils/sessionManager.js";
+import { markPoolUnfit, clearPoolUnfit } from "../services/proxyPoolFitness.js";
+
+// Pool-scoped failure retry: when an executor tags an error as belonging to a
+// proxy pool (region gate, dead proxy, …), re-resolve the proxy config
+// excluding that pool and retry instead of failing the whole account.
+const MAX_POOL_RETRIES = 2;
 
 /**
  * Core chat handler - shared between SSE and Worker
@@ -57,7 +63,7 @@ export function stripContinuityFields(body) {
   return body;
 }
 
-export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking }) {
+export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking, resolveProxyConfig }) {
   const { provider, model } = modelInfo;
   const requestStartTime = Date.now();
   // Stable per-session color so all lines of one CLI conversation share a tag
@@ -291,16 +297,23 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
     log, provider, model, reqTag
   });
 
-  const proxyOptions = {
-    connectionProxyEnabled: credentials?.providerSpecificData?.connectionProxyEnabled === true,
-    connectionProxyUrl: credentials?.providerSpecificData?.connectionProxyUrl || "",
-    connectionNoProxy: credentials?.providerSpecificData?.connectionNoProxy || "",
-    vercelRelayUrl: credentials?.providerSpecificData?.vercelRelayUrl || "",
-  };
+  // Build proxy options from the resolved provider-specific data. `strictProxy`
+  // is forced for freebuff so a dead/limited pool can never leak the request
+  // to the caller's real IP (the freebuff session tier is per-egress-IP).
+  const buildProxyOptions = (psd = {}) => ({
+    connectionProxyEnabled: psd?.connectionProxyEnabled === true,
+    connectionProxyUrl: psd?.connectionProxyUrl || "",
+    connectionNoProxy: psd?.connectionNoProxy || "",
+    vercelRelayUrl: psd?.vercelRelayUrl || "",
+    strictProxy: psd?.strictProxy === true || provider === "freebuff",
+    proxyPoolId: psd?.proxyPoolId || psd?.connectionProxyPoolId || null,
+  });
+
+  let proxyOptions = buildProxyOptions(credentials?.providerSpecificData || {});
 
   if (proxyOptions.vercelRelayUrl) {
     const connectionName = credentials?.connectionName || credentials?.connectionId || "unknown";
-    const poolId = credentials?.providerSpecificData?.connectionProxyPoolId || "none";
+    const poolId = proxyOptions.proxyPoolId || "none";
     log?.info?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | pool=${poolId} | vercel-relay=${proxyOptions.vercelRelayUrl}`);
   } else if (proxyOptions.connectionProxyEnabled && proxyOptions.connectionProxyUrl) {
     let maskedProxyUrl = proxyOptions.connectionProxyUrl;
@@ -314,7 +327,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
       // Keep raw if URL parsing fails
     }
 
-    const poolId = credentials?.providerSpecificData?.connectionProxyPoolId || "none";
+    const poolId = proxyOptions.proxyPoolId || "none";
     const connectionName = credentials?.connectionName || credentials?.connectionId || "unknown";
     log?.info?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | pool=${poolId} | url=${maskedProxyUrl}`);
   }
@@ -324,13 +337,65 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
     log?.debug?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | no_proxy=${proxyOptions.connectionNoProxy}`);
   }
 
+  // Execute request — with pool-scoped retry: a failed pool (region gate,
+  // per-IP limit, dead proxy) is marked unfit and the request retried via
+  // another pool instead of failing the account. Covers both thrown errors
+  // (executor.execute) and non-ok responses declared poolScoped via
+  // parseError — poolId/scope are completed here from proxyOptions.
+  const proxyScope = `${provider}::${model}`;
+  let parsedNonOk = null;
+
+  const tryNextPool = async (poolScoped, reasonMsg) => {
+    const failed = {
+      poolId: poolScoped?.poolId || proxyOptions.proxyPoolId || null,
+      scope: poolScoped?.scope || proxyScope,
+      reason: poolScoped?.reason || "pool-scoped",
+    };
+    markPoolUnfit(failed.poolId, failed.scope, undefined, failed.reason);
+    log?.warn?.("PROXY", `${provider.toUpperCase()} | pool ${failed.poolId || "?"} unfit for ${failed.scope} (${failed.reason}) — retry with another pool. ${reasonMsg || ""}`);
+    try {
+      const resolved = await resolveProxyConfig(credentials, [failed.poolId]);
+      if (resolved?.proxyPoolId) {
+        credentials.providerSpecificData = { ...(credentials.providerSpecificData || {}), ...resolved };
+        proxyOptions = buildProxyOptions(credentials.providerSpecificData);
+        return true;
+      }
+    } catch (resolverError) {
+      // A resolver failure must not mask the original pool error.
+      log?.warn?.("PROXY", `${provider.toUpperCase()} | pool re-resolve failed: ${resolverError.message}`);
+    }
+    return false;
+  };
+
+  const executeWithPoolFallback = async (attempt = 0) => {
+    let result;
+    try {
+      result = await executor.execute({ model, body: translatedBody, stream, credentials, signal: streamController.signal, log, proxyOptions });
+    } catch (error) {
+      if (error?.poolScoped && typeof resolveProxyConfig === "function" && attempt < MAX_POOL_RETRIES) {
+        if (await tryNextPool(error.poolScoped, error.message)) return executeWithPoolFallback(attempt + 1);
+      }
+      throw error;
+    }
+    // Non-ok response that the executor declared pool/IP-scoped (e.g. opencode
+    // free per-IP limit) — parse once, retry via another pool when possible.
+    if (!result.response.ok) {
+      const parsed = await parseUpstreamError(result.response, executor);
+      if (parsed.poolScoped && typeof resolveProxyConfig === "function" && attempt < MAX_POOL_RETRIES) {
+        if (await tryNextPool(parsed.poolScoped, parsed.message)) return executeWithPoolFallback(attempt + 1);
+      }
+      parsedNonOk = parsed;
+    }
+    return result;
+  };
+
   // Execute request
   let providerResponse, providerUrl, providerHeaders, finalBody;
   // Most executors return their registry format. Cursor AgentService is an
   // exception: it is decoded by the executor into OpenAI-compatible output.
   let providerResponseFormat = targetFormat;
   try {
-    const result = await executor.execute({ model, body: translatedBody, stream, credentials, signal: streamController.signal, log, proxyOptions });
+    const result = await executeWithPoolFallback();
     providerResponse = result.response;
     providerUrl = result.url;
     providerHeaders = result.headers;
@@ -359,7 +424,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
     if (log?.errorLine) {
       log.errorLine(reqTag, "✗", `ERROR 502 · ${provider}/${model} · ${Date.now() - requestStartTime}ms\n    ${errMsg}${error.stack ? `\n    ${error.stack}` : ""}`);
     }
-    return createErrorResult(HTTP_STATUS.BAD_GATEWAY, errMsg);
+    return createErrorResult(HTTP_STATUS.BAD_GATEWAY, errMsg, error?.resetsAtMs || undefined);
   }
 
   // Handle 401/403 - try token refresh (skip for noAuth providers)
@@ -402,7 +467,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
   // Provider returned error
   if (!providerResponse.ok) {
     trackPendingRequest(model, provider, connectionId, false, true);
-    const { statusCode, message, resetsAtMs } = await parseUpstreamError(providerResponse, executor);
+    const { statusCode, message, resetsAtMs } = parsedNonOk || await parseUpstreamError(providerResponse, executor);
     appendRequestLog({ model, provider, connectionId, status: `FAILED ${statusCode}` }).catch(() => { });
     saveRequestDetail(buildRequestDetail({
       provider, model, connectionId,

open-sse/services/proxyPoolFitness.js

@@ -0,0 +1,126 @@
+// Pool fitness registry — shared, in-memory, engine layer.
+//
+// Rotation strategies can opt into region/provider-aware pool selection: an
+// executor that learns a pool's egress is unfit for a provider/model (e.g.
+// Freebuff limited-mode IP) marks it here, and the pool picker skips it for
+// that scope until the cooldown expires.
+//
+// Scope format: `provider::model` (e.g. "freebuff::openai/gpt-5.6-luna").
+// All functions are fail-open: unknown pool/scope ⇒ fit.
+
+// Scope format: `provider::model` (e.g. "freebuff::openai/gpt-5.6-luna").
+// All functions are fail-open: unknown pool/scope ⇒ fit.
+//
+// State lives on globalThis so Next dev (Turbopack) never splits one Map into
+// several per-bundle copies — the executor/chatCore marks and the
+// /api/proxy-pools/fitness reader must share the SAME registry.
+
+const FITNESS_STATE_KEY = "__9routerPoolFitness__";
+const fitness = (globalThis[FITNESS_STATE_KEY] ??= new Map()); // poolId -> Map<scope, { until, reason }>
+
+export const POOL_UNFIT_MS = 5 * 60 * 1000;
+
+export function markPoolUnfit(poolId, scope, until = Date.now() + POOL_UNFIT_MS, reason = "") {
+  if (!poolId || !scope) return;
+  const byScope = fitness.get(poolId) || new Map();
+  byScope.set(scope, { until, reason });
+  fitness.set(poolId, byScope);
+}
+
+export function clearPoolUnfit(poolId, scope) {
+  const byScope = fitness.get(poolId);
+  if (!byScope) return;
+  byScope.delete(scope);
+  if (byScope.size === 0) fitness.delete(poolId);
+}
+
+// "provider::model" -> "provider::*" (null when the scope has no provider part)
+function providerWildcardScope(scope) {
+  const sep = String(scope || "").indexOf("::");
+  if (sep < 0) return null;
+  return `${scope.slice(0, sep)}::*`;
+}
+
+export function isPoolFit(poolId, scope, now = Date.now()) {
+  if (!poolId) return true;
+  const byScope = fitness.get(poolId);
+  if (!byScope) return true;
+  // A provider-wide mark ("provider::*", e.g. manual blocks) also covers any
+  // model lookup for that provider.
+  const candidates = [scope, providerWildcardScope(scope)];
+  for (const key of candidates) {
+    if (!key) continue;
+    const entry = byScope.get(key);
+    if (!entry) continue;
+    if (entry.until <= now) {
+      byScope.delete(key);
+      if (byScope.size === 0) fitness.delete(poolId);
+      continue;
+    }
+    return false;
+  }
+  return true;
+}
+
+// Keep only pool ids that are not in cooldown for the scope.
+export function fitPoolIds(poolIds, scope, now = Date.now()) {
+  return (poolIds || []).filter((id) => isPoolFit(id, scope, now));
+}
+
+// Clear every mark — or only scopes belonging to one provider (`provider::*`).
+export function clearAllPoolUnfit(provider = null) {
+  if (provider) {
+    const prefix = `${provider}::`;
+    for (const [poolId, byScope] of fitness) {
+      for (const scope of [...byScope.keys()]) {
+        if (scope.startsWith(prefix)) byScope.delete(scope);
+      }
+      if (byScope.size === 0) fitness.delete(poolId);
+    }
+    return;
+  }
+  fitness.clear();
+}
+
+// Snapshot of live (non-expired) marks — expired entries are pruned here so
+// consumers never see stale data and memory stays bounded.
+// Test helper: drop all marks (module state is globalThis-backed).
+export function resetPoolFitness() {
+  fitness.clear();
+}
+
+// Sweep all expired marks. Returns how many scope entries were removed.
+export function pruneExpired(now = Date.now()) {
+  let removed = 0;
+  for (const [poolId, byScope] of fitness) {
+    for (const [scope, entry] of byScope) {
+      if (entry.until <= now) {
+        byScope.delete(scope);
+        removed += 1;
+      }
+    }
+    if (byScope.size === 0) fitness.delete(poolId);
+  }
+  return removed;
+}
+
+export function poolFitnessSnapshot(now = Date.now()) {
+  const out = {};
+  for (const [poolId, byScope] of fitness) {
+    let pruned = false;
+    for (const [scope, entry] of byScope) {
+      if (entry.until <= now) {
+        byScope.delete(scope);
+        pruned = true;
+      }
+    }
+    if (byScope.size === 0) {
+      fitness.delete(poolId);
+      continue;
+    }
+    if (pruned || byScope.size > 0) {
+      out[poolId] = Object.fromEntries(byScope);
+    }
+  }
+  return out;
+}

open-sse/utils/error.js

@@ -69,7 +69,14 @@ export async function parseUpstreamError(response, executor = null) {
       const parsed = executor.parseError(response, bodyText);
       if (parsed && typeof parsed === "object") {
         const msg = parsed.message || DEFAULT_ERROR_MESSAGES[response.status] || `Upstream error: ${response.status}`;
-        return { statusCode: parsed.status || response.status, message: msg, resetsAtMs: parsed.resetsAtMs };
+        return {
+          statusCode: parsed.status || response.status,
+          message: msg,
+          resetsAtMs: parsed.resetsAtMs,
+          // Executors declare IP/pool-scoped failures (e.g. per-IP rate limits)
+          // here; chatCore completes poolId/scope and retries via another pool.
+          poolScoped: parsed.poolScoped,
+        };
       }
     } catch { /* fall through to default parsing */ }
   }