feat(proxy-fitness): pool fitness registry + Smart rotation + pool-scoped retry
Pool/IP fitness registry (globalThis-backed, provider::model scopes, 5-min cooldown, provider::* wildcard) fed by pool-scoped failures: freebuff limited-IP / model-locked gates and opencode free per-IP limits. chatCore retries a failed pool via another pool without locking the account; new Smart rotation strategy (per-connection + no-auth providers) skips unfit pools. Proxy Fitness dashboard page: active-block table with provider/IP filters, per-record Clear and provider-scoped Clear All; egress column via pool geo enrichment. Unit tests for the registry.
This commit is contained in:
1 parent
90ff8191ff
commit
fd22290a34
17 files changed
+1000
-35
No files matched your search
@@ -8,6 +8,7 @@ import {
|
|||||||
DEFAULT_RETRY_CONFIG,
|
DEFAULT_RETRY_CONFIG,
|
||||||
resolveRetryEntry,
|
resolveRetryEntry,
|
||||||
} from "../config/runtimeConfig.js";
|
} from "../config/runtimeConfig.js";
|
||||||
|
import { markPoolUnfit, clearPoolUnfit } from "../services/proxyPoolFitness.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Freebuff Executor — OpenAI-compatible chat completions on
|
* Freebuff Executor — OpenAI-compatible chat completions on
|
||||||
@@ -87,8 +88,110 @@ const FREE_ROOT_AGENT_BY_MODEL = {
|
|||||||
// don't share one session row). Re-claims are driven by the cache expiring or
|
// don't share one session row). Re-claims are driven by the cache expiring or
|
||||||
// by a 428 from chat — no early re-claim, so we never POST /session while our
|
// by a 428 from chat — no early re-claim, so we never POST /session while our
|
||||||
// own row is still active (which could come back as a spurious model_locked).
|
// own row is still active (which could come back as a spurious model_locked).
|
||||||
const sessionCache = new Map(); // `${token}::${model}` -> { instanceId, expiresAt }
|
// All state lives on globalThis so Next dev (Turbopack) bundles share ONE copy.
|
||||||
const inflight = new Map(); // dedupe concurrent claims for the same key
|
const FB_STATE_KEY = "__9routerFreebuffState__";
|
||||||
|
const fbState = (globalThis[FB_STATE_KEY] ??= {
|
||||||
|
sessionCache: new Map(), // `${token}::${model}` -> { instanceId, expiresAt }
|
||||||
|
inflight: new Map(), // dedupe concurrent claims for the same key
|
||||||
|
modelLockCooldowns: new Map(), // `${token}::${model}` -> expiresAt (ms)
|
||||||
|
poolLimitCooldowns: new Map(), // `${proxyKey}::${model}` -> expiresAt (ms)
|
||||||
|
});
|
||||||
|
const sessionCache = fbState.sessionCache;
|
||||||
|
const inflight = fbState.inflight;
|
||||||
|
const modelLockCooldowns = fbState.modelLockCooldowns;
|
||||||
|
const poolLimitCooldowns = fbState.poolLimitCooldowns;
|
||||||
|
|
||||||
|
const MODEL_LOCK_COOLDOWN_MS = 10 * 60 * 1000; // session bound to another model (~1h) — re-check every 10 min
|
||||||
|
const POOL_LIMITED_COOLDOWN_MS = 5 * 60 * 1000; // IP tier refuses this model — try a different pool/relay
|
||||||
|
|
||||||
|
// Cooldown maps need pruning: expired entries are cleared on write (sweep) and
|
||||||
|
// on read, so long-running servers don't accumulate one entry per (account,model)
|
||||||
|
// / (proxy,model) forever.
|
||||||
|
function setCooldown(map, key, until) {
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [k, v] of map) {
|
||||||
|
if (v <= now) map.delete(k);
|
||||||
|
}
|
||||||
|
map.set(key, until);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getCooldown(map, key) {
|
||||||
|
const until = map.get(key);
|
||||||
|
if (until == null) return null;
|
||||||
|
if (until <= Date.now()) {
|
||||||
|
map.delete(key);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return until;
|
||||||
|
}
|
||||||
|
|
||||||
|
function proxyKeyOf(proxyOptions) {
|
||||||
|
return proxyOptions?.vercelRelayUrl || proxyOptions?.connectionProxyUrl || "direct";
|
||||||
|
}
|
||||||
|
|
||||||
|
function sessionGateFromText(text) {
|
||||||
|
let parsed = {};
|
||||||
|
try { parsed = JSON.parse(String(text || "")); } catch { parsed = {}; }
|
||||||
|
return classifySessionGate(parsed.error || parsed.error_type || "", parsed.message || "", parsed.currentModel || null);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse a 409/428/410 body into { kind, currentModel }. `msg` may be a whole
|
||||||
|
// error string containing a JSON tail (requestSession errors embed the body).
|
||||||
|
function sessionGateFromError(error) {
|
||||||
|
const msg = String(error?.message || "");
|
||||||
|
const start = msg.indexOf("{");
|
||||||
|
if (start < 0) return null;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(msg.slice(start));
|
||||||
|
return classifySessionGate(parsed.error || "", parsed.message || "", parsed.currentModel || null);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function classifySessionGate(code, message, currentModel) {
|
||||||
|
if (code === "session_superseded") return { kind: "superseded" };
|
||||||
|
if (code === "model_locked") return { kind: "model_locked", currentModel };
|
||||||
|
// session_model_mismatch with the limited-tier message is an IP-tier refusal;
|
||||||
|
// without it (or unknown) treat it as a model lock so we don't reclaim in a loop.
|
||||||
|
if (code === "session_model_mismatch") {
|
||||||
|
return /limited/i.test(String(message || ""))
|
||||||
|
? { kind: "limited_ip" }
|
||||||
|
: { kind: "model_locked", currentModel };
|
||||||
|
}
|
||||||
|
return { kind: "stale" }; // 428/410/unknown → reclaim
|
||||||
|
}
|
||||||
|
|
||||||
|
// Applies cooldowns and throws for non-reclaimable gates. Never returns for them.
|
||||||
|
function throwSessionGateError(gate, { token, model, proxyKey, poolId, log }) {
|
||||||
|
if (gate.kind === "model_locked") {
|
||||||
|
const until = Date.now() + MODEL_LOCK_COOLDOWN_MS;
|
||||||
|
setCooldown(modelLockCooldowns, `${token}::${model}`, until);
|
||||||
|
const label = gate.currentModel ? `"${gate.currentModel}"` : "another model";
|
||||||
|
const err = new Error(
|
||||||
|
`Freebuff session is locked to ${label} — it cannot serve ${model}. End the session on freebuff.com or wait for it to expire (~1h).`,
|
||||||
|
);
|
||||||
|
err.status = 409;
|
||||||
|
err.resetsAtMs = until;
|
||||||
|
log?.warn?.("AUTH", `Freebuff model_locked (session=${label}, requested=${model}) — model cooldown ${MODEL_LOCK_COOLDOWN_MS / 60000}min`);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
if (gate.kind === "limited_ip") {
|
||||||
|
const until = Date.now() + POOL_LIMITED_COOLDOWN_MS;
|
||||||
|
setCooldown(poolLimitCooldowns, `${proxyKey}::${model}`, until);
|
||||||
|
const scope = `freebuff::${model}`;
|
||||||
|
if (poolId) markPoolUnfit(poolId, scope, until, "limited_ip");
|
||||||
|
// Pool-scoped, not account-scoped: the caller retries via another pool
|
||||||
|
// instead of locking the account (resetsAtMs intentionally absent).
|
||||||
|
const err = new Error(
|
||||||
|
`Freebuff limited-mode IP rejected ${model} — this IP only allows DeepSeek V4 Flash / MiMo 2.5. Use a full-access proxy or a different model.`,
|
||||||
|
);
|
||||||
|
err.status = 409;
|
||||||
|
err.poolScoped = { poolId, scope, reason: "limited_ip" };
|
||||||
|
log?.warn?.("AUTH", `Freebuff limited-IP refused ${model} (proxy=${proxyKey.slice(0, 40)}…) — cooldown ${POOL_LIMITED_COOLDOWN_MS / 60000}min`);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function sessionOrigin() {
|
function sessionOrigin() {
|
||||||
return new URL(PROVIDERS.freebuff.baseUrl).origin; // https://www.codebuff.com
|
return new URL(PROVIDERS.freebuff.baseUrl).origin; // https://www.codebuff.com
|
||||||
@@ -185,7 +288,11 @@ async function requestSession(token, model, proxyOptions) {
|
|||||||
|
|
||||||
async function ensureSession(token, model, proxyOptions, force = false) {
|
async function ensureSession(token, model, proxyOptions, force = false) {
|
||||||
const key = sessionCacheKey(token, model);
|
const key = sessionCacheKey(token, model);
|
||||||
|
// Lazy prune: drop stale rows so the cache never accumulates expired entries.
|
||||||
const cached = sessionCache.get(key);
|
const cached = sessionCache.get(key);
|
||||||
|
if (cached && cached.expiresAt <= Date.now()) {
|
||||||
|
sessionCache.delete(key);
|
||||||
|
}
|
||||||
if (!force && cached && cached.expiresAt > Date.now()) {
|
if (!force && cached && cached.expiresAt > Date.now()) {
|
||||||
return { instanceId: cached.instanceId, status: "active" };
|
return { instanceId: cached.instanceId, status: "active" };
|
||||||
}
|
}
|
||||||
@@ -262,6 +369,32 @@ export function resetSessionCache() {
|
|||||||
inflight.clear();
|
inflight.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Periodic sweeper: drop stale session rows + expired cooldowns so long-running
|
||||||
|
// servers never accumulate state for accounts/models no longer in use.
|
||||||
|
// Returns how many entries were removed.
|
||||||
|
export function pruneSessionState(now = Date.now()) {
|
||||||
|
let removed = 0;
|
||||||
|
for (const [key, entry] of sessionCache) {
|
||||||
|
if (entry?.expiresAt && entry.expiresAt <= now) {
|
||||||
|
sessionCache.delete(key);
|
||||||
|
removed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const [key, until] of modelLockCooldowns) {
|
||||||
|
if (until <= now) {
|
||||||
|
modelLockCooldowns.delete(key);
|
||||||
|
removed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const [key, until] of poolLimitCooldowns) {
|
||||||
|
if (until <= now) {
|
||||||
|
poolLimitCooldowns.delete(key);
|
||||||
|
removed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
|
||||||
export class FreebuffExecutor extends BaseExecutor {
|
export class FreebuffExecutor extends BaseExecutor {
|
||||||
constructor() {
|
constructor() {
|
||||||
super("freebuff", PROVIDERS.freebuff);
|
super("freebuff", PROVIDERS.freebuff);
|
||||||
@@ -282,6 +415,12 @@ export class FreebuffExecutor extends BaseExecutor {
|
|||||||
cost_mode: "free",
|
cost_mode: "free",
|
||||||
};
|
};
|
||||||
body.provider = { allow_fallbacks: false };
|
body.provider = { allow_fallbacks: false };
|
||||||
|
// Freebuff agents (base2-free-*) own reasoning: the backend applies the
|
||||||
|
// agent's reasoningOptions.effort server-side, so a client-sent
|
||||||
|
// reasoning_effort / reasoning.effort collides with that default →
|
||||||
|
// 400 "both provided with conflicting values". Mirror the CLI: send none.
|
||||||
|
delete body.reasoning_effort;
|
||||||
|
delete body.reasoning;
|
||||||
// Free-tier gate: first system message must open with the CLI marker.
|
// Free-tier gate: first system message must open with the CLI marker.
|
||||||
return injectFreebuffMarker(body);
|
return injectFreebuffMarker(body);
|
||||||
}
|
}
|
||||||
@@ -292,10 +431,32 @@ export class FreebuffExecutor extends BaseExecutor {
|
|||||||
throw new Error("Freebuff requires a connected Freebuff login (no access token found)");
|
throw new Error("Freebuff requires a connected Freebuff login (no access token found)");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Fail fast while a known-dead (account,model) / (proxy,model) pair is in
|
||||||
|
// cooldown — no session claim, no run registration, no upstream spam.
|
||||||
|
const proxyKey = proxyKeyOf(proxyOptions);
|
||||||
|
const poolId = proxyOptions?.proxyPoolId || null;
|
||||||
|
const scope = `freebuff::${model}`;
|
||||||
|
const lockUntil = getCooldown(modelLockCooldowns, `${token}::${model}`);
|
||||||
|
if (lockUntil) {
|
||||||
|
const err = new Error(`Freebuff session locked to another model — retry after ${new Date(lockUntil).toLocaleTimeString()}`);
|
||||||
|
err.status = 409;
|
||||||
|
err.resetsAtMs = lockUntil;
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
const poolUntil = getCooldown(poolLimitCooldowns, `${proxyKey}::${model}`);
|
||||||
|
if (poolUntil) {
|
||||||
|
const err = new Error(`Freebuff limited-mode IP rejected ${model} — retry with a full-access proxy after ${new Date(poolUntil).toLocaleTimeString()}`);
|
||||||
|
err.status = 409;
|
||||||
|
err.poolScoped = { poolId, scope, reason: "limited_ip" };
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
let session;
|
let session;
|
||||||
try {
|
try {
|
||||||
session = await ensureSession(token, model, proxyOptions);
|
session = await ensureSession(token, model, proxyOptions);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
const gate = sessionGateFromError(error);
|
||||||
|
if (gate) throwSessionGateError(gate, { token, model, proxyKey, poolId, log });
|
||||||
log?.error?.("AUTH", `Freebuff session failed: ${error.message}`);
|
log?.error?.("AUTH", `Freebuff session failed: ${error.message}`);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
@@ -396,9 +557,17 @@ export class FreebuffExecutor extends BaseExecutor {
|
|||||||
// 409 session_superseded — another instance took over the session
|
// 409 session_superseded — another instance took over the session
|
||||||
// 409 session_model_mismatch — session bound to a different model
|
// 409 session_model_mismatch — session bound to a different model
|
||||||
// 410 session_expired — the active session's expires_at passed
|
// 410 session_expired — the active session's expires_at passed
|
||||||
// In every case: abandon the run, force a fresh session claim + a fresh
|
// model_locked / limited-tier mismatches are NOT reclaimable — the server
|
||||||
// run, then retry exactly once.
|
// keeps refusing until the session expires or the IP tier changes, so we
|
||||||
|
// set a cooldown and fail fast instead of force re-claiming in a loop.
|
||||||
if (SESSION_STALE_CODES.has(response.status)) {
|
if (SESSION_STALE_CODES.has(response.status)) {
|
||||||
|
const text = await response.text().catch(() => "");
|
||||||
|
const gate = sessionGateFromText(text);
|
||||||
|
if (gate.kind === "model_locked" || gate.kind === "limited_ip") {
|
||||||
|
markFinished("cancelled");
|
||||||
|
throwSessionGateError(gate, { token, model, proxyKey, poolId, log });
|
||||||
|
}
|
||||||
|
|
||||||
log?.debug?.("AUTH", `Freebuff ${response.status} session gate — re-claiming session`);
|
log?.debug?.("AUTH", `Freebuff ${response.status} session gate — re-claiming session`);
|
||||||
markFinished("cancelled");
|
markFinished("cancelled");
|
||||||
try {
|
try {
|
||||||
@@ -406,21 +575,34 @@ export class FreebuffExecutor extends BaseExecutor {
|
|||||||
runId = await startRun(token, model, proxyOptions);
|
runId = await startRun(token, model, proxyOptions);
|
||||||
activeRunId = runId;
|
activeRunId = runId;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
const gate2 = sessionGateFromError(error);
|
||||||
|
if (gate2) throwSessionGateError(gate2, { token, model, proxyKey, poolId, log });
|
||||||
log?.error?.("AUTH", `Freebuff session re-claim failed: ${error.message}`);
|
log?.error?.("AUTH", `Freebuff session re-claim failed: ${error.message}`);
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
({ response, transformedBody } = await doChat());
|
({ response, transformedBody } = await doChat());
|
||||||
|
|
||||||
if (SESSION_STALE_CODES.has(response.status)) {
|
if (SESSION_STALE_CODES.has(response.status)) {
|
||||||
const text = await response.text().catch(() => "");
|
const text2 = await response.text().catch(() => "");
|
||||||
|
const gate3 = sessionGateFromText(text2);
|
||||||
|
if (gate3.kind === "model_locked" || gate3.kind === "limited_ip") {
|
||||||
|
throwSessionGateError(gate3, { token, model, proxyKey, poolId, log });
|
||||||
|
}
|
||||||
const err = new Error(
|
const err = new Error(
|
||||||
`Freebuff session gate refused (${response.status}) — another freebuff instance may be holding the session. ${text.slice(0, 160)}`,
|
`Freebuff session gate refused (${response.status}) — another freebuff instance may be holding the session. ${text2.slice(0, 160)}`,
|
||||||
);
|
);
|
||||||
err.status = response.status;
|
err.status = response.status;
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A successful chat means the pair is healthy again — lift any cooldowns.
|
||||||
|
if (response.ok) {
|
||||||
|
modelLockCooldowns.delete(`${token}::${model}`);
|
||||||
|
poolLimitCooldowns.delete(`${proxyKey}::${model}`);
|
||||||
|
if (poolId) clearPoolUnfit(poolId, scope);
|
||||||
|
}
|
||||||
|
|
||||||
// The authToken has no refresh path — when it dies, the user re-logs in.
|
// The authToken has no refresh path — when it dies, the user re-logs in.
|
||||||
// Drop the cached session for this token so a re-login starts clean.
|
// Drop the cached session for this token so a re-login starts clean.
|
||||||
if (response.status === 401) {
|
if (response.status === 401) {
|
||||||
|
|||||||
@@ -5,6 +5,12 @@ import { injectReasoningContent } from "../utils/reasoningContentInjector.js";
|
|||||||
// Models that use /zen/v1/messages (claude format)
|
// Models that use /zen/v1/messages (claude format)
|
||||||
const MESSAGES_MODELS = new Set();
|
const MESSAGES_MODELS = new Set();
|
||||||
|
|
||||||
|
// OpenCode free tier is limited per egress IP — a 429/403 with a limit-ish
|
||||||
|
// body means the POOL's IP is exhausted, not the account. Declare it
|
||||||
|
// pool-scoped so chatCore marks the pool unfit, retries via another pool, and
|
||||||
|
// it shows up (clearable) on the Proxy Fitness page.
|
||||||
|
const IP_LIMIT_BODY = /limit|rate|quota|exhausted|capacity|too many|retry/i;
|
||||||
|
|
||||||
export class OpenCodeExecutor extends BaseExecutor {
|
export class OpenCodeExecutor extends BaseExecutor {
|
||||||
constructor() {
|
constructor() {
|
||||||
super("opencode", PROVIDERS.opencode);
|
super("opencode", PROVIDERS.opencode);
|
||||||
@@ -29,4 +35,17 @@ export class OpenCodeExecutor extends BaseExecutor {
|
|||||||
"Accept": "text/event-stream"
|
"Accept": "text/event-stream"
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
parseError(response, bodyText) {
|
||||||
|
const status = response?.status || 0;
|
||||||
|
const text = String(bodyText || "");
|
||||||
|
if ((status === 429 || status === 403) && IP_LIMIT_BODY.test(text)) {
|
||||||
|
return {
|
||||||
|
status,
|
||||||
|
message: text.slice(0, 300) || `OpenCode free limit (${status})`,
|
||||||
|
poolScoped: { reason: "ip-limit" },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return null; // fall through to default parsing
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -29,6 +29,12 @@ import { getCapabilitiesForModel } from "../providers/capabilities.js";
|
|||||||
import { stripUnsupportedModalities } from "../translator/concerns/modality.js";
|
import { stripUnsupportedModalities } from "../translator/concerns/modality.js";
|
||||||
import { prefetchRemoteImages } from "../translator/concerns/prefetch.js";
|
import { prefetchRemoteImages } from "../translator/concerns/prefetch.js";
|
||||||
import { resolveSessionId } from "../utils/sessionManager.js";
|
import { resolveSessionId } from "../utils/sessionManager.js";
|
||||||
|
import { markPoolUnfit, clearPoolUnfit } from "../services/proxyPoolFitness.js";
|
||||||
|
|
||||||
|
// Pool-scoped failure retry: when an executor tags an error as belonging to a
|
||||||
|
// proxy pool (region gate, dead proxy, …), re-resolve the proxy config
|
||||||
|
// excluding that pool and retry instead of failing the whole account.
|
||||||
|
const MAX_POOL_RETRIES = 2;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Core chat handler - shared between SSE and Worker
|
* Core chat handler - shared between SSE and Worker
|
||||||
@@ -57,7 +63,7 @@ export function stripContinuityFields(body) {
|
|||||||
return body;
|
return body;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking }) {
|
export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking, resolveProxyConfig }) {
|
||||||
const { provider, model } = modelInfo;
|
const { provider, model } = modelInfo;
|
||||||
const requestStartTime = Date.now();
|
const requestStartTime = Date.now();
|
||||||
// Stable per-session color so all lines of one CLI conversation share a tag
|
// Stable per-session color so all lines of one CLI conversation share a tag
|
||||||
@@ -291,16 +297,23 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
|||||||
log, provider, model, reqTag
|
log, provider, model, reqTag
|
||||||
});
|
});
|
||||||
|
|
||||||
const proxyOptions = {
|
// Build proxy options from the resolved provider-specific data. `strictProxy`
|
||||||
connectionProxyEnabled: credentials?.providerSpecificData?.connectionProxyEnabled === true,
|
// is forced for freebuff so a dead/limited pool can never leak the request
|
||||||
connectionProxyUrl: credentials?.providerSpecificData?.connectionProxyUrl || "",
|
// to the caller's real IP (the freebuff session tier is per-egress-IP).
|
||||||
connectionNoProxy: credentials?.providerSpecificData?.connectionNoProxy || "",
|
const buildProxyOptions = (psd = {}) => ({
|
||||||
vercelRelayUrl: credentials?.providerSpecificData?.vercelRelayUrl || "",
|
connectionProxyEnabled: psd?.connectionProxyEnabled === true,
|
||||||
};
|
connectionProxyUrl: psd?.connectionProxyUrl || "",
|
||||||
|
connectionNoProxy: psd?.connectionNoProxy || "",
|
||||||
|
vercelRelayUrl: psd?.vercelRelayUrl || "",
|
||||||
|
strictProxy: psd?.strictProxy === true || provider === "freebuff",
|
||||||
|
proxyPoolId: psd?.proxyPoolId || psd?.connectionProxyPoolId || null,
|
||||||
|
});
|
||||||
|
|
||||||
|
let proxyOptions = buildProxyOptions(credentials?.providerSpecificData || {});
|
||||||
|
|
||||||
if (proxyOptions.vercelRelayUrl) {
|
if (proxyOptions.vercelRelayUrl) {
|
||||||
const connectionName = credentials?.connectionName || credentials?.connectionId || "unknown";
|
const connectionName = credentials?.connectionName || credentials?.connectionId || "unknown";
|
||||||
const poolId = credentials?.providerSpecificData?.connectionProxyPoolId || "none";
|
const poolId = proxyOptions.proxyPoolId || "none";
|
||||||
log?.info?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | pool=${poolId} | vercel-relay=${proxyOptions.vercelRelayUrl}`);
|
log?.info?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | pool=${poolId} | vercel-relay=${proxyOptions.vercelRelayUrl}`);
|
||||||
} else if (proxyOptions.connectionProxyEnabled && proxyOptions.connectionProxyUrl) {
|
} else if (proxyOptions.connectionProxyEnabled && proxyOptions.connectionProxyUrl) {
|
||||||
let maskedProxyUrl = proxyOptions.connectionProxyUrl;
|
let maskedProxyUrl = proxyOptions.connectionProxyUrl;
|
||||||
@@ -314,7 +327,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
|||||||
// Keep raw if URL parsing fails
|
// Keep raw if URL parsing fails
|
||||||
}
|
}
|
||||||
|
|
||||||
const poolId = credentials?.providerSpecificData?.connectionProxyPoolId || "none";
|
const poolId = proxyOptions.proxyPoolId || "none";
|
||||||
const connectionName = credentials?.connectionName || credentials?.connectionId || "unknown";
|
const connectionName = credentials?.connectionName || credentials?.connectionId || "unknown";
|
||||||
log?.info?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | pool=${poolId} | url=${maskedProxyUrl}`);
|
log?.info?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | pool=${poolId} | url=${maskedProxyUrl}`);
|
||||||
}
|
}
|
||||||
@@ -324,13 +337,65 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
|||||||
log?.debug?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | no_proxy=${proxyOptions.connectionNoProxy}`);
|
log?.debug?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | no_proxy=${proxyOptions.connectionNoProxy}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Execute request — with pool-scoped retry: a failed pool (region gate,
|
||||||
|
// per-IP limit, dead proxy) is marked unfit and the request retried via
|
||||||
|
// another pool instead of failing the account. Covers both thrown errors
|
||||||
|
// (executor.execute) and non-ok responses declared poolScoped via
|
||||||
|
// parseError — poolId/scope are completed here from proxyOptions.
|
||||||
|
const proxyScope = `${provider}::${model}`;
|
||||||
|
let parsedNonOk = null;
|
||||||
|
|
||||||
|
const tryNextPool = async (poolScoped, reasonMsg) => {
|
||||||
|
const failed = {
|
||||||
|
poolId: poolScoped?.poolId || proxyOptions.proxyPoolId || null,
|
||||||
|
scope: poolScoped?.scope || proxyScope,
|
||||||
|
reason: poolScoped?.reason || "pool-scoped",
|
||||||
|
};
|
||||||
|
markPoolUnfit(failed.poolId, failed.scope, undefined, failed.reason);
|
||||||
|
log?.warn?.("PROXY", `${provider.toUpperCase()} | pool ${failed.poolId || "?"} unfit for ${failed.scope} (${failed.reason}) — retry with another pool. ${reasonMsg || ""}`);
|
||||||
|
try {
|
||||||
|
const resolved = await resolveProxyConfig(credentials, [failed.poolId]);
|
||||||
|
if (resolved?.proxyPoolId) {
|
||||||
|
credentials.providerSpecificData = { ...(credentials.providerSpecificData || {}), ...resolved };
|
||||||
|
proxyOptions = buildProxyOptions(credentials.providerSpecificData);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} catch (resolverError) {
|
||||||
|
// A resolver failure must not mask the original pool error.
|
||||||
|
log?.warn?.("PROXY", `${provider.toUpperCase()} | pool re-resolve failed: ${resolverError.message}`);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
const executeWithPoolFallback = async (attempt = 0) => {
|
||||||
|
let result;
|
||||||
|
try {
|
||||||
|
result = await executor.execute({ model, body: translatedBody, stream, credentials, signal: streamController.signal, log, proxyOptions });
|
||||||
|
} catch (error) {
|
||||||
|
if (error?.poolScoped && typeof resolveProxyConfig === "function" && attempt < MAX_POOL_RETRIES) {
|
||||||
|
if (await tryNextPool(error.poolScoped, error.message)) return executeWithPoolFallback(attempt + 1);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
// Non-ok response that the executor declared pool/IP-scoped (e.g. opencode
|
||||||
|
// free per-IP limit) — parse once, retry via another pool when possible.
|
||||||
|
if (!result.response.ok) {
|
||||||
|
const parsed = await parseUpstreamError(result.response, executor);
|
||||||
|
if (parsed.poolScoped && typeof resolveProxyConfig === "function" && attempt < MAX_POOL_RETRIES) {
|
||||||
|
if (await tryNextPool(parsed.poolScoped, parsed.message)) return executeWithPoolFallback(attempt + 1);
|
||||||
|
}
|
||||||
|
parsedNonOk = parsed;
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
// Execute request
|
// Execute request
|
||||||
let providerResponse, providerUrl, providerHeaders, finalBody;
|
let providerResponse, providerUrl, providerHeaders, finalBody;
|
||||||
// Most executors return their registry format. Cursor AgentService is an
|
// Most executors return their registry format. Cursor AgentService is an
|
||||||
// exception: it is decoded by the executor into OpenAI-compatible output.
|
// exception: it is decoded by the executor into OpenAI-compatible output.
|
||||||
let providerResponseFormat = targetFormat;
|
let providerResponseFormat = targetFormat;
|
||||||
try {
|
try {
|
||||||
const result = await executor.execute({ model, body: translatedBody, stream, credentials, signal: streamController.signal, log, proxyOptions });
|
const result = await executeWithPoolFallback();
|
||||||
providerResponse = result.response;
|
providerResponse = result.response;
|
||||||
providerUrl = result.url;
|
providerUrl = result.url;
|
||||||
providerHeaders = result.headers;
|
providerHeaders = result.headers;
|
||||||
@@ -359,7 +424,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
|||||||
if (log?.errorLine) {
|
if (log?.errorLine) {
|
||||||
log.errorLine(reqTag, "✗", `ERROR 502 · ${provider}/${model} · ${Date.now() - requestStartTime}ms\n ${errMsg}${error.stack ? `\n ${error.stack}` : ""}`);
|
log.errorLine(reqTag, "✗", `ERROR 502 · ${provider}/${model} · ${Date.now() - requestStartTime}ms\n ${errMsg}${error.stack ? `\n ${error.stack}` : ""}`);
|
||||||
}
|
}
|
||||||
return createErrorResult(HTTP_STATUS.BAD_GATEWAY, errMsg);
|
return createErrorResult(HTTP_STATUS.BAD_GATEWAY, errMsg, error?.resetsAtMs || undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle 401/403 - try token refresh (skip for noAuth providers)
|
// Handle 401/403 - try token refresh (skip for noAuth providers)
|
||||||
@@ -402,7 +467,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
|||||||
// Provider returned error
|
// Provider returned error
|
||||||
if (!providerResponse.ok) {
|
if (!providerResponse.ok) {
|
||||||
trackPendingRequest(model, provider, connectionId, false, true);
|
trackPendingRequest(model, provider, connectionId, false, true);
|
||||||
const { statusCode, message, resetsAtMs } = await parseUpstreamError(providerResponse, executor);
|
const { statusCode, message, resetsAtMs } = parsedNonOk || await parseUpstreamError(providerResponse, executor);
|
||||||
appendRequestLog({ model, provider, connectionId, status: `FAILED ${statusCode}` }).catch(() => { });
|
appendRequestLog({ model, provider, connectionId, status: `FAILED ${statusCode}` }).catch(() => { });
|
||||||
saveRequestDetail(buildRequestDetail({
|
saveRequestDetail(buildRequestDetail({
|
||||||
provider, model, connectionId,
|
provider, model, connectionId,
|
||||||
|
|||||||
@@ -0,0 +1,126 @@
|
|||||||
|
// Pool fitness registry — shared, in-memory, engine layer.
|
||||||
|
//
|
||||||
|
// Rotation strategies can opt into region/provider-aware pool selection: an
|
||||||
|
// executor that learns a pool's egress is unfit for a provider/model (e.g.
|
||||||
|
// Freebuff limited-mode IP) marks it here, and the pool picker skips it for
|
||||||
|
// that scope until the cooldown expires.
|
||||||
|
//
|
||||||
|
// Scope format: `provider::model` (e.g. "freebuff::openai/gpt-5.6-luna").
|
||||||
|
// All functions are fail-open: unknown pool/scope ⇒ fit.
|
||||||
|
|
||||||
|
// Scope format: `provider::model` (e.g. "freebuff::openai/gpt-5.6-luna").
|
||||||
|
// All functions are fail-open: unknown pool/scope ⇒ fit.
|
||||||
|
//
|
||||||
|
// State lives on globalThis so Next dev (Turbopack) never splits one Map into
|
||||||
|
// several per-bundle copies — the executor/chatCore marks and the
|
||||||
|
// /api/proxy-pools/fitness reader must share the SAME registry.
|
||||||
|
|
||||||
|
const FITNESS_STATE_KEY = "__9routerPoolFitness__";
|
||||||
|
const fitness = (globalThis[FITNESS_STATE_KEY] ??= new Map()); // poolId -> Map<scope, { until, reason }>
|
||||||
|
|
||||||
|
export const POOL_UNFIT_MS = 5 * 60 * 1000;
|
||||||
|
|
||||||
|
export function markPoolUnfit(poolId, scope, until = Date.now() + POOL_UNFIT_MS, reason = "") {
|
||||||
|
if (!poolId || !scope) return;
|
||||||
|
const byScope = fitness.get(poolId) || new Map();
|
||||||
|
byScope.set(scope, { until, reason });
|
||||||
|
fitness.set(poolId, byScope);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clearPoolUnfit(poolId, scope) {
|
||||||
|
const byScope = fitness.get(poolId);
|
||||||
|
if (!byScope) return;
|
||||||
|
byScope.delete(scope);
|
||||||
|
if (byScope.size === 0) fitness.delete(poolId);
|
||||||
|
}
|
||||||
|
|
||||||
|
// "provider::model" -> "provider::*" (null when the scope has no provider part)
|
||||||
|
function providerWildcardScope(scope) {
|
||||||
|
const sep = String(scope || "").indexOf("::");
|
||||||
|
if (sep < 0) return null;
|
||||||
|
return `${scope.slice(0, sep)}::*`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isPoolFit(poolId, scope, now = Date.now()) {
|
||||||
|
if (!poolId) return true;
|
||||||
|
const byScope = fitness.get(poolId);
|
||||||
|
if (!byScope) return true;
|
||||||
|
// A provider-wide mark ("provider::*", e.g. manual blocks) also covers any
|
||||||
|
// model lookup for that provider.
|
||||||
|
const candidates = [scope, providerWildcardScope(scope)];
|
||||||
|
for (const key of candidates) {
|
||||||
|
if (!key) continue;
|
||||||
|
const entry = byScope.get(key);
|
||||||
|
if (!entry) continue;
|
||||||
|
if (entry.until <= now) {
|
||||||
|
byScope.delete(key);
|
||||||
|
if (byScope.size === 0) fitness.delete(poolId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep only pool ids that are not in cooldown for the scope.
|
||||||
|
export function fitPoolIds(poolIds, scope, now = Date.now()) {
|
||||||
|
return (poolIds || []).filter((id) => isPoolFit(id, scope, now));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear every mark — or only scopes belonging to one provider (`provider::*`).
|
||||||
|
export function clearAllPoolUnfit(provider = null) {
|
||||||
|
if (provider) {
|
||||||
|
const prefix = `${provider}::`;
|
||||||
|
for (const [poolId, byScope] of fitness) {
|
||||||
|
for (const scope of [...byScope.keys()]) {
|
||||||
|
if (scope.startsWith(prefix)) byScope.delete(scope);
|
||||||
|
}
|
||||||
|
if (byScope.size === 0) fitness.delete(poolId);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
fitness.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot of live (non-expired) marks — expired entries are pruned here so
|
||||||
|
// consumers never see stale data and memory stays bounded.
|
||||||
|
// Test helper: drop all marks (module state is globalThis-backed).
|
||||||
|
export function resetPoolFitness() {
|
||||||
|
fitness.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sweep all expired marks. Returns how many scope entries were removed.
|
||||||
|
export function pruneExpired(now = Date.now()) {
|
||||||
|
let removed = 0;
|
||||||
|
for (const [poolId, byScope] of fitness) {
|
||||||
|
for (const [scope, entry] of byScope) {
|
||||||
|
if (entry.until <= now) {
|
||||||
|
byScope.delete(scope);
|
||||||
|
removed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (byScope.size === 0) fitness.delete(poolId);
|
||||||
|
}
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function poolFitnessSnapshot(now = Date.now()) {
|
||||||
|
const out = {};
|
||||||
|
for (const [poolId, byScope] of fitness) {
|
||||||
|
let pruned = false;
|
||||||
|
for (const [scope, entry] of byScope) {
|
||||||
|
if (entry.until <= now) {
|
||||||
|
byScope.delete(scope);
|
||||||
|
pruned = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (byScope.size === 0) {
|
||||||
|
fitness.delete(poolId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (pruned || byScope.size > 0) {
|
||||||
|
out[poolId] = Object.fromEntries(byScope);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
@@ -69,7 +69,14 @@ export async function parseUpstreamError(response, executor = null) {
|
|||||||
const parsed = executor.parseError(response, bodyText);
|
const parsed = executor.parseError(response, bodyText);
|
||||||
if (parsed && typeof parsed === "object") {
|
if (parsed && typeof parsed === "object") {
|
||||||
const msg = parsed.message || DEFAULT_ERROR_MESSAGES[response.status] || `Upstream error: ${response.status}`;
|
const msg = parsed.message || DEFAULT_ERROR_MESSAGES[response.status] || `Upstream error: ${response.status}`;
|
||||||
return { statusCode: parsed.status || response.status, message: msg, resetsAtMs: parsed.resetsAtMs };
|
return {
|
||||||
|
statusCode: parsed.status || response.status,
|
||||||
|
message: msg,
|
||||||
|
resetsAtMs: parsed.resetsAtMs,
|
||||||
|
// Executors declare IP/pool-scoped failures (e.g. per-IP rate limits)
|
||||||
|
// here; chatCore completes poolId/scope and retries via another pool.
|
||||||
|
poolScoped: parsed.poolScoped,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
} catch { /* fall through to default parsing */ }
|
} catch { /* fall through to default parsing */ }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ export default function ConnectionRow({ connection, proxyPools, isOAuth, isFirst
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (selectedProxyIds.length > 1) {
|
if (selectedProxyIds.length > 1) {
|
||||||
const strategyLabel = rotationStrategy === "random" ? "Random" : rotationStrategy === "round-robin" ? "Round Robin" : rotationStrategy === "failover" ? "Failover" : "Multiple";
|
const strategyLabel = rotationStrategy === "random" ? "Random" : rotationStrategy === "round-robin" ? "Round Robin" : rotationStrategy === "failover" ? "Failover" : rotationStrategy === "smart" ? "Smart" : "Multiple";
|
||||||
return `${selectedProxyIds.length} pools (${strategyLabel})`;
|
return `${selectedProxyIds.length} pools (${strategyLabel})`;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -318,7 +318,7 @@ export default function ConnectionRow({ connection, proxyPools, isOAuth, isFirst
|
|||||||
<span className="text-[10px] leading-tight">Proxy</span>
|
<span className="text-[10px] leading-tight">Proxy</span>
|
||||||
</button>
|
</button>
|
||||||
{showProxyDropdown && (
|
{showProxyDropdown && (
|
||||||
<div className="absolute right-0 top-full z-50 mt-1 max-w-[90vw] min-w-[280px] rounded-lg border border-border bg-bg shadow-lg">
|
<div className="absolute left-0 top-full z-50 mt-1 max-w-[calc(100vw-2rem)] min-w-[280px] rounded-lg border border-border bg-bg shadow-lg sm:left-auto sm:right-0">
|
||||||
{/* Rotation Strategy Selector */}
|
{/* Rotation Strategy Selector */}
|
||||||
<div className="border-b border-border p-3">
|
<div className="border-b border-border p-3">
|
||||||
<label className="block text-xs font-medium text-text-muted mb-2">Rotation Strategy</label>
|
<label className="block text-xs font-medium text-text-muted mb-2">Rotation Strategy</label>
|
||||||
@@ -331,18 +331,20 @@ export default function ConnectionRow({ connection, proxyPools, isOAuth, isFirst
|
|||||||
<option value="random">Random</option>
|
<option value="random">Random</option>
|
||||||
<option value="round-robin">Round Robin</option>
|
<option value="round-robin">Round Robin</option>
|
||||||
<option value="failover">Failover</option>
|
<option value="failover">Failover</option>
|
||||||
|
<option value="smart">Smart</option>
|
||||||
</select>
|
</select>
|
||||||
{rotationStrategy !== "none" && (
|
{rotationStrategy !== "none" && (
|
||||||
<p className="mt-1 text-[10px] text-text-muted">
|
<p className="mt-1 text-[10px] text-text-muted">
|
||||||
{rotationStrategy === "random" && "Randomly select proxy on each request"}
|
{rotationStrategy === "random" && "Randomly select proxy on each request"}
|
||||||
{rotationStrategy === "round-robin" && "Rotate proxies in order across requests"}
|
{rotationStrategy === "round-robin" && "Rotate proxies in order across requests"}
|
||||||
{rotationStrategy === "failover" && "Try next proxy on failure"}
|
{rotationStrategy === "failover" && "Try next proxy on failure"}
|
||||||
|
{rotationStrategy === "smart" && "Skip pools whose egress IP is blocked for this provider/model (e.g. Freebuff limited-IP). See Proxy Fitness to clear/block."}
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Proxy Pool Selection */}
|
{/* Proxy Pool Selection */}
|
||||||
<div className="max-h-[60vh] overflow-y-auto py-1">
|
<div className="max-h-[200px] overflow-y-auto py-1">
|
||||||
<button
|
<button
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
setSelectedProxyIds([]);
|
setSelectedProxyIds([]);
|
||||||
|
|||||||
@@ -0,0 +1,384 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||||
|
import { Badge, Button, Card, CardSkeleton, Input, ConfirmModal } from "@/shared/components";
|
||||||
|
import ProviderIcon from "@/shared/components/ProviderIcon";
|
||||||
|
import { useNotificationStore } from "@/store/notificationStore";
|
||||||
|
|
||||||
|
function fmtTime(value) {
|
||||||
|
if (!value) return "-";
|
||||||
|
const d = new Date(value);
|
||||||
|
return Number.isNaN(d.getTime()) ? "-" : d.toLocaleTimeString();
|
||||||
|
}
|
||||||
|
|
||||||
|
// "freebuff::openai/gpt-5.6-luna" -> { provider: "freebuff", model: "openai/gpt-5.6-luna" }
|
||||||
|
// "freebuff::*" -> { provider: "freebuff", model: null }
|
||||||
|
function parseScope(scope) {
|
||||||
|
const sep = String(scope || "").indexOf("::");
|
||||||
|
if (sep < 0) return { provider: String(scope || ""), model: null };
|
||||||
|
const provider = scope.slice(0, sep);
|
||||||
|
const model = scope.slice(sep + 2);
|
||||||
|
return { provider, model: model === "*" || model === "" ? null : model };
|
||||||
|
}
|
||||||
|
|
||||||
|
function maskProxyUrl(url) {
|
||||||
|
try {
|
||||||
|
const parsed = new URL(url);
|
||||||
|
const host = parsed.hostname || "";
|
||||||
|
const port = parsed.port ? `:${parsed.port}` : "";
|
||||||
|
return `${parsed.protocol}//${host}${port}`;
|
||||||
|
} catch {
|
||||||
|
return String(url || "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flatten fitness snapshot into one record per (pool, scope); drops expired marks.
|
||||||
|
function buildRecords(fitness, pools, now = Date.now()) {
|
||||||
|
const poolById = new Map((pools || []).map((p) => [p.id, p]));
|
||||||
|
const records = [];
|
||||||
|
for (const [poolId, byScope] of Object.entries(fitness || {})) {
|
||||||
|
const pool = poolById.get(poolId);
|
||||||
|
for (const [scope, info] of Object.entries(byScope || {})) {
|
||||||
|
const until = info?.until ? new Date(info.until).getTime() : 0;
|
||||||
|
if (!until || until <= now) continue;
|
||||||
|
const { provider, model } = parseScope(scope);
|
||||||
|
records.push({
|
||||||
|
poolId,
|
||||||
|
scope,
|
||||||
|
provider,
|
||||||
|
model,
|
||||||
|
until,
|
||||||
|
reason: info?.reason || "blocked",
|
||||||
|
poolName: pool?.name || poolId.slice(0, 8),
|
||||||
|
proxyUrl: pool?.proxyUrl || "",
|
||||||
|
egress: pool?.egress || null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return records;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function ProxyFitnessPage() {
|
||||||
|
const [pools, setPools] = useState([]);
|
||||||
|
const [fitness, setFitness] = useState({});
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [providerFilter, setProviderFilter] = useState("all");
|
||||||
|
const [search, setSearch] = useState("");
|
||||||
|
const [providerMenuOpen, setProviderMenuOpen] = useState(false);
|
||||||
|
const [providerMenuDropUp, setProviderMenuDropUp] = useState(false);
|
||||||
|
const [clearingScope, setClearingScope] = useState(null); // poolId::scope while clearing
|
||||||
|
const [confirmClearAll, setConfirmClearAll] = useState(false);
|
||||||
|
const [clearingAll, setClearingAll] = useState(false);
|
||||||
|
const providerMenuRef = useRef(null);
|
||||||
|
const notify = useNotificationStore();
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const handleClickOutside = (e) => {
|
||||||
|
if (providerMenuRef.current && !providerMenuRef.current.contains(e.target)) {
|
||||||
|
setProviderMenuOpen(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if (providerMenuOpen) {
|
||||||
|
document.addEventListener("mousedown", handleClickOutside);
|
||||||
|
}
|
||||||
|
return () => document.removeEventListener("mousedown", handleClickOutside);
|
||||||
|
}, [providerMenuOpen]);
|
||||||
|
|
||||||
|
const fetchAll = useCallback(async () => {
|
||||||
|
try {
|
||||||
|
const [poolRes, fitRes] = await Promise.all([
|
||||||
|
fetch("/api/proxy-pools?includeUsage=true", { cache: "no-store" }),
|
||||||
|
fetch("/api/proxy-pools/fitness", { cache: "no-store" }),
|
||||||
|
]);
|
||||||
|
const poolData = await poolRes.json().catch(() => ({ proxyPools: [] }));
|
||||||
|
setPools(poolData.proxyPools || []);
|
||||||
|
if (fitRes.ok) {
|
||||||
|
const fitData = await fitRes.json().catch(() => ({}));
|
||||||
|
setFitness(fitData.pools || fitData.fitness || {});
|
||||||
|
} else {
|
||||||
|
setFitness({});
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.log("Error fetching proxy fitness:", error);
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
fetchAll();
|
||||||
|
}, [fetchAll]);
|
||||||
|
|
||||||
|
const records = useMemo(() => buildRecords(fitness, pools), [fitness, pools]);
|
||||||
|
|
||||||
|
// Providers present in the fitness data (filter options)
|
||||||
|
const providerOptions = useMemo(() => {
|
||||||
|
const set = new Set();
|
||||||
|
for (const rec of records) {
|
||||||
|
if (rec.provider) set.add(rec.provider);
|
||||||
|
}
|
||||||
|
return Array.from(set).sort();
|
||||||
|
}, [records]);
|
||||||
|
|
||||||
|
const filteredRecords = useMemo(() => {
|
||||||
|
const q = search.trim().toLowerCase();
|
||||||
|
return records.filter((rec) => {
|
||||||
|
if (providerFilter !== "all" && rec.provider !== providerFilter) return false;
|
||||||
|
if (q) {
|
||||||
|
const hay = [rec.proxyUrl, rec.egress?.ip, rec.egress?.country, rec.egress?.region, rec.poolName]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join(" ")
|
||||||
|
.toLowerCase();
|
||||||
|
if (!hay.includes(q)) return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
}, [records, providerFilter, search]);
|
||||||
|
|
||||||
|
const handleClear = async (rec) => {
|
||||||
|
setClearingScope(rec.scope);
|
||||||
|
try {
|
||||||
|
const res = await fetch(`/api/proxy-pools/${rec.poolId}/fitness/clear`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ scope: rec.scope }),
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||||
|
notify.success(`Cleared ${rec.scope}`);
|
||||||
|
fetchAll();
|
||||||
|
} catch (err) {
|
||||||
|
notify.error(`Clear failed: ${err.message}`);
|
||||||
|
} finally {
|
||||||
|
setClearingScope(null);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleClearAll = async () => {
|
||||||
|
setClearingAll(true);
|
||||||
|
try {
|
||||||
|
const res = await fetch("/api/proxy-pools/fitness/clear-all", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify(providerFilter !== "all" ? { provider: providerFilter } : {}),
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||||
|
notify.success(providerFilter !== "all" ? `Cleared all ${providerFilter} blocks` : "Cleared all blocks");
|
||||||
|
setConfirmClearAll(false);
|
||||||
|
fetchAll();
|
||||||
|
} catch (err) {
|
||||||
|
notify.error(`Clear all failed: ${err.message}`);
|
||||||
|
} finally {
|
||||||
|
setClearingAll(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const selectedProviderLabel = providerFilter === "all" ? "All providers" : providerFilter;
|
||||||
|
|
||||||
|
// Open the provider menu as an overlay anchored to its container. Flips to
|
||||||
|
// drop-up when there isn't enough viewport space below (menu max-h ~288px).
|
||||||
|
const toggleProviderMenu = () => {
|
||||||
|
if (providerMenuOpen) {
|
||||||
|
setProviderMenuOpen(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const el = providerMenuRef.current;
|
||||||
|
if (el) {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
setProviderMenuDropUp(window.innerHeight - rect.bottom < 300);
|
||||||
|
}
|
||||||
|
setProviderMenuOpen(true);
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="mx-auto w-full max-w-5xl space-y-4 p-4 sm:p-6">
|
||||||
|
{loading ? (
|
||||||
|
<CardSkeleton />
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
{/* Header */}
|
||||||
|
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<h1 className="text-lg font-semibold text-text-main">Proxy Fitness</h1>
|
||||||
|
<Badge variant={records.length > 0 ? "error" : "default"} size="sm">
|
||||||
|
{records.length} active block{records.length === 1 ? "" : "s"}
|
||||||
|
</Badge>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
{records.length > 0 && (
|
||||||
|
<Button
|
||||||
|
variant="danger"
|
||||||
|
size="sm"
|
||||||
|
icon="delete_sweep"
|
||||||
|
onClick={() => setConfirmClearAll(true)}
|
||||||
|
disabled={clearingAll}
|
||||||
|
>
|
||||||
|
Clear All{providerFilter !== "all" ? ` (${providerFilter})` : ""}
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
<Button variant="secondary" size="sm" icon="refresh" onClick={fetchAll}>Refresh</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p className="text-sm text-text-muted">
|
||||||
|
Shows which provider is blocked on which proxy IP (from provider region gates, per-IP
|
||||||
|
limits, or manual marks). Smart rotation skips these pools while the block is active.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{/* Filters */}
|
||||||
|
<div className="flex flex-wrap items-end gap-3 rounded-lg border border-border-subtle bg-surface p-3">
|
||||||
|
<div className="relative flex w-64 max-w-full flex-col" ref={providerMenuRef}>
|
||||||
|
<label className="mb-1 text-xs font-medium text-text-muted">Provider</label>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={toggleProviderMenu}
|
||||||
|
className="flex h-8 w-full items-center justify-between gap-1 rounded-lg border border-black/10 bg-black/[0.02] px-2 text-xs text-text-main transition-colors hover:bg-black/5 dark:border-white/10 dark:bg-white/[0.03] dark:hover:bg-white/10"
|
||||||
|
aria-haspopup="menu"
|
||||||
|
aria-expanded={providerMenuOpen}
|
||||||
|
title="Filter by provider"
|
||||||
|
>
|
||||||
|
<span className="flex min-w-0 items-center gap-1.5">
|
||||||
|
{providerFilter === "all" ? (
|
||||||
|
<span className="material-symbols-outlined text-[14px] text-text-muted">apps</span>
|
||||||
|
) : (
|
||||||
|
<ProviderIcon providerId={providerFilter} size={18} className="size-[18px] rounded object-contain" fallbackText={providerFilter.slice(0, 2).toUpperCase()} />
|
||||||
|
)}
|
||||||
|
<span className="truncate capitalize">{selectedProviderLabel}</span>
|
||||||
|
</span>
|
||||||
|
<span className="material-symbols-outlined text-[14px] text-text-muted">expand_more</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{providerMenuOpen && (
|
||||||
|
<div className={`absolute left-0 z-20 w-full max-h-72 overflow-y-auto rounded-lg border border-border-subtle bg-surface p-1 shadow-lg ${providerMenuDropUp ? "bottom-full mb-1" : "top-full mt-1"}`}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => { setProviderFilter("all"); setProviderMenuOpen(false); }}
|
||||||
|
className={`flex w-full items-center gap-3 rounded-xl px-3 py-2.5 text-left text-sm transition-colors ${providerFilter === "all" ? "bg-primary/10 text-primary" : "text-text-main hover:bg-black/5 dark:hover:bg-white/10"}`}
|
||||||
|
>
|
||||||
|
<span className="material-symbols-outlined text-[22px]">apps</span>
|
||||||
|
<span className="font-medium">All providers</span>
|
||||||
|
{providerFilter === "all" && <span className="material-symbols-outlined ml-auto text-[20px]">check</span>}
|
||||||
|
</button>
|
||||||
|
<div className="my-1 h-px bg-black/10 dark:bg-white/10" />
|
||||||
|
{providerOptions.map((provider) => (
|
||||||
|
<button
|
||||||
|
key={provider}
|
||||||
|
type="button"
|
||||||
|
onClick={() => { setProviderFilter(provider); setProviderMenuOpen(false); }}
|
||||||
|
className={`flex w-full items-center gap-3 rounded-xl px-3 py-2.5 text-left text-sm transition-colors ${providerFilter === provider ? "bg-primary/10 text-primary" : "text-text-main hover:bg-black/5 dark:hover:bg-white/10"}`}
|
||||||
|
>
|
||||||
|
<ProviderIcon providerId={provider} size={24} className="size-6 rounded-md object-contain" fallbackText={provider.slice(0, 2).toUpperCase()} />
|
||||||
|
<span className="font-medium capitalize">{provider}</span>
|
||||||
|
{providerFilter === provider && <span className="material-symbols-outlined ml-auto text-[20px]">check</span>}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex min-w-48 flex-1 flex-col">
|
||||||
|
<label className="mb-1 text-xs font-medium text-text-muted">IP / Proxy / Pool</label>
|
||||||
|
<Input
|
||||||
|
value={search}
|
||||||
|
onChange={(e) => setSearch(e.target.value)}
|
||||||
|
placeholder="e.g. 104.28, vercel-relay…"
|
||||||
|
icon="search"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Records table */}
|
||||||
|
<Card className="p-0">
|
||||||
|
<div className="overflow-x-auto">
|
||||||
|
<table className="w-full min-w-[700px] border-collapse text-sm">
|
||||||
|
<thead>
|
||||||
|
<tr className="border-b border-border-subtle text-left text-xs uppercase tracking-wider text-text-muted">
|
||||||
|
<th className="px-4 py-2 font-semibold">Provider</th>
|
||||||
|
<th className="px-4 py-2 font-semibold">Model</th>
|
||||||
|
<th className="px-4 py-2 font-semibold">IP / Proxy</th>
|
||||||
|
<th className="px-4 py-2 font-semibold">Pool</th>
|
||||||
|
<th className="px-4 py-2 font-semibold">Reason</th>
|
||||||
|
<th className="px-4 py-2 font-semibold">Until</th>
|
||||||
|
<th className="px-4 py-2 text-right font-semibold">Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{filteredRecords.length === 0 ? (
|
||||||
|
<tr>
|
||||||
|
<td colSpan={7} className="px-4 py-10 text-center text-text-muted">
|
||||||
|
{records.length === 0
|
||||||
|
? "No active blocks. Blocks appear here when a provider region-gates a proxy IP."
|
||||||
|
: "No blocks match the current filters."}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
) : (
|
||||||
|
filteredRecords.map((rec) => (
|
||||||
|
<tr key={`${rec.poolId}::${rec.scope}`} className="border-b border-border-subtle last:border-0 align-top hover:bg-surface-2/40">
|
||||||
|
<td className="px-4 py-3">
|
||||||
|
<span className="inline-flex items-center gap-2 rounded bg-red-500/10 px-2 py-0.5 text-xs font-medium text-red-600 dark:text-red-400">
|
||||||
|
<span className="material-symbols-outlined text-[14px]">block</span>
|
||||||
|
<span className="capitalize">{rec.provider}</span>
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td className="max-w-56 px-4 py-3">
|
||||||
|
<span className="truncate text-text-main">{rec.model || <em className="text-text-muted">all models</em>}</span>
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-3">
|
||||||
|
<div className="flex min-w-0 flex-col gap-0.5">
|
||||||
|
<code className="truncate font-mono text-xs text-text-main">{maskProxyUrl(rec.proxyUrl)}</code>
|
||||||
|
{rec.egress?.ip && (
|
||||||
|
<span className="flex items-center gap-1 text-[11px] text-text-muted">
|
||||||
|
<span className="truncate">egress {rec.egress.ip}{rec.egress.country ? ` · ${rec.egress.country}` : ""}</span>
|
||||||
|
{rec.egress.isUnstable && (
|
||||||
|
<span
|
||||||
|
className="inline-flex shrink-0 items-center gap-0.5 rounded bg-amber-500/15 px-1 py-0.5 text-[10px] font-medium text-amber-600 dark:text-amber-400"
|
||||||
|
title={rec.egress.ipCount >= 2 ? `Egress IP changed ${rec.egress.ipCount}× — relay egress is not stable` : "Egress is unstable"}
|
||||||
|
>
|
||||||
|
unstable
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td className="px-4 py-3 text-text-muted">{rec.poolName}</td>
|
||||||
|
<td className="px-4 py-3 text-text-muted">{rec.reason}</td>
|
||||||
|
<td className="px-4 py-3 text-text-muted">{fmtTime(rec.until)}</td>
|
||||||
|
<td className="px-4 py-3">
|
||||||
|
<div className="flex justify-end">
|
||||||
|
<Button
|
||||||
|
variant="ghost"
|
||||||
|
size="sm"
|
||||||
|
icon="close"
|
||||||
|
onClick={() => handleClear(rec)}
|
||||||
|
disabled={clearingScope === rec.scope}
|
||||||
|
title="Clear this block"
|
||||||
|
>
|
||||||
|
{clearingScope === rec.scope ? "Clearing..." : "Clear"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))
|
||||||
|
)}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<ConfirmModal
|
||||||
|
isOpen={confirmClearAll}
|
||||||
|
onClose={() => setConfirmClearAll(false)}
|
||||||
|
onConfirm={handleClearAll}
|
||||||
|
title="Clear all blocks"
|
||||||
|
message={providerFilter !== "all"
|
||||||
|
? `Clear all active blocks for provider "${providerFilter}"? This lets those pools be selected again immediately.`
|
||||||
|
: "Clear all active proxy blocks? This lets all pools be selected again immediately."}
|
||||||
|
confirmText={clearingAll ? "Clearing..." : "Clear All"}
|
||||||
|
cancelText="Cancel"
|
||||||
|
variant="danger"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { clearPoolUnfit } from "open-sse/services/proxyPoolFitness.js";
|
||||||
|
|
||||||
|
// POST /api/proxy-pools/[id]/fitness/clear
|
||||||
|
// Body: { scope: "provider::model" } — clears the mark for this pool + scope.
|
||||||
|
export async function POST(request, { params }) {
|
||||||
|
try {
|
||||||
|
const { id } = await params;
|
||||||
|
let body = {};
|
||||||
|
try {
|
||||||
|
body = await request.json();
|
||||||
|
} catch {
|
||||||
|
body = {};
|
||||||
|
}
|
||||||
|
const scope = typeof body?.scope === "string" ? body.scope.trim() : "";
|
||||||
|
if (!id || !scope) {
|
||||||
|
return NextResponse.json({ error: "pool id and scope are required" }, { status: 400 });
|
||||||
|
}
|
||||||
|
clearPoolUnfit(id, scope);
|
||||||
|
return NextResponse.json({ ok: true, poolId: id, scope });
|
||||||
|
} catch (error) {
|
||||||
|
console.log("Error clearing pool fitness:", error);
|
||||||
|
return NextResponse.json({ error: "Failed to clear pool fitness" }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { clearAllPoolUnfit } from "open-sse/services/proxyPoolFitness.js";
|
||||||
|
|
||||||
|
// POST /api/proxy-pools/fitness/clear-all
|
||||||
|
// Body: { provider?: string } — clears every mark, or only marks scoped to the
|
||||||
|
// given provider ("provider::*") when provided.
|
||||||
|
export async function POST(request) {
|
||||||
|
try {
|
||||||
|
let body = {};
|
||||||
|
try {
|
||||||
|
body = await request.json();
|
||||||
|
} catch {
|
||||||
|
body = {};
|
||||||
|
}
|
||||||
|
const provider = typeof body?.provider === "string" && body.provider.trim() ? body.provider.trim() : null;
|
||||||
|
clearAllPoolUnfit(provider);
|
||||||
|
return NextResponse.json({ ok: true, provider: provider || null });
|
||||||
|
} catch (error) {
|
||||||
|
console.log("Error clearing proxy fitness:", error);
|
||||||
|
return NextResponse.json({ error: "Failed to clear proxy fitness" }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { poolFitnessSnapshot } from "open-sse/services/proxyPoolFitness.js";
|
||||||
|
|
||||||
|
// GET /api/proxy-pools/fitness — in-memory snapshot of pool fitness marks.
|
||||||
|
// Returns { pools: { [poolId]: { [scope]: { until, reason } } } }.
|
||||||
|
export async function GET() {
|
||||||
|
try {
|
||||||
|
return NextResponse.json({ pools: poolFitnessSnapshot() });
|
||||||
|
} catch (error) {
|
||||||
|
console.log("Error reading proxy fitness:", error);
|
||||||
|
return NextResponse.json({ error: "Failed to read proxy fitness" }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import { NextResponse } from "next/server";
|
import { NextResponse } from "next/server";
|
||||||
import { createProxyPool, getProviderConnections, getProxyPools } from "@/models";
|
import { createProxyPool, getProviderConnections, getProxyPools } from "@/models";
|
||||||
|
import { getPoolGeo } from "open-sse/services/poolGeo.js";
|
||||||
|
|
||||||
function toBoolean(value) {
|
function toBoolean(value) {
|
||||||
if (value === "true") return true;
|
if (value === "true") return true;
|
||||||
@@ -65,6 +66,8 @@ export async function GET(request) {
|
|||||||
const enrichedProxyPools = proxyPools.map((pool) => ({
|
const enrichedProxyPools = proxyPools.map((pool) => ({
|
||||||
...pool,
|
...pool,
|
||||||
boundConnectionCount: usageMap.get(pool.id) || 0,
|
boundConnectionCount: usageMap.get(pool.id) || 0,
|
||||||
|
// Egress geo from the background probe cache (null until first probe).
|
||||||
|
egress: getPoolGeo(pool.id) || null,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
return NextResponse.json({ proxyPools: enrichedProxyPools });
|
return NextResponse.json({ proxyPools: enrichedProxyPools });
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { getProxyPoolById } from "@/models";
|
import { getProxyPoolById } from "@/models";
|
||||||
|
import { fitPoolIds } from "open-sse/services/proxyPoolFitness.js";
|
||||||
|
|
||||||
// Safely normalize any value into a trimmed string.
|
// Safely normalize any value into a trimmed string.
|
||||||
function normalizeString(value) {
|
function normalizeString(value) {
|
||||||
@@ -13,24 +14,41 @@ const rotateState = new Map(); // providerId → { index }
|
|||||||
* Pick one proxy pool ID from a list based on strategy.
|
* Pick one proxy pool ID from a list based on strategy.
|
||||||
* round-robin: cycle sequentially (in-memory, resets on restart)
|
* round-robin: cycle sequentially (in-memory, resets on restart)
|
||||||
* random: uniform random pick
|
* random: uniform random pick
|
||||||
|
* smart: region-aware — skip pools unfit for `scope`, round-robin on the fit subset
|
||||||
* none/single: return first entry
|
* none/single: return first entry
|
||||||
|
* @param {string[]} poolIds
|
||||||
|
* @param {string} strategy
|
||||||
|
* @param {string} providerId
|
||||||
|
* @param {{ scope?: string, excludeIds?: string[] }} [opts]
|
||||||
*/
|
*/
|
||||||
export function pickProxyPoolId(poolIds, strategy, providerId) {
|
export function pickProxyPoolId(poolIds, strategy, providerId, opts = {}) {
|
||||||
if (!poolIds || poolIds.length === 0) return null;
|
if (!poolIds || poolIds.length === 0) return null;
|
||||||
if (poolIds.length === 1) return poolIds[0];
|
const { scope = null, excludeIds = [] } = opts || {};
|
||||||
|
|
||||||
if (strategy === "round-robin") {
|
let eligible = poolIds.filter((id) => !(excludeIds || []).includes(id));
|
||||||
|
// Region-aware filtering is opt-in via the "smart" strategy.
|
||||||
|
if (strategy === "smart" && scope) eligible = fitPoolIds(eligible, scope);
|
||||||
|
|
||||||
|
if (eligible.length === 0) {
|
||||||
|
// Every candidate is unfit/excluded — fall back to the first non-excluded
|
||||||
|
// pool rather than deadlocking; its egress may have recovered.
|
||||||
|
eligible = poolIds.filter((id) => !(excludeIds || []).includes(id));
|
||||||
|
if (eligible.length === 0) return null;
|
||||||
|
}
|
||||||
|
if (eligible.length === 1) return eligible[0];
|
||||||
|
|
||||||
|
if (strategy === "round-robin" || strategy === "smart") {
|
||||||
const state = rotateState.get(providerId) || { index: -1 };
|
const state = rotateState.get(providerId) || { index: -1 };
|
||||||
state.index = (state.index + 1) % poolIds.length;
|
state.index = (state.index + 1) % eligible.length;
|
||||||
rotateState.set(providerId, state);
|
rotateState.set(providerId, state);
|
||||||
return poolIds[state.index];
|
return eligible[state.index];
|
||||||
}
|
}
|
||||||
|
|
||||||
if (strategy === "random") {
|
if (strategy === "random") {
|
||||||
return poolIds[Math.floor(Math.random() * poolIds.length)];
|
return eligible[Math.floor(Math.random() * eligible.length)];
|
||||||
}
|
}
|
||||||
|
|
||||||
return poolIds[0]; // "none" or unknown
|
return eligible[0]; // "none" or unknown
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -66,7 +84,8 @@ function normalizeLegacyProxy(providerSpecificData = {}) {
|
|||||||
*/
|
*/
|
||||||
export async function resolveConnectionProxyConfig(
|
export async function resolveConnectionProxyConfig(
|
||||||
providerSpecificData = {},
|
providerSpecificData = {},
|
||||||
connectionId = null
|
connectionId = null,
|
||||||
|
excludePoolIds = null
|
||||||
) {
|
) {
|
||||||
try {
|
try {
|
||||||
// Handle new multi-proxy format
|
// Handle new multi-proxy format
|
||||||
@@ -85,7 +104,8 @@ export async function resolveConnectionProxyConfig(
|
|||||||
* -----------------------------
|
* -----------------------------
|
||||||
*/
|
*/
|
||||||
if (proxyPoolIds.length > 0) {
|
if (proxyPoolIds.length > 0) {
|
||||||
const selectedPoolId = pickProxyPoolId(proxyPoolIds, proxyRotationStrategy, connectionId);
|
const scope = providerSpecificData?.proxyPoolScope || null;
|
||||||
|
const selectedPoolId = pickProxyPoolId(proxyPoolIds, proxyRotationStrategy, connectionId, { scope, excludeIds: excludePoolIds });
|
||||||
|
|
||||||
if (selectedPoolId) {
|
if (selectedPoolId) {
|
||||||
const proxyPool = await getProxyPoolById(selectedPoolId);
|
const proxyPool = await getProxyPoolById(selectedPoolId);
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ const STRATEGIES = [
|
|||||||
{ value: "none", label: "None (single pool)" },
|
{ value: "none", label: "None (single pool)" },
|
||||||
{ value: "round-robin", label: "Round-robin" },
|
{ value: "round-robin", label: "Round-robin" },
|
||||||
{ value: "random", label: "Random" },
|
{ value: "random", label: "Random" },
|
||||||
|
{ value: "smart", label: "Smart" },
|
||||||
];
|
];
|
||||||
|
|
||||||
export default function NoAuthProxyCard({ providerId }) {
|
export default function NoAuthProxyCard({ providerId }) {
|
||||||
@@ -121,7 +122,9 @@ export default function NoAuthProxyCard({ providerId }) {
|
|||||||
: isRotation
|
: isRotation
|
||||||
? rotateStrategy === "round-robin"
|
? rotateStrategy === "round-robin"
|
||||||
? `Rotating through all ${proxyPools.length} active pools in order. State is in-memory (resets on restart).`
|
? `Rotating through all ${proxyPools.length} active pools in order. State is in-memory (resets on restart).`
|
||||||
: `Picking a random pool from ${proxyPools.length} active pools each request.`
|
: rotateStrategy === "smart"
|
||||||
|
? `Skip pools whose egress IP is blocked for this provider (e.g. per-IP limits). See Proxy Fitness to clear/block.`
|
||||||
|
: `Picking a random pool from ${proxyPools.length} active pools each request.`
|
||||||
: `Uses the selected pool above. Set to Round-robin or Random to rotate across all active pools.`}
|
: `Uses the selected pool above. Set to Round-robin or Random to rotate across all active pools.`}
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ const navItems = [
|
|||||||
{ href: "/dashboard/usage", label: "Usage", icon: "bar_chart" },
|
{ href: "/dashboard/usage", label: "Usage", icon: "bar_chart" },
|
||||||
{ href: "/dashboard/quota", label: "Quota Tracker", icon: "data_usage" },
|
{ href: "/dashboard/quota", label: "Quota Tracker", icon: "data_usage" },
|
||||||
{ href: "/dashboard/token-saver", label: "Token Saver", icon: "savings" },
|
{ href: "/dashboard/token-saver", label: "Token Saver", icon: "savings" },
|
||||||
|
{ href: "/dashboard/proxy-fitness", label: "Proxy Fitness", icon: "network_check" },
|
||||||
// { href: "/dashboard/pxpipe", label: "PXPIPE", icon: "image" },
|
// { href: "/dashboard/pxpipe", label: "PXPIPE", icon: "image" },
|
||||||
{ href: "/dashboard/cli-tools", label: "CLI Tools", icon: "terminal" },
|
{ href: "/dashboard/cli-tools", label: "CLI Tools", icon: "terminal" },
|
||||||
];
|
];
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import { errorResponse, unavailableResponse } from "open-sse/utils/error.js";
|
|||||||
import { handleComboChat, handleFusionChat, detectRequiredCapabilities } from "open-sse/services/combo.js";
|
import { handleComboChat, handleFusionChat, detectRequiredCapabilities } from "open-sse/services/combo.js";
|
||||||
import { augmentModelsWithCapacityAdapter, withCapacityAdapterStripping, getActiveAdapterStrategy } from "open-sse/services/capacityAdapter.js";
|
import { augmentModelsWithCapacityAdapter, withCapacityAdapterStripping, getActiveAdapterStrategy } from "open-sse/services/capacityAdapter.js";
|
||||||
import { handleBypassRequest } from "open-sse/utils/bypassHandler.js";
|
import { handleBypassRequest } from "open-sse/utils/bypassHandler.js";
|
||||||
|
import { resolveConnectionProxyConfig } from "@/lib/network/connectionProxy";
|
||||||
import { HTTP_STATUS } from "open-sse/config/runtimeConfig.js";
|
import { HTTP_STATUS } from "open-sse/config/runtimeConfig.js";
|
||||||
import { detectFormatByEndpoint } from "open-sse/translator/formats.js";
|
import { detectFormatByEndpoint } from "open-sse/translator/formats.js";
|
||||||
import * as log from "../utils/logger.js";
|
import * as log from "../utils/logger.js";
|
||||||
@@ -284,6 +285,23 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
|
|||||||
pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null,
|
pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null,
|
||||||
onPxpipeEvent: appendPxpipeEvent,
|
onPxpipeEvent: appendPxpipeEvent,
|
||||||
providerThinking,
|
providerThinking,
|
||||||
|
// Pool-scoped failure recovery: re-resolve proxy config excluding the
|
||||||
|
// failed pool so the request retries via another pool, not a dead end.
|
||||||
|
resolveProxyConfig: async (creds, excludePoolIds = []) => {
|
||||||
|
const psd = { ...(creds?.providerSpecificData || {}) };
|
||||||
|
if (psd.proxyPoolIds?.length) psd.proxyPoolScope = `${provider}::${model}`;
|
||||||
|
const resolved = await resolveConnectionProxyConfig(psd, creds?.connectionId || creds?.id, excludePoolIds);
|
||||||
|
if (!resolved?.proxyPoolId) return null;
|
||||||
|
return {
|
||||||
|
connectionProxyEnabled: resolved.connectionProxyEnabled,
|
||||||
|
connectionProxyUrl: resolved.connectionProxyUrl,
|
||||||
|
connectionNoProxy: resolved.connectionNoProxy,
|
||||||
|
connectionProxyPoolId: resolved.proxyPoolId || null,
|
||||||
|
vercelRelayUrl: resolved.vercelRelayUrl || "",
|
||||||
|
proxyPoolId: resolved.proxyPoolId || null,
|
||||||
|
strictProxy: resolved.strictProxy === true,
|
||||||
|
};
|
||||||
|
},
|
||||||
// Detect source format by endpoint + body
|
// Detect source format by endpoint + body
|
||||||
sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null,
|
sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null,
|
||||||
onCredentialsRefreshed: async (newCreds) => {
|
onCredentialsRefreshed: async (newCreds) => {
|
||||||
|
|||||||
@@ -47,10 +47,16 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
|
|||||||
const override = (settings.providerStrategies || {})[providerId] || {};
|
const override = (settings.providerStrategies || {})[providerId] || {};
|
||||||
const strategy = override.rotateStrategy || "none";
|
const strategy = override.rotateStrategy || "none";
|
||||||
let pickedId = override.proxyPoolId || null;
|
let pickedId = override.proxyPoolId || null;
|
||||||
|
let poolIds = [];
|
||||||
if (strategy !== "none") {
|
if (strategy !== "none") {
|
||||||
const allPools = await getProxyPools({ isActive: true });
|
const allPools = await getProxyPools({ isActive: true });
|
||||||
const poolIds = allPools.filter(p => p.proxyUrl).map(p => p.id);
|
poolIds = allPools.filter(p => p.proxyUrl).map(p => p.id);
|
||||||
pickedId = pickProxyPoolId(poolIds, strategy, providerId);
|
// Scope region-aware ("smart") filtering to this provider/model so
|
||||||
|
// pools marked unfit here are skipped.
|
||||||
|
const scope = `${providerId}::${model || "*"}`;
|
||||||
|
pickedId = pickProxyPoolId(poolIds, strategy, providerId, { scope });
|
||||||
|
} else if (override.proxyPoolId) {
|
||||||
|
poolIds = [override.proxyPoolId];
|
||||||
}
|
}
|
||||||
const resolvedProxy = await resolveConnectionProxyConfig({ proxyPoolId: pickedId || "" });
|
const resolvedProxy = await resolveConnectionProxyConfig({ proxyPoolId: pickedId || "" });
|
||||||
return {
|
return {
|
||||||
@@ -64,6 +70,13 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
|
|||||||
connectionNoProxy: resolvedProxy.connectionNoProxy,
|
connectionNoProxy: resolvedProxy.connectionNoProxy,
|
||||||
connectionProxyPoolId: resolvedProxy.proxyPoolId || null,
|
connectionProxyPoolId: resolvedProxy.proxyPoolId || null,
|
||||||
vercelRelayUrl: resolvedProxy.vercelRelayUrl || "",
|
vercelRelayUrl: resolvedProxy.vercelRelayUrl || "",
|
||||||
|
proxyPoolId: resolvedProxy.proxyPoolId || null,
|
||||||
|
strictProxy: resolvedProxy.strictProxy === true,
|
||||||
|
// Let chatCore's pool-scoped retry rotate across the same candidate
|
||||||
|
// pool set (excluding the failed pool) instead of reusing it — this
|
||||||
|
// is what makes per-IP limit retries work for no-auth providers.
|
||||||
|
proxyPoolIds: poolIds,
|
||||||
|
proxyRotationStrategy: strategy,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -172,7 +185,11 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
|
|||||||
connection = availableConnections[0];
|
connection = availableConnections[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
const resolvedProxy = await resolveConnectionProxyConfig(connection.providerSpecificData || {}, connection.id);
|
// Scope the region-aware picker to this provider/model (e.g. freebuff::gpt-5.6-luna)
|
||||||
|
const psdForProxy = connection.providerSpecificData?.proxyPoolIds?.length
|
||||||
|
? { ...connection.providerSpecificData, proxyPoolScope: `${providerId}::${model || ""}` }
|
||||||
|
: connection.providerSpecificData;
|
||||||
|
const resolvedProxy = await resolveConnectionProxyConfig(psdForProxy || {}, connection.id);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
authType: connection.authType,
|
authType: connection.authType,
|
||||||
@@ -193,6 +210,8 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
|
|||||||
connectionNoProxy: resolvedProxy.connectionNoProxy,
|
connectionNoProxy: resolvedProxy.connectionNoProxy,
|
||||||
connectionProxyPoolId: resolvedProxy.proxyPoolId || null,
|
connectionProxyPoolId: resolvedProxy.proxyPoolId || null,
|
||||||
vercelRelayUrl: resolvedProxy.vercelRelayUrl || "",
|
vercelRelayUrl: resolvedProxy.vercelRelayUrl || "",
|
||||||
|
proxyPoolId: resolvedProxy.proxyPoolId || null,
|
||||||
|
strictProxy: resolvedProxy.strictProxy === true,
|
||||||
},
|
},
|
||||||
connectionId: connection.id,
|
connectionId: connection.id,
|
||||||
// Include current status for optimization check
|
// Include current status for optimization check
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
import { describe, it, expect, beforeEach } from "vitest";
|
||||||
|
import {
|
||||||
|
markPoolUnfit,
|
||||||
|
clearPoolUnfit,
|
||||||
|
clearAllPoolUnfit,
|
||||||
|
isPoolFit,
|
||||||
|
fitPoolIds,
|
||||||
|
poolFitnessSnapshot,
|
||||||
|
pruneExpired,
|
||||||
|
resetPoolFitness,
|
||||||
|
} from "open-sse/services/proxyPoolFitness.js";
|
||||||
|
|
||||||
|
describe("proxy pool fitness registry", () => {
|
||||||
|
beforeEach(() => resetPoolFitness());
|
||||||
|
|
||||||
|
it("marks a pool unfit for a scope and prunes on expiry", () => {
|
||||||
|
markPoolUnfit("p1", "freebuff::gpt-5.6-luna", Date.now() + 60_000, "limited_ip");
|
||||||
|
expect(isPoolFit("p1", "freebuff::gpt-5.6-luna")).toBe(false);
|
||||||
|
expect(isPoolFit("p1", "freebuff::other-model")).toBe(true);
|
||||||
|
expect(isPoolFit("p2", "freebuff::gpt-5.6-luna")).toBe(true);
|
||||||
|
|
||||||
|
markPoolUnfit("p1", "freebuff::gpt-5.6-luna", Date.now() - 1000); // expired
|
||||||
|
expect(isPoolFit("p1", "freebuff::gpt-5.6-luna")).toBe(true); // pruned on read
|
||||||
|
});
|
||||||
|
|
||||||
|
it("provider-wide mark (provider::*) covers any model lookup", () => {
|
||||||
|
markPoolUnfit("p1", "opencode::*", Date.now() + 60_000, "manual");
|
||||||
|
expect(isPoolFit("p1", "opencode::sonnet-4.6")).toBe(false);
|
||||||
|
expect(isPoolFit("p1", "freebuff::gpt-5.6-luna")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fitPoolIds filters unfit pools; snapshot drops expired marks", () => {
|
||||||
|
markPoolUnfit("p1", "fb::m1", Date.now() + 60_000);
|
||||||
|
markPoolUnfit("p1", "fb::m2", Date.now() - 1000); // expired
|
||||||
|
expect(fitPoolIds(["p1", "p2"], "fb::m1")).toEqual(["p2"]);
|
||||||
|
|
||||||
|
const snap = poolFitnessSnapshot();
|
||||||
|
expect(snap.p1["fb::m1"]).toBeDefined();
|
||||||
|
expect(snap.p1["fb::m2"]).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("clear per scope, clear-all per provider, clear-all global, pruneExpired", () => {
|
||||||
|
markPoolUnfit("p1", "freebuff::m1", Date.now() + 60_000);
|
||||||
|
markPoolUnfit("p1", "kiro::m3", Date.now() + 60_000);
|
||||||
|
|
||||||
|
clearPoolUnfit("p1", "freebuff::m1");
|
||||||
|
expect(isPoolFit("p1", "freebuff::m1")).toBe(true);
|
||||||
|
|
||||||
|
clearAllPoolUnfit("kiro");
|
||||||
|
expect(poolFitnessSnapshot().p1).toBeUndefined();
|
||||||
|
|
||||||
|
markPoolUnfit("p2", "x::y", Date.now() - 1000);
|
||||||
|
expect(pruneExpired()).toBe(1);
|
||||||
|
expect(poolFitnessSnapshot()).toEqual({});
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in new issue
Block a user