feat(proxy-fitness): pool fitness registry + Smart rotation + pool-scoped retry
Pool/IP fitness registry (globalThis-backed, provider::model scopes, 5-min cooldown, provider::* wildcard) fed by pool-scoped failures: freebuff limited-IP / model-locked gates and opencode free per-IP limits. chatCore retries a failed pool via another pool without locking the account; new Smart rotation strategy (per-connection + no-auth providers) skips unfit pools. Proxy Fitness dashboard page: active-block table with provider/IP filters, per-record Clear and provider-scoped Clear All; egress column via pool geo enrichment. Unit tests for the registry.
This commit is contained in:
MUH. IQRAM BAHRING
@@ -8,6 +8,7 @@ import {
|
||||
DEFAULT_RETRY_CONFIG,
|
||||
resolveRetryEntry,
|
||||
} from "../config/runtimeConfig.js";
|
||||
import { markPoolUnfit, clearPoolUnfit } from "../services/proxyPoolFitness.js";
|
||||
|
||||
/**
|
||||
* Freebuff Executor — OpenAI-compatible chat completions on
|
||||
@@ -87,8 +88,110 @@ const FREE_ROOT_AGENT_BY_MODEL = {
|
||||
// don't share one session row). Re-claims are driven by the cache expiring or
|
||||
// by a 428 from chat — no early re-claim, so we never POST /session while our
|
||||
// own row is still active (which could come back as a spurious model_locked).
|
||||
const sessionCache = new Map(); // `${token}::${model}` -> { instanceId, expiresAt }
|
||||
const inflight = new Map(); // dedupe concurrent claims for the same key
|
||||
// All state lives on globalThis so Next dev (Turbopack) bundles share ONE copy.
|
||||
const FB_STATE_KEY = "__9routerFreebuffState__";
|
||||
const fbState = (globalThis[FB_STATE_KEY] ??= {
|
||||
sessionCache: new Map(), // `${token}::${model}` -> { instanceId, expiresAt }
|
||||
inflight: new Map(), // dedupe concurrent claims for the same key
|
||||
modelLockCooldowns: new Map(), // `${token}::${model}` -> expiresAt (ms)
|
||||
poolLimitCooldowns: new Map(), // `${proxyKey}::${model}` -> expiresAt (ms)
|
||||
});
|
||||
const sessionCache = fbState.sessionCache;
|
||||
const inflight = fbState.inflight;
|
||||
const modelLockCooldowns = fbState.modelLockCooldowns;
|
||||
const poolLimitCooldowns = fbState.poolLimitCooldowns;
|
||||
|
||||
const MODEL_LOCK_COOLDOWN_MS = 10 * 60 * 1000; // session bound to another model (~1h) — re-check every 10 min
|
||||
const POOL_LIMITED_COOLDOWN_MS = 5 * 60 * 1000; // IP tier refuses this model — try a different pool/relay
|
||||
|
||||
// Cooldown maps need pruning: expired entries are cleared on write (sweep) and
|
||||
// on read, so long-running servers don't accumulate one entry per (account,model)
|
||||
// / (proxy,model) forever.
|
||||
function setCooldown(map, key, until) {
|
||||
const now = Date.now();
|
||||
for (const [k, v] of map) {
|
||||
if (v <= now) map.delete(k);
|
||||
}
|
||||
map.set(key, until);
|
||||
}
|
||||
|
||||
function getCooldown(map, key) {
|
||||
const until = map.get(key);
|
||||
if (until == null) return null;
|
||||
if (until <= Date.now()) {
|
||||
map.delete(key);
|
||||
return null;
|
||||
}
|
||||
return until;
|
||||
}
|
||||
|
||||
function proxyKeyOf(proxyOptions) {
|
||||
return proxyOptions?.vercelRelayUrl || proxyOptions?.connectionProxyUrl || "direct";
|
||||
}
|
||||
|
||||
function sessionGateFromText(text) {
|
||||
let parsed = {};
|
||||
try { parsed = JSON.parse(String(text || "")); } catch { parsed = {}; }
|
||||
return classifySessionGate(parsed.error || parsed.error_type || "", parsed.message || "", parsed.currentModel || null);
|
||||
}
|
||||
|
||||
// Parse a 409/428/410 body into { kind, currentModel }. `msg` may be a whole
|
||||
// error string containing a JSON tail (requestSession errors embed the body).
|
||||
function sessionGateFromError(error) {
|
||||
const msg = String(error?.message || "");
|
||||
const start = msg.indexOf("{");
|
||||
if (start < 0) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(msg.slice(start));
|
||||
return classifySessionGate(parsed.error || "", parsed.message || "", parsed.currentModel || null);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function classifySessionGate(code, message, currentModel) {
|
||||
if (code === "session_superseded") return { kind: "superseded" };
|
||||
if (code === "model_locked") return { kind: "model_locked", currentModel };
|
||||
// session_model_mismatch with the limited-tier message is an IP-tier refusal;
|
||||
// without it (or unknown) treat it as a model lock so we don't reclaim in a loop.
|
||||
if (code === "session_model_mismatch") {
|
||||
return /limited/i.test(String(message || ""))
|
||||
? { kind: "limited_ip" }
|
||||
: { kind: "model_locked", currentModel };
|
||||
}
|
||||
return { kind: "stale" }; // 428/410/unknown → reclaim
|
||||
}
|
||||
|
||||
// Applies cooldowns and throws for non-reclaimable gates. Never returns for them.
|
||||
function throwSessionGateError(gate, { token, model, proxyKey, poolId, log }) {
|
||||
if (gate.kind === "model_locked") {
|
||||
const until = Date.now() + MODEL_LOCK_COOLDOWN_MS;
|
||||
setCooldown(modelLockCooldowns, `${token}::${model}`, until);
|
||||
const label = gate.currentModel ? `"${gate.currentModel}"` : "another model";
|
||||
const err = new Error(
|
||||
`Freebuff session is locked to ${label} — it cannot serve ${model}. End the session on freebuff.com or wait for it to expire (~1h).`,
|
||||
);
|
||||
err.status = 409;
|
||||
err.resetsAtMs = until;
|
||||
log?.warn?.("AUTH", `Freebuff model_locked (session=${label}, requested=${model}) — model cooldown ${MODEL_LOCK_COOLDOWN_MS / 60000}min`);
|
||||
throw err;
|
||||
}
|
||||
if (gate.kind === "limited_ip") {
|
||||
const until = Date.now() + POOL_LIMITED_COOLDOWN_MS;
|
||||
setCooldown(poolLimitCooldowns, `${proxyKey}::${model}`, until);
|
||||
const scope = `freebuff::${model}`;
|
||||
if (poolId) markPoolUnfit(poolId, scope, until, "limited_ip");
|
||||
// Pool-scoped, not account-scoped: the caller retries via another pool
|
||||
// instead of locking the account (resetsAtMs intentionally absent).
|
||||
const err = new Error(
|
||||
`Freebuff limited-mode IP rejected ${model} — this IP only allows DeepSeek V4 Flash / MiMo 2.5. Use a full-access proxy or a different model.`,
|
||||
);
|
||||
err.status = 409;
|
||||
err.poolScoped = { poolId, scope, reason: "limited_ip" };
|
||||
log?.warn?.("AUTH", `Freebuff limited-IP refused ${model} (proxy=${proxyKey.slice(0, 40)}…) — cooldown ${POOL_LIMITED_COOLDOWN_MS / 60000}min`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function sessionOrigin() {
|
||||
return new URL(PROVIDERS.freebuff.baseUrl).origin; // https://www.codebuff.com
|
||||
@@ -185,7 +288,11 @@ async function requestSession(token, model, proxyOptions) {
|
||||
|
||||
async function ensureSession(token, model, proxyOptions, force = false) {
|
||||
const key = sessionCacheKey(token, model);
|
||||
// Lazy prune: drop stale rows so the cache never accumulates expired entries.
|
||||
const cached = sessionCache.get(key);
|
||||
if (cached && cached.expiresAt <= Date.now()) {
|
||||
sessionCache.delete(key);
|
||||
}
|
||||
if (!force && cached && cached.expiresAt > Date.now()) {
|
||||
return { instanceId: cached.instanceId, status: "active" };
|
||||
}
|
||||
@@ -262,6 +369,32 @@ export function resetSessionCache() {
|
||||
inflight.clear();
|
||||
}
|
||||
|
||||
// Periodic sweeper: drop stale session rows + expired cooldowns so long-running
|
||||
// servers never accumulate state for accounts/models no longer in use.
|
||||
// Returns how many entries were removed.
|
||||
export function pruneSessionState(now = Date.now()) {
|
||||
let removed = 0;
|
||||
for (const [key, entry] of sessionCache) {
|
||||
if (entry?.expiresAt && entry.expiresAt <= now) {
|
||||
sessionCache.delete(key);
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
for (const [key, until] of modelLockCooldowns) {
|
||||
if (until <= now) {
|
||||
modelLockCooldowns.delete(key);
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
for (const [key, until] of poolLimitCooldowns) {
|
||||
if (until <= now) {
|
||||
poolLimitCooldowns.delete(key);
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
export class FreebuffExecutor extends BaseExecutor {
|
||||
constructor() {
|
||||
super("freebuff", PROVIDERS.freebuff);
|
||||
@@ -282,6 +415,12 @@ export class FreebuffExecutor extends BaseExecutor {
|
||||
cost_mode: "free",
|
||||
};
|
||||
body.provider = { allow_fallbacks: false };
|
||||
// Freebuff agents (base2-free-*) own reasoning: the backend applies the
|
||||
// agent's reasoningOptions.effort server-side, so a client-sent
|
||||
// reasoning_effort / reasoning.effort collides with that default →
|
||||
// 400 "both provided with conflicting values". Mirror the CLI: send none.
|
||||
delete body.reasoning_effort;
|
||||
delete body.reasoning;
|
||||
// Free-tier gate: first system message must open with the CLI marker.
|
||||
return injectFreebuffMarker(body);
|
||||
}
|
||||
@@ -292,10 +431,32 @@ export class FreebuffExecutor extends BaseExecutor {
|
||||
throw new Error("Freebuff requires a connected Freebuff login (no access token found)");
|
||||
}
|
||||
|
||||
// Fail fast while a known-dead (account,model) / (proxy,model) pair is in
|
||||
// cooldown — no session claim, no run registration, no upstream spam.
|
||||
const proxyKey = proxyKeyOf(proxyOptions);
|
||||
const poolId = proxyOptions?.proxyPoolId || null;
|
||||
const scope = `freebuff::${model}`;
|
||||
const lockUntil = getCooldown(modelLockCooldowns, `${token}::${model}`);
|
||||
if (lockUntil) {
|
||||
const err = new Error(`Freebuff session locked to another model — retry after ${new Date(lockUntil).toLocaleTimeString()}`);
|
||||
err.status = 409;
|
||||
err.resetsAtMs = lockUntil;
|
||||
throw err;
|
||||
}
|
||||
const poolUntil = getCooldown(poolLimitCooldowns, `${proxyKey}::${model}`);
|
||||
if (poolUntil) {
|
||||
const err = new Error(`Freebuff limited-mode IP rejected ${model} — retry with a full-access proxy after ${new Date(poolUntil).toLocaleTimeString()}`);
|
||||
err.status = 409;
|
||||
err.poolScoped = { poolId, scope, reason: "limited_ip" };
|
||||
throw err;
|
||||
}
|
||||
|
||||
let session;
|
||||
try {
|
||||
session = await ensureSession(token, model, proxyOptions);
|
||||
} catch (error) {
|
||||
const gate = sessionGateFromError(error);
|
||||
if (gate) throwSessionGateError(gate, { token, model, proxyKey, poolId, log });
|
||||
log?.error?.("AUTH", `Freebuff session failed: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
@@ -396,9 +557,17 @@ export class FreebuffExecutor extends BaseExecutor {
|
||||
// 409 session_superseded — another instance took over the session
|
||||
// 409 session_model_mismatch — session bound to a different model
|
||||
// 410 session_expired — the active session's expires_at passed
|
||||
// In every case: abandon the run, force a fresh session claim + a fresh
|
||||
// run, then retry exactly once.
|
||||
// model_locked / limited-tier mismatches are NOT reclaimable — the server
|
||||
// keeps refusing until the session expires or the IP tier changes, so we
|
||||
// set a cooldown and fail fast instead of force re-claiming in a loop.
|
||||
if (SESSION_STALE_CODES.has(response.status)) {
|
||||
const text = await response.text().catch(() => "");
|
||||
const gate = sessionGateFromText(text);
|
||||
if (gate.kind === "model_locked" || gate.kind === "limited_ip") {
|
||||
markFinished("cancelled");
|
||||
throwSessionGateError(gate, { token, model, proxyKey, poolId, log });
|
||||
}
|
||||
|
||||
log?.debug?.("AUTH", `Freebuff ${response.status} session gate — re-claiming session`);
|
||||
markFinished("cancelled");
|
||||
try {
|
||||
@@ -406,21 +575,34 @@ export class FreebuffExecutor extends BaseExecutor {
|
||||
runId = await startRun(token, model, proxyOptions);
|
||||
activeRunId = runId;
|
||||
} catch (error) {
|
||||
const gate2 = sessionGateFromError(error);
|
||||
if (gate2) throwSessionGateError(gate2, { token, model, proxyKey, poolId, log });
|
||||
log?.error?.("AUTH", `Freebuff session re-claim failed: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
({ response, transformedBody } = await doChat());
|
||||
|
||||
if (SESSION_STALE_CODES.has(response.status)) {
|
||||
const text = await response.text().catch(() => "");
|
||||
const text2 = await response.text().catch(() => "");
|
||||
const gate3 = sessionGateFromText(text2);
|
||||
if (gate3.kind === "model_locked" || gate3.kind === "limited_ip") {
|
||||
throwSessionGateError(gate3, { token, model, proxyKey, poolId, log });
|
||||
}
|
||||
const err = new Error(
|
||||
`Freebuff session gate refused (${response.status}) — another freebuff instance may be holding the session. ${text.slice(0, 160)}`,
|
||||
`Freebuff session gate refused (${response.status}) — another freebuff instance may be holding the session. ${text2.slice(0, 160)}`,
|
||||
);
|
||||
err.status = response.status;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// A successful chat means the pair is healthy again — lift any cooldowns.
|
||||
if (response.ok) {
|
||||
modelLockCooldowns.delete(`${token}::${model}`);
|
||||
poolLimitCooldowns.delete(`${proxyKey}::${model}`);
|
||||
if (poolId) clearPoolUnfit(poolId, scope);
|
||||
}
|
||||
|
||||
// The authToken has no refresh path — when it dies, the user re-logs in.
|
||||
// Drop the cached session for this token so a re-login starts clean.
|
||||
if (response.status === 401) {
|
||||
|
||||
@@ -5,6 +5,12 @@ import { injectReasoningContent } from "../utils/reasoningContentInjector.js";
|
||||
// Models that use /zen/v1/messages (claude format)
|
||||
const MESSAGES_MODELS = new Set();
|
||||
|
||||
// OpenCode free tier is limited per egress IP — a 429/403 with a limit-ish
|
||||
// body means the POOL's IP is exhausted, not the account. Declare it
|
||||
// pool-scoped so chatCore marks the pool unfit, retries via another pool, and
|
||||
// it shows up (clearable) on the Proxy Fitness page.
|
||||
const IP_LIMIT_BODY = /limit|rate|quota|exhausted|capacity|too many|retry/i;
|
||||
|
||||
export class OpenCodeExecutor extends BaseExecutor {
|
||||
constructor() {
|
||||
super("opencode", PROVIDERS.opencode);
|
||||
@@ -29,4 +35,17 @@ export class OpenCodeExecutor extends BaseExecutor {
|
||||
"Accept": "text/event-stream"
|
||||
};
|
||||
}
|
||||
|
||||
parseError(response, bodyText) {
|
||||
const status = response?.status || 0;
|
||||
const text = String(bodyText || "");
|
||||
if ((status === 429 || status === 403) && IP_LIMIT_BODY.test(text)) {
|
||||
return {
|
||||
status,
|
||||
message: text.slice(0, 300) || `OpenCode free limit (${status})`,
|
||||
poolScoped: { reason: "ip-limit" },
|
||||
};
|
||||
}
|
||||
return null; // fall through to default parsing
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,6 +29,12 @@ import { getCapabilitiesForModel } from "../providers/capabilities.js";
|
||||
import { stripUnsupportedModalities } from "../translator/concerns/modality.js";
|
||||
import { prefetchRemoteImages } from "../translator/concerns/prefetch.js";
|
||||
import { resolveSessionId } from "../utils/sessionManager.js";
|
||||
import { markPoolUnfit, clearPoolUnfit } from "../services/proxyPoolFitness.js";
|
||||
|
||||
// Pool-scoped failure retry: when an executor tags an error as belonging to a
|
||||
// proxy pool (region gate, dead proxy, …), re-resolve the proxy config
|
||||
// excluding that pool and retry instead of failing the whole account.
|
||||
const MAX_POOL_RETRIES = 2;
|
||||
|
||||
/**
|
||||
* Core chat handler - shared between SSE and Worker
|
||||
@@ -57,7 +63,7 @@ export function stripContinuityFields(body) {
|
||||
return body;
|
||||
}
|
||||
|
||||
export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking }) {
|
||||
export async function handleChatCore({ body, modelInfo, credentials, log, onCredentialsRefreshed, onRequestSuccess, onDisconnect, clientRawRequest, connectionId, userAgent, apiKey, ccFilterNaming, rtkEnabled, headroomEnabled, headroomUrl, headroomCompressUserMessages, cavemanEnabled, cavemanLevel, ponytailEnabled, ponytailLevel, pxpipeEnabled, pxpipeMinChars, pxpipeTimeoutMs, pxpipeTransform, onPxpipeEvent, sourceFormatOverride, providerThinking, resolveProxyConfig }) {
|
||||
const { provider, model } = modelInfo;
|
||||
const requestStartTime = Date.now();
|
||||
// Stable per-session color so all lines of one CLI conversation share a tag
|
||||
@@ -291,16 +297,23 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
||||
log, provider, model, reqTag
|
||||
});
|
||||
|
||||
const proxyOptions = {
|
||||
connectionProxyEnabled: credentials?.providerSpecificData?.connectionProxyEnabled === true,
|
||||
connectionProxyUrl: credentials?.providerSpecificData?.connectionProxyUrl || "",
|
||||
connectionNoProxy: credentials?.providerSpecificData?.connectionNoProxy || "",
|
||||
vercelRelayUrl: credentials?.providerSpecificData?.vercelRelayUrl || "",
|
||||
};
|
||||
// Build proxy options from the resolved provider-specific data. `strictProxy`
|
||||
// is forced for freebuff so a dead/limited pool can never leak the request
|
||||
// to the caller's real IP (the freebuff session tier is per-egress-IP).
|
||||
const buildProxyOptions = (psd = {}) => ({
|
||||
connectionProxyEnabled: psd?.connectionProxyEnabled === true,
|
||||
connectionProxyUrl: psd?.connectionProxyUrl || "",
|
||||
connectionNoProxy: psd?.connectionNoProxy || "",
|
||||
vercelRelayUrl: psd?.vercelRelayUrl || "",
|
||||
strictProxy: psd?.strictProxy === true || provider === "freebuff",
|
||||
proxyPoolId: psd?.proxyPoolId || psd?.connectionProxyPoolId || null,
|
||||
});
|
||||
|
||||
let proxyOptions = buildProxyOptions(credentials?.providerSpecificData || {});
|
||||
|
||||
if (proxyOptions.vercelRelayUrl) {
|
||||
const connectionName = credentials?.connectionName || credentials?.connectionId || "unknown";
|
||||
const poolId = credentials?.providerSpecificData?.connectionProxyPoolId || "none";
|
||||
const poolId = proxyOptions.proxyPoolId || "none";
|
||||
log?.info?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | pool=${poolId} | vercel-relay=${proxyOptions.vercelRelayUrl}`);
|
||||
} else if (proxyOptions.connectionProxyEnabled && proxyOptions.connectionProxyUrl) {
|
||||
let maskedProxyUrl = proxyOptions.connectionProxyUrl;
|
||||
@@ -314,7 +327,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
||||
// Keep raw if URL parsing fails
|
||||
}
|
||||
|
||||
const poolId = credentials?.providerSpecificData?.connectionProxyPoolId || "none";
|
||||
const poolId = proxyOptions.proxyPoolId || "none";
|
||||
const connectionName = credentials?.connectionName || credentials?.connectionId || "unknown";
|
||||
log?.info?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | pool=${poolId} | url=${maskedProxyUrl}`);
|
||||
}
|
||||
@@ -324,13 +337,65 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
||||
log?.debug?.("PROXY", `${provider.toUpperCase()} | ${model} | conn=${connectionName} | no_proxy=${proxyOptions.connectionNoProxy}`);
|
||||
}
|
||||
|
||||
// Execute request — with pool-scoped retry: a failed pool (region gate,
|
||||
// per-IP limit, dead proxy) is marked unfit and the request retried via
|
||||
// another pool instead of failing the account. Covers both thrown errors
|
||||
// (executor.execute) and non-ok responses declared poolScoped via
|
||||
// parseError — poolId/scope are completed here from proxyOptions.
|
||||
const proxyScope = `${provider}::${model}`;
|
||||
let parsedNonOk = null;
|
||||
|
||||
const tryNextPool = async (poolScoped, reasonMsg) => {
|
||||
const failed = {
|
||||
poolId: poolScoped?.poolId || proxyOptions.proxyPoolId || null,
|
||||
scope: poolScoped?.scope || proxyScope,
|
||||
reason: poolScoped?.reason || "pool-scoped",
|
||||
};
|
||||
markPoolUnfit(failed.poolId, failed.scope, undefined, failed.reason);
|
||||
log?.warn?.("PROXY", `${provider.toUpperCase()} | pool ${failed.poolId || "?"} unfit for ${failed.scope} (${failed.reason}) — retry with another pool. ${reasonMsg || ""}`);
|
||||
try {
|
||||
const resolved = await resolveProxyConfig(credentials, [failed.poolId]);
|
||||
if (resolved?.proxyPoolId) {
|
||||
credentials.providerSpecificData = { ...(credentials.providerSpecificData || {}), ...resolved };
|
||||
proxyOptions = buildProxyOptions(credentials.providerSpecificData);
|
||||
return true;
|
||||
}
|
||||
} catch (resolverError) {
|
||||
// A resolver failure must not mask the original pool error.
|
||||
log?.warn?.("PROXY", `${provider.toUpperCase()} | pool re-resolve failed: ${resolverError.message}`);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const executeWithPoolFallback = async (attempt = 0) => {
|
||||
let result;
|
||||
try {
|
||||
result = await executor.execute({ model, body: translatedBody, stream, credentials, signal: streamController.signal, log, proxyOptions });
|
||||
} catch (error) {
|
||||
if (error?.poolScoped && typeof resolveProxyConfig === "function" && attempt < MAX_POOL_RETRIES) {
|
||||
if (await tryNextPool(error.poolScoped, error.message)) return executeWithPoolFallback(attempt + 1);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
// Non-ok response that the executor declared pool/IP-scoped (e.g. opencode
|
||||
// free per-IP limit) — parse once, retry via another pool when possible.
|
||||
if (!result.response.ok) {
|
||||
const parsed = await parseUpstreamError(result.response, executor);
|
||||
if (parsed.poolScoped && typeof resolveProxyConfig === "function" && attempt < MAX_POOL_RETRIES) {
|
||||
if (await tryNextPool(parsed.poolScoped, parsed.message)) return executeWithPoolFallback(attempt + 1);
|
||||
}
|
||||
parsedNonOk = parsed;
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
// Execute request
|
||||
let providerResponse, providerUrl, providerHeaders, finalBody;
|
||||
// Most executors return their registry format. Cursor AgentService is an
|
||||
// exception: it is decoded by the executor into OpenAI-compatible output.
|
||||
let providerResponseFormat = targetFormat;
|
||||
try {
|
||||
const result = await executor.execute({ model, body: translatedBody, stream, credentials, signal: streamController.signal, log, proxyOptions });
|
||||
const result = await executeWithPoolFallback();
|
||||
providerResponse = result.response;
|
||||
providerUrl = result.url;
|
||||
providerHeaders = result.headers;
|
||||
@@ -359,7 +424,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
||||
if (log?.errorLine) {
|
||||
log.errorLine(reqTag, "✗", `ERROR 502 · ${provider}/${model} · ${Date.now() - requestStartTime}ms\n ${errMsg}${error.stack ? `\n ${error.stack}` : ""}`);
|
||||
}
|
||||
return createErrorResult(HTTP_STATUS.BAD_GATEWAY, errMsg);
|
||||
return createErrorResult(HTTP_STATUS.BAD_GATEWAY, errMsg, error?.resetsAtMs || undefined);
|
||||
}
|
||||
|
||||
// Handle 401/403 - try token refresh (skip for noAuth providers)
|
||||
@@ -402,7 +467,7 @@ export async function handleChatCore({ body, modelInfo, credentials, log, onCred
|
||||
// Provider returned error
|
||||
if (!providerResponse.ok) {
|
||||
trackPendingRequest(model, provider, connectionId, false, true);
|
||||
const { statusCode, message, resetsAtMs } = await parseUpstreamError(providerResponse, executor);
|
||||
const { statusCode, message, resetsAtMs } = parsedNonOk || await parseUpstreamError(providerResponse, executor);
|
||||
appendRequestLog({ model, provider, connectionId, status: `FAILED ${statusCode}` }).catch(() => { });
|
||||
saveRequestDetail(buildRequestDetail({
|
||||
provider, model, connectionId,
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
// Pool fitness registry — shared, in-memory, engine layer.
|
||||
//
|
||||
// Rotation strategies can opt into region/provider-aware pool selection: an
|
||||
// executor that learns a pool's egress is unfit for a provider/model (e.g.
|
||||
// Freebuff limited-mode IP) marks it here, and the pool picker skips it for
|
||||
// that scope until the cooldown expires.
|
||||
//
|
||||
// Scope format: `provider::model` (e.g. "freebuff::openai/gpt-5.6-luna").
|
||||
// All functions are fail-open: unknown pool/scope ⇒ fit.
|
||||
|
||||
// Scope format: `provider::model` (e.g. "freebuff::openai/gpt-5.6-luna").
|
||||
// All functions are fail-open: unknown pool/scope ⇒ fit.
|
||||
//
|
||||
// State lives on globalThis so Next dev (Turbopack) never splits one Map into
|
||||
// several per-bundle copies — the executor/chatCore marks and the
|
||||
// /api/proxy-pools/fitness reader must share the SAME registry.
|
||||
|
||||
const FITNESS_STATE_KEY = "__9routerPoolFitness__";
|
||||
const fitness = (globalThis[FITNESS_STATE_KEY] ??= new Map()); // poolId -> Map<scope, { until, reason }>
|
||||
|
||||
export const POOL_UNFIT_MS = 5 * 60 * 1000;
|
||||
|
||||
export function markPoolUnfit(poolId, scope, until = Date.now() + POOL_UNFIT_MS, reason = "") {
|
||||
if (!poolId || !scope) return;
|
||||
const byScope = fitness.get(poolId) || new Map();
|
||||
byScope.set(scope, { until, reason });
|
||||
fitness.set(poolId, byScope);
|
||||
}
|
||||
|
||||
export function clearPoolUnfit(poolId, scope) {
|
||||
const byScope = fitness.get(poolId);
|
||||
if (!byScope) return;
|
||||
byScope.delete(scope);
|
||||
if (byScope.size === 0) fitness.delete(poolId);
|
||||
}
|
||||
|
||||
// "provider::model" -> "provider::*" (null when the scope has no provider part)
|
||||
function providerWildcardScope(scope) {
|
||||
const sep = String(scope || "").indexOf("::");
|
||||
if (sep < 0) return null;
|
||||
return `${scope.slice(0, sep)}::*`;
|
||||
}
|
||||
|
||||
export function isPoolFit(poolId, scope, now = Date.now()) {
|
||||
if (!poolId) return true;
|
||||
const byScope = fitness.get(poolId);
|
||||
if (!byScope) return true;
|
||||
// A provider-wide mark ("provider::*", e.g. manual blocks) also covers any
|
||||
// model lookup for that provider.
|
||||
const candidates = [scope, providerWildcardScope(scope)];
|
||||
for (const key of candidates) {
|
||||
if (!key) continue;
|
||||
const entry = byScope.get(key);
|
||||
if (!entry) continue;
|
||||
if (entry.until <= now) {
|
||||
byScope.delete(key);
|
||||
if (byScope.size === 0) fitness.delete(poolId);
|
||||
continue;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Keep only pool ids that are not in cooldown for the scope.
|
||||
export function fitPoolIds(poolIds, scope, now = Date.now()) {
|
||||
return (poolIds || []).filter((id) => isPoolFit(id, scope, now));
|
||||
}
|
||||
|
||||
// Clear every mark — or only scopes belonging to one provider (`provider::*`).
|
||||
export function clearAllPoolUnfit(provider = null) {
|
||||
if (provider) {
|
||||
const prefix = `${provider}::`;
|
||||
for (const [poolId, byScope] of fitness) {
|
||||
for (const scope of [...byScope.keys()]) {
|
||||
if (scope.startsWith(prefix)) byScope.delete(scope);
|
||||
}
|
||||
if (byScope.size === 0) fitness.delete(poolId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
fitness.clear();
|
||||
}
|
||||
|
||||
// Snapshot of live (non-expired) marks — expired entries are pruned here so
|
||||
// consumers never see stale data and memory stays bounded.
|
||||
// Test helper: drop all marks (module state is globalThis-backed).
|
||||
export function resetPoolFitness() {
|
||||
fitness.clear();
|
||||
}
|
||||
|
||||
// Sweep all expired marks. Returns how many scope entries were removed.
|
||||
export function pruneExpired(now = Date.now()) {
|
||||
let removed = 0;
|
||||
for (const [poolId, byScope] of fitness) {
|
||||
for (const [scope, entry] of byScope) {
|
||||
if (entry.until <= now) {
|
||||
byScope.delete(scope);
|
||||
removed += 1;
|
||||
}
|
||||
}
|
||||
if (byScope.size === 0) fitness.delete(poolId);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
export function poolFitnessSnapshot(now = Date.now()) {
|
||||
const out = {};
|
||||
for (const [poolId, byScope] of fitness) {
|
||||
let pruned = false;
|
||||
for (const [scope, entry] of byScope) {
|
||||
if (entry.until <= now) {
|
||||
byScope.delete(scope);
|
||||
pruned = true;
|
||||
}
|
||||
}
|
||||
if (byScope.size === 0) {
|
||||
fitness.delete(poolId);
|
||||
continue;
|
||||
}
|
||||
if (pruned || byScope.size > 0) {
|
||||
out[poolId] = Object.fromEntries(byScope);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -69,7 +69,14 @@ export async function parseUpstreamError(response, executor = null) {
|
||||
const parsed = executor.parseError(response, bodyText);
|
||||
if (parsed && typeof parsed === "object") {
|
||||
const msg = parsed.message || DEFAULT_ERROR_MESSAGES[response.status] || `Upstream error: ${response.status}`;
|
||||
return { statusCode: parsed.status || response.status, message: msg, resetsAtMs: parsed.resetsAtMs };
|
||||
return {
|
||||
statusCode: parsed.status || response.status,
|
||||
message: msg,
|
||||
resetsAtMs: parsed.resetsAtMs,
|
||||
// Executors declare IP/pool-scoped failures (e.g. per-IP rate limits)
|
||||
// here; chatCore completes poolId/scope and retries via another pool.
|
||||
poolScoped: parsed.poolScoped,
|
||||
};
|
||||
}
|
||||
} catch { /* fall through to default parsing */ }
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ export default function ConnectionRow({ connection, proxyPools, isOAuth, isFirst
|
||||
}
|
||||
|
||||
if (selectedProxyIds.length > 1) {
|
||||
const strategyLabel = rotationStrategy === "random" ? "Random" : rotationStrategy === "round-robin" ? "Round Robin" : rotationStrategy === "failover" ? "Failover" : "Multiple";
|
||||
const strategyLabel = rotationStrategy === "random" ? "Random" : rotationStrategy === "round-robin" ? "Round Robin" : rotationStrategy === "failover" ? "Failover" : rotationStrategy === "smart" ? "Smart" : "Multiple";
|
||||
return `${selectedProxyIds.length} pools (${strategyLabel})`;
|
||||
}
|
||||
|
||||
@@ -318,7 +318,7 @@ export default function ConnectionRow({ connection, proxyPools, isOAuth, isFirst
|
||||
<span className="text-[10px] leading-tight">Proxy</span>
|
||||
</button>
|
||||
{showProxyDropdown && (
|
||||
<div className="absolute right-0 top-full z-50 mt-1 max-w-[90vw] min-w-[280px] rounded-lg border border-border bg-bg shadow-lg">
|
||||
<div className="absolute left-0 top-full z-50 mt-1 max-w-[calc(100vw-2rem)] min-w-[280px] rounded-lg border border-border bg-bg shadow-lg sm:left-auto sm:right-0">
|
||||
{/* Rotation Strategy Selector */}
|
||||
<div className="border-b border-border p-3">
|
||||
<label className="block text-xs font-medium text-text-muted mb-2">Rotation Strategy</label>
|
||||
@@ -331,18 +331,20 @@ export default function ConnectionRow({ connection, proxyPools, isOAuth, isFirst
|
||||
<option value="random">Random</option>
|
||||
<option value="round-robin">Round Robin</option>
|
||||
<option value="failover">Failover</option>
|
||||
<option value="smart">Smart</option>
|
||||
</select>
|
||||
{rotationStrategy !== "none" && (
|
||||
<p className="mt-1 text-[10px] text-text-muted">
|
||||
{rotationStrategy === "random" && "Randomly select proxy on each request"}
|
||||
{rotationStrategy === "round-robin" && "Rotate proxies in order across requests"}
|
||||
{rotationStrategy === "failover" && "Try next proxy on failure"}
|
||||
{rotationStrategy === "smart" && "Skip pools whose egress IP is blocked for this provider/model (e.g. Freebuff limited-IP). See Proxy Fitness to clear/block."}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Proxy Pool Selection */}
|
||||
<div className="max-h-[60vh] overflow-y-auto py-1">
|
||||
<div className="max-h-[200px] overflow-y-auto py-1">
|
||||
<button
|
||||
onClick={() => {
|
||||
setSelectedProxyIds([]);
|
||||
|
||||
@@ -0,0 +1,384 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { Badge, Button, Card, CardSkeleton, Input, ConfirmModal } from "@/shared/components";
|
||||
import ProviderIcon from "@/shared/components/ProviderIcon";
|
||||
import { useNotificationStore } from "@/store/notificationStore";
|
||||
|
||||
function fmtTime(value) {
|
||||
if (!value) return "-";
|
||||
const d = new Date(value);
|
||||
return Number.isNaN(d.getTime()) ? "-" : d.toLocaleTimeString();
|
||||
}
|
||||
|
||||
// "freebuff::openai/gpt-5.6-luna" -> { provider: "freebuff", model: "openai/gpt-5.6-luna" }
|
||||
// "freebuff::*" -> { provider: "freebuff", model: null }
|
||||
function parseScope(scope) {
|
||||
const sep = String(scope || "").indexOf("::");
|
||||
if (sep < 0) return { provider: String(scope || ""), model: null };
|
||||
const provider = scope.slice(0, sep);
|
||||
const model = scope.slice(sep + 2);
|
||||
return { provider, model: model === "*" || model === "" ? null : model };
|
||||
}
|
||||
|
||||
function maskProxyUrl(url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
const host = parsed.hostname || "";
|
||||
const port = parsed.port ? `:${parsed.port}` : "";
|
||||
return `${parsed.protocol}//${host}${port}`;
|
||||
} catch {
|
||||
return String(url || "");
|
||||
}
|
||||
}
|
||||
|
||||
// Flatten fitness snapshot into one record per (pool, scope); drops expired marks.
|
||||
function buildRecords(fitness, pools, now = Date.now()) {
|
||||
const poolById = new Map((pools || []).map((p) => [p.id, p]));
|
||||
const records = [];
|
||||
for (const [poolId, byScope] of Object.entries(fitness || {})) {
|
||||
const pool = poolById.get(poolId);
|
||||
for (const [scope, info] of Object.entries(byScope || {})) {
|
||||
const until = info?.until ? new Date(info.until).getTime() : 0;
|
||||
if (!until || until <= now) continue;
|
||||
const { provider, model } = parseScope(scope);
|
||||
records.push({
|
||||
poolId,
|
||||
scope,
|
||||
provider,
|
||||
model,
|
||||
until,
|
||||
reason: info?.reason || "blocked",
|
||||
poolName: pool?.name || poolId.slice(0, 8),
|
||||
proxyUrl: pool?.proxyUrl || "",
|
||||
egress: pool?.egress || null,
|
||||
});
|
||||
}
|
||||
}
|
||||
return records;
|
||||
}
|
||||
|
||||
export default function ProxyFitnessPage() {
|
||||
const [pools, setPools] = useState([]);
|
||||
const [fitness, setFitness] = useState({});
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [providerFilter, setProviderFilter] = useState("all");
|
||||
const [search, setSearch] = useState("");
|
||||
const [providerMenuOpen, setProviderMenuOpen] = useState(false);
|
||||
const [providerMenuDropUp, setProviderMenuDropUp] = useState(false);
|
||||
const [clearingScope, setClearingScope] = useState(null); // poolId::scope while clearing
|
||||
const [confirmClearAll, setConfirmClearAll] = useState(false);
|
||||
const [clearingAll, setClearingAll] = useState(false);
|
||||
const providerMenuRef = useRef(null);
|
||||
const notify = useNotificationStore();
|
||||
|
||||
useEffect(() => {
|
||||
const handleClickOutside = (e) => {
|
||||
if (providerMenuRef.current && !providerMenuRef.current.contains(e.target)) {
|
||||
setProviderMenuOpen(false);
|
||||
}
|
||||
};
|
||||
if (providerMenuOpen) {
|
||||
document.addEventListener("mousedown", handleClickOutside);
|
||||
}
|
||||
return () => document.removeEventListener("mousedown", handleClickOutside);
|
||||
}, [providerMenuOpen]);
|
||||
|
||||
const fetchAll = useCallback(async () => {
|
||||
try {
|
||||
const [poolRes, fitRes] = await Promise.all([
|
||||
fetch("/api/proxy-pools?includeUsage=true", { cache: "no-store" }),
|
||||
fetch("/api/proxy-pools/fitness", { cache: "no-store" }),
|
||||
]);
|
||||
const poolData = await poolRes.json().catch(() => ({ proxyPools: [] }));
|
||||
setPools(poolData.proxyPools || []);
|
||||
if (fitRes.ok) {
|
||||
const fitData = await fitRes.json().catch(() => ({}));
|
||||
setFitness(fitData.pools || fitData.fitness || {});
|
||||
} else {
|
||||
setFitness({});
|
||||
}
|
||||
} catch (error) {
|
||||
console.log("Error fetching proxy fitness:", error);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
fetchAll();
|
||||
}, [fetchAll]);
|
||||
|
||||
const records = useMemo(() => buildRecords(fitness, pools), [fitness, pools]);
|
||||
|
||||
// Providers present in the fitness data (filter options)
|
||||
const providerOptions = useMemo(() => {
|
||||
const set = new Set();
|
||||
for (const rec of records) {
|
||||
if (rec.provider) set.add(rec.provider);
|
||||
}
|
||||
return Array.from(set).sort();
|
||||
}, [records]);
|
||||
|
||||
const filteredRecords = useMemo(() => {
|
||||
const q = search.trim().toLowerCase();
|
||||
return records.filter((rec) => {
|
||||
if (providerFilter !== "all" && rec.provider !== providerFilter) return false;
|
||||
if (q) {
|
||||
const hay = [rec.proxyUrl, rec.egress?.ip, rec.egress?.country, rec.egress?.region, rec.poolName]
|
||||
.filter(Boolean)
|
||||
.join(" ")
|
||||
.toLowerCase();
|
||||
if (!hay.includes(q)) return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
}, [records, providerFilter, search]);
|
||||
|
||||
const handleClear = async (rec) => {
|
||||
setClearingScope(rec.scope);
|
||||
try {
|
||||
const res = await fetch(`/api/proxy-pools/${rec.poolId}/fitness/clear`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ scope: rec.scope }),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
notify.success(`Cleared ${rec.scope}`);
|
||||
fetchAll();
|
||||
} catch (err) {
|
||||
notify.error(`Clear failed: ${err.message}`);
|
||||
} finally {
|
||||
setClearingScope(null);
|
||||
}
|
||||
};
|
||||
|
||||
const handleClearAll = async () => {
|
||||
setClearingAll(true);
|
||||
try {
|
||||
const res = await fetch("/api/proxy-pools/fitness/clear-all", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(providerFilter !== "all" ? { provider: providerFilter } : {}),
|
||||
});
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
||||
notify.success(providerFilter !== "all" ? `Cleared all ${providerFilter} blocks` : "Cleared all blocks");
|
||||
setConfirmClearAll(false);
|
||||
fetchAll();
|
||||
} catch (err) {
|
||||
notify.error(`Clear all failed: ${err.message}`);
|
||||
} finally {
|
||||
setClearingAll(false);
|
||||
}
|
||||
};
|
||||
|
||||
const selectedProviderLabel = providerFilter === "all" ? "All providers" : providerFilter;
|
||||
|
||||
// Open the provider menu as an overlay anchored to its container. Flips to
|
||||
// drop-up when there isn't enough viewport space below (menu max-h ~288px).
|
||||
const toggleProviderMenu = () => {
|
||||
if (providerMenuOpen) {
|
||||
setProviderMenuOpen(false);
|
||||
return;
|
||||
}
|
||||
const el = providerMenuRef.current;
|
||||
if (el) {
|
||||
const rect = el.getBoundingClientRect();
|
||||
setProviderMenuDropUp(window.innerHeight - rect.bottom < 300);
|
||||
}
|
||||
setProviderMenuOpen(true);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="mx-auto w-full max-w-5xl space-y-4 p-4 sm:p-6">
|
||||
{loading ? (
|
||||
<CardSkeleton />
|
||||
) : (
|
||||
<>
|
||||
{/* Header */}
|
||||
<div className="flex flex-wrap items-center justify-between gap-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<h1 className="text-lg font-semibold text-text-main">Proxy Fitness</h1>
|
||||
<Badge variant={records.length > 0 ? "error" : "default"} size="sm">
|
||||
{records.length} active block{records.length === 1 ? "" : "s"}
|
||||
</Badge>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
{records.length > 0 && (
|
||||
<Button
|
||||
variant="danger"
|
||||
size="sm"
|
||||
icon="delete_sweep"
|
||||
onClick={() => setConfirmClearAll(true)}
|
||||
disabled={clearingAll}
|
||||
>
|
||||
Clear All{providerFilter !== "all" ? ` (${providerFilter})` : ""}
|
||||
</Button>
|
||||
)}
|
||||
<Button variant="secondary" size="sm" icon="refresh" onClick={fetchAll}>Refresh</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="text-sm text-text-muted">
|
||||
Shows which provider is blocked on which proxy IP (from provider region gates, per-IP
|
||||
limits, or manual marks). Smart rotation skips these pools while the block is active.
|
||||
</p>
|
||||
|
||||
{/* Filters */}
|
||||
<div className="flex flex-wrap items-end gap-3 rounded-lg border border-border-subtle bg-surface p-3">
|
||||
<div className="relative flex w-64 max-w-full flex-col" ref={providerMenuRef}>
|
||||
<label className="mb-1 text-xs font-medium text-text-muted">Provider</label>
|
||||
<button
|
||||
type="button"
|
||||
onClick={toggleProviderMenu}
|
||||
className="flex h-8 w-full items-center justify-between gap-1 rounded-lg border border-black/10 bg-black/[0.02] px-2 text-xs text-text-main transition-colors hover:bg-black/5 dark:border-white/10 dark:bg-white/[0.03] dark:hover:bg-white/10"
|
||||
aria-haspopup="menu"
|
||||
aria-expanded={providerMenuOpen}
|
||||
title="Filter by provider"
|
||||
>
|
||||
<span className="flex min-w-0 items-center gap-1.5">
|
||||
{providerFilter === "all" ? (
|
||||
<span className="material-symbols-outlined text-[14px] text-text-muted">apps</span>
|
||||
) : (
|
||||
<ProviderIcon providerId={providerFilter} size={18} className="size-[18px] rounded object-contain" fallbackText={providerFilter.slice(0, 2).toUpperCase()} />
|
||||
)}
|
||||
<span className="truncate capitalize">{selectedProviderLabel}</span>
|
||||
</span>
|
||||
<span className="material-symbols-outlined text-[14px] text-text-muted">expand_more</span>
|
||||
</button>
|
||||
|
||||
{providerMenuOpen && (
|
||||
<div className={`absolute left-0 z-20 w-full max-h-72 overflow-y-auto rounded-lg border border-border-subtle bg-surface p-1 shadow-lg ${providerMenuDropUp ? "bottom-full mb-1" : "top-full mt-1"}`}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => { setProviderFilter("all"); setProviderMenuOpen(false); }}
|
||||
className={`flex w-full items-center gap-3 rounded-xl px-3 py-2.5 text-left text-sm transition-colors ${providerFilter === "all" ? "bg-primary/10 text-primary" : "text-text-main hover:bg-black/5 dark:hover:bg-white/10"}`}
|
||||
>
|
||||
<span className="material-symbols-outlined text-[22px]">apps</span>
|
||||
<span className="font-medium">All providers</span>
|
||||
{providerFilter === "all" && <span className="material-symbols-outlined ml-auto text-[20px]">check</span>}
|
||||
</button>
|
||||
<div className="my-1 h-px bg-black/10 dark:bg-white/10" />
|
||||
{providerOptions.map((provider) => (
|
||||
<button
|
||||
key={provider}
|
||||
type="button"
|
||||
onClick={() => { setProviderFilter(provider); setProviderMenuOpen(false); }}
|
||||
className={`flex w-full items-center gap-3 rounded-xl px-3 py-2.5 text-left text-sm transition-colors ${providerFilter === provider ? "bg-primary/10 text-primary" : "text-text-main hover:bg-black/5 dark:hover:bg-white/10"}`}
|
||||
>
|
||||
<ProviderIcon providerId={provider} size={24} className="size-6 rounded-md object-contain" fallbackText={provider.slice(0, 2).toUpperCase()} />
|
||||
<span className="font-medium capitalize">{provider}</span>
|
||||
{providerFilter === provider && <span className="material-symbols-outlined ml-auto text-[20px]">check</span>}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex min-w-48 flex-1 flex-col">
|
||||
<label className="mb-1 text-xs font-medium text-text-muted">IP / Proxy / Pool</label>
|
||||
<Input
|
||||
value={search}
|
||||
onChange={(e) => setSearch(e.target.value)}
|
||||
placeholder="e.g. 104.28, vercel-relay…"
|
||||
icon="search"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Records table */}
|
||||
<Card className="p-0">
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full min-w-[700px] border-collapse text-sm">
|
||||
<thead>
|
||||
<tr className="border-b border-border-subtle text-left text-xs uppercase tracking-wider text-text-muted">
|
||||
<th className="px-4 py-2 font-semibold">Provider</th>
|
||||
<th className="px-4 py-2 font-semibold">Model</th>
|
||||
<th className="px-4 py-2 font-semibold">IP / Proxy</th>
|
||||
<th className="px-4 py-2 font-semibold">Pool</th>
|
||||
<th className="px-4 py-2 font-semibold">Reason</th>
|
||||
<th className="px-4 py-2 font-semibold">Until</th>
|
||||
<th className="px-4 py-2 text-right font-semibold">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{filteredRecords.length === 0 ? (
|
||||
<tr>
|
||||
<td colSpan={7} className="px-4 py-10 text-center text-text-muted">
|
||||
{records.length === 0
|
||||
? "No active blocks. Blocks appear here when a provider region-gates a proxy IP."
|
||||
: "No blocks match the current filters."}
|
||||
</td>
|
||||
</tr>
|
||||
) : (
|
||||
filteredRecords.map((rec) => (
|
||||
<tr key={`${rec.poolId}::${rec.scope}`} className="border-b border-border-subtle last:border-0 align-top hover:bg-surface-2/40">
|
||||
<td className="px-4 py-3">
|
||||
<span className="inline-flex items-center gap-2 rounded bg-red-500/10 px-2 py-0.5 text-xs font-medium text-red-600 dark:text-red-400">
|
||||
<span className="material-symbols-outlined text-[14px]">block</span>
|
||||
<span className="capitalize">{rec.provider}</span>
|
||||
</span>
|
||||
</td>
|
||||
<td className="max-w-56 px-4 py-3">
|
||||
<span className="truncate text-text-main">{rec.model || <em className="text-text-muted">all models</em>}</span>
|
||||
</td>
|
||||
<td className="px-4 py-3">
|
||||
<div className="flex min-w-0 flex-col gap-0.5">
|
||||
<code className="truncate font-mono text-xs text-text-main">{maskProxyUrl(rec.proxyUrl)}</code>
|
||||
{rec.egress?.ip && (
|
||||
<span className="flex items-center gap-1 text-[11px] text-text-muted">
|
||||
<span className="truncate">egress {rec.egress.ip}{rec.egress.country ? ` · ${rec.egress.country}` : ""}</span>
|
||||
{rec.egress.isUnstable && (
|
||||
<span
|
||||
className="inline-flex shrink-0 items-center gap-0.5 rounded bg-amber-500/15 px-1 py-0.5 text-[10px] font-medium text-amber-600 dark:text-amber-400"
|
||||
title={rec.egress.ipCount >= 2 ? `Egress IP changed ${rec.egress.ipCount}× — relay egress is not stable` : "Egress is unstable"}
|
||||
>
|
||||
unstable
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-text-muted">{rec.poolName}</td>
|
||||
<td className="px-4 py-3 text-text-muted">{rec.reason}</td>
|
||||
<td className="px-4 py-3 text-text-muted">{fmtTime(rec.until)}</td>
|
||||
<td className="px-4 py-3">
|
||||
<div className="flex justify-end">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
icon="close"
|
||||
onClick={() => handleClear(rec)}
|
||||
disabled={clearingScope === rec.scope}
|
||||
title="Clear this block"
|
||||
>
|
||||
{clearingScope === rec.scope ? "Clearing..." : "Clear"}
|
||||
</Button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
))
|
||||
)}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</Card>
|
||||
</>
|
||||
)}
|
||||
|
||||
<ConfirmModal
|
||||
isOpen={confirmClearAll}
|
||||
onClose={() => setConfirmClearAll(false)}
|
||||
onConfirm={handleClearAll}
|
||||
title="Clear all blocks"
|
||||
message={providerFilter !== "all"
|
||||
? `Clear all active blocks for provider "${providerFilter}"? This lets those pools be selected again immediately.`
|
||||
: "Clear all active proxy blocks? This lets all pools be selected again immediately."}
|
||||
confirmText={clearingAll ? "Clearing..." : "Clear All"}
|
||||
cancelText="Cancel"
|
||||
variant="danger"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { clearPoolUnfit } from "open-sse/services/proxyPoolFitness.js";
|
||||
|
||||
// POST /api/proxy-pools/[id]/fitness/clear
|
||||
// Body: { scope: "provider::model" } — clears the mark for this pool + scope.
|
||||
export async function POST(request, { params }) {
|
||||
try {
|
||||
const { id } = await params;
|
||||
let body = {};
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
body = {};
|
||||
}
|
||||
const scope = typeof body?.scope === "string" ? body.scope.trim() : "";
|
||||
if (!id || !scope) {
|
||||
return NextResponse.json({ error: "pool id and scope are required" }, { status: 400 });
|
||||
}
|
||||
clearPoolUnfit(id, scope);
|
||||
return NextResponse.json({ ok: true, poolId: id, scope });
|
||||
} catch (error) {
|
||||
console.log("Error clearing pool fitness:", error);
|
||||
return NextResponse.json({ error: "Failed to clear pool fitness" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { clearAllPoolUnfit } from "open-sse/services/proxyPoolFitness.js";
|
||||
|
||||
// POST /api/proxy-pools/fitness/clear-all
|
||||
// Body: { provider?: string } — clears every mark, or only marks scoped to the
|
||||
// given provider ("provider::*") when provided.
|
||||
export async function POST(request) {
|
||||
try {
|
||||
let body = {};
|
||||
try {
|
||||
body = await request.json();
|
||||
} catch {
|
||||
body = {};
|
||||
}
|
||||
const provider = typeof body?.provider === "string" && body.provider.trim() ? body.provider.trim() : null;
|
||||
clearAllPoolUnfit(provider);
|
||||
return NextResponse.json({ ok: true, provider: provider || null });
|
||||
} catch (error) {
|
||||
console.log("Error clearing proxy fitness:", error);
|
||||
return NextResponse.json({ error: "Failed to clear proxy fitness" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { poolFitnessSnapshot } from "open-sse/services/proxyPoolFitness.js";
|
||||
|
||||
// GET /api/proxy-pools/fitness — in-memory snapshot of pool fitness marks.
|
||||
// Returns { pools: { [poolId]: { [scope]: { until, reason } } } }.
|
||||
export async function GET() {
|
||||
try {
|
||||
return NextResponse.json({ pools: poolFitnessSnapshot() });
|
||||
} catch (error) {
|
||||
console.log("Error reading proxy fitness:", error);
|
||||
return NextResponse.json({ error: "Failed to read proxy fitness" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { createProxyPool, getProviderConnections, getProxyPools } from "@/models";
|
||||
import { getPoolGeo } from "open-sse/services/poolGeo.js";
|
||||
|
||||
function toBoolean(value) {
|
||||
if (value === "true") return true;
|
||||
@@ -65,6 +66,8 @@ export async function GET(request) {
|
||||
const enrichedProxyPools = proxyPools.map((pool) => ({
|
||||
...pool,
|
||||
boundConnectionCount: usageMap.get(pool.id) || 0,
|
||||
// Egress geo from the background probe cache (null until first probe).
|
||||
egress: getPoolGeo(pool.id) || null,
|
||||
}));
|
||||
|
||||
return NextResponse.json({ proxyPools: enrichedProxyPools });
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { getProxyPoolById } from "@/models";
|
||||
import { fitPoolIds } from "open-sse/services/proxyPoolFitness.js";
|
||||
|
||||
// Safely normalize any value into a trimmed string.
|
||||
function normalizeString(value) {
|
||||
@@ -13,24 +14,41 @@ const rotateState = new Map(); // providerId → { index }
|
||||
* Pick one proxy pool ID from a list based on strategy.
|
||||
* round-robin: cycle sequentially (in-memory, resets on restart)
|
||||
* random: uniform random pick
|
||||
* smart: region-aware — skip pools unfit for `scope`, round-robin on the fit subset
|
||||
* none/single: return first entry
|
||||
* @param {string[]} poolIds
|
||||
* @param {string} strategy
|
||||
* @param {string} providerId
|
||||
* @param {{ scope?: string, excludeIds?: string[] }} [opts]
|
||||
*/
|
||||
export function pickProxyPoolId(poolIds, strategy, providerId) {
|
||||
export function pickProxyPoolId(poolIds, strategy, providerId, opts = {}) {
|
||||
if (!poolIds || poolIds.length === 0) return null;
|
||||
if (poolIds.length === 1) return poolIds[0];
|
||||
const { scope = null, excludeIds = [] } = opts || {};
|
||||
|
||||
if (strategy === "round-robin") {
|
||||
let eligible = poolIds.filter((id) => !(excludeIds || []).includes(id));
|
||||
// Region-aware filtering is opt-in via the "smart" strategy.
|
||||
if (strategy === "smart" && scope) eligible = fitPoolIds(eligible, scope);
|
||||
|
||||
if (eligible.length === 0) {
|
||||
// Every candidate is unfit/excluded — fall back to the first non-excluded
|
||||
// pool rather than deadlocking; its egress may have recovered.
|
||||
eligible = poolIds.filter((id) => !(excludeIds || []).includes(id));
|
||||
if (eligible.length === 0) return null;
|
||||
}
|
||||
if (eligible.length === 1) return eligible[0];
|
||||
|
||||
if (strategy === "round-robin" || strategy === "smart") {
|
||||
const state = rotateState.get(providerId) || { index: -1 };
|
||||
state.index = (state.index + 1) % poolIds.length;
|
||||
state.index = (state.index + 1) % eligible.length;
|
||||
rotateState.set(providerId, state);
|
||||
return poolIds[state.index];
|
||||
return eligible[state.index];
|
||||
}
|
||||
|
||||
if (strategy === "random") {
|
||||
return poolIds[Math.floor(Math.random() * poolIds.length)];
|
||||
return eligible[Math.floor(Math.random() * eligible.length)];
|
||||
}
|
||||
|
||||
return poolIds[0]; // "none" or unknown
|
||||
return eligible[0]; // "none" or unknown
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -66,7 +84,8 @@ function normalizeLegacyProxy(providerSpecificData = {}) {
|
||||
*/
|
||||
export async function resolveConnectionProxyConfig(
|
||||
providerSpecificData = {},
|
||||
connectionId = null
|
||||
connectionId = null,
|
||||
excludePoolIds = null
|
||||
) {
|
||||
try {
|
||||
// Handle new multi-proxy format
|
||||
@@ -85,7 +104,8 @@ export async function resolveConnectionProxyConfig(
|
||||
* -----------------------------
|
||||
*/
|
||||
if (proxyPoolIds.length > 0) {
|
||||
const selectedPoolId = pickProxyPoolId(proxyPoolIds, proxyRotationStrategy, connectionId);
|
||||
const scope = providerSpecificData?.proxyPoolScope || null;
|
||||
const selectedPoolId = pickProxyPoolId(proxyPoolIds, proxyRotationStrategy, connectionId, { scope, excludeIds: excludePoolIds });
|
||||
|
||||
if (selectedPoolId) {
|
||||
const proxyPool = await getProxyPoolById(selectedPoolId);
|
||||
|
||||
@@ -11,6 +11,7 @@ const STRATEGIES = [
|
||||
{ value: "none", label: "None (single pool)" },
|
||||
{ value: "round-robin", label: "Round-robin" },
|
||||
{ value: "random", label: "Random" },
|
||||
{ value: "smart", label: "Smart" },
|
||||
];
|
||||
|
||||
export default function NoAuthProxyCard({ providerId }) {
|
||||
@@ -121,7 +122,9 @@ export default function NoAuthProxyCard({ providerId }) {
|
||||
: isRotation
|
||||
? rotateStrategy === "round-robin"
|
||||
? `Rotating through all ${proxyPools.length} active pools in order. State is in-memory (resets on restart).`
|
||||
: `Picking a random pool from ${proxyPools.length} active pools each request.`
|
||||
: rotateStrategy === "smart"
|
||||
? `Skip pools whose egress IP is blocked for this provider (e.g. per-IP limits). See Proxy Fitness to clear/block.`
|
||||
: `Picking a random pool from ${proxyPools.length} active pools each request.`
|
||||
: `Uses the selected pool above. Set to Round-robin or Random to rotate across all active pools.`}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -25,6 +25,7 @@ const navItems = [
|
||||
{ href: "/dashboard/usage", label: "Usage", icon: "bar_chart" },
|
||||
{ href: "/dashboard/quota", label: "Quota Tracker", icon: "data_usage" },
|
||||
{ href: "/dashboard/token-saver", label: "Token Saver", icon: "savings" },
|
||||
{ href: "/dashboard/proxy-fitness", label: "Proxy Fitness", icon: "network_check" },
|
||||
// { href: "/dashboard/pxpipe", label: "PXPIPE", icon: "image" },
|
||||
{ href: "/dashboard/cli-tools", label: "CLI Tools", icon: "terminal" },
|
||||
];
|
||||
|
||||
@@ -17,6 +17,7 @@ import { errorResponse, unavailableResponse } from "open-sse/utils/error.js";
|
||||
import { handleComboChat, handleFusionChat, detectRequiredCapabilities } from "open-sse/services/combo.js";
|
||||
import { augmentModelsWithCapacityAdapter, withCapacityAdapterStripping, getActiveAdapterStrategy } from "open-sse/services/capacityAdapter.js";
|
||||
import { handleBypassRequest } from "open-sse/utils/bypassHandler.js";
|
||||
import { resolveConnectionProxyConfig } from "@/lib/network/connectionProxy";
|
||||
import { HTTP_STATUS } from "open-sse/config/runtimeConfig.js";
|
||||
import { detectFormatByEndpoint } from "open-sse/translator/formats.js";
|
||||
import * as log from "../utils/logger.js";
|
||||
@@ -284,6 +285,23 @@ async function handleSingleModelChat(body, modelStr, clientRawRequest = null, re
|
||||
pxpipeTransform: chatSettings.pxpipeEnabled ? await getPxpipeTransform() : null,
|
||||
onPxpipeEvent: appendPxpipeEvent,
|
||||
providerThinking,
|
||||
// Pool-scoped failure recovery: re-resolve proxy config excluding the
|
||||
// failed pool so the request retries via another pool, not a dead end.
|
||||
resolveProxyConfig: async (creds, excludePoolIds = []) => {
|
||||
const psd = { ...(creds?.providerSpecificData || {}) };
|
||||
if (psd.proxyPoolIds?.length) psd.proxyPoolScope = `${provider}::${model}`;
|
||||
const resolved = await resolveConnectionProxyConfig(psd, creds?.connectionId || creds?.id, excludePoolIds);
|
||||
if (!resolved?.proxyPoolId) return null;
|
||||
return {
|
||||
connectionProxyEnabled: resolved.connectionProxyEnabled,
|
||||
connectionProxyUrl: resolved.connectionProxyUrl,
|
||||
connectionNoProxy: resolved.connectionNoProxy,
|
||||
connectionProxyPoolId: resolved.proxyPoolId || null,
|
||||
vercelRelayUrl: resolved.vercelRelayUrl || "",
|
||||
proxyPoolId: resolved.proxyPoolId || null,
|
||||
strictProxy: resolved.strictProxy === true,
|
||||
};
|
||||
},
|
||||
// Detect source format by endpoint + body
|
||||
sourceFormatOverride: request?.url ? detectFormatByEndpoint(new URL(request.url).pathname, body) : null,
|
||||
onCredentialsRefreshed: async (newCreds) => {
|
||||
|
||||
@@ -47,10 +47,16 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
|
||||
const override = (settings.providerStrategies || {})[providerId] || {};
|
||||
const strategy = override.rotateStrategy || "none";
|
||||
let pickedId = override.proxyPoolId || null;
|
||||
let poolIds = [];
|
||||
if (strategy !== "none") {
|
||||
const allPools = await getProxyPools({ isActive: true });
|
||||
const poolIds = allPools.filter(p => p.proxyUrl).map(p => p.id);
|
||||
pickedId = pickProxyPoolId(poolIds, strategy, providerId);
|
||||
poolIds = allPools.filter(p => p.proxyUrl).map(p => p.id);
|
||||
// Scope region-aware ("smart") filtering to this provider/model so
|
||||
// pools marked unfit here are skipped.
|
||||
const scope = `${providerId}::${model || "*"}`;
|
||||
pickedId = pickProxyPoolId(poolIds, strategy, providerId, { scope });
|
||||
} else if (override.proxyPoolId) {
|
||||
poolIds = [override.proxyPoolId];
|
||||
}
|
||||
const resolvedProxy = await resolveConnectionProxyConfig({ proxyPoolId: pickedId || "" });
|
||||
return {
|
||||
@@ -64,6 +70,13 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
|
||||
connectionNoProxy: resolvedProxy.connectionNoProxy,
|
||||
connectionProxyPoolId: resolvedProxy.proxyPoolId || null,
|
||||
vercelRelayUrl: resolvedProxy.vercelRelayUrl || "",
|
||||
proxyPoolId: resolvedProxy.proxyPoolId || null,
|
||||
strictProxy: resolvedProxy.strictProxy === true,
|
||||
// Let chatCore's pool-scoped retry rotate across the same candidate
|
||||
// pool set (excluding the failed pool) instead of reusing it — this
|
||||
// is what makes per-IP limit retries work for no-auth providers.
|
||||
proxyPoolIds: poolIds,
|
||||
proxyRotationStrategy: strategy,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -172,7 +185,11 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
|
||||
connection = availableConnections[0];
|
||||
}
|
||||
|
||||
const resolvedProxy = await resolveConnectionProxyConfig(connection.providerSpecificData || {}, connection.id);
|
||||
// Scope the region-aware picker to this provider/model (e.g. freebuff::gpt-5.6-luna)
|
||||
const psdForProxy = connection.providerSpecificData?.proxyPoolIds?.length
|
||||
? { ...connection.providerSpecificData, proxyPoolScope: `${providerId}::${model || ""}` }
|
||||
: connection.providerSpecificData;
|
||||
const resolvedProxy = await resolveConnectionProxyConfig(psdForProxy || {}, connection.id);
|
||||
|
||||
return {
|
||||
authType: connection.authType,
|
||||
@@ -193,6 +210,8 @@ export async function getProviderCredentials(provider, excludeConnectionIds = nu
|
||||
connectionNoProxy: resolvedProxy.connectionNoProxy,
|
||||
connectionProxyPoolId: resolvedProxy.proxyPoolId || null,
|
||||
vercelRelayUrl: resolvedProxy.vercelRelayUrl || "",
|
||||
proxyPoolId: resolvedProxy.proxyPoolId || null,
|
||||
strictProxy: resolvedProxy.strictProxy === true,
|
||||
},
|
||||
connectionId: connection.id,
|
||||
// Include current status for optimization check
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { describe, it, expect, beforeEach } from "vitest";
|
||||
import {
|
||||
markPoolUnfit,
|
||||
clearPoolUnfit,
|
||||
clearAllPoolUnfit,
|
||||
isPoolFit,
|
||||
fitPoolIds,
|
||||
poolFitnessSnapshot,
|
||||
pruneExpired,
|
||||
resetPoolFitness,
|
||||
} from "open-sse/services/proxyPoolFitness.js";
|
||||
|
||||
describe("proxy pool fitness registry", () => {
|
||||
beforeEach(() => resetPoolFitness());
|
||||
|
||||
it("marks a pool unfit for a scope and prunes on expiry", () => {
|
||||
markPoolUnfit("p1", "freebuff::gpt-5.6-luna", Date.now() + 60_000, "limited_ip");
|
||||
expect(isPoolFit("p1", "freebuff::gpt-5.6-luna")).toBe(false);
|
||||
expect(isPoolFit("p1", "freebuff::other-model")).toBe(true);
|
||||
expect(isPoolFit("p2", "freebuff::gpt-5.6-luna")).toBe(true);
|
||||
|
||||
markPoolUnfit("p1", "freebuff::gpt-5.6-luna", Date.now() - 1000); // expired
|
||||
expect(isPoolFit("p1", "freebuff::gpt-5.6-luna")).toBe(true); // pruned on read
|
||||
});
|
||||
|
||||
it("provider-wide mark (provider::*) covers any model lookup", () => {
|
||||
markPoolUnfit("p1", "opencode::*", Date.now() + 60_000, "manual");
|
||||
expect(isPoolFit("p1", "opencode::sonnet-4.6")).toBe(false);
|
||||
expect(isPoolFit("p1", "freebuff::gpt-5.6-luna")).toBe(true);
|
||||
});
|
||||
|
||||
it("fitPoolIds filters unfit pools; snapshot drops expired marks", () => {
|
||||
markPoolUnfit("p1", "fb::m1", Date.now() + 60_000);
|
||||
markPoolUnfit("p1", "fb::m2", Date.now() - 1000); // expired
|
||||
expect(fitPoolIds(["p1", "p2"], "fb::m1")).toEqual(["p2"]);
|
||||
|
||||
const snap = poolFitnessSnapshot();
|
||||
expect(snap.p1["fb::m1"]).toBeDefined();
|
||||
expect(snap.p1["fb::m2"]).toBeUndefined();
|
||||
});
|
||||
|
||||
it("clear per scope, clear-all per provider, clear-all global, pruneExpired", () => {
|
||||
markPoolUnfit("p1", "freebuff::m1", Date.now() + 60_000);
|
||||
markPoolUnfit("p1", "kiro::m3", Date.now() + 60_000);
|
||||
|
||||
clearPoolUnfit("p1", "freebuff::m1");
|
||||
expect(isPoolFit("p1", "freebuff::m1")).toBe(true);
|
||||
|
||||
clearAllPoolUnfit("kiro");
|
||||
expect(poolFitnessSnapshot().p1).toBeUndefined();
|
||||
|
||||
markPoolUnfit("p2", "x::y", Date.now() - 1000);
|
||||
expect(pruneExpired()).toBe(1);
|
||||
expect(poolFitnessSnapshot()).toEqual({});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user