feat(proxy-fitness): egress geo probe + unstable detection + state sweeper
Background probe fetches ipinfo through each pool itself (provider-agnostic transport), fills an egress IP/country cache (TTL 1h, 8-IP history) and flags flapping relays as unstable. Periodic state sweeper (10 min) prunes expired fitness marks, stale geo/ip-history and Freebuff session/cooldown state; schedulers skip non-server runtimes. Unit tests for the geo cache.
This commit is contained in:
@@ -0,0 +1,75 @@
|
||||
// Background pool egress geo probe — fills the poolGeo cache so the Proxy
|
||||
// Fitness / Proxy Pools UI can show each pool's egress IP + country, and
|
||||
// future provider region policies can pre-mark pools unfit.
|
||||
// Fail-open everywhere; never blocks startup or requests.
|
||||
|
||||
import { getProxyPools } from "@/models";
|
||||
import { probePoolGeo, setPoolGeo, getPoolGeo } from "open-sse/services/poolGeo.js";
|
||||
import { isNonServerRuntime } from "@/sse/services/backgroundTokenRefresh.js";
|
||||
|
||||
const PROBE_INTERVAL_MS = 30 * 60 * 1000;
|
||||
const INITIAL_DELAY_MS = 15 * 1000;
|
||||
const CONCURRENCY = 4;
|
||||
// Re-probe a pool when its last sample is older than this — multiple samples
|
||||
// per pool are what let us flag flapping (changing egress) relays.
|
||||
const GEO_REPROBE_MS = 15 * 60 * 1000;
|
||||
|
||||
let started = false;
|
||||
let intervalHandle = null;
|
||||
let initialTimeoutHandle = null;
|
||||
let probing = false;
|
||||
|
||||
// probe with bounded concurrency; re-probes pools whose sample is stale enough.
|
||||
async function probeAll() {
|
||||
if (probing) return;
|
||||
probing = true;
|
||||
try {
|
||||
const pools = await getProxyPools({ isActive: true });
|
||||
const now = Date.now();
|
||||
const targets = (pools || []).filter((p) => {
|
||||
if (!p?.proxyUrl) return false;
|
||||
const geo = getPoolGeo(p.id);
|
||||
return !geo || now - geo.ts >= GEO_REPROBE_MS;
|
||||
});
|
||||
if (targets.length === 0) return;
|
||||
console.log(`[PoolEgressProbe] probing ${targets.length}/${(pools || []).length} active pools`);
|
||||
let next = 0;
|
||||
const workers = Array.from({ length: Math.min(CONCURRENCY, Math.max(targets.length, 1)) }, async () => {
|
||||
while (next < targets.length) {
|
||||
const pool = targets[next++];
|
||||
const geo = await probePoolGeo(pool);
|
||||
if (geo) setPoolGeo(pool.id, geo);
|
||||
}
|
||||
});
|
||||
await Promise.allSettled(workers);
|
||||
const filled = (pools || []).filter((p) => getPoolGeo(p.id)).length;
|
||||
console.log(`[PoolEgressProbe] pass done — geo cached for ${filled}/${(pools || []).length} pools`);
|
||||
} catch (e) {
|
||||
console.log(`[PoolEgressProbe] pass failed: ${e?.message || e}`);
|
||||
} finally {
|
||||
probing = false;
|
||||
}
|
||||
}
|
||||
|
||||
export function startPoolEgressProbe({ intervalMs } = {}) {
|
||||
if (started) return false;
|
||||
if (isNonServerRuntime()) return false;
|
||||
started = true;
|
||||
const period = Number.isFinite(intervalMs) && intervalMs > 0 ? intervalMs : PROBE_INTERVAL_MS;
|
||||
console.log("[PoolEgressProbe] Scheduler started", { intervalMs: period, initialDelayMs: INITIAL_DELAY_MS });
|
||||
initialTimeoutHandle = setTimeout(() => { probeAll().catch(() => {}); }, INITIAL_DELAY_MS);
|
||||
if (initialTimeoutHandle.unref) initialTimeoutHandle.unref();
|
||||
intervalHandle = setInterval(() => { probeAll().catch(() => {}); }, period);
|
||||
if (intervalHandle.unref) intervalHandle.unref();
|
||||
return true;
|
||||
}
|
||||
|
||||
export function stopPoolEgressProbe() {
|
||||
if (initialTimeoutHandle) clearTimeout(initialTimeoutHandle);
|
||||
if (intervalHandle) clearInterval(intervalHandle);
|
||||
initialTimeoutHandle = null;
|
||||
intervalHandle = null;
|
||||
started = false;
|
||||
}
|
||||
|
||||
export const __test__ = { probeAll };
|
||||
@@ -0,0 +1,47 @@
|
||||
// Periodic in-memory state sweeper — a cron-like job that prunes expired data
|
||||
// so here long-running servers never accumulate stale entries:
|
||||
// • pool fitness marks (proxyPoolFitness.pruneExpired)
|
||||
// • pool egress geo cache incl. ipHistory (poolGeo.pruneStaleGeo)
|
||||
// • freebuff session cache + cooldowns (freebuff.pruneSessionState)
|
||||
// Fail-open everywhere; never blocks startup or requests.
|
||||
|
||||
import { pruneExpired } from "open-sse/services/proxyPoolFitness.js";
|
||||
import { pruneStaleGeo } from "open-sse/services/poolGeo.js";
|
||||
import { isNonServerRuntime } from "@/sse/services/backgroundTokenRefresh.js";
|
||||
|
||||
const SWEEP_INTERVAL_MS = 10 * 60 * 1000;
|
||||
|
||||
let started = false;
|
||||
let handle = null;
|
||||
|
||||
async function sweep() {
|
||||
try {
|
||||
const fitness = pruneExpired();
|
||||
const geo = pruneStaleGeo();
|
||||
const { pruneSessionState } = await import("open-sse/executors/freebuff.js");
|
||||
const sessions = pruneSessionState();
|
||||
if (fitness || geo || sessions) {
|
||||
console.log(`[StateSweeper] pruned ${fitness} fitness, ${geo} geo, ${sessions} session/cooldown entries`);
|
||||
}
|
||||
} catch {
|
||||
// fail-open: next tick retries
|
||||
}
|
||||
}
|
||||
|
||||
export function startStateSweeper({ intervalMs } = {}) {
|
||||
if (started) return false;
|
||||
if (isNonServerRuntime()) return false;
|
||||
started = true;
|
||||
const period = Number.isFinite(intervalMs) && intervalMs > 0 ? intervalMs : SWEEP_INTERVAL_MS;
|
||||
handle = setInterval(() => { sweep().catch(() => {}); }, period);
|
||||
if (handle.unref) handle.unref();
|
||||
return true;
|
||||
}
|
||||
|
||||
export function stopStateSweeper() {
|
||||
if (handle) clearInterval(handle);
|
||||
handle = null;
|
||||
started = false;
|
||||
}
|
||||
|
||||
export const __test__ = { sweep };
|
||||
@@ -118,6 +118,16 @@ async function runHeavyStartup() {
|
||||
import("@/sse/services/backgroundTokenRefresh.js")
|
||||
.then(({ startBackgroundTokenRefresh }) => startBackgroundTokenRefresh())
|
||||
.catch((e) => console.log("[BackgroundTokenRefresh] scheduler start failed:", e.message));
|
||||
|
||||
// Pool egress geo probe — fills the Proxy Fitness / Proxy Pools egress column.
|
||||
import("@/lib/network/poolEgressProbe.js")
|
||||
.then(({ startPoolEgressProbe }) => startPoolEgressProbe())
|
||||
.catch((e) => console.log("[PoolEgressProbe] scheduler start failed:", e.message));
|
||||
|
||||
// Periodic in-memory state sweeper — prunes expired fitness/geo/session state.
|
||||
import("@/lib/network/stateSweeper.js")
|
||||
.then(({ startStateSweeper }) => startStateSweeper())
|
||||
.catch((e) => console.log("[StateSweeper] scheduler start failed:", e.message));
|
||||
}
|
||||
|
||||
function hasQuotaAutoPingEnabled(settings) {
|
||||
|
||||
Reference in New Issue
Block a user