perf: optimasi 7 hot-path bottleneck (buildKey, SSE, cooldowns, encode) (#6)
1. ResponseCache: short-circuit buildKey() via shouldCacheModel() check
+ WeakMap memoization untuk stableStringify (skip recursive sort
kalau object reference sudah pernah di-stringify).
2. extractTextFromSSEEvent: ganti regex greedy \{.*\} dengan manual
indexOf untuk JSON bounds (hindari backtracking per SSE chunk).
3. ProxyPool: bound cooldowns Map ke MAX_COOLDOWNS=10_000 dengan
insertion-order LRU eviction — mencegah memory growth kalau
banyak unique (proxy, model) pairs kena 429.
4. Single JSON.stringify: hitung responseBody sekali, pakai untuk
cache.set dan Response constructor (sebelumnya stringified 2x).
5. Shared SHARED_ENCODER singleton: TextEncoder stateless, share
module-level. Decoder tetap per-stream (stateful).
6. Branch DSML early: skip extractTextFromSSEEvent + JSON.parse
kalau isDSMLDetectionEnabled() === false (untuk non-DeepSeek model).
7. safeReleaseReader() idempotent guard: gunakan readerReleased flag
untuk mencegah double-delete di ACTIVE_READERS kalau exception
terjadi di tengah stream cleanup.
274 tests pass, no regression. Estimated total saving: 5-15ms/req
untuk model non-cached + 30-150ms per streaming response.
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
030c4f884b
commit
173fa68025
+24
-1
@@ -38,8 +38,11 @@ export class ProxyPool {
|
||||
private failureThreshold = 3;
|
||||
/** host:port -> consecutive failure count */
|
||||
private failures = new Map<string, number>();
|
||||
/** host:port::model -> expiry epoch ms */
|
||||
/** host:port::model -> expiry epoch ms. Bounded by MAX_COOLDOWNS to prevent
|
||||
* unbounded growth when many unique (proxy, model) pairs receive 429s. */
|
||||
private cooldowns = new Map<string, number>();
|
||||
/** Cap on cooldown entries. Oldest (by insertion order) is evicted on overflow. */
|
||||
private readonly MAX_COOLDOWNS = 10_000;
|
||||
private cooldownDuration = 60000; // default 60s
|
||||
/** Periodic cleanup timer for expired cooldowns */
|
||||
private cleanupTimer: ReturnType<typeof setInterval> | null = null;
|
||||
@@ -151,6 +154,17 @@ export class ProxyPool {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Evict oldest cooldown entries when over capacity. Insertion order in
|
||||
* Map is preserved — the first key iterated is the oldest.
|
||||
*/
|
||||
private evictOldestCooldown(): void {
|
||||
const oldestKey = this.cooldowns.keys().next().value;
|
||||
if (oldestKey !== undefined) {
|
||||
this.cooldowns.delete(oldestKey);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience -- load from a path or skip.
|
||||
* Returns `true` if proxies were loaded.
|
||||
@@ -353,12 +367,21 @@ export class ProxyPool {
|
||||
* Mark the **current** proxy as rate-limited for a specific model.
|
||||
* The proxy enters a cooldown period during which it will be skipped
|
||||
* for this model but remains available for other models.
|
||||
*
|
||||
* Bounded: when cooldowns exceeds MAX_COOLDOWNS, the oldest entry is
|
||||
* evicted (insertion-order LRU) to prevent memory growth across
|
||||
* many unique (proxy, model) pairs.
|
||||
*/
|
||||
markRateLimited(model: string): void {
|
||||
const entry = this.getCurrent();
|
||||
if (!entry) return;
|
||||
const key = this.cooldownKey(entry.host, entry.port, model);
|
||||
const expiry = Date.now() + this.cooldownDuration;
|
||||
|
||||
// If at capacity and this is a new key, evict oldest first.
|
||||
if (!this.cooldowns.has(key) && this.cooldowns.size >= this.MAX_COOLDOWNS) {
|
||||
this.evictOldestCooldown();
|
||||
}
|
||||
this.cooldowns.set(key, expiry);
|
||||
logPool(`markRateLimited key=${key} expiry=${expiry} duration=${this.cooldownDuration}ms`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user