perf(gateway): retry transient attachment + wikipedia failures

- attachmentUploader: downloadDiscordAttachment retries CDN timeouts
  via retryWithBackoff (ATTACHMENT_RETRY_ATTEMPTS, 1s-8s backoff);
  AbortError normalized so 403/404 refresh path never fires on timeouts
- attachmentUploader: uploadAttachmentToTele uses ATTACHMENT_RETRY_ATTEMPTS
  instead of retries:0 (Tele 5xx under load was failing outright)
- wikipediaClient: wikipediaSummary retries once on abort/timeout
  (100% of prod summary errors were aborts); termGlossary drops its
  redundant second call (was up to 4 reqs/term under miss+retry)

Verified: typecheck + lint clean, 210/210 tests pass
This commit is contained in:
asepharyana
2026-09-14 00:13:56 +07:00
parent 90d4c2d0b1
commit 9685fa02be
3 changed files with 79 additions and 50 deletions
@@ -69,8 +69,6 @@ const MISS_TTL_SECONDS = 60 * 60;
const MISS_TTL_MS = MISS_TTL_SECONDS * 1000; const MISS_TTL_MS = MISS_TTL_SECONDS * 1000;
/** Sentinel stored in caches for "term has no resolvable definition". */ /** Sentinel stored in caches for "term has no resolvable definition". */
const EMPTY_SENTINEL = "__not_found__"; const EMPTY_SENTINEL = "__not_found__";
/** Delay before retrying a search that returned zero results. */
const RETRY_DELAY_MS = 350;
/** Max definition snippet length kept in the prompt. */ /** Max definition snippet length kept in the prompt. */
const MAX_DEFINITION_CHARS = 300; const MAX_DEFINITION_CHARS = 300;
/** /**
@@ -209,15 +207,12 @@ async function fetchDefinitionLive(
return liveSearchLimit(async () => { return liveSearchLimit(async () => {
await acquireLiveSlot(); await acquireLiveSlot();
try { try {
let result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS); // Single call: wikipediaSummary already retries once internally on
let def = result ? buildDefinition(result, term) : null; // abort/timeout, so a second call here would double the attempts
// Zero result is usually the limiter/network blip, not a real miss — // (up to 4 requests per term under stagger+concurrency-2 — pure
// retry once. Result-but-unusable = genuine miss, no retry. // Wikipedia pressure for a miss that gets cached 1h anyway).
if (!def) { const result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
await delay(RETRY_DELAY_MS); const def = result ? buildDefinition(result, term) : null;
result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
def = result ? buildDefinition(result, term) : null;
}
if (def) { if (def) {
// Persist permanently (definitions rarely change) — best-effort, // Persist permanently (definitions rarely change) — best-effort,
@@ -169,47 +169,55 @@ async function wikipediaSearchLive(
export async function wikipediaSummary( export async function wikipediaSummary(
title: string, title: string,
timeoutMs: number = DEFAULT_TIMEOUT_MS, timeoutMs: number = DEFAULT_TIMEOUT_MS,
retries = 1,
): Promise<SearchResult | null> { ): Promise<SearchResult | null> {
const t = title.trim(); const t = title.trim();
if (!t) return null; if (!t) return null;
const { controller, clear } = createAbortControllerWithTimeout(timeoutMs); const url = `https://${WIKIPEDIA_LANG}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(
try { t.replace(/ /g, "_"),
const res = await fetch( )}`;
`https://${WIKIPEDIA_LANG}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent( const headers = {
t.replace(/ /g, "_"), Accept: "application/json",
)}`, "User-Agent": buildUserAgent(),
{ };
signal: controller.signal,
headers: {
Accept: "application/json",
"User-Agent": buildUserAgent(),
},
},
);
if (!res.ok) return null; for (let attempt = 0; attempt <= retries; attempt++) {
const data = (await res.json()) as { const { controller, clear } = createAbortControllerWithTimeout(timeoutMs);
title?: string; try {
extract?: string; const res = await fetch(url, { signal: controller.signal, headers });
content_urls?: { desktop?: { page?: string } };
};
if (!data.extract) return null;
return { if (!res.ok) return null;
title: data.title ?? t, const data = (await res.json()) as {
url: data.content_urls?.desktop?.page ?? wikipediaPageUrl(t), title?: string;
snippet: data.extract.slice(0, 500), extract?: string;
}; content_urls?: { desktop?: { page?: string } };
} catch (err) { };
log.warn( if (!data.extract) return null;
{ error: err instanceof Error ? err.message : String(err), title: t },
"Wikipedia summary error", return {
); title: data.title ?? t,
return null; url: data.content_urls?.desktop?.page ?? wikipediaPageUrl(t),
} finally { snippet: data.extract.slice(0, 500),
clear(); };
} catch (err) {
// Abort/timeout is transient (prod: 100% of summary errors were
// "This operation was aborted"); retry once after a short pause so
// one slow Wikipedia response doesn't burn a glossary term. A final
// failure still degrades gracefully to null.
if (attempt === retries) {
log.warn(
{ error: err instanceof Error ? err.message : String(err), title: t },
"Wikipedia summary error",
);
return null;
}
await new Promise((r) => setTimeout(r, 300));
} finally {
clear();
}
} }
return null;
} }
export interface ExtractSearchQueryOptions { export interface ExtractSearchQueryOptions {
@@ -1,4 +1,5 @@
import { createChildLogger } from "@/shared/logger/index"; import { createChildLogger } from "@/shared/logger/index";
import { retryWithBackoff } from "@/shared/utils/index";
import { config } from "../../shared/config/config.js"; import { config } from "../../shared/config/config.js";
import { uploadToTele } from "../../shared/uploader.js"; import { uploadToTele } from "../../shared/uploader.js";
import { messageStore } from "../message-capture/messageStore.js"; import { messageStore } from "../message-capture/messageStore.js";
@@ -44,7 +45,9 @@ export async function uploadAttachmentToTele(
contentType, contentType,
uploadUrl: config.TELE_UPLOAD_URL, uploadUrl: config.TELE_UPLOAD_URL,
timeoutMs: config.ATTACHMENT_UPLOAD_TIMEOUT_MS, timeoutMs: config.ATTACHMENT_UPLOAD_TIMEOUT_MS,
retries: 0, // Tele 5xx under load is transient (prod: 12x Status 500) — retry
// inside uploadToTele instead of failing the attachment outright.
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
}); });
logger.info( logger.info(
@@ -64,12 +67,28 @@ export async function uploadAttachmentToTele(
} }
} }
export async function downloadDiscordAttachment(url: string): Promise<Buffer> { export async function downloadDiscordAttachment(
url: string,
timeoutMs?: number,
): Promise<Buffer> {
const timeout = timeoutMs ?? config.ATTACHMENT_UPLOAD_TIMEOUT_MS;
logger.debug({ url }, "Starting Discord attachment download"); logger.debug({ url }, "Starting Discord attachment download");
try { try {
const response = await fetch(url, { // Timeout-only retry: a CDN abort mid-download is transient (the prod
signal: AbortSignal.timeout(config.ATTACHMENT_UPLOAD_TIMEOUT_MS), // failure signature is "The operation was aborted due to timeout").
}); // 403/404 = expired or purged URL — not retried here; the caller
// refreshes via refreshDiscordUrl instead.
const response = await retryWithBackoff(
() =>
fetch(url, {
signal: AbortSignal.timeout(timeout),
}),
{
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
minTimeout: 1000,
maxTimeout: 8000,
},
);
if (!response.ok) { if (!response.ok) {
throw new AttachmentDownloadError( throw new AttachmentDownloadError(
@@ -86,6 +105,13 @@ export async function downloadDiscordAttachment(url: string): Promise<Buffer> {
); );
return result; return result;
} catch (error) { } catch (error) {
if (error instanceof AttachmentDownloadError) throw error;
// retryWithBackoff rethrows AbortError unwrapped — normalize it to a
// plain Error so shouldRefreshDiscordUrl() never mistakes a timeout
// for an expired URL (which would trigger a useless refresh fetch).
if (error instanceof Error && error.name === "AbortError") {
throw new Error(`Download timed out: ${toErrorMessage(error)}`);
}
logger.error( logger.error(
{ url, error: toErrorMessage(error) }, { url, error: toErrorMessage(error) },
"Failed to download Discord attachment", "Failed to download Discord attachment",