perf(gateway): retry transient attachment + wikipedia failures

- attachmentUploader: downloadDiscordAttachment retries CDN timeouts
  via retryWithBackoff (ATTACHMENT_RETRY_ATTEMPTS, 1s-8s backoff);
  AbortError normalized so 403/404 refresh path never fires on timeouts
- attachmentUploader: uploadAttachmentToTele uses ATTACHMENT_RETRY_ATTEMPTS
  instead of retries:0 (Tele 5xx under load was failing outright)
- wikipediaClient: wikipediaSummary retries once on abort/timeout
  (100% of prod summary errors were aborts); termGlossary drops its
  redundant second call (was up to 4 reqs/term under miss+retry)

Verified: typecheck + lint clean, 210/210 tests pass
This commit is contained in:
asepharyana
2026-09-14 00:13:56 +07:00
parent 90d4c2d0b1
commit 9685fa02be
3 changed files with 79 additions and 50 deletions
@@ -69,8 +69,6 @@ const MISS_TTL_SECONDS = 60 * 60;
const MISS_TTL_MS = MISS_TTL_SECONDS * 1000;
/** Sentinel stored in caches for "term has no resolvable definition". */
const EMPTY_SENTINEL = "__not_found__";
/** Delay before retrying a search that returned zero results. */
const RETRY_DELAY_MS = 350;
/** Max definition snippet length kept in the prompt. */
const MAX_DEFINITION_CHARS = 300;
/**
@@ -209,15 +207,12 @@ async function fetchDefinitionLive(
return liveSearchLimit(async () => {
await acquireLiveSlot();
try {
let result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
let def = result ? buildDefinition(result, term) : null;
// Zero result is usually the limiter/network blip, not a real miss —
// retry once. Result-but-unusable = genuine miss, no retry.
if (!def) {
await delay(RETRY_DELAY_MS);
result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
def = result ? buildDefinition(result, term) : null;
}
// Single call: wikipediaSummary already retries once internally on
// abort/timeout, so a second call here would double the attempts
// (up to 4 requests per term under stagger+concurrency-2 — pure
// Wikipedia pressure for a miss that gets cached 1h anyway).
const result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
const def = result ? buildDefinition(result, term) : null;
if (def) {
// Persist permanently (definitions rarely change) — best-effort,
@@ -169,47 +169,55 @@ async function wikipediaSearchLive(
export async function wikipediaSummary(
title: string,
timeoutMs: number = DEFAULT_TIMEOUT_MS,
retries = 1,
): Promise<SearchResult | null> {
const t = title.trim();
if (!t) return null;
const { controller, clear } = createAbortControllerWithTimeout(timeoutMs);
try {
const res = await fetch(
`https://${WIKIPEDIA_LANG}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(
t.replace(/ /g, "_"),
)}`,
{
signal: controller.signal,
headers: {
Accept: "application/json",
"User-Agent": buildUserAgent(),
},
},
);
const url = `https://${WIKIPEDIA_LANG}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(
t.replace(/ /g, "_"),
)}`;
const headers = {
Accept: "application/json",
"User-Agent": buildUserAgent(),
};
if (!res.ok) return null;
const data = (await res.json()) as {
title?: string;
extract?: string;
content_urls?: { desktop?: { page?: string } };
};
if (!data.extract) return null;
for (let attempt = 0; attempt <= retries; attempt++) {
const { controller, clear } = createAbortControllerWithTimeout(timeoutMs);
try {
const res = await fetch(url, { signal: controller.signal, headers });
return {
title: data.title ?? t,
url: data.content_urls?.desktop?.page ?? wikipediaPageUrl(t),
snippet: data.extract.slice(0, 500),
};
} catch (err) {
log.warn(
{ error: err instanceof Error ? err.message : String(err), title: t },
"Wikipedia summary error",
);
return null;
} finally {
clear();
if (!res.ok) return null;
const data = (await res.json()) as {
title?: string;
extract?: string;
content_urls?: { desktop?: { page?: string } };
};
if (!data.extract) return null;
return {
title: data.title ?? t,
url: data.content_urls?.desktop?.page ?? wikipediaPageUrl(t),
snippet: data.extract.slice(0, 500),
};
} catch (err) {
// Abort/timeout is transient (prod: 100% of summary errors were
// "This operation was aborted"); retry once after a short pause so
// one slow Wikipedia response doesn't burn a glossary term. A final
// failure still degrades gracefully to null.
if (attempt === retries) {
log.warn(
{ error: err instanceof Error ? err.message : String(err), title: t },
"Wikipedia summary error",
);
return null;
}
await new Promise((r) => setTimeout(r, 300));
} finally {
clear();
}
}
return null;
}
export interface ExtractSearchQueryOptions {
@@ -1,4 +1,5 @@
import { createChildLogger } from "@/shared/logger/index";
import { retryWithBackoff } from "@/shared/utils/index";
import { config } from "../../shared/config/config.js";
import { uploadToTele } from "../../shared/uploader.js";
import { messageStore } from "../message-capture/messageStore.js";
@@ -44,7 +45,9 @@ export async function uploadAttachmentToTele(
contentType,
uploadUrl: config.TELE_UPLOAD_URL,
timeoutMs: config.ATTACHMENT_UPLOAD_TIMEOUT_MS,
retries: 0,
// Tele 5xx under load is transient (prod: 12x Status 500) — retry
// inside uploadToTele instead of failing the attachment outright.
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
});
logger.info(
@@ -64,12 +67,28 @@ export async function uploadAttachmentToTele(
}
}
export async function downloadDiscordAttachment(url: string): Promise<Buffer> {
export async function downloadDiscordAttachment(
url: string,
timeoutMs?: number,
): Promise<Buffer> {
const timeout = timeoutMs ?? config.ATTACHMENT_UPLOAD_TIMEOUT_MS;
logger.debug({ url }, "Starting Discord attachment download");
try {
const response = await fetch(url, {
signal: AbortSignal.timeout(config.ATTACHMENT_UPLOAD_TIMEOUT_MS),
});
// Timeout-only retry: a CDN abort mid-download is transient (the prod
// failure signature is "The operation was aborted due to timeout").
// 403/404 = expired or purged URL — not retried here; the caller
// refreshes via refreshDiscordUrl instead.
const response = await retryWithBackoff(
() =>
fetch(url, {
signal: AbortSignal.timeout(timeout),
}),
{
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
minTimeout: 1000,
maxTimeout: 8000,
},
);
if (!response.ok) {
throw new AttachmentDownloadError(
@@ -86,6 +105,13 @@ export async function downloadDiscordAttachment(url: string): Promise<Buffer> {
);
return result;
} catch (error) {
if (error instanceof AttachmentDownloadError) throw error;
// retryWithBackoff rethrows AbortError unwrapped — normalize it to a
// plain Error so shouldRefreshDiscordUrl() never mistakes a timeout
// for an expired URL (which would trigger a useless refresh fetch).
if (error instanceof Error && error.name === "AbortError") {
throw new Error(`Download timed out: ${toErrorMessage(error)}`);
}
logger.error(
{ url, error: toErrorMessage(error) },
"Failed to download Discord attachment",