perf(gateway): retry transient attachment + wikipedia failures
- attachmentUploader: downloadDiscordAttachment retries CDN timeouts via retryWithBackoff (ATTACHMENT_RETRY_ATTEMPTS, 1s-8s backoff); AbortError normalized so 403/404 refresh path never fires on timeouts - attachmentUploader: uploadAttachmentToTele uses ATTACHMENT_RETRY_ATTEMPTS instead of retries:0 (Tele 5xx under load was failing outright) - wikipediaClient: wikipediaSummary retries once on abort/timeout (100% of prod summary errors were aborts); termGlossary drops its redundant second call (was up to 4 reqs/term under miss+retry) Verified: typecheck + lint clean, 210/210 tests pass
This commit is contained in:
@@ -69,8 +69,6 @@ const MISS_TTL_SECONDS = 60 * 60;
|
||||
const MISS_TTL_MS = MISS_TTL_SECONDS * 1000;
|
||||
/** Sentinel stored in caches for "term has no resolvable definition". */
|
||||
const EMPTY_SENTINEL = "__not_found__";
|
||||
/** Delay before retrying a search that returned zero results. */
|
||||
const RETRY_DELAY_MS = 350;
|
||||
/** Max definition snippet length kept in the prompt. */
|
||||
const MAX_DEFINITION_CHARS = 300;
|
||||
/**
|
||||
@@ -209,15 +207,12 @@ async function fetchDefinitionLive(
|
||||
return liveSearchLimit(async () => {
|
||||
await acquireLiveSlot();
|
||||
try {
|
||||
let result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
|
||||
let def = result ? buildDefinition(result, term) : null;
|
||||
// Zero result is usually the limiter/network blip, not a real miss —
|
||||
// retry once. Result-but-unusable = genuine miss, no retry.
|
||||
if (!def) {
|
||||
await delay(RETRY_DELAY_MS);
|
||||
result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
|
||||
def = result ? buildDefinition(result, term) : null;
|
||||
}
|
||||
// Single call: wikipediaSummary already retries once internally on
|
||||
// abort/timeout, so a second call here would double the attempts
|
||||
// (up to 4 requests per term under stagger+concurrency-2 — pure
|
||||
// Wikipedia pressure for a miss that gets cached 1h anyway).
|
||||
const result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
|
||||
const def = result ? buildDefinition(result, term) : null;
|
||||
|
||||
if (def) {
|
||||
// Persist permanently (definitions rarely change) — best-effort,
|
||||
|
||||
@@ -169,47 +169,55 @@ async function wikipediaSearchLive(
|
||||
export async function wikipediaSummary(
|
||||
title: string,
|
||||
timeoutMs: number = DEFAULT_TIMEOUT_MS,
|
||||
retries = 1,
|
||||
): Promise<SearchResult | null> {
|
||||
const t = title.trim();
|
||||
if (!t) return null;
|
||||
|
||||
const { controller, clear } = createAbortControllerWithTimeout(timeoutMs);
|
||||
try {
|
||||
const res = await fetch(
|
||||
`https://${WIKIPEDIA_LANG}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(
|
||||
t.replace(/ /g, "_"),
|
||||
)}`,
|
||||
{
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"User-Agent": buildUserAgent(),
|
||||
},
|
||||
},
|
||||
);
|
||||
const url = `https://${WIKIPEDIA_LANG}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(
|
||||
t.replace(/ /g, "_"),
|
||||
)}`;
|
||||
const headers = {
|
||||
Accept: "application/json",
|
||||
"User-Agent": buildUserAgent(),
|
||||
};
|
||||
|
||||
if (!res.ok) return null;
|
||||
const data = (await res.json()) as {
|
||||
title?: string;
|
||||
extract?: string;
|
||||
content_urls?: { desktop?: { page?: string } };
|
||||
};
|
||||
if (!data.extract) return null;
|
||||
for (let attempt = 0; attempt <= retries; attempt++) {
|
||||
const { controller, clear } = createAbortControllerWithTimeout(timeoutMs);
|
||||
try {
|
||||
const res = await fetch(url, { signal: controller.signal, headers });
|
||||
|
||||
return {
|
||||
title: data.title ?? t,
|
||||
url: data.content_urls?.desktop?.page ?? wikipediaPageUrl(t),
|
||||
snippet: data.extract.slice(0, 500),
|
||||
};
|
||||
} catch (err) {
|
||||
log.warn(
|
||||
{ error: err instanceof Error ? err.message : String(err), title: t },
|
||||
"Wikipedia summary error",
|
||||
);
|
||||
return null;
|
||||
} finally {
|
||||
clear();
|
||||
if (!res.ok) return null;
|
||||
const data = (await res.json()) as {
|
||||
title?: string;
|
||||
extract?: string;
|
||||
content_urls?: { desktop?: { page?: string } };
|
||||
};
|
||||
if (!data.extract) return null;
|
||||
|
||||
return {
|
||||
title: data.title ?? t,
|
||||
url: data.content_urls?.desktop?.page ?? wikipediaPageUrl(t),
|
||||
snippet: data.extract.slice(0, 500),
|
||||
};
|
||||
} catch (err) {
|
||||
// Abort/timeout is transient (prod: 100% of summary errors were
|
||||
// "This operation was aborted"); retry once after a short pause so
|
||||
// one slow Wikipedia response doesn't burn a glossary term. A final
|
||||
// failure still degrades gracefully to null.
|
||||
if (attempt === retries) {
|
||||
log.warn(
|
||||
{ error: err instanceof Error ? err.message : String(err), title: t },
|
||||
"Wikipedia summary error",
|
||||
);
|
||||
return null;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
} finally {
|
||||
clear();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface ExtractSearchQueryOptions {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createChildLogger } from "@/shared/logger/index";
|
||||
import { retryWithBackoff } from "@/shared/utils/index";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import { uploadToTele } from "../../shared/uploader.js";
|
||||
import { messageStore } from "../message-capture/messageStore.js";
|
||||
@@ -44,7 +45,9 @@ export async function uploadAttachmentToTele(
|
||||
contentType,
|
||||
uploadUrl: config.TELE_UPLOAD_URL,
|
||||
timeoutMs: config.ATTACHMENT_UPLOAD_TIMEOUT_MS,
|
||||
retries: 0,
|
||||
// Tele 5xx under load is transient (prod: 12x Status 500) — retry
|
||||
// inside uploadToTele instead of failing the attachment outright.
|
||||
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
@@ -64,12 +67,28 @@ export async function uploadAttachmentToTele(
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadDiscordAttachment(url: string): Promise<Buffer> {
|
||||
export async function downloadDiscordAttachment(
|
||||
url: string,
|
||||
timeoutMs?: number,
|
||||
): Promise<Buffer> {
|
||||
const timeout = timeoutMs ?? config.ATTACHMENT_UPLOAD_TIMEOUT_MS;
|
||||
logger.debug({ url }, "Starting Discord attachment download");
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
signal: AbortSignal.timeout(config.ATTACHMENT_UPLOAD_TIMEOUT_MS),
|
||||
});
|
||||
// Timeout-only retry: a CDN abort mid-download is transient (the prod
|
||||
// failure signature is "The operation was aborted due to timeout").
|
||||
// 403/404 = expired or purged URL — not retried here; the caller
|
||||
// refreshes via refreshDiscordUrl instead.
|
||||
const response = await retryWithBackoff(
|
||||
() =>
|
||||
fetch(url, {
|
||||
signal: AbortSignal.timeout(timeout),
|
||||
}),
|
||||
{
|
||||
retries: config.ATTACHMENT_RETRY_ATTEMPTS,
|
||||
minTimeout: 1000,
|
||||
maxTimeout: 8000,
|
||||
},
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new AttachmentDownloadError(
|
||||
@@ -86,6 +105,13 @@ export async function downloadDiscordAttachment(url: string): Promise<Buffer> {
|
||||
);
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (error instanceof AttachmentDownloadError) throw error;
|
||||
// retryWithBackoff rethrows AbortError unwrapped — normalize it to a
|
||||
// plain Error so shouldRefreshDiscordUrl() never mistakes a timeout
|
||||
// for an expired URL (which would trigger a useless refresh fetch).
|
||||
if (error instanceof Error && error.name === "AbortError") {
|
||||
throw new Error(`Download timed out: ${toErrorMessage(error)}`);
|
||||
}
|
||||
logger.error(
|
||||
{ url, error: toErrorMessage(error) },
|
||||
"Failed to download Discord attachment",
|
||||
|
||||
Reference in New Issue
Block a user