fix: S3→Telegram upload pipeline — OOM, queue limits, shutdown drain, timeouts
Deploy FileDrop / deploy (push) Successful in 46s

CRITICAL:
- Content-MD5 no longer loads entire file via arrayBuffer() — MD5 computed
  incrementally in streamBodyToTemp alongside SHA-256 (fixes OOM for GB files)

HIGH:
- Add 120s timeout to Telegraf API calls via Promise.race in executeWithBotRetry
  (prevents queue slot exhaustion from hung Telegram connections)
- Add queue size limit (1000 pending max) — reject new tasks when full
- Add graceful shutdown drain — waitForQueue with 30s timeout before exit
- Fix temp file leak when findFileByBucketAndKey throws (wrap in try-catch)
- Fix createReadStream fd leak — destroy stream on forwardToStorage error
- writer.end() wrapped in silent try-catch to prevent error swallowing
- writer.end() result ignored, writerFailed flag prevents double-end

MEDIUM:
- Remove 'retry after' from isTransientError patterns to stop double-retry
  layering (was causing up to 96 bot attempts per chunk)

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude
2026-07-29 08:40:50 +07:00
parent de7d276245
commit d8da2044b2
4 changed files with 110 additions and 42 deletions
+20 -2
View File
@@ -42,7 +42,9 @@ const sleep = (ms: number): Promise<void> => {
const isTransientError = (error: unknown): boolean => {
const str = error instanceof Error ? error.message : String(error);
const transientPatterns = [
'retry after',
// 'retry after' is deliberately omitted — 429 is handled by
// executeWithBotRetry at a deeper layer. Including it here would
// cause double-retry (up to 96 attempts per chunk).
'timeout',
'Timed out',
'etimedout',
@@ -76,6 +78,12 @@ const isTransientError = (error: unknown): boolean => {
*/
const MAX_TRANSIENT_RETRIES = 3;
/**
* Timeout in milliseconds for individual Telegram API calls.
* 120 seconds to accommodate large document uploads.
*/
const TELEGRAM_API_TIMEOUT_MS = 120_000;
/**
* Manages a pool of Telegram bots with automatic rotation and rate-limit handling.
*
@@ -129,7 +137,17 @@ export class BotPool implements ITelegramService {
const currentBot = this.bots[botIndex];
const currentToken = this.botTokens[botIndex];
try {
return await action(currentBot, currentToken);
// Add timeout to prevent hung API calls from occupying queue slots
const result = await Promise.race([
action(currentBot, currentToken),
new Promise<never>((_, reject) =>
setTimeout(
() => reject(new Error(`Telegram API timeout after ${TELEGRAM_API_TIMEOUT_MS}ms`)),
TELEGRAM_API_TIMEOUT_MS,
),
),
]);
return result;
} catch (error: unknown) {
const errorStr = error instanceof Error ? error.message : String(error);
const match = errorStr.match(/retry after (\d+)/i);
+20 -1
View File
@@ -2,6 +2,13 @@ import PQueue from 'p-queue';
import { config } from '../../env';
import logger from '../../shared/logger/index';
/**
* Maximum number of pending (queued + in-flight) upload tasks before
* new submissions are rejected. Prevents unbounded memory growth when
* Telegram is slow or unavailable.
*/
const MAX_QUEUE_PENDING = 1000;
/**
* P-queue instance for serialising Telegram upload tasks.
*
@@ -16,7 +23,11 @@ const uploadQueue = new PQueue({
uploadQueue.on('add', () => {
const stats = getQueueStats();
if (stats.size > 5) {
logger.warn('Upload queue building up', { pending: stats.pending, size: stats.size });
logger.warn('Upload queue building up', {
pending: stats.pending,
size: stats.size,
max: MAX_QUEUE_PENDING,
});
}
});
@@ -34,6 +45,14 @@ uploadQueue.on('next', () => {
* @returns A promise that resolves with the task's result.
*/
export const enqueueUpload = <T>(task: () => Promise<T>): Promise<T> => {
const stats = getQueueStats();
if (stats.pending + stats.size > MAX_QUEUE_PENDING) {
return Promise.reject(
new Error(
`Upload queue full (${stats.pending + stats.size} pending, max ${MAX_QUEUE_PENDING})`,
),
);
}
return uploadQueue.add(task);
};