refactor: per-bot queue with selectBot() and rate-limit tracking

Each bot has its own PQueue (concurrency=1). Uploads are assigned to
the least-loaded available bot. On 429, the bot is marked rate-limited
and the upload retries on the next available bot.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

src/infrastructure/telegram/bot-pool.ts

@@ -1,3 +1,4 @@
+import PQueue from 'p-queue';
 import { Telegraf } from 'telegraf';
 import type {
   ForwardResult,
@@ -14,36 +15,11 @@ import {
   type TelegramMessageResult,
 } from './types';
 
-/**
+const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
- * Sleep for a given number of milliseconds.
- *
- * Used as a backoff mechanism when all bots in the pool are rate-limited
- * or when retrying transient Telegram API errors.
- *
- * @param ms - Number of milliseconds to sleep.
- * @returns A promise that resolves after the specified delay.
- */
-const sleep = (ms: number): Promise<void> => {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-};
 
-/**
- * Determines whether an error from the Telegram API is likely transient
- * and worth retrying.
- *
- * Transient telegrams errors include: network timeouts, 5xx server errors,
- * and "Too Many Requests" (429) which is already handled by bot rotation
- * but is also transient at the network level.
- *
- * @param error - The caught error object.
- * @returns True if the error is likely transient and worth retrying.
- */
 const isTransientError = (error: unknown): boolean => {
   const str = error instanceof Error ? error.message : String(error);
   const transientPatterns = [
-    // 'retry after' is deliberately omitted — 429 is handled by
-    // executeWithBotRetry at a deeper layer. Including it here would
-    // cause double-retry (up to 96 attempts per chunk).
     'timeout',
     'Timed out',
     'etimedout',
@@ -71,74 +47,69 @@ const isTransientError = (error: unknown): boolean => {
   return transientPatterns.some((p) => str.toLowerCase().includes(p.toLowerCase()));
 };
 
-/**
- * Maximum number of retries for transient Telegram API errors
- * before giving up and propagating the error to the caller.
- */
 const MAX_TRANSIENT_RETRIES = 3;
-
-/**
- * Timeout in milliseconds for individual Telegram API calls.
- * 120 seconds to accommodate large document uploads.
- */
 const TELEGRAM_API_TIMEOUT_MS = 120_000;
+const PER_BOT_CONCURRENCY = 1;
+
+interface BotEntry {
+  index: number;
+  token: string;
+  instance: Telegraf;
+  queue: PQueue;
+  rateLimitedUntil: number; // 0 = not rate-limited
+}
 
-/**
- * Manages a pool of Telegram bots with automatic rotation and rate-limit handling.
- *
- * Distributes uploads across multiple bot tokens to maximise throughput.
- * When a bot receives a 429 (rate-limit) error, the pool instantly rotates
- * to the next available bot. If all bots are rate-limited, a coordinated
- * sleep is performed before retrying.
- *
- * Implements the {@link ITelegramService} contract.
- */
 export class BotPool implements ITelegramService {
-  private readonly bots: Telegraf[];
+  private readonly bots: BotEntry[] = [];
-  private readonly botTokens: string[];
-  private nextBotIndex = 0;
 
-  /** Create a new BotPool from the application configuration. */
   constructor() {
-    this.botTokens = Array.from(new Set([config.botToken, ...config.additionalBotTokens]));
+    const tokens = Array.from(new Set([config.botToken, ...config.additionalBotTokens]));
-    this.bots = this.botTokens.map((token) => new Telegraf(token));
+    this.bots = tokens.map((token, index) => ({
+      index,
+      token,
+      instance: new Telegraf(token),
+      queue: new PQueue({ concurrency: PER_BOT_CONCURRENCY }),
+      rateLimitedUntil: 0,
+    }));
+  }
+
+  /** Number of bots in the pool */
+  get size(): number {
+    return this.bots.length;
   }
 
   /**
-   * Claim the next bot index using round-robin rotation.
+   * Select the bot with the fewest pending tasks that isn't rate-limited
-   *
+   * or in the skip set.
-   * @returns The index of the selected bot.
    */
-  private claimBotIndex(): number {
+  private selectBot(skipIndexes?: Set<number>): BotEntry | null {
-    const botIndex = this.nextBotIndex;
+    let best: BotEntry | null = null;
-    this.nextBotIndex = (this.nextBotIndex + 1) % this.bots.length;
+    let bestPending = Infinity;
-    return botIndex;
+
+    for (const bot of this.bots) {
+      if (skipIndexes?.has(bot.index)) continue;
+      if (bot.rateLimitedUntil > Date.now()) continue;
+
+      const pending = bot.queue.pending + bot.queue.size;
+      if (pending < bestPending) {
+        bestPending = pending;
+        best = bot;
+      }
+    }
+
+    return best;
   }
 
   /**
-   * Execute a Telegram API action with automatic retry and bot rotation.
+   * Execute a Telegram API action on a specific bot entry.
-   *
+   * Wraps with timeout.
-   * On 429 errors the pool either:
-   * 1. Rotates to the next bot immediately (if another bot is available), or
-   * 2. Sleeps for the required duration after all bots are exhausted, then retries.
-   *
-   * @param action - The action to execute on a bot instance.
-   * @param retries - Number of full-pool retry cycles remaining.
-   * @param attemptedBots - Number of bots attempted in the current cycle.
-   * @returns The result of the action.
    */
-  private async executeWithBotRetry<T>(
+  private async executeBotAction<T>(
-    action: (botInstance: Telegraf, botToken: string) => Promise<T>,
+    bot: BotEntry,
-    retries = 5,
+    action: (instance: Telegraf, token: string) => Promise<T>,
-    attemptedBots = 0,
   ): Promise<T> {
-    const botIndex = this.claimBotIndex();
+    return Promise.race([
-    const currentBot = this.bots[botIndex];
+      action(bot.instance, bot.token),
-    const currentToken = this.botTokens[botIndex];
-    try {
-      // Add timeout to prevent hung API calls from occupying queue slots
-      const result = await Promise.race([
-        action(currentBot, currentToken),
       new Promise<never>((_, reject) =>
         setTimeout(
           () => reject(new Error(`Telegram API timeout after ${TELEGRAM_API_TIMEOUT_MS}ms`)),
@@ -146,44 +117,14 @@ export class BotPool implements ITelegramService {
         ),
       ),
     ]);
-      return result;
-    } catch (error: unknown) {
-      const errorStr = error instanceof Error ? error.message : String(error);
-      const match = errorStr.match(/retry after (\d+)/i);
-
-      if (match) {
-        const nextIndex = this.nextBotIndex;
-        const nextAttemptedBots = attemptedBots + 1;
-
-        if (nextAttemptedBots < this.bots.length) {
-          logger.info(
-            `Bot Index ${botIndex} hit 429. Instantly rotating to Bot Index ${nextIndex}...`,
-          );
-          return this.executeWithBotRetry(action, retries, nextAttemptedBots);
-        }
-
-        if (retries > 0) {
-          const seconds = parseInt(match[1], 10);
-          logger.warn(`All bots in the pool are rate-limited. Sleeping for ${seconds} seconds...`, {
-            error: errorStr,
-          });
-          await sleep(seconds);
-          return this.executeWithBotRetry(action, retries - 1, 0);
-        }
-      }
-      throw error;
-    }
   }
 
   /**
    * Forward a file chunk to the configured Telegram storage chat.
    *
-   * The upload is executed with automatic bot rotation on rate-limit errors.
+   * The upload is submitted to the least-loaded bot's queue. If the bot
-   *
+   * returns 429, it is marked rate-limited and the upload retries on the
-   * @param fileChunk - The file data (ReadStream, Buffer, or file path).
+   * next available bot. If all bots are rate-limited, sleeps before retrying.
-   * @param fileName - The original file name.
-   * @param fileType - The file type classification (e.g. "document", "photo").
-   * @returns The Telegram identifiers of the stored file.
    */
   async forwardToStorage(
     fileChunk: unknown,
@@ -191,46 +132,100 @@ export class BotPool implements ITelegramService {
     fileType: string,
   ): Promise<ForwardResult> {
     let lastError: unknown;
-    let attempt = 0;
+    const attemptedIndexes = new Set<number>();
 
-    while (attempt <= MAX_TRANSIENT_RETRIES) {
+    // Outer retry loop — up to 10 attempts across all bots
-      attempt++;
+    for (let attempt = 0; attempt < 10; attempt++) {
+      const bot = this.selectBot(attemptedIndexes);
+
+      if (!bot) {
+        // No available bots — either all rate-limited or all attempted
+        if (attemptedIndexes.size > 0) {
+          // All non-rate-limited bots were tried and failed — wait & reset
+          logger.warn('All available bots exhausted, sleeping 5s before retry');
+          await sleep(5000 + Math.random() * 1000);
+          attemptedIndexes.clear();
+          continue;
+        }
+        // All bots rate-limited — wait for the shortest cooldown
+        const earliestCooldown = Math.min(...this.bots.map((b) => b.rateLimitedUntil || Infinity));
+        const waitMs = Math.max(1000, earliestCooldown - Date.now() + 500);
+        logger.warn('All bots rate-limited, waiting', { waitMs });
+        await sleep(waitMs);
+        attemptedIndexes.clear();
+        continue;
+      }
+
+      attemptedIndexes.add(bot.index);
+
+      try {
+        const result = await bot.queue.add(async () => {
+          // Inner transient retry loop inside the queue
+          for (let innerRetry = 0; innerRetry <= MAX_TRANSIENT_RETRIES; innerRetry++) {
             try {
               const filePayload = { source: fileChunk, filename: fileName };
               const sendMethodName = sendMethodMap[fileType] || 'sendDocument';
               const payload = buildSendPayload(fileType, fileName);
 
-        const result = await this.executeWithBotRetry<TelegramMessageResult>((activeBot) => {
+              const tgResult = await this.executeBotAction<TelegramMessageResult>(
+                bot,
+                (activeBot) => {
                   const telegram = activeBot.telegram as unknown as Record<string, SendMethod>;
                   return telegram[sendMethodName](config.storageChatId, filePayload, payload);
-        });
+                },
-
+              );
-        const uploadedFile = extractUploadedFile(result, fileType);
-        logger.info('File forwarded to storage', { fileName, message: result.message_id });
 
+              const uploadedFile = extractUploadedFile(tgResult, fileType);
               return {
                 telegramFileId: uploadedFile?.file_id || '',
                 telegramFileUniqueId: uploadedFile?.file_unique_id || '',
-          storageMessageId: result.message_id,
+                storageMessageId: tgResult.message_id,
               };
             } catch (error: unknown) {
-        lastError = error;
               const errorStr = error instanceof Error ? error.message : String(error);
+              const retryAfterMatch = errorStr.match(/retry after (\d+)/i);
 
-        if (attempt <= MAX_TRANSIENT_RETRIES && isTransientError(error)) {
+              if (retryAfterMatch) {
-          const backoffMs = Math.min(1000 * 2 ** attempt, 10_000);
+                // 429 — mark bot rate-limited, throw to outer loop for retry on different bot
-          logger.warn(
+                const seconds = parseInt(retryAfterMatch[1], 10);
-            `Transient error forwarding file, retrying (${attempt}/${MAX_TRANSIENT_RETRIES})`,
+                bot.rateLimitedUntil = Date.now() + seconds * 1000;
-            {
+                logger.info(`Bot #${bot.index} rate-limited for ${seconds}s`, {
                   fileName,
-              error: errorStr,
+                  attempt,
-              backoffMs,
+                });
-            },
+                throw error; // caught by outer retry loop
+              }
+
+              if (innerRetry < MAX_TRANSIENT_RETRIES && isTransientError(error)) {
+                const backoffMs = Math.min(1000 * 2 ** innerRetry, 10_000);
+                logger.warn(
+                  `Transient error on bot #${bot.index}, retrying (${innerRetry + 1}/${MAX_TRANSIENT_RETRIES})`,
+                  { fileName, error: errorStr, backoffMs },
                 );
                 await sleep(backoffMs);
                 continue;
               }
 
+              throw error; // non-transient — propagate
+            }
+          }
+
+          throw new Error(`Exhausted transient retries on bot #${bot.index}`);
+        });
+
+        logger.info('File forwarded to storage', { fileName, message: result.storageMessageId });
+        return result;
+      } catch (error: unknown) {
+        lastError = error;
+        const errorStr = error instanceof Error ? error.message : String(error);
+        const retryAfterMatch = errorStr.match(/retry after (\d+)/i);
+
+        if (retryAfterMatch) {
+          // Bot was rate-limited — already marked, try next bot
+          continue;
+        }
+
+        // Non-transient — give up
         logger.error('Failed to forward file to storage', {
           fileName,
           error: errorStr,
@@ -240,59 +235,45 @@ export class BotPool implements ITelegramService {
       }
     }
 
-    // Should not reach here — last iteration throws above
+    throw lastError || new Error('Failed to forward file after all retries');
-    throw lastError;
+  }
+
+  /** Get total effective concurrency across all bots */
+  getEffectiveConcurrency(): number {
+    return this.bots.length * PER_BOT_CONCURRENCY;
   }
 
-  /**
-   * Retrieve file metadata from Telegram by file ID.
-   *
-   * Tries all configured bots sequentially; returns info from the first
-   * bot that can retrieve the file. Errors indicating the file belongs
-   * to a different bot are silently skipped.
-   *
-   * @param telegramFileId - The Telegram file_id to look up.
-   * @returns Metadata including size, MIME type, download path, and bot token.
-   */
   async getFileInfo(telegramFileId: string): Promise<TelegramFileInfo> {
     let lastError: unknown;
-    for (const activeBot of this.bots) {
+    for (const bot of this.bots) {
       for (let retry = 0; retry <= MAX_TRANSIENT_RETRIES; retry++) {
         try {
-          const result = await activeBot.telegram.getFile(telegramFileId);
+          const result = await bot.instance.telegram.getFile(telegramFileId);
           const fileData = result as unknown as Omit<TelegramFileInfo, 'bot_token'>;
           return {
             file_size: fileData.file_size || 0,
             mime_type: fileData.mime_type || 'application/octet-stream',
             file_path: fileData.file_path || '',
-            bot_token: activeBot.telegram.token,
+            bot_token: bot.token,
           };
         } catch (error: unknown) {
           lastError = error;
           const errorStr = error instanceof Error ? error.message : String(error);
-          // Belongs to a different bot — skip to next bot immediately
           if (
             errorStr.includes('wrong file_id') ||
             errorStr.includes('file is temporarily unavailable')
           ) {
-            break; // skip to next bot
+            break;
           }
-          // Transient — retry on the same bot
           if (retry < MAX_TRANSIENT_RETRIES && isTransientError(error)) {
             const backoffMs = Math.min(1000 * 2 ** (retry + 1), 5_000);
-            logger.warn(
-              `Transient error getting file info, retrying bot ${activeBot.telegram.token.slice(0, 8)}... (${retry + 1}/${MAX_TRANSIENT_RETRIES})`,
-              { telegramFileId, error: errorStr, backoffMs },
-            );
             await sleep(backoffMs);
             continue;
           }
-          // Non-transient or exhausted retries — try next bot
           break;
         }
       }
     }
-
     logger.error('Failed to get file info from any bot', {
       error: lastError instanceof Error ? lastError.message : String(lastError),
     });
@@ -300,7 +281,4 @@ export class BotPool implements ITelegramService {
   }
 }
 
-/**
- * Singleton BotPool instance initialised from application configuration.
- */
 export const botPool = new BotPool();