perf: parallelize backlog sync with concurrency limit

- Sync channels in parallel with concurrency limit of 3 instead of sequentially
- Reduces backlog sync time from O(n) to O(n/3) for n channels
- Prevents overwhelming Discord API with too many concurrent requests
- Maintains per-channel message batching for memory efficiency

This resolves:
- Slow backlog sync performance
- Sequential channel processing bottleneck

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-05-14 03:52:21 +07:00
parent eb27d36cce
commit 54534fe84c

View File

@@ -140,11 +140,19 @@ export async function syncBacklogMessages(
"Watchable channels collected for backlog sync", "Watchable channels collected for backlog sync",
); );
for (const channel of channels) { // Sync channels in parallel with concurrency limit of 3
const concurrency = 3;
const queue = [...channels];
const active: Promise<number>[] = [];
while (queue.length > 0 || active.length > 0) {
while (active.length < concurrency && queue.length > 0) {
const channel = queue.shift()!;
const promise = (async () => {
try { try {
const count = await syncChannelMessages(db, channel as any, cutoffTime); const count = await syncChannelMessages(db, channel as any, cutoffTime);
total += count;
logger.info({ channelId: channel.id, count }, "Backlog channel sync completed"); logger.info({ channelId: channel.id, count }, "Backlog channel sync completed");
return count;
} catch (error) { } catch (error) {
logger.warn( logger.warn(
{ {
@@ -153,6 +161,16 @@ export async function syncBacklogMessages(
}, },
"Backlog channel sync failed", "Backlog channel sync failed",
); );
return 0;
}
})();
active.push(promise);
}
if (active.length > 0) {
const result = await Promise.race(active);
total += result;
active.splice(active.findIndex((p) => p === Promise.resolve(result)), 1);
} }
} }