perf: parallelize backlog sync with concurrency limit

- Sync channels in parallel with concurrency limit of 3 instead of sequentially
- Reduces backlog sync time from O(n) to O(n/3) for n channels
- Prevents overwhelming Discord API with too many concurrent requests
- Maintains per-channel message batching for memory efficiency

This resolves:
- Slow backlog sync performance
- Sequential channel processing bottleneck

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-05-14 03:52:21 +07:00
parent eb27d36cce
commit 54534fe84c

View File

@@ -140,19 +140,37 @@ export async function syncBacklogMessages(
"Watchable channels collected for backlog sync",
);
for (const channel of channels) {
try {
const count = await syncChannelMessages(db, channel as any, cutoffTime);
total += count;
logger.info({ channelId: channel.id, count }, "Backlog channel sync completed");
} catch (error) {
logger.warn(
{
channelId: channel.id,
error: error instanceof Error ? error.message : String(error),
},
"Backlog channel sync failed",
);
// Sync channels in parallel with concurrency limit of 3
const concurrency = 3;
const queue = [...channels];
const active: Promise<number>[] = [];
while (queue.length > 0 || active.length > 0) {
while (active.length < concurrency && queue.length > 0) {
const channel = queue.shift()!;
const promise = (async () => {
try {
const count = await syncChannelMessages(db, channel as any, cutoffTime);
logger.info({ channelId: channel.id, count }, "Backlog channel sync completed");
return count;
} catch (error) {
logger.warn(
{
channelId: channel.id,
error: error instanceof Error ? error.message : String(error),
},
"Backlog channel sync failed",
);
return 0;
}
})();
active.push(promise);
}
if (active.length > 0) {
const result = await Promise.race(active);
total += result;
active.splice(active.findIndex((p) => p === Promise.resolve(result)), 1);
}
}