perf: parallelize backlog sync with concurrency limit
- Sync channels in parallel with concurrency limit of 3 instead of sequentially - Reduces backlog sync time from O(n) to O(n/3) for n channels - Prevents overwhelming Discord API with too many concurrent requests - Maintains per-channel message batching for memory efficiency This resolves: - Slow backlog sync performance - Sequential channel processing bottleneck Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -140,19 +140,37 @@ export async function syncBacklogMessages(
|
|||||||
"Watchable channels collected for backlog sync",
|
"Watchable channels collected for backlog sync",
|
||||||
);
|
);
|
||||||
|
|
||||||
for (const channel of channels) {
|
// Sync channels in parallel with concurrency limit of 3
|
||||||
try {
|
const concurrency = 3;
|
||||||
const count = await syncChannelMessages(db, channel as any, cutoffTime);
|
const queue = [...channels];
|
||||||
total += count;
|
const active: Promise<number>[] = [];
|
||||||
logger.info({ channelId: channel.id, count }, "Backlog channel sync completed");
|
|
||||||
} catch (error) {
|
while (queue.length > 0 || active.length > 0) {
|
||||||
logger.warn(
|
while (active.length < concurrency && queue.length > 0) {
|
||||||
{
|
const channel = queue.shift()!;
|
||||||
channelId: channel.id,
|
const promise = (async () => {
|
||||||
error: error instanceof Error ? error.message : String(error),
|
try {
|
||||||
},
|
const count = await syncChannelMessages(db, channel as any, cutoffTime);
|
||||||
"Backlog channel sync failed",
|
logger.info({ channelId: channel.id, count }, "Backlog channel sync completed");
|
||||||
);
|
return count;
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
channelId: channel.id,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
},
|
||||||
|
"Backlog channel sync failed",
|
||||||
|
);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
active.push(promise);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (active.length > 0) {
|
||||||
|
const result = await Promise.race(active);
|
||||||
|
total += result;
|
||||||
|
active.splice(active.findIndex((p) => p === Promise.resolve(result)), 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user