There are no staged changes to commit. The staging area is empty — stage some files first with git add and I can help craft the message.
This commit is contained in:
@@ -908,10 +908,51 @@ async function runTextOnlyBatch(
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
// Split into sub-batches if needed (R6)
|
// ── Group identical short messages (< 20 chars) to reduce redundant analysis ──
|
||||||
|
// Messages with identical normalized content share a single representative.
|
||||||
|
// Results are fanned out to all group members after the LLM call.
|
||||||
|
const shortContentGroups = new Map<string, MessageRecord[]>();
|
||||||
|
const deduplicatedTargets: MessageRecord[] = [];
|
||||||
|
const groupMapping = new Map<string, string[]>(); // representativeId → [all memberIds]
|
||||||
|
|
||||||
|
for (const msg of targets) {
|
||||||
|
const rawContent = (msg.edited_content ?? msg.content).trim();
|
||||||
|
if (rawContent.length > 0 && rawContent.length < 20) {
|
||||||
|
const groupKey = rawContent.toLowerCase();
|
||||||
|
if (shortContentGroups.has(groupKey)) {
|
||||||
|
shortContentGroups.get(groupKey)!.push(msg);
|
||||||
|
} else {
|
||||||
|
shortContentGroups.set(groupKey, [msg]);
|
||||||
|
deduplicatedTargets.push(msg); // first occurrence = representative
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
deduplicatedTargets.push(msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build group mapping for results fan-out
|
||||||
|
for (const [, members] of shortContentGroups) {
|
||||||
|
if (members.length > 1) {
|
||||||
|
const rep = members[0];
|
||||||
|
groupMapping.set(rep.id, members.map((m) => m.id));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (groupMapping.size > 0) {
|
||||||
|
log.info(
|
||||||
|
{
|
||||||
|
originalCount: targets.length,
|
||||||
|
deduplicatedCount: deduplicatedTargets.length,
|
||||||
|
groupsFormed: groupMapping.size,
|
||||||
|
},
|
||||||
|
"Grouped identical short messages for text batch",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Split into sub-batches if needed (R6) — using deduplicated targets
|
||||||
const subBatches: MessageRecord[][] = [];
|
const subBatches: MessageRecord[][] = [];
|
||||||
for (let i = 0; i < targets.length; i += maxBatchSize) {
|
for (let i = 0; i < deduplicatedTargets.length; i += maxBatchSize) {
|
||||||
subBatches.push(targets.slice(i, i + maxBatchSize));
|
subBatches.push(deduplicatedTargets.slice(i, i + maxBatchSize));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (subBatches.length > 1) {
|
if (subBatches.length > 1) {
|
||||||
|
|||||||
Reference in New Issue
Block a user