feat(moderation): update auto-delete logic to include 'warn' status for message eligibility
This commit is contained in:
@@ -40,7 +40,7 @@ function getModerationBroadcaster(): ModerationBroadcaster | undefined {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function scheduleAutoDelete(row: MessageRecord): void {
|
function scheduleAutoDelete(row: MessageRecord): void {
|
||||||
if (row.ai_status !== "flagged") return;
|
if (row.ai_status !== "flagged" && row.ai_status !== "warn") return;
|
||||||
const run = () => {
|
const run = () => {
|
||||||
attemptAutoDeleteFlaggedMessage(moderationClient, row).catch((error) => {
|
attemptAutoDeleteFlaggedMessage(moderationClient, row).catch((error) => {
|
||||||
logger.error(
|
logger.error(
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ const parseStringList = (value?: string | null): string[] => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
function isAutoDeleteEligible(message: MessageRecord): boolean {
|
function isAutoDeleteEligible(message: MessageRecord): boolean {
|
||||||
if (message.ai_status !== "flagged") return false;
|
if (message.ai_status !== "flagged" && message.ai_status !== "warn") return false;
|
||||||
|
|
||||||
const confidence = message.ai_confidence ?? message.ai_moderation_score ?? 0;
|
const confidence = message.ai_confidence ?? message.ai_moderation_score ?? 0;
|
||||||
if (confidence < config.AUTO_DELETE_MIN_CONFIDENCE) {
|
if (confidence < config.AUTO_DELETE_MIN_CONFIDENCE) {
|
||||||
@@ -31,7 +31,7 @@ function isAutoDeleteEligible(message: MessageRecord): boolean {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
const allowedSeverities = config.AUTO_DELETE_ALLOWED_SEVERITIES
|
const allowedSeverities = (config.AUTO_DELETE_ALLOWED_SEVERITIES || "")
|
||||||
.split(",")
|
.split(",")
|
||||||
.map((s) => s.trim())
|
.map((s) => s.trim())
|
||||||
.filter(Boolean);
|
.filter(Boolean);
|
||||||
@@ -159,8 +159,8 @@ export async function attemptAutoDeleteFlaggedMessage(
|
|||||||
return { deleted: false, skipped: true, reason: "disabled" };
|
return { deleted: false, skipped: true, reason: "disabled" };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (message.ai_status !== "flagged") {
|
if (message.ai_status !== "flagged" && message.ai_status !== "warn") {
|
||||||
const result = { deleted: false, skipped: true, reason: "not_flagged" } as AutoDeleteResult;
|
const result = { deleted: false, skipped: true, reason: "not_flagged_or_warn" } as AutoDeleteResult;
|
||||||
await logAutoDeleteAttempt(message, result);
|
await logAutoDeleteAttempt(message, result);
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,11 @@ vi.mock("../../src/config", () => ({
|
|||||||
AUTO_DELETE_FLAGGED_ENABLED: true,
|
AUTO_DELETE_FLAGGED_ENABLED: true,
|
||||||
AUTO_DELETE_FLAGGED_DRY_RUN: false,
|
AUTO_DELETE_FLAGGED_DRY_RUN: false,
|
||||||
AUTO_DELETE_FLAGGED_DELAY_MS: 0,
|
AUTO_DELETE_FLAGGED_DELAY_MS: 0,
|
||||||
|
AUTO_DELETE_MIN_CONFIDENCE: 0,
|
||||||
|
AUTO_DELETE_ALLOWED_SEVERITIES: "",
|
||||||
|
AUTO_DELETE_ALLOWED_CATEGORIES: "",
|
||||||
|
AUTO_DELETE_EXCLUDED_CHANNEL_IDS: "",
|
||||||
|
AUTO_DELETE_EXCLUDED_USER_IDS: "",
|
||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -36,6 +41,7 @@ function createMessage(overrides: Partial<MessageRecord> = {}): MessageRecord {
|
|||||||
type: "text",
|
type: "text",
|
||||||
metadata: null,
|
metadata: null,
|
||||||
ai_status: "flagged",
|
ai_status: "flagged",
|
||||||
|
ai_recommended_action: "delete",
|
||||||
...overrides,
|
...overrides,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -83,7 +89,7 @@ describe("attemptAutoDeleteFlaggedMessage", () => {
|
|||||||
createMessage({ ai_status: "clean" }),
|
createMessage({ ai_status: "clean" }),
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result.reason).toBe("not_flagged");
|
expect(result.reason).toBe("not_flagged_or_warn");
|
||||||
expect(deleteMock).not.toHaveBeenCalled();
|
expect(deleteMock).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -104,7 +110,21 @@ describe("attemptAutoDeleteFlaggedMessage", () => {
|
|||||||
});
|
});
|
||||||
const result = await attemptAutoDeleteFlaggedMessage(
|
const result = await attemptAutoDeleteFlaggedMessage(
|
||||||
client as any,
|
client as any,
|
||||||
createMessage(),
|
createMessage(), // defaults to flagged
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result).toEqual({ deleted: true, skipped: false, reason: "deleted" });
|
||||||
|
expect(fetchMessageMock).toHaveBeenCalledWith("m1");
|
||||||
|
expect(deleteMock).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("deletes when message is warn and permission exists", async () => {
|
||||||
|
const { client, fetchMessageMock, deleteMock } = createClient({
|
||||||
|
canManageMessages: true,
|
||||||
|
});
|
||||||
|
const result = await attemptAutoDeleteFlaggedMessage(
|
||||||
|
client as any,
|
||||||
|
createMessage({ ai_status: "warn" }),
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result).toEqual({ deleted: true, skipped: false, reason: "deleted" });
|
expect(result).toEqual({ deleted: true, skipped: false, reason: "deleted" });
|
||||||
|
|||||||
Reference in New Issue
Block a user