feat(ai-moderation): introduce user reputation and channel culture context
Implements a context-aware moderation system by tracking user behavior and channel-specific norms to improve AI decision-making accuracy. - Adds `user_reputations` table to track trust scores, clean streaks, and infraction history. - Adds `channel_cultures` table to store AI-generated summaries of channel-specific norms and slang. - Implements `userReputationStore` to autonomously update user scores based on moderation outcomes (clean vs. flagged). - Implements `cultureLearner` and `channelCultureStore` to manage evolving channel contexts. - Enhances LLM prompts to inject user reputation (trust scores, history) and channel culture summaries, enabling "wisdom-based" moderation (e.g., giving benefit of the doubt to high-trust users). - Integrates reputation and culture updates into the existing `aiAnalyzer` pipeline.
This commit is contained in:
@@ -488,6 +488,17 @@ async function processIndividualFallback(
|
||||
broadcastAnalysisCompleted(row);
|
||||
invalidateAnalyticsCache(row.guild_id);
|
||||
scheduleAutoDelete(row);
|
||||
|
||||
// Update reputation autonomously (Belajar & Kebijaksanaan)
|
||||
if (row.ai_status === "clean") {
|
||||
import("./userReputationStore.js")
|
||||
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
|
||||
.catch((e) => logger.error({ error: e }, "Failed to record clean message streak in fallback"));
|
||||
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
|
||||
import("./userReputationStore.js")
|
||||
.then((store) => store.recordInfraction(row.user_id, row.guild_id, row.ai_severity as "low"|"medium"|"high"|"critical"))
|
||||
.catch((e) => logger.error({ error: e }, "Failed to record infraction penalty in fallback"));
|
||||
}
|
||||
}
|
||||
|
||||
const resultSummary = analysisResult.results[0];
|
||||
@@ -701,6 +712,17 @@ async function processBatch(
|
||||
if (!isApiFailure) {
|
||||
broadcastAnalysisCompleted(row);
|
||||
scheduleAutoDelete(row);
|
||||
|
||||
// Update reputation autonomously (Belajar & Kebijaksanaan)
|
||||
if (row.ai_status === "clean") {
|
||||
import("./userReputationStore.js")
|
||||
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
|
||||
.catch((e) => logger.error({ error: e }, "Failed to record clean message streak"));
|
||||
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
|
||||
import("./userReputationStore.js")
|
||||
.then((store) => store.recordInfraction(row.user_id, row.guild_id, row.ai_severity as "low"|"medium"|"high"|"critical"))
|
||||
.catch((e) => logger.error({ error: e }, "Failed to record infraction penalty"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1097,6 +1119,8 @@ export function startPendingAIAnalysisWorker(
|
||||
_redisEventBroadcaster = eventBroadcaster;
|
||||
if (!config.AI_ANALYSIS_ENABLED) return;
|
||||
|
||||
import("./cultureLearner.js").then(m => m.startCultureLearnerWorker()).catch(console.error);
|
||||
|
||||
setInterval(() => {
|
||||
revertStuckProcessingMessages(300000).catch((err: unknown) => {
|
||||
logger.error(
|
||||
|
||||
Reference in New Issue
Block a user