feat(gmw): public features #7-14 — scam domains, top channels, hourly heatmap, category drill-down, coverage stats, channel culture glossary, term KB, edit history

ALSO fixes: dashboard.repository still JOINed dropped user_reputations table (listUsers/getUserDetail crash).
This commit is contained in:
asepharyana
2026-08-18 20:45:12 +07:00
parent 2a8f6d9062
commit 00e8d68ce5
38 changed files with 1542 additions and 25 deletions
@@ -5,7 +5,6 @@ import {
pgChannelCulturesTable,
pgMessagesTable,
pgUserProfilesTable,
pgUserReputationsTable,
pgVoiceRecordingsTable,
} from "../../shared/index.js";
import type { ListUsersQuery } from "./dashboard.service.js";
@@ -156,8 +155,7 @@ export class DashboardRepository {
p.profile_summary,
m.total_messages,
m.flagged_count,
m.last_message_at,
r.trust_score
m.last_message_at
FROM (
SELECT
user_id,
@@ -170,7 +168,6 @@ export class DashboardRepository {
GROUP BY user_id, username, avatar_url
) m
LEFT JOIN ${pgUserProfilesTable} p ON p.user_id = m.user_id
LEFT JOIN ${pgUserReputationsTable} r ON r.user_id = m.user_id
${whereClause}
ORDER BY m.last_message_at DESC NULLS LAST
LIMIT ${limit + 1}
@@ -186,10 +183,6 @@ export class DashboardRepository {
total_messages: Number(r.total_messages),
flagged_count: Number(r.flagged_count),
last_message_at: r.last_message_at ? Number(r.last_message_at) : null,
trust_score:
r.trust_score !== null && r.trust_score !== undefined
? Number(r.trust_score)
: null,
}));
const lastRow = rows[limit - 1] as Record<string, unknown> | undefined;
@@ -437,10 +430,7 @@ export class DashboardRepository {
m.flagged_count,
m.clean_count,
p.profile_summary,
p.last_analyzed_at,
r.trust_score,
r.clean_message_streak,
r.total_infractions
p.last_analyzed_at
FROM (
SELECT
user_id,
@@ -454,7 +444,6 @@ export class DashboardRepository {
GROUP BY user_id, username, avatar_url
) m
LEFT JOIN ${pgUserProfilesTable} p ON p.user_id = m.user_id
LEFT JOIN ${pgUserReputationsTable} r ON r.user_id = m.user_id
`);
const row = userResult.rows[0] as Record<string, unknown> | undefined;
@@ -481,13 +470,6 @@ export class DashboardRepository {
last_analyzed_at: row.last_analyzed_at
? Number(row.last_analyzed_at)
: null,
trust_score: row.trust_score != null ? Number(row.trust_score) : null,
clean_message_streak:
row.clean_message_streak != null
? Number(row.clean_message_streak)
: null,
total_infractions:
row.total_infractions != null ? Number(row.total_infractions) : null,
recent_messages: (recent.rows as Record<string, unknown>[]).map((r) => ({
id: String(r.id),
content: String(r.content),
@@ -0,0 +1,99 @@
import { sql } from "drizzle-orm";
import { getDatabase } from "../../shared/database/index.js";
export interface ChannelCultureRow {
channel_id: string;
guild_id: string | null;
channel_name: string | null;
culture_summary: string | null;
last_analyzed_at: number | null;
}
export interface GlossaryRow {
term: string;
definition: string;
source_url: string;
resolved_at: number;
hit_count: number;
}
export interface EditHistoryRow {
id: string;
message_id: string;
old_content: string;
edited_at: number;
channel_id: string | null;
channel_name: string | null;
username: string | null;
}
export class KnowledgeRepository {
/** Public read-only channel culture glossary (AI-generated norms/slang). */
async listChannelCultures(limit = 50, search?: string) {
const db = getDatabase();
const conditions: string[] = [];
if (search) {
conditions.push(
`(c.channel_id ILIKE '%${search.replace(/'/g, "''")}%' OR c.culture_summary ILIKE '%${search.replace(/'/g, "''")}%')`,
);
}
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
const result = await db.execute(
sql.raw(`
SELECT
c.channel_id,
c.guild_id,
COALESCE(NULLIF((
SELECT (metadata::jsonb -> 'channel' ->> 'channelName')
FROM messages WHERE channel_id = c.channel_id AND metadata IS NOT NULL
LIMIT 1
), ''), c.channel_id) AS channel_name,
c.culture_summary,
c.last_analyzed_at
FROM channel_cultures c
${where}
ORDER BY c.last_analyzed_at DESC NULLS LAST
LIMIT ${limit}
`),
);
const rows = (result.rows as Record<string, unknown>[]) || [];
return rows.map((r) => ({
channel_id: String(r.channel_id),
guild_id: r.guild_id ? String(r.guild_id) : null,
channel_name: r.channel_name ? String(r.channel_name) : null,
culture_summary: r.culture_summary ? String(r.culture_summary) : null,
last_analyzed_at: r.last_analyzed_at ? Number(r.last_analyzed_at) : null,
}));
}
/** Public read-only term knowledge base (resolved via Wikipedia/SearXNG). */
async listGlossary(limit = 50, search?: string) {
const db = getDatabase();
const conditions: string[] = [];
if (search) {
conditions.push(
`(term ILIKE '%${search.replace(/'/g, "''")}%' OR definition ILIKE '%${search.replace(/'/g, "''")}%')`,
);
}
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
const result = await db.execute(
sql.raw(`
SELECT term, definition, source_url, resolved_at, hit_count
FROM term_glossary_cache
${where}
ORDER BY hit_count DESC, resolved_at DESC
LIMIT ${limit}
`),
);
const rows = (result.rows as Record<string, unknown>[]) || [];
return rows.map((r) => ({
term: String(r.term),
definition: String(r.definition ?? ""),
source_url: r.source_url ? String(r.source_url) : "",
resolved_at: r.resolved_at ? Number(r.resolved_at) : 0,
hit_count: Number(r.hit_count ?? 0),
}));
}
}
export const knowledgeRepository = new KnowledgeRepository();
@@ -0,0 +1,18 @@
import { createChildLogger } from "../../shared/logger/index.js";
import { knowledgeRepository } from "./knowledge.repository.js";
const logger = createChildLogger("knowledge.service");
export class KnowledgeService {
async listChannelCultures(limit = 50, search?: string) {
logger.debug({ limit, search }, "Listing channel cultures");
return knowledgeRepository.listChannelCultures(limit, search);
}
async listGlossary(limit = 50, search?: string) {
logger.debug({ limit, search }, "Listing glossary terms");
return knowledgeRepository.listGlossary(limit, search);
}
}
export const knowledgeService = new KnowledgeService();
@@ -484,6 +484,44 @@ export class MessagesRepository {
count: Number(r.c ?? 0),
}));
}
/**
* Recent message edits across the server (evasion-signal tracker).
* Public, read-only. Joins message_edits → messages for context.
*/
async getRecentEdits(limit = 50, channelId?: string) {
const db = getDatabase();
const where = channelId
? `WHERE m.channel_id = '${channelId.replace(/'/g, "''")}'`
: "";
const result = await db.execute(
sql.raw(`
SELECT
e.id,
e.message_id,
e.old_content,
e.edited_at,
m.channel_id,
COALESCE(NULLIF((m.metadata::jsonb -> 'channel' ->> 'channelName'), ''), m.channel_id) AS channel_name,
m.username
FROM message_edits e
JOIN messages m ON m.id = e.message_id
${where}
ORDER BY e.edited_at DESC
LIMIT ${limit}
`),
);
const rows = (result.rows as Record<string, unknown>[]) || [];
return rows.map((r) => ({
id: String(r.id),
message_id: String(r.message_id),
old_content: r.old_content ? String(r.old_content) : "",
edited_at: r.edited_at ? Number(r.edited_at) : 0,
channel_id: r.channel_id ? String(r.channel_id) : null,
channel_name: r.channel_name ? String(r.channel_name) : null,
username: r.username ? String(r.username) : null,
}));
}
}
export const messagesRepository = new MessagesRepository();
@@ -105,6 +105,11 @@ export class MessagesService {
async getActivity(days = 30) {
return messagesRepository.getActivity(days);
}
async getRecentEdits(limit = 50, channelId?: string) {
logger.debug({ limit, channelId }, "Getting recent message edits");
return messagesRepository.getRecentEdits(limit, channelId);
}
}
/** Shape returned to the frontend (text + metadata from the archive payload). */
@@ -218,6 +218,166 @@ export class ModerationRepository {
})),
};
}
/**
* Top flagged domains over the last `days` days.
* Extracts the host from any URL in `content`/`reason`/`evidence` and ranks
* by how often it appears in moderation actions. Powers the Scam Domain panel.
*/
async getTopFlaggedDomains(days: number) {
const db = getDatabase();
const since = Date.now() - days * 24 * 60 * 60 * 1000;
const result = await db.execute(sql`
SELECT host, COUNT(*)::int AS c
FROM (
SELECT DISTINCT a.id,
(regexp_matches(COALESCE(a.content,'') || ' ' || COALESCE(a.reason,'') || ' ' || COALESCE(a.evidence,''), 'https?://([^/\s?#]+)', 'g'))[1] AS host
FROM moderation_actions a
WHERE a.created_at >= ${since}
AND (a.content IS NOT NULL OR a.reason IS NOT NULL OR a.evidence IS NOT NULL)
) sub
WHERE host IS NOT NULL
GROUP BY host
ORDER BY c DESC
LIMIT 20
`);
const rows = (result.rows as Record<string, unknown>[]) || [];
return rows.map((r) => ({
domain: String(r.host).toLowerCase(),
count: Number(r.c ?? 0),
}));
}
/**
* Top flagged channels over the last `days` days.
* Joins moderation_actions → messages to attribute each action to a channel.
* Powers the Top Flagged Channels panel.
*/
async getTopFlaggedChannels(days: number) {
const db = getDatabase();
const since = Date.now() - days * 24 * 60 * 60 * 1000;
const result = await db.execute(sql`
SELECT
m.channel_id,
COALESCE(NULLIF((m.metadata::jsonb -> 'channel' ->> 'channelName'), ''), m.channel_id) AS channel_name,
COUNT(*)::int AS flagged_count
FROM moderation_actions a
LEFT JOIN messages m ON m.id = a.message_id
WHERE a.created_at >= ${since} AND m.channel_id IS NOT NULL
GROUP BY m.channel_id, (m.metadata::jsonb -> 'channel' ->> 'channelName')
ORDER BY flagged_count DESC
LIMIT 15
`);
const rows = (result.rows as Record<string, unknown>[]) || [];
return rows.map((r) => ({
channel_id: String(r.channel_id),
channel_name: r.channel_name ? String(r.channel_name) : null,
flagged_count: Number(r.flagged_count),
}));
}
/**
* Hour-of-day distribution of moderation actions over the last `days` days.
* 24 rows (hour 0..23), with total + flagged-by-severity counts.
* Powers the Moderation Heatmap by Hour panel.
*/
async getHourlyModeration(days: number) {
const db = getDatabase();
const since = Date.now() - days * 24 * 60 * 60 * 1000;
const result = await db.execute(sql`
SELECT
EXTRACT(HOUR FROM to_timestamp(created_at / 1000))::int AS hour,
COUNT(*)::int AS total
FROM moderation_actions
WHERE created_at >= ${since}
GROUP BY hour
ORDER BY hour
`);
const rows = (result.rows as Record<string, unknown>[]) || [];
const byHour = new Map<number, number>();
for (const r of rows) byHour.set(Number(r.hour), Number(r.total));
return Array.from({ length: 24 }, (_, h) => ({
hour: h,
total: byHour.get(h) ?? 0,
}));
}
/**
* Moderation actions filtered to a single category (drill-down).
* Powers the Flag Category Drill-down panel.
*/
async getByCategory(days: number, category: string, limit = 50) {
const db = getDatabase();
const since = Date.now() - days * 24 * 60 * 60 * 1000;
const result = await db.execute(
sql.raw(`
SELECT
a.id, a.message_id, a.user_id, a.guild_id, a.action_type,
a.reason, a.status, a.created_at, a.severity, a.confidence, a.score,
m.username, LEFT(m.content, 300) AS content
FROM moderation_actions a
LEFT JOIN messages m ON m.id = a.message_id
WHERE a.created_at >= ${since}
AND a.categories IS NOT NULL
AND a.categories::jsonb @> ${JSON.stringify([category])}::jsonb
ORDER BY a.created_at DESC
LIMIT ${limit}
`),
);
const rows = (result.rows as Record<string, unknown>[]) || [];
return rows.map((r) => ({
id: String(r.id ?? ""),
message_id: r.message_id ? String(r.message_id) : null,
user_id: r.user_id ? String(r.user_id) : null,
guild_id: String(r.guild_id ?? ""),
action_type: String(r.action_type ?? "unknown"),
reason: r.reason ? String(r.reason) : null,
status: String(r.status ?? "unknown"),
created_at: r.created_at ? Number(r.created_at) : null,
severity: r.severity ? String(r.severity) : null,
confidence: r.confidence != null ? Number(r.confidence) : null,
score: r.score != null ? Number(r.score) : null,
username: r.username ? String(r.username) : null,
content: r.content ? String(r.content) : null,
}));
}
/**
* Auto-moderation coverage over the last `days` days.
* Run completion rate from ai_analysis_runs — what fraction of analysis runs
* completed (vs failed/pending). Public "how much is automated" trust metric.
*/
async getCoverage(days: number) {
const db = getDatabase();
const since = Date.now() - days * 24 * 60 * 60 * 1000;
const result = await db.execute(sql`
SELECT status, COUNT(*)::int AS c
FROM ai_analysis_runs
WHERE created_at >= ${since}
GROUP BY status
`);
const rows = (result.rows as Record<string, unknown>[]) || [];
const counts: Record<string, number> = {};
let total = 0;
for (const r of rows) {
const s = String(r.status);
const c = Number(r.c ?? 0);
counts[s] = c;
total += c;
}
const completed = counts.completed ?? 0;
const failed = counts.failed ?? 0;
const pending = (counts.pending ?? 0) + (counts.processing ?? 0);
return {
total,
completed,
failed,
pending,
coverage_rate:
total > 0 ? Number(((completed / total) * 100).toFixed(1)) : 0,
failed_rate: total > 0 ? Number(((failed / total) * 100).toFixed(1)) : 0,
};
}
}
export const moderationRepository = new ModerationRepository();
@@ -15,6 +15,26 @@ export class ModerationService {
return moderationRepository.getTrends(days);
}
async getTopFlaggedDomains(days = 30) {
return moderationRepository.getTopFlaggedDomains(days);
}
async getTopFlaggedChannels(days = 30) {
return moderationRepository.getTopFlaggedChannels(days);
}
async getHourlyModeration(days = 30) {
return moderationRepository.getHourlyModeration(days);
}
async getByCategory(days = 30, category: string) {
return moderationRepository.getByCategory(days, category);
}
async getCoverage(days = 30) {
return moderationRepository.getCoverage(days);
}
async listActions(query: ListModerationQuery) {
logger.debug({ query }, "Listing moderation actions");
return moderationRepository.listActions(query);
+81 -1
View File
@@ -5,6 +5,7 @@ import { chatRequestSchema } from "../modules/chatbot/chatbot.schema";
import { chatbotService } from "../modules/chatbot/chatbot.service";
// ── Service imports ──────────────────────────────────────────────
import { dashboardService } from "../modules/dashboard/dashboard.service";
import { knowledgeService } from "../modules/knowledge/knowledge.service";
import {
mediaLoopSchema,
mediaQueueSchema,
@@ -151,6 +152,17 @@ const messagesRouter = {
}),
)
.handler(({ input }) => messagesService.getActivity(input.days)),
// Public, read-only recent message edits (evasion tracker).
editHistory: os
.input(
z.object({
limit: z.coerce.number().int().positive().default(50),
channelId: z.string().optional(),
}),
)
.handler(({ input }) =>
messagesService.getRecentEdits(input.limit, input.channelId),
),
};
// ── Moderation ───────────────────────────────────────────────────
@@ -180,6 +192,51 @@ const moderationRouter = {
}),
)
.handler(({ input }) => moderationService.getTrends(input.days)),
// Flagged link / scam domain ranking (public Scam Domain panel).
topDomains: os
.input(
z.object({
days: z.coerce.number().int().positive().max(365).default(30),
}),
)
.handler(({ input }) => moderationService.getTopFlaggedDomains(input.days)),
// Top flagged channels (join moderation_actions → messages).
topChannels: os
.input(
z.object({
days: z.coerce.number().int().positive().max(365).default(30),
}),
)
.handler(({ input }) =>
moderationService.getTopFlaggedChannels(input.days),
),
// Hour-of-day moderation distribution (heatmap by hour).
byHour: os
.input(
z.object({
days: z.coerce.number().int().positive().max(365).default(30),
}),
)
.handler(({ input }) => moderationService.getHourlyModeration(input.days)),
// Flag category drill-down (list actions for one category).
byCategory: os
.input(
z.object({
days: z.coerce.number().int().positive().max(365).default(30),
category: z.string().min(1),
}),
)
.handler(({ input }) =>
moderationService.getByCategory(input.days, input.category),
),
// Auto-moderation coverage (analysis run completion rate).
coverage: os
.input(
z.object({
days: z.coerce.number().int().positive().max(365).default(30),
}),
)
.handler(({ input }) => moderationService.getCoverage(input.days)),
};
// ── Media ────────────────────────────────────────────────────────
@@ -321,7 +378,29 @@ const chatbotRouter = {
}),
};
// ── Config (public dashboard config snapshot) ──────────────────────
// ── Knowledge (public read-only culture glossary + term KB) ───────
const knowledgeRouter = {
channelCultures: os
.input(
z.object({
limit: z.coerce.number().int().positive().default(50),
search: z.string().optional(),
}),
)
.handler(({ input }) =>
knowledgeService.listChannelCultures(input.limit, input.search),
),
glossary: os
.input(
z.object({
limit: z.coerce.number().int().positive().default(50),
search: z.string().optional(),
}),
)
.handler(({ input }) =>
knowledgeService.listGlossary(input.limit, input.search),
),
};
const configRouter = {
get: os.handler(() => ({
monitorGuildId: config.MONITOR_GUILD_ID || null,
@@ -360,6 +439,7 @@ export const appRouter = {
chatbot: chatbotRouter,
config: configRouter,
uiState: uiStateRouter,
knowledge: knowledgeRouter,
};
export type AppRouter = typeof appRouter;