MythEclipse/GMW
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: comprehensive codebase cleanup and architecture hardening

Browse Source 
- Sprint 1 (Quick Wins): Remove dead analytics modules, fix 4 unresolved
  imports, replace 3 console.warn with logger, remove mock-crc import
- Sprint 2 (Architecture): Create MascotChatRepository, AnalysisRepository,
  3 Zod schemas (mascot-chat, analysis, voice), deduplicate error classes,
  move 3 SQL queries from routes to repository
- Sprint 3 (Complexity): Replace 7 any types with proper interfaces,
  extract 6 helpers from prepareMediaMessage (CC 85 -> ~15)
- Sprint 4 (Config): Remove 22 dead env vars from .env, add 30 missing
  vars to .env.example, standardize naming

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MythEclipse
2026-06-09 10:16:04 +07:00
co-authored by Claude Opus 4.8
parent d0d9e1669e
commit 4becf0d6f1
89 changed files with 1260 additions and 5499 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/discord-gateway/src/modules/ai-moderation/aiAnalyzer.ts
-2
View File
@@ -7,7 +7,6 @@ import { LRUCache } from "lru-cache";
import { Piscina } from "piscina";
import { config } from "../../shared/config/config.js";
import type { EventBroadcaster } from "../event-broadcaster/index.js";
import { invalidateAnalyticsCache } from "../message-capture/analyticsStore.js";
import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js";
import {
getConversationKeysWithIncompleteAnalysis,
@@ -500,7 +499,6 @@ async function processIndividualFallback(
const rows = await updateMessagesAIAnalysisBulk(updates);
for (const row of rows) {
broadcastAnalysisCompleted(row);
invalidateAnalyticsCache(row.guild_id);
scheduleAutoDelete(row);
// Update reputation autonomously (Belajar & Kebijaksanaan)
services/discord-gateway/src/modules/ai-moderation/channelCultureStore.ts
+1 -1
View File
@@ -1,8 +1,8 @@
import { eq } from "drizzle-orm";
import { getDatabase } from "../../shared/database/drizzle.js";
import {
channelCulturesTable,
ChannelCulture,
channelCulturesTable,
} from "../../shared/database/schema.js";
/**
services/discord-gateway/src/modules/ai-moderation/cultureLearner.ts
+5 -5
View File
@@ -1,13 +1,13 @@
import { eq, desc, sql, and } from "drizzle-orm";
import { createChildLogger } from "@bete/shared/logger";
import { and, desc, eq, sql } from "drizzle-orm";
import { config } from "../../shared/config/config.js";
import { getDatabase } from "../../shared/database/drizzle.js";
import {
messagesTable,
channelCulturesTable,
messagesTable,
} from "../../shared/database/schema.js";
import { config } from "../../shared/config/config.js";
import { createChildLogger } from "@bete/shared/logger";
import { llmChat } from "./llmClient.js";
import { updateChannelCulture } from "./channelCultureStore.js";
import { llmChat } from "./llmClient.js";
const CULTURE_LEARNING_INTERVAL = 1000 * 60 * 60 * 12; // 12 hours
const log = createChildLogger("cultureLearner");
services/discord-gateway/src/modules/ai-moderation/llmClient.ts
+9 -5
View File
@@ -84,14 +84,14 @@ export async function llmChat(
signal,
} = opts;
const params: any = {
const params = {
model,
messages,
};
...(stream !== undefined ? { stream } : {}),
} as OpenAI.Chat.Completions.ChatCompletionCreateParams;
// Attach optional parameters only if explicitly provided to maintain
// maximum compatibility with various LLM providers and local APIs.
if (stream !== undefined) params.stream = stream;
if (temperature !== undefined) params.temperature = temperature;
if (top_p !== undefined) params.top_p = top_p;
if (max_tokens !== undefined) params.max_tokens = max_tokens;
@@ -103,7 +103,9 @@ export async function llmChat(
return retryWithBackoff(
async () => {
return withLlmConcurrency(async () => {
const execute = async (currentParams: any) => {
const execute = async (
currentParams: OpenAI.Chat.Completions.ChatCompletionCreateParams,
) => {
const response = await client.chat.completions.create(currentParams, {
signal,
});
@@ -161,7 +163,9 @@ export async function llmChat(
{ model },
"Provider rejected non-streaming request. Fallback to stream: true initiated.",
);
params.stream = true;
(
params as unknown as OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming
).stream = true;
return await execute(params);
}
services/discord-gateway/src/modules/ai-moderation/llmModerationClient.ts
+287 -261
View File
@@ -11,11 +11,9 @@ import type {
AttachmentRecord,
MessageRecord,
} from "../message-capture/types.js";
import { getChannelCulture } from "./channelCultureStore.js";
import { llmChat, llmVision } from "./llmClient.js";
import { buildSystemPrompt as buildSystemPromptModular } from "./moderationPrompt.js";
import { initializeUserReputation } from "./userReputationStore.js";
import { getChannelCulture } from "./channelCultureStore.js";
import { logModerationAnalysis, logModerationError } from "./responseLogger.js";
import {
getStickerFromCache,
@@ -46,6 +44,7 @@ import {
upsertCachedMediaByPhash,
} from "./textCacheStore.js";
import { extractUrlsFromText, fetchUrlSafely } from "./urlFetcher.js";
import { initializeUserReputation } from "./userReputationStore.js";
const SeveritySchema = z.enum(["none", "low", "medium", "high", "critical"]);
const RecommendedActionSchema = z.enum([
@@ -170,7 +169,7 @@ function deriveRecommendedAction(
/**
* Helper to extract JSON from a potentially conversational or markdown-wrapped string.
*/
export function extractJson(content: string): any {
export function extractJson(content: string): unknown {
const codeBlockRegex = /```(?:json)?\s*([\s\S]*?)\s*```/g;
const matches = content.matchAll(codeBlockRegex);
for (const match of matches) {
@@ -1046,7 +1045,7 @@ async function runTextOnlyBatch(
if (rawContent.length > 0 && rawContent.length < 20) {
const groupKey = rawContent.toLowerCase();
if (shortContentGroups.has(groupKey)) {
shortContentGroups.get(groupKey)!.push(msg);
shortContentGroups.get(groupKey)?.push(msg);
} else {
shortContentGroups.set(groupKey, [msg]);
deduplicatedTargets.push(msg); // first occurrence = representative
@@ -1278,9 +1277,6 @@ async function prepareMediaMessage(
const webTextMap = new Map<string, string[]>();
const mediaAnalysisMap = new Map<string, string[]>();
const getAttachmentImageUrl = (att: AttachmentRecord): string | null =>
att.uploaded_url ?? att.discord_url ?? null;
const maxDimension = config.AI_LLM_IMAGE_MAX_DIMENSION ?? 1024;
const content = getAnalysisContent(target);
@@ -1292,93 +1288,14 @@ async function prepareMediaMessage(
.filter(
(att) =>
att.message_id === targetId &&
getAttachmentImageUrl(att) &&
(att.uploaded_url ?? att.discord_url ?? null) &&
att.type.startsWith("image/"),
)
.slice(0, 8);
for (const att of msgAttachments) {
downloadPromises.push(
(async () => {
const urlToUse = getAttachmentImageUrl(att);
if (!urlToUse) {
log.warn(
{ attachmentId: att.id, messageId: att.message_id },
"Skipping attachment: no uploaded URL available",
);
return;
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 15000);
try {
const res = await fetch(urlToUse, { signal: controller.signal });
if (!res.ok || !res.body) {
log.warn(
{ attachmentId: att.id, url: urlToUse, status: res.status },
"Failed to download attachment: HTTP error or no body",
);
return;
}
let totalBytes = 0;
const chunks: Uint8Array[] = [];
const reader = res.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) {
totalBytes += value.length;
if (totalBytes > 10 * 1024 * 1024) {
log.warn(
{ attachmentId: att.id, totalBytes },
"Attachment too large (>10MB) — skipping",
);
reader.cancel();
return;
}
chunks.push(value);
}
}
const imageBytes = Buffer.concat(chunks);
const sniffedMime = sniffImageMimeType(imageBytes);
if (!sniffedMime) {
log.warn(
{ attachmentId: att.id },
"Skipping attachment: not a recognised image format",
);
return;
}
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(imageBytes, maxDimension);
const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`;
const part: MessageImagePart = {
type: "image_url",
image_url: { url: dataUrl },
sourceLabel: `[gambar di atas adalah attachment ${att.filename} dari pesan id=${att.message_id}]`,
};
const existing = imageMap.get(targetId) ?? [];
if (existing.length < 8) {
existing.push(part);
imageMap.set(targetId, existing);
}
} catch (err) {
log.warn(
{
attachmentId: att.id,
error: err instanceof Error ? err.message : String(err),
},
"Error downloading attachment",
);
} finally {
clearTimeout(timeoutId);
}
})(),
downloadSingleAttachment(att, targetId, maxDimension, imageMap),
);
}
@@ -1388,186 +1305,23 @@ async function prepareMediaMessage(
for (const url of urls) {
downloadPromises.push(
(async () => {
const result = await fetchUrlSafely(url);
if (result.type === "image" && result.data && result.mimeType) {
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(result.data, maxDimension);
const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`;
const part: MessageImagePart = {
type: "image_url",
image_url: { url: dataUrl },
sourceLabel: `[gambar di atas berasal dari link ${url} pada pesan id=${targetId}]`,
};
const existing = imageMap.get(targetId) ?? [];
if (existing.length < 8) {
existing.push(part);
imageMap.set(targetId, existing);
}
} else if (result.type === "text" && result.textContent) {
urlWebTexts.push(`[Isi Web dari ${url}]: ${result.textContent}`);
}
})(),
fetchUrlInline(url, targetId, maxDimension, imageMap, urlWebTexts),
);
}
// ── Sticker / embed / custom emoji download promises ──
const mediaEvidence = extractMessageMediaEvidence(target.metadata);
const mediaCandidates: Array<{
messageId: string;
url: string;
label: string;
stickerName?: string;
customEmojiId?: string;
customEmojiName?: string;
}> = [
...mediaEvidence.stickers
.filter((s) => s.url)
.map((s) => ({
messageId: targetId,
url: s.url,
label: `[gambar di atas adalah sticker "${s.name}" dari pesan id=${targetId}]`,
stickerName: s.name,
})),
...mediaEvidence.embeds.flatMap((embed) =>
[
embed.image
? {
messageId: targetId,
url: embed.image,
label: `[gambar di atas berasal dari embed image pada pesan id=${targetId}]`,
}
: null,
embed.thumbnail
? {
messageId: targetId,
url: embed.thumbnail,
label: `[gambar di atas berasal dari embed thumbnail pada pesan id=${targetId}]`,
}
: null,
].filter(
(
c,
): c is {
messageId: string;
url: string;
label: string;
stickerName?: string;
customEmojiId?: string;
customEmojiName?: string;
} => c !== null,
),
),
...mediaEvidence.customEmojis.map((emoji) => ({
messageId: targetId,
url: emoji.url,
label: `[gambar di atas adalah custom emoji "${emoji.name}" dari pesan id=${targetId}]`,
customEmojiId: emoji.id,
customEmojiName: emoji.name,
})),
];
const mediaCandidates = buildMediaCandidates(targetId, mediaEvidence);
for (const candidate of mediaCandidates) {
downloadPromises.push(
(async () => {
if ((imageMap.get(targetId)?.length ?? 0) >= 8) return;
if (candidate.customEmojiId || candidate.stickerName) {
const visionCacheKey = candidate.customEmojiId
? makeCustomEmojiCacheKey(candidate.customEmojiId)
: makeStickerCacheKey(candidate.stickerName!);
const cachedVision = await getCachedMediaAnalysis(visionCacheKey);
if (cachedVision) {
log.debug(
{ cacheKey: visionCacheKey },
"Vision cache HIT for media candidate — skipped download",
);
const analysisText = `[Media analysis for message ${candidate.messageId}] ${candidate.label}: ${cachedVision}`;
const existing = mediaAnalysisMap.get(targetId) ?? [];
existing.push(analysisText);
mediaAnalysisMap.set(targetId, existing);
return;
}
}
if (candidate.stickerName && isStickerCacheReady()) {
try {
const cached = await getStickerFromCache(candidate.stickerName);
if (cached && cached.imageUrl) {
const part: MessageImagePart = {
type: "image_url",
image_url: { url: cached.imageUrl },
sourceLabel: candidate.label,
stickerName: candidate.stickerName,
};
const existing = imageMap.get(targetId) ?? [];
if (existing.length < 8) {
existing.push(part);
imageMap.set(targetId, existing);
}
return;
}
} catch (stickerErr) {
log.warn(
{
stickerName: candidate.stickerName,
error:
stickerErr instanceof Error
? stickerErr.message
: String(stickerErr),
},
"Sticker cache lookup failed — falling through to network fetch",
);
}
}
const result = await fetchUrlSafely(candidate.url);
if (result.type !== "image" || !result.data || !result.mimeType) {
log.warn(
{
url: candidate.url,
resultType: result.type,
resultHasData: !!result.data,
messageId: candidate.messageId,
label: candidate.stickerName
? `sticker:${candidate.stickerName}`
: candidate.customEmojiName
? `emoji:${candidate.customEmojiName}`
: "embed/other",
},
"Media candidate fetch did not return a usable image — skipping",
);
return;
}
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(result.data, maxDimension);
const base64 = resizedBuffer.toString("base64");
if (candidate.stickerName) {
uploadAndCacheSticker(
candidate.stickerName,
resizedBuffer,
resizedMime,
).catch(() => {});
}
const part: MessageImagePart = {
type: "image_url",
image_url: { url: `data:${resizedMime};base64,${base64}` },
sourceLabel: candidate.label,
stickerName: candidate.stickerName,
customEmojiId: candidate.customEmojiId,
customEmojiName: candidate.customEmojiName,
};
const existing = imageMap.get(targetId) ?? [];
if (existing.length < 8) {
existing.push(part);
imageMap.set(targetId, existing);
}
})(),
downloadMediaCandidate(
candidate,
targetId,
maxDimension,
imageMap,
mediaAnalysisMap,
),
);
}
@@ -2132,3 +1886,275 @@ Kategori: spam`;
: [],
};
}
// ---------------------------------------------------------------------------
// Refactored helpers for prepareMediaMessage (extracted to reduce CC)
// ---------------------------------------------------------------------------
async function downloadSingleAttachment(
att: AttachmentRecord,
targetId: string,
maxDimension: number,
imageMap: Map<string, MessageImagePart[]>,
): Promise<void> {
const urlToUse = att.uploaded_url ?? att.discord_url ?? null;
if (!urlToUse) {
log.warn(
{ attachmentId: att.id, messageId: att.message_id },
"Skipping attachment: no uploaded URL available",
);
return;
}
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 15000);
try {
const res = await fetch(urlToUse, { signal: controller.signal });
if (!res.ok || !res.body) {
log.warn(
{ attachmentId: att.id, url: urlToUse, status: res.status },
"Failed to download attachment: HTTP error or no body",
);
return;
}
let totalBytes = 0;
const chunks: Uint8Array[] = [];
const reader = res.body.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) {
totalBytes += value.length;
if (totalBytes > 10 * 1024 * 1024) {
log.warn(
{ attachmentId: att.id, totalBytes },
"Attachment too large (>10MB) — skipping",
);
reader.cancel();
return;
}
chunks.push(value);
}
}
const imageBytes = Buffer.concat(chunks);
const sniffedMime = sniffImageMimeType(imageBytes);
if (!sniffedMime) {
log.warn(
{ attachmentId: att.id },
"Skipping attachment: not a recognised image format",
);
return;
}
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(imageBytes, maxDimension);
const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`;
const part: MessageImagePart = {
type: "image_url",
image_url: { url: dataUrl },
sourceLabel: `[gambar di atas adalah attachment ${att.filename} dari pesan id=${att.message_id}]`,
};
addImageToMap(imageMap, targetId, part);
} catch (err) {
log.warn(
{
attachmentId: att.id,
error: err instanceof Error ? err.message : String(err),
},
"Error downloading attachment",
);
} finally {
clearTimeout(timeoutId);
}
}
async function downloadMediaCandidate(
candidate: MediaCandidate,
targetId: string,
maxDimension: number,
imageMap: Map<string, MessageImagePart[]>,
mediaAnalysisMap: Map<string, string[]>,
): Promise<void> {
if ((imageMap.get(targetId)?.length ?? 0) >= 8) return;
if (candidate.customEmojiId || candidate.stickerName) {
const visionCacheKey = candidate.customEmojiId
? makeCustomEmojiCacheKey(candidate.customEmojiId)
: makeStickerCacheKey(candidate.stickerName!);
const cachedVision = await getCachedMediaAnalysis(visionCacheKey);
if (cachedVision) {
log.debug(
{ cacheKey: visionCacheKey },
"Vision cache HIT for media candidate — skipped download",
);
const analysisText = `[Media analysis for message ${candidate.messageId}] ${candidate.label}: ${cachedVision}`;
const existing = mediaAnalysisMap.get(targetId) ?? [];
existing.push(analysisText);
mediaAnalysisMap.set(targetId, existing);
return;
}
}
if (candidate.stickerName && isStickerCacheReady()) {
try {
const cached = await getStickerFromCache(candidate.stickerName);
if (cached && cached.imageUrl) {
const part: MessageImagePart = {
type: "image_url",
image_url: { url: cached.imageUrl },
sourceLabel: candidate.label,
stickerName: candidate.stickerName,
};
addImageToMap(imageMap, targetId, part);
return;
}
} catch (stickerErr) {
log.warn(
{
stickerName: candidate.stickerName,
error:
stickerErr instanceof Error
? stickerErr.message
: String(stickerErr),
},
"Sticker cache lookup failed — falling through to network fetch",
);
}
}
const result = await fetchUrlSafely(candidate.url);
if (result.type !== "image" || !result.data || !result.mimeType) {
log.warn(
{
url: candidate.url,
resultType: result.type,
resultHasData: !!result.data,
messageId: candidate.messageId,
label: candidate.stickerName
? `sticker:${candidate.stickerName}`
: candidate.customEmojiName
? `emoji:${candidate.customEmojiName}`
: "embed/other",
},
"Media candidate fetch did not return a usable image — skipping",
);
return;
}
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(result.data, maxDimension);
const base64 = resizedBuffer.toString("base64");
if (candidate.stickerName) {
uploadAndCacheSticker(
candidate.stickerName,
resizedBuffer,
resizedMime,
).catch(() => {});
}
const part: MessageImagePart = {
type: "image_url",
image_url: { url: `data:${resizedMime};base64,${base64}` },
sourceLabel: candidate.label,
stickerName: candidate.stickerName,
customEmojiId: candidate.customEmojiId,
customEmojiName: candidate.customEmojiName,
};
addImageToMap(imageMap, targetId, part);
}
async function fetchUrlInline(
url: string,
targetId: string,
maxDimension: number,
imageMap: Map<string, MessageImagePart[]>,
urlWebTexts: string[],
): Promise<void> {
const result = await fetchUrlSafely(url);
if (result.type === "image" && result.data && result.mimeType) {
const { data: resizedBuffer, mimeType: resizedMime } =
await resizeImageForVision(result.data, maxDimension);
const dataUrl = `data:${resizedMime};base64,${resizedBuffer.toString("base64")}`;
const part: MessageImagePart = {
type: "image_url",
image_url: { url: dataUrl },
sourceLabel: `[gambar di atas berasal dari link ${url} pada pesan id=${targetId}]`,
};
addImageToMap(imageMap, targetId, part);
} else if (result.type === "text" && result.textContent) {
urlWebTexts.push(`[Isi Web dari ${url}]: ${result.textContent}`);
}
}
function addImageToMap(
imageMap: Map<string, MessageImagePart[]>,
targetId: string,
part: MessageImagePart,
): void {
const existing = imageMap.get(targetId) ?? [];
if (existing.length < 8) {
existing.push(part);
imageMap.set(targetId, existing);
}
}
interface MediaCandidate {
messageId: string;
url: string;
label: string;
stickerName?: string;
customEmojiId?: string;
customEmojiName?: string;
}
function buildMediaCandidates(
targetId: string,
mediaEvidence: ReturnType<typeof extractMessageMediaEvidence>,
): MediaCandidate[] {
return [
...mediaEvidence.stickers
.filter((s) => s.url)
.map(
(s): MediaCandidate => ({
messageId: targetId,
url: s.url,
label: `[gambar di atas adalah sticker "${s.name}" dari pesan id=${targetId}]`,
stickerName: s.name,
}),
),
...mediaEvidence.embeds.flatMap((embed): MediaCandidate[] =>
[
embed.image
? ({
messageId: targetId,
url: embed.image,
label: `[gambar di atas berasal dari embed image pada pesan id=${targetId}]`,
} as MediaCandidate)
: null,
embed.thumbnail
? ({
messageId: targetId,
url: embed.thumbnail,
label: `[gambar di atas berasal dari embed thumbnail pada pesan id=${targetId}]`,
} as MediaCandidate)
: null,
].filter((c): c is MediaCandidate => c !== null),
),
...mediaEvidence.customEmojis.map(
(emoji): MediaCandidate => ({
messageId: targetId,
url: emoji.url,
label: `[gambar di atas adalah custom emoji "${emoji.name}" dari pesan id=${targetId}]`,
customEmojiId: emoji.id,
customEmojiName: emoji.name,
}),
),
];
}
services/discord-gateway/src/modules/ai-moderation/stickerCache.ts
+1 -1
View File
@@ -1,7 +1,7 @@
import { createChildLogger } from "@bete/shared/logger";
import { config } from "../../shared/config/config.js";
import { uploadToTele } from "../attachment-upload/teleUpload.js";
import { executeAll, executeGet } from "../../shared/database/drizzle.js";
import { uploadToTele } from "../attachment-upload/teleUpload.js";
const logger = createChildLogger("sticker-cache");
services/discord-gateway/src/modules/ai-moderation/userReputationStore.ts
+2 -2
View File
@@ -1,9 +1,9 @@
import { eq, and, desc } from "drizzle-orm";
import { and, desc, eq } from "drizzle-orm";
import { getDatabase } from "../../shared/database/drizzle.js";
import {
userReputationsTable,
messagesTable,
UserReputation,
userReputationsTable,
} from "../../shared/database/schema.js";
/**