MythEclipse/GMW
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: rombak cache AI analisis image — pakai CDN URL langsung, hapus phash+sha

Browse Source 
- Cache key image = CDN URL (query params stripped), bukan SHA data URL
  → re-analysis SAME attachment selalu cache-hit, berbeda attachment tidak kolisi
- Hapus perceptual hash (imghash dep + phash get/upsert/compute) sepenuhnya
- Hapus makeImageCacheKey hashing, ganti makeImageCacheKey yang return CDN URL
- textCacheStore, visionAnalyzer, mediaCache, mediaAnalysisClient updated
- imghash dependency removed from package.json
- Purge 82 stale cache rows (image: + phash:) dari DB
This commit is contained in:
asepharyana
2026-08-12 22:31:08 +07:00
parent 37787cc4f0
commit d3cb5f6756
5 changed files with 24 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/discord-gateway/package.json
-1
View File
@@ -28,7 +28,6 @@
"discord.js-selfbot-v13": "^3.7.1",
"dotenv": "^17.4.2",
"drizzle-orm": "^0.45.2",
"imghash": "^1.1.4",
"ioredis": "^5.11.0",
"libsodium-wrappers": "^0.8.4",
"lru-cache": "^11.5.1",
services/discord-gateway/src/modules/ai-moderation/mediaAnalysisClient.ts
-1
View File
@@ -6,7 +6,6 @@
*/
export {
acquireMediaAnalysisLock,
computeImagePhash,
deleteCachedMediaAnalysis,
getCachedMediaAnalysis,
setCachedMediaAnalysis,
services/discord-gateway/src/modules/ai-moderation/mediaCache.ts
-6
View File
@@ -7,28 +7,22 @@
import { LRUCache } from "lru-cache";
import {
acquireMediaAnalysisLock,
computeImagePhash,
deleteCachedMediaAnalysis,
getCachedMediaAnalysis,
getCachedMediaByPhash,
makeCustomEmojiCacheKey,
makeImageCacheKey,
makeStickerCacheKey,
upsertCachedMediaAnalysis,
upsertCachedMediaByPhash,
} from "./textCacheStore.js";
export {
acquireMediaAnalysisLock,
computeImagePhash,
deleteCachedMediaAnalysis,
getCachedMediaAnalysis,
getCachedMediaByPhash,
makeCustomEmojiCacheKey,
makeImageCacheKey,
makeStickerCacheKey,
upsertCachedMediaAnalysis,
upsertCachedMediaByPhash,
};
/** Convenience alias for upsertCachedMediaAnalysis. */
services/discord-gateway/src/modules/ai-moderation/textCacheStore.ts
+22 -68
View File
@@ -62,17 +62,29 @@ export function makeCustomEmojiCacheKey(emojiId: string): string {
}
/**
* Generate a deterministic cache key for an image data URL.
* Hashes the FULL data URL — only hashing a prefix (e.g. first 128 chars)
* causes hash collisions for images that share the same MIME prefix +
* identical base64 header bytes (common when images are resized to the same
* dimensions), which makes every image incorrectly reuse the same cached
* vision analysis. Hashing the entire data URL guarantees uniqueness per
* actual pixel content.
* Generate a deterministic cache key for an image from its source URL
* (Discord CDN / embed URL / inline URL).
*
* The CDN URL is the stable identity of an attachment: re-analysis of the
* same message (recovery worker, retries) always hits the cache regardless
* of resize/encoding output. Query params are stripped (Discord signed
* tokens `?ex=&is=&hm=` and render variants `?format=&width=`) so the same
* attachment resolves to the same key even when fetched with different
* signatures or sizes.
*
* No SHA/phash — the CDN URL is the cache key itself. This makes
* re-analysis of the SAME attachment cache-hit, while different attachments
* (different URLs) never collide.
*/
export function makeImageCacheKey(dataUrl: string): string {
const hash = createHash("sha256").update(dataUrl).digest("hex").slice(0, 16);
return `image:${hash}`;
export function makeImageCacheKey(imageUrl: string): string {
try {
const u = new URL(imageUrl);
u.search = "";
u.hash = "";
return `image:${u.toString()}`;
} catch {
return `image:${imageUrl}`;
}
}
/**
@@ -518,64 +530,6 @@ export async function setCachedTextModeration(
}
}
// ---------------------------------------------------------------------------
// Perceptual hash helpers for image deduplication
// ---------------------------------------------------------------------------
/**
* Generate a deterministic cache key for a perceptual hash.
* The phash value is a string like "a1b2c3d4e5f6..." from the imghash library.
*/
export function makePhashCacheKey(phash: string): string {
return `phash:${phash.slice(0, 16)}`;
}
/**
* Look up a cached media analysis by perceptual hash.
* Returns the cached analysis string or null if not found/expired.
*/
export async function getCachedMediaByPhash(
phash: string,
): Promise<string | null> {
const cacheKey = makePhashCacheKey(phash);
return getCachedMediaAnalysis(cacheKey);
}
/**
* Store a media analysis result keyed by perceptual hash.
*/
export async function upsertCachedMediaByPhash(
phash: string,
analysisResult: string,
source: "vision_llm",
expiresAt: number,
): Promise<void> {
const cacheKey = makePhashCacheKey(phash);
return upsertCachedMediaAnalysis(cacheKey, analysisResult, source, expiresAt);
}
/**
* Compute perceptual hash from image buffer using imghash.
* Returns a hexadecimal string representation of the hash.
* Returns null if hashing fails (e.g., invalid image data).
*/
export async function computeImagePhash(
buffer: Buffer,
): Promise<string | null> {
try {
// Dynamic import — imghash is ESM with a default export containing { hash, hashRaw, ... }
const imghashModule: {
default?: { hash?: (buf: Buffer) => Promise<string> };
} = await import("imghash");
const hashFn = imghashModule.default?.hash;
if (typeof hashFn !== "function") return null;
const hash = await hashFn(buffer);
return hash;
} catch {
return null;
}
}
// ---------------------------------------------------------------------------
// Corrected Moderation (false-positive) helpers for dynamic few-shot injection
// ---------------------------------------------------------------------------
services/discord-gateway/src/modules/ai-moderation/visionAnalyzer.ts
+2 -47
View File
@@ -16,17 +16,14 @@ import type {
import { llmVision } from "./llmClient.js";
import {
acquireMediaAnalysisLock,
computeImagePhash,
deleteCachedMediaAnalysis,
FAILED_ANALYSIS_PREFIX,
getCachedMediaAnalysis,
getCachedMediaByPhash,
inFlightVisionCalls,
makeCustomEmojiCacheKey,
makeImageCacheKey,
makeStickerCacheKey,
upsertCachedMediaAnalysis,
upsertCachedMediaByPhash,
visionLruCache,
} from "./mediaCache.js";
@@ -206,39 +203,6 @@ export const analyzeSingleMediaImage = async (
return FAILED_ANALYSIS_PREFIX;
}
// phash check
let phash: string | null = null;
if (image.image_url.url.startsWith("data:")) {
try {
const base64Data = image.image_url.url.split(",")[1];
if (base64Data) {
const imgBuffer = Buffer.from(base64Data, "base64");
phash = await computeImagePhash(imgBuffer);
if (phash) {
const phashCached = await getCachedMediaByPhash(phash);
if (phashCached && !isNoImageSeenText(phashCached)) {
visionLruCache.set(cacheKey, phashCached);
await upsertCachedMediaAnalysis(
cacheKey,
phashCached,
"vision_llm",
Date.now() + 24 * 60 * 60 * 1000,
).catch(() => {});
return phashCached;
}
if (phashCached) {
log.warn(
{ phash, cacheKey },
"phash cache HIT was no-image-seen — ignoring",
);
}
}
}
} catch {
phash = null;
}
}
// Vision API call
let lastError: Error | null = null;
for (let attempt = 0; attempt < 3; attempt++) {
@@ -249,7 +213,7 @@ export const analyzeSingleMediaImage = async (
// if the SAME analysis text is being stored for DIFFERENT cache keys
// (which would indicate the vision model is returning duplicates).
log.debug(
{ cacheKey, phash, messageId, contentLen: content.length },
{ cacheKey, messageId, contentLen: content.length },
"Vision analysis cached (new entry)",
);
await upsertCachedMediaAnalysis(
@@ -259,20 +223,11 @@ export const analyzeSingleMediaImage = async (
Date.now() + 24 * 60 * 60 * 1000,
);
visionLruCache.set(cacheKey, content);
if (phash) {
upsertCachedMediaByPhash(
phash,
content,
"vision_llm",
Date.now() + 7 * 24 * 60 * 60 * 1000,
).catch(() => {});
}
return content;
}
if (content) {
// Model claims it saw no image — same as a null response: NOT a
// valid analysis, and caching it would poison the key for every
// re-analysis of the same image (phash TTL is 7 days).
// valid analysis, and caching it would poison the cache key.
log.warn(
{ messageId, cacheKey },
"Vision returned no-image-seen text — not caching",