diff --git a/services/discord-gateway/package.json b/services/discord-gateway/package.json index e10ea27..b27c0c1 100644 --- a/services/discord-gateway/package.json +++ b/services/discord-gateway/package.json @@ -28,7 +28,6 @@ "discord.js-selfbot-v13": "^3.7.1", "dotenv": "^17.4.2", "drizzle-orm": "^0.45.2", - "imghash": "^1.1.4", "ioredis": "^5.11.0", "libsodium-wrappers": "^0.8.4", "lru-cache": "^11.5.1", diff --git a/services/discord-gateway/src/modules/ai-moderation/mediaAnalysisClient.ts b/services/discord-gateway/src/modules/ai-moderation/mediaAnalysisClient.ts index e125be3..5250dbe 100644 --- a/services/discord-gateway/src/modules/ai-moderation/mediaAnalysisClient.ts +++ b/services/discord-gateway/src/modules/ai-moderation/mediaAnalysisClient.ts @@ -6,7 +6,6 @@ */ export { acquireMediaAnalysisLock, - computeImagePhash, deleteCachedMediaAnalysis, getCachedMediaAnalysis, setCachedMediaAnalysis, diff --git a/services/discord-gateway/src/modules/ai-moderation/mediaCache.ts b/services/discord-gateway/src/modules/ai-moderation/mediaCache.ts index d7f075b..e97dc13 100644 --- a/services/discord-gateway/src/modules/ai-moderation/mediaCache.ts +++ b/services/discord-gateway/src/modules/ai-moderation/mediaCache.ts @@ -7,28 +7,22 @@ import { LRUCache } from "lru-cache"; import { acquireMediaAnalysisLock, - computeImagePhash, deleteCachedMediaAnalysis, getCachedMediaAnalysis, - getCachedMediaByPhash, makeCustomEmojiCacheKey, makeImageCacheKey, makeStickerCacheKey, upsertCachedMediaAnalysis, - upsertCachedMediaByPhash, } from "./textCacheStore.js"; export { acquireMediaAnalysisLock, - computeImagePhash, deleteCachedMediaAnalysis, getCachedMediaAnalysis, - getCachedMediaByPhash, makeCustomEmojiCacheKey, makeImageCacheKey, makeStickerCacheKey, upsertCachedMediaAnalysis, - upsertCachedMediaByPhash, }; /** Convenience alias for upsertCachedMediaAnalysis. */ diff --git a/services/discord-gateway/src/modules/ai-moderation/textCacheStore.ts b/services/discord-gateway/src/modules/ai-moderation/textCacheStore.ts index 191d37b..522f319 100644 --- a/services/discord-gateway/src/modules/ai-moderation/textCacheStore.ts +++ b/services/discord-gateway/src/modules/ai-moderation/textCacheStore.ts @@ -62,17 +62,29 @@ export function makeCustomEmojiCacheKey(emojiId: string): string { } /** - * Generate a deterministic cache key for an image data URL. - * Hashes the FULL data URL — only hashing a prefix (e.g. first 128 chars) - * causes hash collisions for images that share the same MIME prefix + - * identical base64 header bytes (common when images are resized to the same - * dimensions), which makes every image incorrectly reuse the same cached - * vision analysis. Hashing the entire data URL guarantees uniqueness per - * actual pixel content. + * Generate a deterministic cache key for an image from its source URL + * (Discord CDN / embed URL / inline URL). + * + * The CDN URL is the stable identity of an attachment: re-analysis of the + * same message (recovery worker, retries) always hits the cache regardless + * of resize/encoding output. Query params are stripped (Discord signed + * tokens `?ex=&is=&hm=` and render variants `?format=&width=`) so the same + * attachment resolves to the same key even when fetched with different + * signatures or sizes. + * + * No SHA/phash — the CDN URL is the cache key itself. This makes + * re-analysis of the SAME attachment cache-hit, while different attachments + * (different URLs) never collide. */ -export function makeImageCacheKey(dataUrl: string): string { - const hash = createHash("sha256").update(dataUrl).digest("hex").slice(0, 16); - return `image:${hash}`; +export function makeImageCacheKey(imageUrl: string): string { + try { + const u = new URL(imageUrl); + u.search = ""; + u.hash = ""; + return `image:${u.toString()}`; + } catch { + return `image:${imageUrl}`; + } } /** @@ -518,64 +530,6 @@ export async function setCachedTextModeration( } } -// --------------------------------------------------------------------------- -// Perceptual hash helpers for image deduplication -// --------------------------------------------------------------------------- - -/** - * Generate a deterministic cache key for a perceptual hash. - * The phash value is a string like "a1b2c3d4e5f6..." from the imghash library. - */ -export function makePhashCacheKey(phash: string): string { - return `phash:${phash.slice(0, 16)}`; -} - -/** - * Look up a cached media analysis by perceptual hash. - * Returns the cached analysis string or null if not found/expired. - */ -export async function getCachedMediaByPhash( - phash: string, -): Promise { - const cacheKey = makePhashCacheKey(phash); - return getCachedMediaAnalysis(cacheKey); -} - -/** - * Store a media analysis result keyed by perceptual hash. - */ -export async function upsertCachedMediaByPhash( - phash: string, - analysisResult: string, - source: "vision_llm", - expiresAt: number, -): Promise { - const cacheKey = makePhashCacheKey(phash); - return upsertCachedMediaAnalysis(cacheKey, analysisResult, source, expiresAt); -} - -/** - * Compute perceptual hash from image buffer using imghash. - * Returns a hexadecimal string representation of the hash. - * Returns null if hashing fails (e.g., invalid image data). - */ -export async function computeImagePhash( - buffer: Buffer, -): Promise { - try { - // Dynamic import — imghash is ESM with a default export containing { hash, hashRaw, ... } - const imghashModule: { - default?: { hash?: (buf: Buffer) => Promise }; - } = await import("imghash"); - const hashFn = imghashModule.default?.hash; - if (typeof hashFn !== "function") return null; - const hash = await hashFn(buffer); - return hash; - } catch { - return null; - } -} - // --------------------------------------------------------------------------- // Corrected Moderation (false-positive) helpers for dynamic few-shot injection // --------------------------------------------------------------------------- diff --git a/services/discord-gateway/src/modules/ai-moderation/visionAnalyzer.ts b/services/discord-gateway/src/modules/ai-moderation/visionAnalyzer.ts index c1b1963..11e723e 100644 --- a/services/discord-gateway/src/modules/ai-moderation/visionAnalyzer.ts +++ b/services/discord-gateway/src/modules/ai-moderation/visionAnalyzer.ts @@ -16,17 +16,14 @@ import type { import { llmVision } from "./llmClient.js"; import { acquireMediaAnalysisLock, - computeImagePhash, deleteCachedMediaAnalysis, FAILED_ANALYSIS_PREFIX, getCachedMediaAnalysis, - getCachedMediaByPhash, inFlightVisionCalls, makeCustomEmojiCacheKey, makeImageCacheKey, makeStickerCacheKey, upsertCachedMediaAnalysis, - upsertCachedMediaByPhash, visionLruCache, } from "./mediaCache.js"; @@ -206,39 +203,6 @@ export const analyzeSingleMediaImage = async ( return FAILED_ANALYSIS_PREFIX; } - // phash check - let phash: string | null = null; - if (image.image_url.url.startsWith("data:")) { - try { - const base64Data = image.image_url.url.split(",")[1]; - if (base64Data) { - const imgBuffer = Buffer.from(base64Data, "base64"); - phash = await computeImagePhash(imgBuffer); - if (phash) { - const phashCached = await getCachedMediaByPhash(phash); - if (phashCached && !isNoImageSeenText(phashCached)) { - visionLruCache.set(cacheKey, phashCached); - await upsertCachedMediaAnalysis( - cacheKey, - phashCached, - "vision_llm", - Date.now() + 24 * 60 * 60 * 1000, - ).catch(() => {}); - return phashCached; - } - if (phashCached) { - log.warn( - { phash, cacheKey }, - "phash cache HIT was no-image-seen — ignoring", - ); - } - } - } - } catch { - phash = null; - } - } - // Vision API call let lastError: Error | null = null; for (let attempt = 0; attempt < 3; attempt++) { @@ -249,7 +213,7 @@ export const analyzeSingleMediaImage = async ( // if the SAME analysis text is being stored for DIFFERENT cache keys // (which would indicate the vision model is returning duplicates). log.debug( - { cacheKey, phash, messageId, contentLen: content.length }, + { cacheKey, messageId, contentLen: content.length }, "Vision analysis cached (new entry)", ); await upsertCachedMediaAnalysis( @@ -259,20 +223,11 @@ export const analyzeSingleMediaImage = async ( Date.now() + 24 * 60 * 60 * 1000, ); visionLruCache.set(cacheKey, content); - if (phash) { - upsertCachedMediaByPhash( - phash, - content, - "vision_llm", - Date.now() + 7 * 24 * 60 * 60 * 1000, - ).catch(() => {}); - } return content; } if (content) { // Model claims it saw no image — same as a null response: NOT a - // valid analysis, and caching it would poison the key for every - // re-analysis of the same image (phash TTL is 7 days). + // valid analysis, and caching it would poison the cache key. log.warn( { messageId, cacheKey }, "Vision returned no-image-seen text — not caching",