import { resolve } from "node:dns/promises"; import { isIP } from "node:net"; import { createChildLogger } from "@/shared/logger/index"; import { createAbortControllerWithTimeout } from "@/shared/utils/index"; const _log = createChildLogger("urlFetcher"); export interface FetchedUrlContext { url: string; type: "image" | "text" | "error"; data?: Buffer; mimeType?: string; textContent?: string; error?: string; } const MAX_FETCH_SIZE = 5 * 1024 * 1024; // 5 MB const FETCH_TIMEOUT_MS = 8000; const URL_REGEX = /https?:\/\/[^\s<]+[^<.,:;"')\]\s]/gi; /** * Basic SSRF protection. * Note: A sophisticated attacker could still use DNS rebinding. */ async function isSafeUrl(urlStr: string): Promise { try { const parsed = new URL(urlStr); const host = parsed.hostname; // Block obvious local IPs/hostnames if ( host === "localhost" || host === "127.0.0.1" || host === "::1" || host.startsWith("192.168.") || host.startsWith("10.") || /^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(host) ) { return false; } // Try resolving to check if it resolves to a local IP if (!isIP(host)) { try { const addresses = await resolve(host); for (const ip of addresses) { if ( ip === "127.0.0.1" || ip.startsWith("192.168.") || ip.startsWith("10.") || /^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(ip) ) { return false; } } } catch (_err) { // If DNS fails, we can't fetch it anyway return false; } } return true; } catch (_err) { return false; } } function extractOgImage(html: string): string | null { // Look for or const ogRegex = /]*(?:property|name)=["'](?:og:image|twitter:image)["'][^>]*content=["']([^"']+)["']/i; const match = html.match(ogRegex); if (match?.[1]) { // Unescape basic HTML entities return match[1].replace(/&/g, "&").replace(/"/g, '"'); } // Try reversed attribute order: const ogRegexRev = /]*content=["']([^"']+)["'][^>]*(?:property|name)=["'](?:og:image|twitter:image)["']/i; const matchRev = html.match(ogRegexRev); if (matchRev?.[1]) { return matchRev[1].replace(/&/g, "&").replace(/"/g, '"'); } return null; } function truncateAndCleanHtml(html: string, maxLen = 1000): string { // Strip