import { resolve } from "node:dns/promises";
import { isIP } from "node:net";
import { createChildLogger } from "@/shared/logger/index";
import { createAbortControllerWithTimeout } from "@/shared/utils/index";
const _log = createChildLogger("urlFetcher");
export interface FetchedUrlContext {
url: string;
type: "image" | "text" | "error";
data?: Buffer;
mimeType?: string;
textContent?: string;
/** Page title from og:title /
— strong signal for the LLM. */
title?: string;
error?: string;
}
const MAX_FETCH_SIZE = 5 * 1024 * 1024; // 5 MB
const FETCH_TIMEOUT_MS = 8000;
const URL_REGEX = /https?:\/\/[^\s<]+[^<.,:;"')\]\s]/gi;
/**
* Basic SSRF protection.
* Note: A sophisticated attacker could still use DNS rebinding.
*/
async function isSafeUrl(urlStr: string): Promise {
try {
const parsed = new URL(urlStr);
const host = parsed.hostname;
// Block obvious local IPs/hostnames
if (
host === "localhost" ||
host === "127.0.0.1" ||
host === "::1" ||
host.startsWith("192.168.") ||
host.startsWith("10.") ||
/^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(host)
) {
return false;
}
// Try resolving to check if it resolves to a local IP
if (!isIP(host)) {
try {
const addresses = await resolve(host);
for (const ip of addresses) {
if (
ip === "127.0.0.1" ||
ip.startsWith("192.168.") ||
ip.startsWith("10.") ||
/^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(ip)
) {
return false;
}
}
} catch (_err) {
// If DNS fails, we can't fetch it anyway
return false;
}
}
return true;
} catch (_err) {
return false;
}
}
function extractOgImage(html: string): string | null {
// Look for or
const ogRegex =
/]*(?:property|name)=["'](?:og:image|twitter:image)["'][^>]*content=["']([^"']+)["']/i;
const match = html.match(ogRegex);
if (match?.[1]) {
// Unescape basic HTML entities
return match[1].replace(/&/g, "&").replace(/"/g, '"');
}
// Try reversed attribute order:
const ogRegexRev =
/]*content=["']([^"']+)["'][^>]*(?:property|name)=["'](?:og:image|twitter:image)["']/i;
const matchRev = html.match(ogRegexRev);
if (matchRev?.[1]) {
return matchRev[1].replace(/&/g, "&").replace(/"/g, '"');
}
return null;
}
export interface OgMeta {
title: string | null;
description: string | null;
siteName: string | null;
}
/**
* Extracts OpenGraph / twitter meta + from raw HTML. Both attribute
* orders are accepted ( and reversed).
*/
export function extractOgMeta(html: string): OgMeta {
const metaValue = (name: string): string | null => {
const re = new RegExp(
`]*(?:property|name)=["']${name}["'][^>]*content=["']([^"']+)["']`,
"i",
);
const m = html.match(re);
if (m?.[1]) return m[1].replace(/&/g, "&").replace(/"/g, '"');
const reRev = new RegExp(
`]*content=["']([^"']+)["'][^>]*(?:property|name)=["']${name}["']`,
"i",
);
const mRev = html.match(reRev);
return mRev?.[1]
? mRev[1].replace(/&/g, "&").replace(/"/g, '"')
: null;
};
const title =
metaValue("og:title") ||
metaValue("twitter:title") ||
html.match(/]*>([^<]+)<\/title>/i)?.[1]?.trim() ||
null;
const description =
metaValue("og:description") ||
metaValue("twitter:description") ||
metaValue("description") ||
null;
const siteName =
metaValue("og:site_name") || metaValue("application-name") || null;
return { title, description, siteName };
}
function truncateAndCleanHtml(html: string, maxLen = 1000): string {
// Strip