import { afterEach, beforeEach, expect, test } from 'bun:test'; import { checkUrl, isPrivateAddress, webFetchTool } from '../src/tools-net'; import { htmlToMarkdown } from '../src/markdown'; /** The tool takes its fetch from the call context, so a server stands in for the web. */ const run = (input: unknown, fetchImpl?: typeof globalThis.fetch) => Promise.resolve( webFetchTool.execute!(input as never, { toolCallId: 't1', messages: [], ...(fetchImpl ? { experimental_context: { fetch: fetchImpl } } : {}), } as never), ) as Promise; let server: ReturnType | undefined; afterEach(() => { server?.stop(true); server = undefined; }); beforeEach(() => { server = undefined; }); function serve(routes: Record }>) { server = Bun.serve({ port: 0, fetch(req) { const path = new URL(req.url).pathname; const hit = routes[path]; if (!hit) return new Response('not found', { status: 404 }); return new Response(hit.body, { status: hit.status ?? 200, headers: { 'content-type': hit.type ?? 'text/plain', ...(hit.headers ?? {}) }, }); }, }); return server; } test('loopback and private ranges are recognised', () => { for (const host of [ 'localhost', 'api.localhost', '127.0.0.1', '0.0.0.0', '10.1.2.3', '192.168.0.1', '172.16.0.1', '172.31.255.255', '169.254.169.254', '100.64.0.1', '::1', '[::1]', 'fd00::1', 'fe80::1', ]) { expect(isPrivateAddress(host), host).toBe(true); } }); test('public addresses are not treated as private', () => { for (const host of ['example.com', '8.8.8.8', '172.32.0.1', '11.0.0.1', '2606:4700::1111']) { expect(isPrivateAddress(host), host).toBe(false); } }); test('only http and https are fetched', () => { for (const url of ['file:///etc/passwd', 'data:text/html,x', 'ftp://example.com/x']) { const checked = checkUrl(url); expect(checked.ok, url).toBe(false); } }); test('plain http to a public host is refused, https is accepted', () => { expect(checkUrl('http://example.com').ok).toBe(false); expect(checkUrl('https://example.com').ok).toBe(true); }); test('https to a private host is refused', () => { // The metadata endpoint is the reason this exists: a URL from model output must // not be able to read cloud credentials. expect(checkUrl('https://169.254.169.254/latest/meta-data/').ok).toBe(false); expect(checkUrl('https://localhost/admin').ok).toBe(false); }); test('http to localhost is allowed, so a local dev server can be read', () => { expect(checkUrl('http://localhost:3000/docs').ok).toBe(true); }); test('a malformed URL is reported rather than fetched', () => { const checked = checkUrl('not a url'); expect(checked.ok).toBe(false); if (!checked.ok) expect(checked.reason).toContain('not a URL'); }); test('a refused URL never reaches the network', async () => { let called = 0; const spy = (async () => { called++; return new Response('x'); }) as unknown as typeof globalThis.fetch; expect(run({ url: 'https://169.254.169.254/' }, spy)).rejects.toThrow(/private or loopback/); await Bun.sleep(20); expect(called).toBe(0); }); test('plain text comes back with the URL that served it', async () => { const s = serve({ '/doc.txt': { body: 'the answer is 42' } }); const out = await run({ url: `${s.url}doc.txt` }); expect(out).toContain('doc.txt'); expect(out).toContain('the answer is 42'); }); test('html is converted, and script and nav are dropped', async () => { const html = `

Install

Run bun install first.

copyright
`; const s = serve({ '/page': { body: html, type: 'text/html; charset=utf-8' } }); const out = await run({ url: `${s.url}page` }); expect(out).toContain('# Install'); expect(out).toContain('`bun install`'); expect(out).not.toContain('window.tracker'); expect(out).not.toContain('color:red'); expect(out).not.toContain('Home'); expect(out).not.toContain('copyright'); }); test('a non-2xx status is reported with the code', async () => { const s = serve({ '/gone': { body: 'nope', status: 503 } }); expect(run({ url: `${s.url}gone` })).rejects.toThrow(/503/); }); test('a binary content type is refused rather than returned as mojibake', async () => { const s = serve({ '/blob': { body: 'PK\u0003\u0004', type: 'application/zip' } }); expect(run({ url: `${s.url}blob` })).rejects.toThrow(/not text/); }); test('a redirect is followed and both URLs are shown', async () => { const s = serve({ '/old': { body: '', status: 302, headers: { location: '/new' } }, '/new': { body: 'moved content' }, }); const out = await run({ url: `${s.url}old` }); expect(out).toContain('->'); expect(out).toContain('moved content'); }); test('a redirect to a private host is refused mid-chain', async () => { // redirect: 'follow' would let a public URL walk to the metadata endpoint with // the check already passed, which is why every hop is re-checked. const s = serve({ '/trap': { body: '', status: 302, headers: { location: 'https://169.254.169.254/latest/' } }, }); expect(run({ url: `${s.url}trap` })).rejects.toThrow(/refused URL/); }); test('a redirect loop stops rather than hanging', async () => { const s = serve({ '/loop': { body: '', status: 302, headers: { location: '/loop' } } }); expect(run({ url: `${s.url}loop` })).rejects.toThrow(/redirects/); }); test('maxChars truncates with a count', async () => { const s = serve({ '/long': { body: 'x'.repeat(5000) } }); const out = await run({ url: `${s.url}long`, maxChars: 500 }); expect(out).toContain('truncated'); expect(out.length).toBeLessThan(1200); }); test('an oversized body is capped and the cap is stated', async () => { const s = serve({ '/huge': { body: 'y'.repeat(700 * 1024) } }); const out = await run({ url: `${s.url}huge` }); expect(out).toContain('capped at'); }); test('entities are decoded so code samples read correctly', () => { const out = htmlToMarkdown('

use <div> & "quotes" 'here'

'); expect(out).toBe(`use
& "quotes" 'here'`); }); test('a pre block becomes a fenced block', () => { const out = htmlToMarkdown('
const a = 1;\nconst b = 2;
'); expect(out).toContain('```'); expect(out).toContain('const a = 1;'); expect(out).toContain('const b = 2;'); }); test('list items become bullets', () => { const out = htmlToMarkdown('
  • first
  • second
'); expect(out).toContain('- first'); expect(out).toContain('- second'); }); test('headings keep their level', () => { expect(htmlToMarkdown('

Third

')).toBe('### Third'); });