import { afterEach, beforeEach, expect, test } from 'bun:test'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { streamText, tool } from 'ai'; import { z } from 'zod'; import { resolveModel, type Config } from '../src/config'; import { fallbackChainOf, type FallbackEvent } from '../src/fallback'; let home: string; const saved: Record = {}; const ENV_KEYS = ['SHIRO_HOME', 'SHIRO_API_KEY', 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY']; beforeEach(() => { for (const k of ENV_KEYS) { saved[k] = process.env[k]; delete process.env[k]; } home = mkdtempSync(join(tmpdir(), 'shiro-live-')); process.env['SHIRO_HOME'] = home; }); afterEach(() => { for (const k of ENV_KEYS) { if (saved[k] === undefined) delete process.env[k]; else process.env[k] = saved[k]; } rmSync(home, { recursive: true, force: true }); }); const sse = (o: unknown) => `data: ${JSON.stringify(o)}\n\n`; const chatDone = [ sse({ id: '1', choices: [{ index: 0, delta: { role: 'assistant', content: 'chat path' } }] }), sse({ id: '1', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage: { prompt_tokens: 4, completion_tokens: 2 } }), 'data: [DONE]\n\n', ].join(''); const REJECT_BODY = JSON.stringify({ error: { message: "Function tools with reasoning_effort are not supported for gpt-5.6-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.", type: 'invalid_request_error', param: 'reasoning_effort', code: null, }, }); /** Minimal /v1/responses SSE conversation ending in a completed response. */ const responsesDone = [ sse({ type: 'response.created', response: { id: 'resp_1', created_at: 1, model: 'gpt-5.6-sol', object: 'response', status: 'in_progress', output: [] }, }), sse({ type: 'response.output_item.added', output_index: 0, item: { type: 'message', id: 'msg_1', role: 'assistant', status: 'in_progress', content: [] } }), sse({ type: 'response.content_part.added', item_id: 'msg_1', output_index: 0, content_index: 0, part: { type: 'output_text', text: '' } }), sse({ type: 'response.output_text.delta', item_id: 'msg_1', output_index: 0, content_index: 0, delta: 'responses path' }), sse({ type: 'response.output_text.done', item_id: 'msg_1', output_index: 0, content_index: 0, text: 'responses path' }), sse({ type: 'response.content_part.done', item_id: 'msg_1', output_index: 0, content_index: 0, part: { type: 'output_text', text: 'responses path' } }), sse({ type: 'response.output_item.done', output_index: 0, item: { type: 'message', id: 'msg_1', role: 'assistant', status: 'completed', content: [{ type: 'output_text', text: 'responses path' }] } }), sse({ type: 'response.completed', response: { id: 'resp_1', created_at: 1, model: 'gpt-5.6-sol', object: 'response', status: 'completed', output: [{ type: 'message', id: 'msg_1', role: 'assistant', status: 'completed', content: [{ type: 'output_text', text: 'responses path' }] }], usage: { input_tokens: 5, output_tokens: 3 }, }, }), ].join(''); const eventStream = (body: string) => new Response(body, { headers: { 'content-type': 'text/event-stream' } }); async function collect(model: Parameters[0]['model'], signalTools: boolean) { const result = streamText({ model, prompt: 'hi', maxRetries: 0, ...(signalTools ? { tools: { ping: tool({ description: 'ping', inputSchema: z.object({}) }) } } : {}), }); void result.responseMessages.then(undefined, () => {}); void result.usage.then(undefined, () => {}); let text = ''; for await (const part of result.stream) if (part.type === 'text-delta') text += part.text; return text; } test('a chat-completions rejection of function tools switches to /v1/responses', async () => { const hits: string[] = []; const server = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch: (req) => { const path = new URL(req.url).pathname; hits.push(path); if (path === '/v1/chat/completions') { return new Response(REJECT_BODY, { status: 400, headers: { 'content-type': 'application/json' } }); } return eventStream(responsesDone); }, }); const events: FallbackEvent[] = []; const cfg: Config = { provider: 'openai', model: 'gpt-5.6-sol', apiKey: 'sk-test', baseURL: `http://127.0.0.1:${server.port}/v1`, }; // resolveModel only chains the fallback for api.openai.com, so drive the wrapper // the same way production does but pointed at the local server. const { withFallback } = await import('../src/fallback'); const { createOpenAI } = await import('@ai-sdk/openai'); const { createOpenAICompatible } = await import('@ai-sdk/openai-compatible'); const chat = createOpenAICompatible({ name: 'openai', apiKey: cfg.apiKey!, baseURL: cfg.baseURL! })(cfg.model); const responses = createOpenAI({ apiKey: cfg.apiKey!, baseURL: cfg.baseURL! }).responses(cfg.model); const text = await collect(withFallback([chat, responses], (e) => events.push(e)), true); server.stop(true); expect(text).toBe('responses path'); expect(hits).toEqual(['/v1/chat/completions', '/v1/responses']); expect(events).toHaveLength(1); expect(events[0]?.reason).toContain('400'); }); test('when chat-completions works, /v1/responses is never touched', async () => { const hits: string[] = []; const server = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch: (req) => { hits.push(new URL(req.url).pathname); return eventStream(chatDone); }, }); const { withFallback } = await import('../src/fallback'); const { createOpenAI } = await import('@ai-sdk/openai'); const { createOpenAICompatible } = await import('@ai-sdk/openai-compatible'); const base = `http://127.0.0.1:${server.port}/v1`; const chat = createOpenAICompatible({ name: 'openai', apiKey: 'sk', baseURL: base })('gpt-5'); const responses = createOpenAI({ apiKey: 'sk', baseURL: base }).responses('gpt-5'); const events: FallbackEvent[] = []; const text = await collect(withFallback([chat, responses], (e) => events.push(e)), false); server.stop(true); expect(text).toBe('chat path'); expect(hits).toEqual(['/v1/chat/completions']); expect(events).toEqual([]); }); test('official OpenAI gets the chat -> responses chain', () => { const model = resolveModel({ provider: 'openai', model: 'gpt-5.6-sol', apiKey: 'sk-x' }); expect(fallbackChainOf(model)).toEqual(['openai.chat/gpt-5.6-sol', 'openai.responses/gpt-5.6-sol']); }); test('a non-OpenAI endpoint gets a plain chat model with no fallback chain', () => { const model = resolveModel({ provider: 'openai', model: 'llama-3.3', apiKey: 'gsk_x', baseURL: 'https://api.groq.com/openai/v1', }); expect(typeof model).not.toBe('string'); expect(fallbackChainOf(model)).toBeUndefined(); }); test('anthropic is left alone: no responses endpoint exists there', () => { const model = resolveModel({ provider: 'anthropic', model: 'claude-sonnet-4-5', apiKey: 'sk-ant' }); expect(fallbackChainOf(model)).toBeUndefined(); }); test('the SDK retries a retryable failure the configured number of times', async () => { let attempts = 0; const server = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch: () => { attempts++; if (attempts <= 2) return new Response('overloaded', { status: 503 }); return eventStream(chatDone); }, }); const { createOpenAICompatible } = await import('@ai-sdk/openai-compatible'); const model = createOpenAICompatible({ name: 'openai', apiKey: 'sk', baseURL: `http://127.0.0.1:${server.port}/v1`, })('gpt-5'); const result = streamText({ model, prompt: 'hi', maxRetries: 3 }); void result.responseMessages.then(undefined, () => {}); void result.usage.then(undefined, () => {}); let text = ''; for await (const part of result.stream) if (part.type === 'text-delta') text += part.text; server.stop(true); expect(attempts).toBe(3); expect(text).toBe('chat path'); }, 20_000);