feat(gateway): tinyfish web search as fallback when wikipedia misses

- new tinyFishSearch module: GET api.search.tinyfish.ai with X-API-Key,
  maps top-3 to SearchResult shape, never throws (all failure modes -> [])
- wikipediaSearch: on wiki miss, one tinyfish attempt; hits cached 6h
  under the same key so fallback latency is paid once
- termGlossary: on summary miss, top tinyfish hit becomes the definition
  (persisted permanently like wiki defs); miss keeps 1h sentinel
- config: TINYFISH_API_KEY (empty = fallback disabled), ENABLED,
  BASE_URL, TIMEOUT_MS, LOCATION, LANGUAGE knobs
- tests: 6 coverage for disabled/mapping/non-OK/network/bad-json

API key NOT committed — set TINYFISH_API_KEY in BWS gmw secrets.

Verified: typecheck + lint clean, 216/216 tests pass, live probe
'gubernur jawa barat' returned 3 mapped results
This commit is contained in:
asepharyana committed 2026-09-14 00:44:38 +07:00
1 parent 9685fa02be
commit 5c133f7302
5 files changed
+303 -1

No files matched your search

@@ -49,6 +49,7 @@ import {
scoreWord,
WORD_RE,
} from "./textSignals.js";
import { isTinyFishEnabled, tinyFishSearchLive } from "./tinyFishSearch.js";
import { wikipediaSummary } from "./wikipediaClient.js";
const log = createChildLogger("term-glossary");
@@ -212,7 +213,17 @@ async function fetchDefinitionLive(
// (up to 4 requests per term under stagger+concurrency-2 — pure
// Wikipedia pressure for a miss that gets cached 1h anyway).
const result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
const def = result ? buildDefinition(result, term) : null;
let def = result ? buildDefinition(result, term) : null;
// Wikipedia miss (null result, no throw): one TinyFish web-search
// attempt — its top hit's snippet becomes the definition. Disabled/
// no-key returns [] instantly. A hit is persisted permanently like a
// Wikipedia definition; a miss keeps the short-TTL miss sentinel.
if (!def && isTinyFishEnabled()) {
const fallback = await tinyFishSearchLive(key);
const top = fallback[0];
if (top) def = buildDefinition(top, term);
}
if (def) {
// Persist permanently (definitions rarely change) — best-effort,
@@ -0,0 +1,111 @@
import { createChildLogger } from "@/shared/logger/index";
import { createAbortControllerWithTimeout } from "@/shared/utils/index";
import { config } from "../../shared/config/config.js";
import type { SearchResult } from "./wikipediaClient.js";
const log = createChildLogger("tinyfish-search");
/**
* Raw TinyFish search result (subset of fields we use).
* Endpoint: GET {base}?query=..&location=..&language=.. with X-API-Key.
*/
interface TinyFishResult {
title?: string;
url?: string;
snippet?: string;
date?: string;
}
interface TinyFishResponse {
query?: string;
results?: TinyFishResult[];
}
/**
* Fallback detection: true only when the provider is reachable AND returns a
* usable answer. Everything else (missing key, disabled, timeout, non-200,
* malformed body, zero results) returns false/null and the caller falls
* through to empty enrichment — never throws.
*/
export function isTinyFishEnabled(): boolean {
return (
config.TINYFISH_SEARCH_ENABLED === true &&
config.TINYFISH_API_KEY.length > 0
);
}
function mapResult(r: TinyFishResult): SearchResult | null {
const title = (r.title ?? "").trim();
const url = (r.url ?? "").trim();
if (!title || !url) return null;
return {
title,
url,
snippet: (r.snippet ?? "").replace(/\s+/g, " ").trim().slice(0, 500),
};
}
/**
* Live TinyFish web search for one query. Returns [] on any failure
* (disabled, no key, network error, non-OK, bad JSON, no results).
* Callers treat [] the same as a Wikipedia miss — enrichment is skipped.
*/
export async function tinyFishSearchLive(
query: string,
timeoutMs: number = config.TINYFISH_SEARCH_TIMEOUT_MS,
): Promise<SearchResult[]> {
const q = query.trim();
if (!q) return [];
if (!isTinyFishEnabled()) return [];
const params = new URLSearchParams({
query: q,
location: config.TINYFISH_SEARCH_LOCATION,
language: config.TINYFISH_SEARCH_LANGUAGE,
});
const { controller, clear } = createAbortControllerWithTimeout(timeoutMs);
try {
const res = await fetch(
`${config.TINYFISH_SEARCH_BASE_URL}?${params.toString()}`,
{
signal: controller.signal,
headers: {
Accept: "application/json",
"X-API-Key": config.TINYFISH_API_KEY,
},
},
);
if (!res.ok) {
log.warn(
{ status: res.status, query: q },
"TinyFish search failed (non-OK) — skipping fallback",
);
return [];
}
let data: TinyFishResponse;
try {
data = (await res.json()) as TinyFishResponse;
} catch {
log.warn({ query: q }, "TinyFish search returned invalid JSON");
return [];
}
const results = Array.isArray(data.results) ? data.results : [];
const mapped = results
.slice(0, 3)
.map(mapResult)
.filter((r): r is SearchResult => r !== null);
log.debug(
{ query: q, resultCount: mapped.length },
"TinyFish search fallback OK",
);
return mapped;
} catch (err) {
log.warn(
{ error: err instanceof Error ? err.message : String(err), query: q },
"TinyFish search error — skipping fallback",
);
return [];
} finally {
clear();
}
}
@@ -28,6 +28,7 @@ import {
isMostlyStopwords,
scoreWord,
} from "./textSignals.js";
import { isTinyFishEnabled, tinyFishSearchLive } from "./tinyFishSearch.js";
const log = createChildLogger("wikipedia-client");
@@ -102,6 +103,19 @@ export async function wikipediaSearch(
const mapped = await wikipediaSearchLive(q, timeoutMs);
if (mapped.length > 0) {
cacheSet(cacheKey, JSON.stringify(mapped), SEARCH_CACHE_TTL_SECONDS);
return mapped;
}
// Wikipedia miss (empty, not error — errors already returned [] the same
// way): one TinyFish web-search attempt as fallback. Disabled/no-key
// returns [] instantly; a hit is cached under the same key so the next
// batch never pays the fallback latency again.
if (isTinyFishEnabled()) {
const fallback = await tinyFishSearchLive(q);
if (fallback.length > 0) {
cacheSet(cacheKey, JSON.stringify(fallback), SEARCH_CACHE_TTL_SECONDS);
return fallback;
}
}
return mapped;
}
@@ -132,6 +132,22 @@ export const configSchema = z
WIKIPEDIA_LANG: z.string().min(1).default("id"),
// Per-request timeout (ms) for Wikipedia API calls.
WIKIPEDIA_TIMEOUT_MS: z.coerce.number().positive().default(8000),
// ── TinyFish web search (fallback when Wikipedia misses) ─────────────
// GET {base}?query=..&location=..&language=.. with X-API-Key header.
// Empty key = fallback disabled (Wikipedia-only, tests stay offline).
TINYFISH_API_KEY: z.string().optional().default(""),
TINYFISH_SEARCH_ENABLED: z
.string()
.optional()
.transform((v) => v === "true")
.default(true),
TINYFISH_SEARCH_BASE_URL: z
.string()
.url()
.default("https://api.search.tinyfish.ai"),
TINYFISH_SEARCH_TIMEOUT_MS: z.coerce.number().positive().default(10000),
TINYFISH_SEARCH_LOCATION: z.string().min(1).default("US"),
TINYFISH_SEARCH_LANGUAGE: z.string().min(1).default("en"),
// ── Voice PCM WebSocket (direct gateway→backend, bypasses Redis) ────
VOICE_PCM_WS_ENABLED: z
.string()