feat(gateway): tinyfish web search as fallback when wikipedia misses
- new tinyFishSearch module: GET api.search.tinyfish.ai with X-API-Key, maps top-3 to SearchResult shape, never throws (all failure modes -> []) - wikipediaSearch: on wiki miss, one tinyfish attempt; hits cached 6h under the same key so fallback latency is paid once - termGlossary: on summary miss, top tinyfish hit becomes the definition (persisted permanently like wiki defs); miss keeps 1h sentinel - config: TINYFISH_API_KEY (empty = fallback disabled), ENABLED, BASE_URL, TIMEOUT_MS, LOCATION, LANGUAGE knobs - tests: 6 coverage for disabled/mapping/non-OK/network/bad-json API key NOT committed — set TINYFISH_API_KEY in BWS gmw secrets. Verified: typecheck + lint clean, 216/216 tests pass, live probe 'gubernur jawa barat' returned 3 mapped results
This commit is contained in:
1 parent
9685fa02be
commit
5c133f7302
5 files changed
+303
-1
No files matched your search
@@ -49,6 +49,7 @@ import {
|
||||
scoreWord,
|
||||
WORD_RE,
|
||||
} from "./textSignals.js";
|
||||
import { isTinyFishEnabled, tinyFishSearchLive } from "./tinyFishSearch.js";
|
||||
import { wikipediaSummary } from "./wikipediaClient.js";
|
||||
|
||||
const log = createChildLogger("term-glossary");
|
||||
@@ -212,7 +213,17 @@ async function fetchDefinitionLive(
|
||||
// (up to 4 requests per term under stagger+concurrency-2 — pure
|
||||
// Wikipedia pressure for a miss that gets cached 1h anyway).
|
||||
const result = await wikipediaSummary(key, GLOSSARY_SEARCH_TIMEOUT_MS);
|
||||
const def = result ? buildDefinition(result, term) : null;
|
||||
let def = result ? buildDefinition(result, term) : null;
|
||||
|
||||
// Wikipedia miss (null result, no throw): one TinyFish web-search
|
||||
// attempt — its top hit's snippet becomes the definition. Disabled/
|
||||
// no-key returns [] instantly. A hit is persisted permanently like a
|
||||
// Wikipedia definition; a miss keeps the short-TTL miss sentinel.
|
||||
if (!def && isTinyFishEnabled()) {
|
||||
const fallback = await tinyFishSearchLive(key);
|
||||
const top = fallback[0];
|
||||
if (top) def = buildDefinition(top, term);
|
||||
}
|
||||
|
||||
if (def) {
|
||||
// Persist permanently (definitions rarely change) — best-effort,
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import { createChildLogger } from "@/shared/logger/index";
|
||||
import { createAbortControllerWithTimeout } from "@/shared/utils/index";
|
||||
import { config } from "../../shared/config/config.js";
|
||||
import type { SearchResult } from "./wikipediaClient.js";
|
||||
|
||||
const log = createChildLogger("tinyfish-search");
|
||||
|
||||
/**
|
||||
* Raw TinyFish search result (subset of fields we use).
|
||||
* Endpoint: GET {base}?query=..&location=..&language=.. with X-API-Key.
|
||||
*/
|
||||
interface TinyFishResult {
|
||||
title?: string;
|
||||
url?: string;
|
||||
snippet?: string;
|
||||
date?: string;
|
||||
}
|
||||
|
||||
interface TinyFishResponse {
|
||||
query?: string;
|
||||
results?: TinyFishResult[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback detection: true only when the provider is reachable AND returns a
|
||||
* usable answer. Everything else (missing key, disabled, timeout, non-200,
|
||||
* malformed body, zero results) returns false/null and the caller falls
|
||||
* through to empty enrichment — never throws.
|
||||
*/
|
||||
export function isTinyFishEnabled(): boolean {
|
||||
return (
|
||||
config.TINYFISH_SEARCH_ENABLED === true &&
|
||||
config.TINYFISH_API_KEY.length > 0
|
||||
);
|
||||
}
|
||||
|
||||
function mapResult(r: TinyFishResult): SearchResult | null {
|
||||
const title = (r.title ?? "").trim();
|
||||
const url = (r.url ?? "").trim();
|
||||
if (!title || !url) return null;
|
||||
return {
|
||||
title,
|
||||
url,
|
||||
snippet: (r.snippet ?? "").replace(/\s+/g, " ").trim().slice(0, 500),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Live TinyFish web search for one query. Returns [] on any failure
|
||||
* (disabled, no key, network error, non-OK, bad JSON, no results).
|
||||
* Callers treat [] the same as a Wikipedia miss — enrichment is skipped.
|
||||
*/
|
||||
export async function tinyFishSearchLive(
|
||||
query: string,
|
||||
timeoutMs: number = config.TINYFISH_SEARCH_TIMEOUT_MS,
|
||||
): Promise<SearchResult[]> {
|
||||
const q = query.trim();
|
||||
if (!q) return [];
|
||||
if (!isTinyFishEnabled()) return [];
|
||||
|
||||
const params = new URLSearchParams({
|
||||
query: q,
|
||||
location: config.TINYFISH_SEARCH_LOCATION,
|
||||
language: config.TINYFISH_SEARCH_LANGUAGE,
|
||||
});
|
||||
const { controller, clear } = createAbortControllerWithTimeout(timeoutMs);
|
||||
try {
|
||||
const res = await fetch(
|
||||
`${config.TINYFISH_SEARCH_BASE_URL}?${params.toString()}`,
|
||||
{
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"X-API-Key": config.TINYFISH_API_KEY,
|
||||
},
|
||||
},
|
||||
);
|
||||
if (!res.ok) {
|
||||
log.warn(
|
||||
{ status: res.status, query: q },
|
||||
"TinyFish search failed (non-OK) — skipping fallback",
|
||||
);
|
||||
return [];
|
||||
}
|
||||
let data: TinyFishResponse;
|
||||
try {
|
||||
data = (await res.json()) as TinyFishResponse;
|
||||
} catch {
|
||||
log.warn({ query: q }, "TinyFish search returned invalid JSON");
|
||||
return [];
|
||||
}
|
||||
const results = Array.isArray(data.results) ? data.results : [];
|
||||
const mapped = results
|
||||
.slice(0, 3)
|
||||
.map(mapResult)
|
||||
.filter((r): r is SearchResult => r !== null);
|
||||
log.debug(
|
||||
{ query: q, resultCount: mapped.length },
|
||||
"TinyFish search fallback OK",
|
||||
);
|
||||
return mapped;
|
||||
} catch (err) {
|
||||
log.warn(
|
||||
{ error: err instanceof Error ? err.message : String(err), query: q },
|
||||
"TinyFish search error — skipping fallback",
|
||||
);
|
||||
return [];
|
||||
} finally {
|
||||
clear();
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
isMostlyStopwords,
|
||||
scoreWord,
|
||||
} from "./textSignals.js";
|
||||
import { isTinyFishEnabled, tinyFishSearchLive } from "./tinyFishSearch.js";
|
||||
|
||||
const log = createChildLogger("wikipedia-client");
|
||||
|
||||
@@ -102,6 +103,19 @@ export async function wikipediaSearch(
|
||||
const mapped = await wikipediaSearchLive(q, timeoutMs);
|
||||
if (mapped.length > 0) {
|
||||
cacheSet(cacheKey, JSON.stringify(mapped), SEARCH_CACHE_TTL_SECONDS);
|
||||
return mapped;
|
||||
}
|
||||
|
||||
// Wikipedia miss (empty, not error — errors already returned [] the same
|
||||
// way): one TinyFish web-search attempt as fallback. Disabled/no-key
|
||||
// returns [] instantly; a hit is cached under the same key so the next
|
||||
// batch never pays the fallback latency again.
|
||||
if (isTinyFishEnabled()) {
|
||||
const fallback = await tinyFishSearchLive(q);
|
||||
if (fallback.length > 0) {
|
||||
cacheSet(cacheKey, JSON.stringify(fallback), SEARCH_CACHE_TTL_SECONDS);
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
return mapped;
|
||||
}
|
||||
|
||||
@@ -132,6 +132,22 @@ export const configSchema = z
|
||||
WIKIPEDIA_LANG: z.string().min(1).default("id"),
|
||||
// Per-request timeout (ms) for Wikipedia API calls.
|
||||
WIKIPEDIA_TIMEOUT_MS: z.coerce.number().positive().default(8000),
|
||||
// ── TinyFish web search (fallback when Wikipedia misses) ─────────────
|
||||
// GET {base}?query=..&location=..&language=.. with X-API-Key header.
|
||||
// Empty key = fallback disabled (Wikipedia-only, tests stay offline).
|
||||
TINYFISH_API_KEY: z.string().optional().default(""),
|
||||
TINYFISH_SEARCH_ENABLED: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((v) => v === "true")
|
||||
.default(true),
|
||||
TINYFISH_SEARCH_BASE_URL: z
|
||||
.string()
|
||||
.url()
|
||||
.default("https://api.search.tinyfish.ai"),
|
||||
TINYFISH_SEARCH_TIMEOUT_MS: z.coerce.number().positive().default(10000),
|
||||
TINYFISH_SEARCH_LOCATION: z.string().min(1).default("US"),
|
||||
TINYFISH_SEARCH_LANGUAGE: z.string().min(1).default("en"),
|
||||
// ── Voice PCM WebSocket (direct gateway→backend, bypasses Redis) ────
|
||||
VOICE_PCM_WS_ENABLED: z
|
||||
.string()
|
||||
|
||||
Reference in new issue
Block a user