feat: add STT support, Gemini TTS, and expand usage tracking

- Speech-to-Text: full pipeline with sttCore handler, /v1/audio/transcriptions endpoint, sttConfig for OpenAI, Gemini, Groq, Deepgram, AssemblyAI, HuggingFace, NVIDIA Parakeet; new 9router-stt skill - Gemini TTS: add gemini provider with 30 prebuilt voices and TTS_PROVIDER_CONFIG - Usage: implement GLM (intl/cn) and MiniMax (intl/cn) quota fetchers; refactor Gemini CLI usage to use retrieveUserQuota with per-model buckets - Disabled models: lowdb-backed disabledModelsDb + /api/models/disabled route - Header search: reusable Zustand store (headerSearchStore) wired into Header - CLI tools: add Claude Cowork tool card and cowork-settings API - Providers: introduce mediaPriority sorting in getProvidersByKind, add Kimi K2.6, reorder hermes, drop qwen STT kind - UI: expand media-providers/[kind]/[id] page (+314), enhance OAuthModal, ModelSelectModal, ProviderTopology, ProxyPools, ProviderLimits - Assets: refresh provider PNGs (alicode, byteplus, cloudflare-ai, nvidia, ollama, vertex, volcengine-ark) and add aws-polly, fal-ai, jina-ai, recraft, runwayml, stability-ai, topaz, black-forest-labs
2026-05-05 10:32:59 +07:00
1 parent bfb7d42164
commit d4bc42e1f5
@@ -12,6 +12,7 @@ import ConnectionsCard from "@/app/(dashboard)/dashboard/providers/components/Co
 import ModelsCard from "@/app/(dashboard)/dashboard/providers/components/ModelsCard";
 import { TTS_PROVIDER_CONFIG } from "@/shared/constants/ttsProviders";
 import { getTtsVoicesForModel } from "open-sse/config/ttsModels.js";
+import { GOOGLE_TTS_LANGUAGES } from "open-sse/config/googleTtsLanguages.js";

 // Shared row layout — defined outside components to avoid re-mount on re-render
 function Row({ label, children }) {
@@ -92,13 +93,6 @@ const KIND_EXAMPLE_CONFIG = {
    extraBody: { prompt: "Describe this image in detail" },
    defaultResponse: `{\n  "text": "A cat sitting on a windowsill...",\n  "model": "..."\n}`,
  },
-  stt: {
-    inputLabel: "Audio URL",
-    inputPlaceholder: "https://example.com/audio.mp3",
-    defaultInput: "",
-    bodyKey: "url",
-    defaultResponse: `{\n  "text": "Hello world...",\n  "model": "..."\n}`,
-  },
  video: {
    inputLabel: "Prompt",
    inputPlaceholder: "A serene lake at sunset",
@@ -394,6 +388,8 @@ function TtsExampleCard({ providerId }) {
  const [modalSearch, setModalSearch]       = useState("");
  const [modalError, setModalError]         = useState("");
  const [byLang, setByLang]                 = useState({});
+  // Language hint (e.g. Gemini): controls the spoken language without affecting voice selection
+  const [languageHint, setLanguageHint]     = useState("");

  useEffect(() => {
    setLocalEndpoint(window.location.origin);
@@ -514,10 +510,15 @@ function TtsExampleCard({ providerId }) {
    return "";
  })();

+  const ttsBody = (() => {
+    const b = { model: modelFull, input };
+    if (config.hasLanguageHint && languageHint) b.language = languageHint;
+    return b;
+  })();
  const curlSnippet = `curl -X POST ${endpoint}/v1/audio/speech${responseFormat === "json" ? "?response_format=json" : ""} \\
  -H "Content-Type: application/json" \\
  -H "Authorization: Bearer ${apiKey || "YOUR_KEY"}" \\
-  -d '{"model": "${modelFull}", "input": "${input}"}' \\
+  -d '${JSON.stringify(ttsBody)}' \\
  ${responseFormat === "json" ? "" : "--output speech.mp3"}`;

  const handleRun = async () => {
@@ -534,7 +535,7 @@ function TtsExampleCard({ providerId }) {
      const res = await fetch(url, {
        method: "POST",
        headers,
-        body: JSON.stringify({ model: modelFull, input: input.trim() }),
+        body: JSON.stringify({ ...ttsBody, input: input.trim() }),
      });
      setLatency(Date.now() - start);
      if (!res.ok) {
@@ -608,6 +609,22 @@ function TtsExampleCard({ providerId }) {
            </Row>
          )}

+          {/* Language hint dropdown (Gemini) — sends body.language to guide pronunciation */}
+          {config.hasLanguageHint && (
+            <Row label="Language">
+              <select
+                value={languageHint}
+                onChange={(e) => setLanguageHint(e.target.value)}
+                className="w-full px-3 py-1.5 text-sm border border-border rounded-lg bg-background focus:outline-none focus:border-primary"
+              >
+                <option value="">Auto-detect</option>
+                {GOOGLE_TTS_LANGUAGES.map((l) => (
+                  <option key={l.id} value={l.name}>{l.name}</option>
+                ))}
+              </select>
+            </Row>
+          )}
+
          {/* Language row + Browse button (edge-tts, local-device, elevenlabs) */}
          {config.hasBrowseButton && (
            <Row label="Language">
@@ -886,7 +903,7 @@ function GenericExampleCard({ providerId, kind }) {
  // Get models for this kind (e.g., type="image")
  const kindModels = getModelsByProviderId(providerId).filter((m) => m.type === kind);
  // Kinds that need a model identifier in the request (image/video/music)
-  const KIND_NEEDS_MODEL = new Set(["image", "video", "music", "stt", "imageToText"]);
+  const KIND_NEEDS_MODEL = new Set(["image", "video", "music", "imageToText"]);
  const needsModel = KIND_NEEDS_MODEL.has(kind);
  const allowManualModel = needsModel && kindModels.length === 0;
  const [selectedModel, setSelectedModel] = useState(kindModels[0]?.id ?? "");
@@ -1344,6 +1361,288 @@ function GenericExampleCard({ providerId, kind }) {
  );
 }

+// ─── STT Example Card ────────────────────────────────────────────────────────
+function SttExampleCard({ providerId }) {
+  const providerAlias = getProviderAlias(providerId);
+  const builtinSttModels = getModelsByProviderId(providerId).filter((m) => m.type === "stt");
+  const [customSttModels, setCustomSttModels] = useState([]);
+  const sttModels = [...builtinSttModels, ...customSttModels];
+
+  const [selectedModel, setSelectedModel] = useState(builtinSttModels[0]?.id ?? "");
+  const selectedModelObj = sttModels.find((m) => m.id === selectedModel);
+  const allowedParams = Array.isArray(selectedModelObj?.params) ? selectedModelObj.params : [];
+
+  const [audioFile, setAudioFile] = useState(null);
+  const [language, setLanguage] = useState("");
+  const [prompt, setPrompt] = useState("");
+  const [responseFormat, setResponseFormat] = useState("json");
+  const [temperature, setTemperature] = useState("");
+  const [apiKey, setApiKey] = useState("");
+  const [useTunnel, setUseTunnel] = useState(false);
+  const [localEndpoint, setLocalEndpoint] = useState("");
+  const [tunnelEndpoint, setTunnelEndpoint] = useState("");
+  const [result, setResult] = useState(null);
+  const [latency, setLatency] = useState(null);
+  const [running, setRunning] = useState(false);
+  const [error, setError] = useState("");
+  const { copied: copiedCurl, copy: copyCurl } = useCopyToClipboard();
+  const { copied: copiedRes, copy: copyRes } = useCopyToClipboard();
+
+  useEffect(() => {
+    setLocalEndpoint(window.location.origin);
+    fetch("/api/keys")
+      .then((r) => r.json())
+      .then((d) => { setApiKey((d.keys || []).find((k) => k.isActive !== false)?.key || ""); })
+      .catch(() => {});
+    fetch("/api/tunnel/status")
+      .then((r) => r.json())
+      .then((d) => { if (d.publicUrl) setTunnelEndpoint(d.publicUrl); })
+      .catch(() => {});
+    const loadCustom = () => {
+      fetch("/api/models/custom", { cache: "no-store" })
+        .then((r) => r.json())
+        .then((d) => {
+          const list = (d.models || []).filter((m) => m.type === "stt" && m.providerAlias === providerAlias);
+          setCustomSttModels(list);
+        })
+        .catch(() => {});
+    };
+    loadCustom();
+    window.addEventListener("focus", loadCustom);
+    window.addEventListener("customModelChanged", loadCustom);
+    return () => {
+      window.removeEventListener("focus", loadCustom);
+      window.removeEventListener("customModelChanged", loadCustom);
+    };
+  }, [providerAlias]);
+
+  const endpoint = useTunnel ? tunnelEndpoint : localEndpoint;
+  const modelFull = selectedModel ? `${providerAlias}/${selectedModel}` : "";
+
+  const curlSnippet = `curl -X POST ${endpoint}/v1/audio/transcriptions \\
+  -H "Authorization: Bearer ${apiKey || "YOUR_KEY"}" \\
+  -F "file=@${audioFile?.name || "audio.mp3"}" \\
+  -F "model=${modelFull}"${allowedParams.includes("language") && language ? ` \\\n  -F "language=${language}"` : ""}${allowedParams.includes("response_format") ? ` \\\n  -F "response_format=${responseFormat}"` : ""}${allowedParams.includes("temperature") && temperature ? ` \\\n  -F "temperature=${temperature}"` : ""}${allowedParams.includes("prompt") && prompt ? ` \\\n  -F "prompt=${prompt}"` : ""}`;
+
+  const handleRun = async () => {
+    if (!audioFile || !modelFull) return;
+    setRunning(true);
+    setError("");
+    setResult(null);
+    const start = Date.now();
+    try {
+      const fd = new FormData();
+      fd.append("file", audioFile);
+      fd.append("model", modelFull);
+      if (allowedParams.includes("language") && language) fd.append("language", language);
+      if (allowedParams.includes("response_format")) fd.append("response_format", responseFormat);
+      if (allowedParams.includes("temperature") && temperature) fd.append("temperature", temperature);
+      if (allowedParams.includes("prompt") && prompt) fd.append("prompt", prompt);
+
+      const headers = {};
+      if (apiKey) headers["Authorization"] = `Bearer ${apiKey}`;
+      const res = await fetch("/api/v1/audio/transcriptions", { method: "POST", headers, body: fd });
+      setLatency(Date.now() - start);
+      const ct = res.headers.get("content-type") || "";
+      const data = ct.includes("application/json") ? await res.json() : await res.text();
+      if (!res.ok) {
+        setError(data?.error?.message || data?.error || data || `HTTP ${res.status}`);
+        return;
+      }
+      setResult(data);
+    } catch (e) {
+      setError(e.message || "Network error");
+    } finally {
+      setRunning(false);
+    }
+  };
+
+  const resultStr = typeof result === "string" ? result : (result ? JSON.stringify(result, null, 2) : `{\n  "text": "Hello world..."\n}`);
+
+  return (
+    <Card>
+      <h2 className="text-lg font-semibold mb-4">Example</h2>
+      <div className="flex flex-col gap-2.5">
+        {/* Model */}
+        {sttModels.length > 0 ? (
+          <Row label="Model">
+            <select
+              value={selectedModel}
+              onChange={(e) => setSelectedModel(e.target.value)}
+              className="w-full px-3 py-1.5 text-sm border border-border rounded-lg bg-background focus:outline-none focus:border-primary"
+            >
+              {sttModels.map((m) => (
+                <option key={m.id} value={m.id}>{m.name || m.id}</option>
+              ))}
+            </select>
+          </Row>
+        ) : (
+          <Row label="Model">
+            <input
+              value={selectedModel}
+              onChange={(e) => setSelectedModel(e.target.value)}
+              placeholder="Enter model id"
+              className="w-full px-3 py-1.5 text-sm border border-border rounded-lg bg-background focus:outline-none focus:border-primary font-mono"
+            />
+          </Row>
+        )}
+
+        {/* Endpoint */}
+        <Row label="Endpoint">
+          <div className="flex w-full flex-col gap-2 sm:w-auto sm:flex-row sm:items-center">
+            <span className="w-full min-w-0 flex-1 px-3 py-1.5 text-sm font-mono text-text-main bg-sidebar rounded-lg truncate">
+              {endpoint}/v1/audio/transcriptions
+            </span>
+            {tunnelEndpoint && (
+              <button
+                onClick={() => setUseTunnel((v) => !v)}
+                title={useTunnel ? "Using tunnel" : "Using local"}
+                className={`flex items-center gap-1 text-xs px-2 py-1.5 rounded-lg border shrink-0 transition-colors ${
+                  useTunnel ? "border-primary/40 bg-primary/10 text-primary" : "border-border text-text-muted hover:text-primary"
+                }`}
+              >
+                <span className="material-symbols-outlined text-[14px]">wifi_tethering</span>
+                Tunnel
+              </button>
+            )}
+          </div>
+        </Row>
+
+        {/* API Key */}
+        <Row label="API Key">
+          <span className="px-3 py-1.5 text-sm font-mono text-text-main bg-sidebar rounded-lg truncate block">
+            {apiKey ? `${apiKey.slice(0, 8)}${"\u2022".repeat(Math.min(20, apiKey.length - 8))}` : <span className="text-text-muted italic">No key configured</span>}
+          </span>
+        </Row>
+
+        {/* Audio file */}
+        <Row label="Audio File">
+          <div className="flex flex-col gap-2">
+            <input
+              type="file"
+              accept="audio/*,video/mp4,.m4a,.mp3,.wav,.ogg,.flac,.webm,.opus"
+              onChange={(e) => setAudioFile(e.target.files?.[0] || null)}
+              className="w-full text-xs text-text-muted file:mr-2 file:py-1 file:px-2.5 file:rounded-lg file:border file:border-border file:bg-background file:text-text-main hover:file:bg-sidebar file:cursor-pointer"
+            />
+            {audioFile && (
+              <span className="text-xs text-text-muted font-mono">
+                {audioFile.name} · {(audioFile.size / 1024).toFixed(1)} KB
+              </span>
+            )}
+          </div>
+        </Row>
+
+        {/* Language (if model supports) */}
+        {allowedParams.includes("language") && (
+          <Row label="Language">
+            <input
+              value={language}
+              onChange={(e) => setLanguage(e.target.value)}
+              placeholder="e.g. en, vi, ja (auto-detect if empty)"
+              className="w-full px-3 py-1.5 text-sm border border-border rounded-lg bg-background focus:outline-none focus:border-primary font-mono"
+            />
+          </Row>
+        )}
+
+        {/* Prompt (if model supports) */}
+        {allowedParams.includes("prompt") && (
+          <Row label="Prompt">
+            <input
+              value={prompt}
+              onChange={(e) => setPrompt(e.target.value)}
+              placeholder="optional context to improve accuracy"
+              className="w-full px-3 py-1.5 text-sm border border-border rounded-lg bg-background focus:outline-none focus:border-primary"
+            />
+          </Row>
+        )}
+
+        {/* Temperature (if model supports) */}
+        {allowedParams.includes("temperature") && (
+          <Row label="Temperature">
+            <input
+              type="number"
+              step="0.1"
+              min="0"
+              max="1"
+              value={temperature}
+              onChange={(e) => setTemperature(e.target.value)}
+              placeholder="0 - 1 (default 0)"
+              className="w-full px-3 py-1.5 text-sm border border-border rounded-lg bg-background focus:outline-none focus:border-primary"
+            />
+          </Row>
+        )}
+
+        {/* Response format (if model supports) */}
+        {allowedParams.includes("response_format") && (
+          <Row label="Response Format">
+            <select
+              value={responseFormat}
+              onChange={(e) => setResponseFormat(e.target.value)}
+              className="w-full px-3 py-1.5 text-sm border border-border rounded-lg bg-background focus:outline-none focus:border-primary"
+            >
+              <option value="json">json</option>
+              <option value="text">text</option>
+              <option value="srt">srt</option>
+              <option value="verbose_json">verbose_json</option>
+              <option value="vtt">vtt</option>
+            </select>
+          </Row>
+        )}
+
+        {/* Curl + Run */}
+        <div className="mt-1">
+          <div className="flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between mb-1.5">
+            <span className="text-xs font-semibold text-text-muted uppercase tracking-wider">Request</span>
+            <div className="flex w-full flex-col gap-2 sm:w-auto sm:flex-row sm:items-center">
+              <button
+                onClick={() => copyCurl(curlSnippet)}
+                className="inline-flex items-center gap-1 text-xs text-text-muted hover:text-primary transition-colors"
+              >
+                <span className="material-symbols-outlined text-[14px]">{copiedCurl ? "check" : "content_copy"}</span>
+                {copiedCurl ? "Copied" : "Copy"}
+              </button>
+              <button
+                onClick={handleRun}
+                disabled={running || !audioFile || !modelFull}
+                className="flex w-full sm:w-auto items-center justify-center gap-1.5 px-3 py-1 rounded-lg bg-primary text-white text-xs font-medium hover:bg-primary/90 transition-colors disabled:opacity-50 disabled:cursor-not-allowed"
+              >
+                <span className="material-symbols-outlined text-[14px]" style={running ? { animation: "spin 1s linear infinite" } : undefined}>
+                  play_arrow
+                </span>
+                {running ? "Transcribing..." : "Run"}
+              </button>
+            </div>
+          </div>
+          <pre className="bg-sidebar rounded-lg px-3 py-2.5 text-xs font-mono text-text-main overflow-x-auto whitespace-pre-wrap break-all">{curlSnippet}</pre>
+        </div>
+
+        {error && <p className="text-xs text-red-500 break-words">{error}</p>}
+
+        {/* Response */}
+        <div>
+          <div className="flex flex-col gap-2 sm:flex-row sm:items-center sm:justify-between mb-1.5">
+            <span className="text-xs font-semibold text-text-muted uppercase tracking-wider">
+              Response {result && latency && <span className="font-normal normal-case">&#9889; {latency}ms</span>}
+            </span>
+            {result && (
+              <button
+                onClick={() => copyRes(resultStr)}
+                className="inline-flex items-center gap-1 text-xs text-text-muted hover:text-primary transition-colors"
+              >
+                <span className="material-symbols-outlined text-[14px]">{copiedRes ? "check" : "content_copy"}</span>
+                {copiedRes ? "Copied" : "Copy"}
+              </button>
+            )}
+          </div>
+          <pre className="bg-sidebar rounded-lg px-3 py-2.5 text-xs font-mono text-text-main overflow-x-auto whitespace-pre-wrap break-all opacity-70">
+            {resultStr}
+          </pre>
+        </div>
+      </div>
+    </Card>
+  );
+}
+
 // MediaProviderDetailPage
 export default function MediaProviderDetailPage() {
  const { kind, id } = useParams();
@@ -1502,11 +1801,12 @@ export default function MediaProviderDetailPage() {
      )}

      {/* Provider Info — config-driven, supports searchConfig, fetchConfig, ttsConfig, embeddingConfig, searchViaChat */}
-      {!isCustom && (provider.searchConfig || provider.fetchConfig || provider.ttsConfig || provider.embeddingConfig || provider.searchViaChat) && (
+      {!isCustom && (provider.searchConfig || provider.fetchConfig || provider.ttsConfig || provider.sttConfig || provider.embeddingConfig || provider.searchViaChat) && (
        <ProviderInfoCard
          config={
            kind === "webFetch" ? provider.fetchConfig
              : kind === "tts" ? provider.ttsConfig
+              : kind === "stt" ? provider.sttConfig
              : kind === "embedding" ? provider.embeddingConfig
              : provider.searchConfig || { mode: "chat-completions", defaultModel: provider.searchViaChat?.defaultModel, pricingUrl: provider.searchViaChat?.pricingUrl, freeTier: provider.searchViaChat?.freeTier }
          }
@@ -1520,6 +1820,7 @@ export default function MediaProviderDetailPage() {
        <EmbeddingExampleCard providerId={id} customAlias={customNode?.prefix} />
      )}
      {kind === "tts" && <TtsExampleCard providerId={id} />}
+      {kind === "stt" && !isCustom && <SttExampleCard providerId={id} />}
      {!isCustom && KIND_EXAMPLE_CONFIG[kind] && <GenericExampleCard providerId={id} kind={kind} />}

      {isCustom && (