feat(db): migrate from lowdb to SQLite with repos pattern
- Add modular DB layer (adapters, migrations, repos, helpers) - Replace localDb/usageDb/requestDetailsDb monoliths with repos - Add Tailscale tunnel integration & status check API - Add /api/cli-tools/all-statuses aggregated endpoint - Add settingsStore (Zustand) and mitm/dbReader - Add DB unit tests (benchmark, concurrent, migration, vs-lowdb)
This commit is contained in:
decolua committed
1 parent
145f588cc0
commit
bee8dad946
63 files changed
+4223
-2330
No files matched your search
@@ -0,0 +1,62 @@
|
||||
import { getAdapter } from "../driver.js";
|
||||
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
|
||||
import { makeKv } from "../helpers/kvStore.js";
|
||||
|
||||
const aliasKv = makeKv("modelAliases");
|
||||
const customKv = makeKv("customModels");
|
||||
const mitmKv = makeKv("mitmAlias");
|
||||
|
||||
// modelAliases: key=alias, value=modelString
|
||||
export async function getModelAliases() {
|
||||
return await aliasKv.getAll();
|
||||
}
|
||||
|
||||
export async function setModelAlias(alias, model) {
|
||||
await aliasKv.set(alias, model);
|
||||
}
|
||||
|
||||
export async function deleteModelAlias(alias) {
|
||||
await aliasKv.remove(alias);
|
||||
}
|
||||
|
||||
// customModels: key=`${providerAlias}|${id}|${type}`, value=full model object
|
||||
function customKey(providerAlias, id, type) {
|
||||
return `${providerAlias}|${id}|${type}`;
|
||||
}
|
||||
|
||||
export async function getCustomModels() {
|
||||
const all = await customKv.getAll();
|
||||
return Object.values(all);
|
||||
}
|
||||
|
||||
// Atomic check-then-insert inside transaction to prevent duplicate races
|
||||
export async function addCustomModel({ providerAlias, id, type = "llm", name }) {
|
||||
const k = customKey(providerAlias, id, type);
|
||||
const db = await getAdapter();
|
||||
let added = false;
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT 1 FROM kv WHERE scope = 'customModels' AND key = ?`, [k]);
|
||||
if (row) return;
|
||||
const value = stringifyJson({ providerAlias, id, type, name: name || id });
|
||||
db.run(`INSERT INTO kv(scope, key, value) VALUES('customModels', ?, ?)`, [k, value]);
|
||||
added = true;
|
||||
});
|
||||
return added;
|
||||
}
|
||||
|
||||
export async function deleteCustomModel({ providerAlias, id, type = "llm" }) {
|
||||
await customKv.remove(customKey(providerAlias, id, type));
|
||||
}
|
||||
|
||||
// mitmAlias: key=toolName, value=mappings object
|
||||
export async function getMitmAlias(toolName) {
|
||||
if (toolName) {
|
||||
const v = await mitmKv.get(toolName);
|
||||
return v || {};
|
||||
}
|
||||
return await mitmKv.getAll();
|
||||
}
|
||||
|
||||
export async function setMitmAliasAll(toolName, mappings) {
|
||||
await mitmKv.set(toolName, mappings || {});
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { getAdapter } from "../driver.js";
|
||||
|
||||
function rowToKey(row) {
|
||||
if (!row) return null;
|
||||
return {
|
||||
id: row.id,
|
||||
key: row.key,
|
||||
name: row.name,
|
||||
machineId: row.machineId,
|
||||
isActive: row.isActive === 1 || row.isActive === true,
|
||||
createdAt: row.createdAt,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getApiKeys() {
|
||||
const db = await getAdapter();
|
||||
const rows = db.all(`SELECT * FROM apiKeys ORDER BY createdAt ASC`);
|
||||
return rows.map(rowToKey);
|
||||
}
|
||||
|
||||
export async function getApiKeyById(id) {
|
||||
const db = await getAdapter();
|
||||
const row = db.get(`SELECT * FROM apiKeys WHERE id = ?`, [id]);
|
||||
return rowToKey(row);
|
||||
}
|
||||
|
||||
export async function createApiKey(name, machineId) {
|
||||
if (!machineId) throw new Error("machineId is required");
|
||||
const db = await getAdapter();
|
||||
const { generateApiKeyWithMachine } = await import("@/shared/utils/apiKey");
|
||||
const result = generateApiKeyWithMachine(machineId);
|
||||
const apiKey = {
|
||||
id: uuidv4(),
|
||||
name,
|
||||
key: result.key,
|
||||
machineId,
|
||||
isActive: true,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
db.run(
|
||||
`INSERT INTO apiKeys(id, key, name, machineId, isActive, createdAt) VALUES(?, ?, ?, ?, ?, ?)`,
|
||||
[apiKey.id, apiKey.key, apiKey.name, apiKey.machineId, 1, apiKey.createdAt]
|
||||
);
|
||||
return apiKey;
|
||||
}
|
||||
|
||||
export async function updateApiKey(id, data) {
|
||||
const db = await getAdapter();
|
||||
let result = null;
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT * FROM apiKeys WHERE id = ?`, [id]);
|
||||
if (!row) return;
|
||||
const merged = { ...rowToKey(row), ...data };
|
||||
db.run(
|
||||
`UPDATE apiKeys SET key = ?, name = ?, machineId = ?, isActive = ? WHERE id = ?`,
|
||||
[merged.key, merged.name, merged.machineId, merged.isActive ? 1 : 0, id]
|
||||
);
|
||||
result = merged;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function deleteApiKey(id) {
|
||||
const db = await getAdapter();
|
||||
const res = db.run(`DELETE FROM apiKeys WHERE id = ?`, [id]);
|
||||
return (res?.changes ?? 0) > 0;
|
||||
}
|
||||
|
||||
export async function validateApiKey(key) {
|
||||
const db = await getAdapter();
|
||||
const row = db.get(`SELECT isActive FROM apiKeys WHERE key = ?`, [key]);
|
||||
if (!row) return false;
|
||||
return row.isActive === 1 || row.isActive === true;
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { getAdapter } from "../driver.js";
|
||||
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
|
||||
|
||||
function rowToCombo(row) {
|
||||
if (!row) return null;
|
||||
return {
|
||||
id: row.id,
|
||||
name: row.name,
|
||||
kind: row.kind,
|
||||
models: parseJson(row.models, []),
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getCombos() {
|
||||
const db = await getAdapter();
|
||||
const rows = db.all(`SELECT * FROM combos ORDER BY createdAt ASC`);
|
||||
return rows.map(rowToCombo);
|
||||
}
|
||||
|
||||
export async function getComboById(id) {
|
||||
const db = await getAdapter();
|
||||
const row = db.get(`SELECT * FROM combos WHERE id = ?`, [id]);
|
||||
return rowToCombo(row);
|
||||
}
|
||||
|
||||
export async function getComboByName(name) {
|
||||
const db = await getAdapter();
|
||||
const row = db.get(`SELECT * FROM combos WHERE name = ?`, [name]);
|
||||
return rowToCombo(row);
|
||||
}
|
||||
|
||||
export async function createCombo(data) {
|
||||
const db = await getAdapter();
|
||||
const now = new Date().toISOString();
|
||||
const combo = {
|
||||
id: uuidv4(),
|
||||
name: data.name,
|
||||
kind: data.kind || null,
|
||||
models: data.models || [],
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
db.run(
|
||||
`INSERT INTO combos(id, name, kind, models, createdAt, updatedAt) VALUES(?, ?, ?, ?, ?, ?)`,
|
||||
[combo.id, combo.name, combo.kind, stringifyJson(combo.models), combo.createdAt, combo.updatedAt]
|
||||
);
|
||||
return combo;
|
||||
}
|
||||
|
||||
export async function updateCombo(id, data) {
|
||||
const db = await getAdapter();
|
||||
let result = null;
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT * FROM combos WHERE id = ?`, [id]);
|
||||
if (!row) return;
|
||||
const merged = { ...rowToCombo(row), ...data, updatedAt: new Date().toISOString() };
|
||||
db.run(
|
||||
`UPDATE combos SET name = ?, kind = ?, models = ?, updatedAt = ? WHERE id = ?`,
|
||||
[merged.name, merged.kind, stringifyJson(merged.models || []), merged.updatedAt, id]
|
||||
);
|
||||
result = merged;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function deleteCombo(id) {
|
||||
const db = await getAdapter();
|
||||
const res = db.run(`DELETE FROM combos WHERE id = ?`, [id]);
|
||||
return (res?.changes ?? 0) > 0;
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { getAdapter } from "../driver.js";
|
||||
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
|
||||
|
||||
const OPTIONAL_FIELDS = [
|
||||
"displayName", "email", "globalPriority", "defaultModel",
|
||||
"accessToken", "refreshToken", "expiresAt", "tokenType",
|
||||
"scope", "projectId", "apiKey", "testStatus",
|
||||
"lastTested", "lastError", "lastErrorAt", "rateLimitedUntil", "expiresIn", "errorCode",
|
||||
"consecutiveUseCount",
|
||||
];
|
||||
|
||||
function rowToConn(row) {
|
||||
if (!row) return null;
|
||||
const extra = parseJson(row.data, {});
|
||||
return {
|
||||
...extra,
|
||||
id: row.id,
|
||||
provider: row.provider,
|
||||
authType: row.authType,
|
||||
name: row.name,
|
||||
email: row.email,
|
||||
priority: row.priority,
|
||||
isActive: row.isActive === 1 || row.isActive === true,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function connToRow(c) {
|
||||
const { id, provider, authType, name, email, priority, isActive, createdAt, updatedAt, ...rest } = c;
|
||||
return {
|
||||
id,
|
||||
provider,
|
||||
authType,
|
||||
name: name ?? null,
|
||||
email: email ?? null,
|
||||
priority: priority ?? null,
|
||||
isActive: isActive === false ? 0 : 1,
|
||||
data: stringifyJson(rest),
|
||||
createdAt,
|
||||
updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function upsert(db, c) {
|
||||
const r = connToRow(c);
|
||||
db.run(
|
||||
`INSERT INTO providerConnections(id, provider, authType, name, email, priority, isActive, data, createdAt, updatedAt)
|
||||
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
provider=excluded.provider, authType=excluded.authType, name=excluded.name,
|
||||
email=excluded.email, priority=excluded.priority, isActive=excluded.isActive,
|
||||
data=excluded.data, updatedAt=excluded.updatedAt`,
|
||||
[r.id, r.provider, r.authType, r.name, r.email, r.priority, r.isActive, r.data, r.createdAt, r.updatedAt]
|
||||
);
|
||||
}
|
||||
|
||||
export async function getProviderConnections(filter = {}) {
|
||||
const db = await getAdapter();
|
||||
const where = [];
|
||||
const params = [];
|
||||
if (filter.provider) { where.push("provider = ?"); params.push(filter.provider); }
|
||||
if (filter.isActive !== undefined) { where.push("isActive = ?"); params.push(filter.isActive ? 1 : 0); }
|
||||
const sql = `SELECT * FROM providerConnections${where.length ? ` WHERE ${where.join(" AND ")}` : ""}`;
|
||||
const rows = db.all(sql, params);
|
||||
const list = rows.map(rowToConn);
|
||||
list.sort((a, b) => (a.priority || 999) - (b.priority || 999));
|
||||
return list;
|
||||
}
|
||||
|
||||
export async function getProviderConnectionById(id) {
|
||||
const db = await getAdapter();
|
||||
const row = db.get(`SELECT * FROM providerConnections WHERE id = ?`, [id]);
|
||||
return rowToConn(row);
|
||||
}
|
||||
|
||||
// Internal sync reorder — must be called INSIDE a transaction
|
||||
function reorderInTx(db, providerId) {
|
||||
const list = db.all(`SELECT * FROM providerConnections WHERE provider = ?`, [providerId]).map(rowToConn);
|
||||
list.sort((a, b) => {
|
||||
const pDiff = (a.priority || 0) - (b.priority || 0);
|
||||
if (pDiff !== 0) return pDiff;
|
||||
return new Date(b.updatedAt || 0) - new Date(a.updatedAt || 0);
|
||||
});
|
||||
list.forEach((c, i) => {
|
||||
db.run(`UPDATE providerConnections SET priority = ? WHERE id = ?`, [i + 1, c.id]);
|
||||
});
|
||||
}
|
||||
|
||||
export async function createProviderConnection(data) {
|
||||
const db = await getAdapter();
|
||||
const now = new Date().toISOString();
|
||||
let result;
|
||||
|
||||
db.transaction(() => {
|
||||
const all = db.all(`SELECT * FROM providerConnections WHERE provider = ?`, [data.provider]).map(rowToConn);
|
||||
|
||||
let existing = null;
|
||||
if (data.authType === "oauth" && data.email) {
|
||||
existing = all.find(c => c.authType === "oauth" && c.email === data.email);
|
||||
} else if (data.authType === "apikey" && data.name) {
|
||||
existing = all.find(c => c.authType === "apikey" && c.name === data.name);
|
||||
}
|
||||
|
||||
if (existing) {
|
||||
const merged = { ...existing, ...data, updatedAt: now };
|
||||
upsert(db, merged);
|
||||
result = merged;
|
||||
return;
|
||||
}
|
||||
|
||||
let connectionName = data.name || null;
|
||||
if (!connectionName && data.authType === "oauth") {
|
||||
connectionName = data.email || `Account ${all.length + 1}`;
|
||||
}
|
||||
let connectionPriority = data.priority;
|
||||
if (!connectionPriority) {
|
||||
connectionPriority = all.reduce((m, c) => Math.max(m, c.priority || 0), 0) + 1;
|
||||
}
|
||||
|
||||
const conn = {
|
||||
id: uuidv4(),
|
||||
provider: data.provider,
|
||||
authType: data.authType || "oauth",
|
||||
name: connectionName,
|
||||
priority: connectionPriority,
|
||||
isActive: data.isActive !== undefined ? data.isActive : true,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
for (const f of OPTIONAL_FIELDS) {
|
||||
if (data[f] !== undefined && data[f] !== null) conn[f] = data[f];
|
||||
}
|
||||
if (data.providerSpecificData && Object.keys(data.providerSpecificData).length > 0) {
|
||||
conn.providerSpecificData = data.providerSpecificData;
|
||||
}
|
||||
if (data.email !== undefined) conn.email = data.email;
|
||||
|
||||
upsert(db, conn);
|
||||
reorderInTx(db, data.provider);
|
||||
result = conn;
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// Critical: OAuth refresh token race — atomic merge inside transaction
|
||||
export async function updateProviderConnection(id, data) {
|
||||
const db = await getAdapter();
|
||||
let result;
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT * FROM providerConnections WHERE id = ?`, [id]);
|
||||
if (!row) { result = null; return; }
|
||||
const existing = rowToConn(row);
|
||||
const merged = { ...existing, ...data, updatedAt: new Date().toISOString() };
|
||||
upsert(db, merged);
|
||||
if (data.priority !== undefined) reorderInTx(db, existing.provider);
|
||||
result = merged;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function deleteProviderConnection(id) {
|
||||
const db = await getAdapter();
|
||||
let ok = false;
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT provider FROM providerConnections WHERE id = ?`, [id]);
|
||||
if (!row) return;
|
||||
db.run(`DELETE FROM providerConnections WHERE id = ?`, [id]);
|
||||
reorderInTx(db, row.provider);
|
||||
ok = true;
|
||||
});
|
||||
return ok;
|
||||
}
|
||||
|
||||
export async function deleteProviderConnectionsByProvider(providerId) {
|
||||
const db = await getAdapter();
|
||||
const before = db.get(`SELECT COUNT(*) AS n FROM providerConnections WHERE provider = ?`, [providerId]);
|
||||
db.run(`DELETE FROM providerConnections WHERE provider = ?`, [providerId]);
|
||||
return before?.n || 0;
|
||||
}
|
||||
|
||||
export async function reorderProviderConnections(providerId) {
|
||||
const db = await getAdapter();
|
||||
db.transaction(() => reorderInTx(db, providerId));
|
||||
}
|
||||
|
||||
export async function cleanupProviderConnections() {
|
||||
const db = await getAdapter();
|
||||
const fieldsToCheck = [
|
||||
"displayName", "email", "globalPriority", "defaultModel",
|
||||
"accessToken", "refreshToken", "expiresAt", "tokenType",
|
||||
"scope", "projectId", "apiKey", "testStatus",
|
||||
"lastTested", "lastError", "lastErrorAt", "rateLimitedUntil", "expiresIn",
|
||||
"consecutiveUseCount",
|
||||
];
|
||||
let cleaned = 0;
|
||||
db.transaction(() => {
|
||||
const rows = db.all(`SELECT * FROM providerConnections`);
|
||||
for (const row of rows) {
|
||||
const conn = rowToConn(row);
|
||||
let dirty = false;
|
||||
for (const f of fieldsToCheck) {
|
||||
if (conn[f] === null || conn[f] === undefined) {
|
||||
if (f in conn) { delete conn[f]; cleaned++; dirty = true; }
|
||||
}
|
||||
}
|
||||
if (conn.providerSpecificData && Object.keys(conn.providerSpecificData).length === 0) {
|
||||
delete conn.providerSpecificData;
|
||||
cleaned++;
|
||||
dirty = true;
|
||||
}
|
||||
if (dirty) upsert(db, conn);
|
||||
}
|
||||
});
|
||||
return cleaned;
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { getAdapter } from "../driver.js";
|
||||
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
|
||||
|
||||
const SCOPE = "disabledModels";
|
||||
|
||||
export async function getDisabledModels() {
|
||||
const db = await getAdapter();
|
||||
const rows = db.all(`SELECT key, value FROM kv WHERE scope = ?`, [SCOPE]);
|
||||
const out = {};
|
||||
for (const r of rows) out[r.key] = parseJson(r.value, []);
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function getDisabledByProvider(providerAlias) {
|
||||
const db = await getAdapter();
|
||||
const row = db.get(`SELECT value FROM kv WHERE scope = ? AND key = ?`, [SCOPE, providerAlias]);
|
||||
return row ? (parseJson(row.value, []) || []) : [];
|
||||
}
|
||||
|
||||
// Atomic read-merge-write inside a transaction (no JS yield mid-transaction).
|
||||
export async function disableModels(providerAlias, ids) {
|
||||
if (!providerAlias || !Array.isArray(ids)) return;
|
||||
const db = await getAdapter();
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT value FROM kv WHERE scope = ? AND key = ?`, [SCOPE, providerAlias]);
|
||||
const current = row ? (parseJson(row.value, []) || []) : [];
|
||||
const merged = [...new Set([...current, ...ids])];
|
||||
db.run(
|
||||
`INSERT INTO kv(scope, key, value) VALUES(?, ?, ?) ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`,
|
||||
[SCOPE, providerAlias, stringifyJson(merged)]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export async function enableModels(providerAlias, ids) {
|
||||
if (!providerAlias) return;
|
||||
const db = await getAdapter();
|
||||
db.transaction(() => {
|
||||
if (!Array.isArray(ids) || ids.length === 0) {
|
||||
db.run(`DELETE FROM kv WHERE scope = ? AND key = ?`, [SCOPE, providerAlias]);
|
||||
return;
|
||||
}
|
||||
const row = db.get(`SELECT value FROM kv WHERE scope = ? AND key = ?`, [SCOPE, providerAlias]);
|
||||
const current = row ? (parseJson(row.value, []) || []) : [];
|
||||
const removeSet = new Set(ids);
|
||||
const next = current.filter((id) => !removeSet.has(id));
|
||||
if (next.length === 0) {
|
||||
db.run(`DELETE FROM kv WHERE scope = ? AND key = ?`, [SCOPE, providerAlias]);
|
||||
} else {
|
||||
db.run(
|
||||
`INSERT INTO kv(scope, key, value) VALUES(?, ?, ?) ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`,
|
||||
[SCOPE, providerAlias, stringifyJson(next)]
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { getAdapter } from "../driver.js";
|
||||
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
|
||||
|
||||
function rowToNode(row) {
|
||||
if (!row) return null;
|
||||
const extra = parseJson(row.data, {});
|
||||
return {
|
||||
...extra,
|
||||
id: row.id,
|
||||
type: row.type,
|
||||
name: row.name,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function nodeToRow(n) {
|
||||
const { id, type, name, createdAt, updatedAt, ...rest } = n;
|
||||
return {
|
||||
id,
|
||||
type: type ?? null,
|
||||
name: name ?? null,
|
||||
data: stringifyJson(rest),
|
||||
createdAt,
|
||||
updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function upsert(db, n) {
|
||||
const r = nodeToRow(n);
|
||||
db.run(
|
||||
`INSERT INTO providerNodes(id, type, name, data, createdAt, updatedAt)
|
||||
VALUES(?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
type=excluded.type, name=excluded.name, data=excluded.data, updatedAt=excluded.updatedAt`,
|
||||
[r.id, r.type, r.name, r.data, r.createdAt, r.updatedAt]
|
||||
);
|
||||
}
|
||||
|
||||
export async function getProviderNodes(filter = {}) {
|
||||
const db = await getAdapter();
|
||||
const where = [];
|
||||
const params = [];
|
||||
if (filter.type) { where.push("type = ?"); params.push(filter.type); }
|
||||
const sql = `SELECT * FROM providerNodes${where.length ? ` WHERE ${where.join(" AND ")}` : ""}`;
|
||||
return db.all(sql, params).map(rowToNode);
|
||||
}
|
||||
|
||||
export async function getProviderNodeById(id) {
|
||||
const db = await getAdapter();
|
||||
return rowToNode(db.get(`SELECT * FROM providerNodes WHERE id = ?`, [id]));
|
||||
}
|
||||
|
||||
export async function createProviderNode(data) {
|
||||
const db = await getAdapter();
|
||||
const now = new Date().toISOString();
|
||||
const node = {
|
||||
id: data.id || uuidv4(),
|
||||
type: data.type,
|
||||
name: data.name,
|
||||
prefix: data.prefix,
|
||||
apiType: data.apiType,
|
||||
baseUrl: data.baseUrl,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
upsert(db, node);
|
||||
return node;
|
||||
}
|
||||
|
||||
export async function updateProviderNode(id, data) {
|
||||
const db = await getAdapter();
|
||||
let result = null;
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT * FROM providerNodes WHERE id = ?`, [id]);
|
||||
if (!row) return;
|
||||
const merged = { ...rowToNode(row), ...data, updatedAt: new Date().toISOString() };
|
||||
upsert(db, merged);
|
||||
result = merged;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function deleteProviderNode(id) {
|
||||
const db = await getAdapter();
|
||||
let removed = null;
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT * FROM providerNodes WHERE id = ?`, [id]);
|
||||
if (!row) return;
|
||||
removed = rowToNode(row);
|
||||
db.run(`DELETE FROM providerNodes WHERE id = ?`, [id]);
|
||||
});
|
||||
return removed;
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
import { getAdapter } from "../driver.js";
|
||||
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
|
||||
import { makeKv } from "../helpers/kvStore.js";
|
||||
|
||||
const pricingKv = makeKv("pricing");
|
||||
const CACHE_TTL_MS = 5000;
|
||||
|
||||
let cache = { value: null, expiresAt: 0 };
|
||||
|
||||
function invalidate() {
|
||||
cache = { value: null, expiresAt: 0 };
|
||||
}
|
||||
|
||||
async function getUserPricing() {
|
||||
return await pricingKv.getAll();
|
||||
}
|
||||
|
||||
export async function getPricing() {
|
||||
const now = Date.now();
|
||||
if (cache.value && cache.expiresAt > now) return cache.value;
|
||||
|
||||
const userPricing = await getUserPricing();
|
||||
const { PROVIDER_PRICING } = await import("@/shared/constants/pricing.js");
|
||||
const merged = {};
|
||||
|
||||
for (const [provider, models] of Object.entries(PROVIDER_PRICING)) {
|
||||
merged[provider] = { ...models };
|
||||
if (userPricing[provider]) {
|
||||
for (const [model, pricing] of Object.entries(userPricing[provider])) {
|
||||
merged[provider][model] = merged[provider][model]
|
||||
? { ...merged[provider][model], ...pricing }
|
||||
: pricing;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [provider, models] of Object.entries(userPricing)) {
|
||||
if (!merged[provider]) {
|
||||
merged[provider] = { ...models };
|
||||
} else {
|
||||
for (const [model, pricing] of Object.entries(models)) {
|
||||
if (!merged[provider][model]) merged[provider][model] = pricing;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cache = { value: merged, expiresAt: now + CACHE_TTL_MS };
|
||||
return merged;
|
||||
}
|
||||
|
||||
export async function getPricingForModel(provider, model) {
|
||||
if (!model) return null;
|
||||
const userPricing = await getUserPricing();
|
||||
if (provider && userPricing[provider]?.[model]) return userPricing[provider][model];
|
||||
const { getPricingForModel: resolveConst } = await import("@/shared/constants/pricing.js");
|
||||
return resolveConst(provider, model);
|
||||
}
|
||||
|
||||
// Atomic merge inside transaction (per-provider read-modify-write)
|
||||
export async function updatePricing(pricingData) {
|
||||
const db = await getAdapter();
|
||||
db.transaction(() => {
|
||||
for (const [provider, models] of Object.entries(pricingData)) {
|
||||
const row = db.get(`SELECT value FROM kv WHERE scope = 'pricing' AND key = ?`, [provider]);
|
||||
const current = row ? (parseJson(row.value, {}) || {}) : {};
|
||||
const merged = { ...current };
|
||||
for (const [model, pricing] of Object.entries(models)) {
|
||||
merged[model] = pricing;
|
||||
}
|
||||
db.run(
|
||||
`INSERT INTO kv(scope, key, value) VALUES('pricing', ?, ?) ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`,
|
||||
[provider, stringifyJson(merged)]
|
||||
);
|
||||
}
|
||||
});
|
||||
invalidate();
|
||||
return await getUserPricing();
|
||||
}
|
||||
|
||||
export async function resetPricing(provider, model) {
|
||||
if (!provider) return await getUserPricing();
|
||||
const db = await getAdapter();
|
||||
db.transaction(() => {
|
||||
if (!model) {
|
||||
db.run(`DELETE FROM kv WHERE scope = 'pricing' AND key = ?`, [provider]);
|
||||
return;
|
||||
}
|
||||
const row = db.get(`SELECT value FROM kv WHERE scope = 'pricing' AND key = ?`, [provider]);
|
||||
const current = row ? (parseJson(row.value, {}) || {}) : {};
|
||||
delete current[model];
|
||||
if (Object.keys(current).length === 0) {
|
||||
db.run(`DELETE FROM kv WHERE scope = 'pricing' AND key = ?`, [provider]);
|
||||
} else {
|
||||
db.run(
|
||||
`INSERT INTO kv(scope, key, value) VALUES('pricing', ?, ?) ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value`,
|
||||
[provider, stringifyJson(current)]
|
||||
);
|
||||
}
|
||||
});
|
||||
invalidate();
|
||||
return await getUserPricing();
|
||||
}
|
||||
|
||||
export async function resetAllPricing() {
|
||||
await pricingKv.clear();
|
||||
invalidate();
|
||||
return {};
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import { getAdapter } from "../driver.js";
|
||||
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
|
||||
|
||||
function rowToPool(row) {
|
||||
if (!row) return null;
|
||||
const extra = parseJson(row.data, {});
|
||||
return {
|
||||
...extra,
|
||||
id: row.id,
|
||||
isActive: row.isActive === 1 || row.isActive === true,
|
||||
testStatus: row.testStatus,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function poolToRow(p) {
|
||||
const { id, isActive, testStatus, createdAt, updatedAt, ...rest } = p;
|
||||
return {
|
||||
id,
|
||||
isActive: isActive === false ? 0 : 1,
|
||||
testStatus: testStatus ?? null,
|
||||
data: stringifyJson(rest),
|
||||
createdAt,
|
||||
updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function upsert(db, p) {
|
||||
const r = poolToRow(p);
|
||||
db.run(
|
||||
`INSERT INTO proxyPools(id, isActive, testStatus, data, createdAt, updatedAt)
|
||||
VALUES(?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
isActive=excluded.isActive, testStatus=excluded.testStatus,
|
||||
data=excluded.data, updatedAt=excluded.updatedAt`,
|
||||
[r.id, r.isActive, r.testStatus, r.data, r.createdAt, r.updatedAt]
|
||||
);
|
||||
}
|
||||
|
||||
export async function getProxyPools(filter = {}) {
|
||||
const db = await getAdapter();
|
||||
const where = [];
|
||||
const params = [];
|
||||
if (filter.isActive !== undefined) { where.push("isActive = ?"); params.push(filter.isActive ? 1 : 0); }
|
||||
if (filter.testStatus) { where.push("testStatus = ?"); params.push(filter.testStatus); }
|
||||
const sql = `SELECT * FROM proxyPools${where.length ? ` WHERE ${where.join(" AND ")}` : ""}`;
|
||||
const list = db.all(sql, params).map(rowToPool);
|
||||
list.sort((a, b) => new Date(b.updatedAt || 0) - new Date(a.updatedAt || 0));
|
||||
return list;
|
||||
}
|
||||
|
||||
export async function getProxyPoolById(id) {
|
||||
const db = await getAdapter();
|
||||
return rowToPool(db.get(`SELECT * FROM proxyPools WHERE id = ?`, [id]));
|
||||
}
|
||||
|
||||
export async function createProxyPool(data) {
|
||||
const db = await getAdapter();
|
||||
const now = new Date().toISOString();
|
||||
const pool = {
|
||||
id: data.id || uuidv4(),
|
||||
name: data.name,
|
||||
proxyUrl: data.proxyUrl,
|
||||
noProxy: data.noProxy || "",
|
||||
type: data.type || "http",
|
||||
isActive: data.isActive !== undefined ? data.isActive : true,
|
||||
strictProxy: data.strictProxy === true,
|
||||
testStatus: data.testStatus || "unknown",
|
||||
lastTestedAt: data.lastTestedAt || null,
|
||||
lastError: data.lastError || null,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
upsert(db, pool);
|
||||
return pool;
|
||||
}
|
||||
|
||||
export async function updateProxyPool(id, data) {
|
||||
const db = await getAdapter();
|
||||
let result = null;
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT * FROM proxyPools WHERE id = ?`, [id]);
|
||||
if (!row) return;
|
||||
const merged = { ...rowToPool(row), ...data, updatedAt: new Date().toISOString() };
|
||||
upsert(db, merged);
|
||||
result = merged;
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function deleteProxyPool(id) {
|
||||
const db = await getAdapter();
|
||||
let removed = null;
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT * FROM proxyPools WHERE id = ?`, [id]);
|
||||
if (!row) return;
|
||||
removed = rowToPool(row);
|
||||
db.run(`DELETE FROM proxyPools WHERE id = ?`, [id]);
|
||||
});
|
||||
return removed;
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
import { getAdapter } from "../driver.js";
|
||||
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
|
||||
|
||||
const DEFAULT_MAX_RECORDS = 200;
|
||||
const DEFAULT_BATCH_SIZE = 20;
|
||||
const DEFAULT_FLUSH_INTERVAL_MS = 5000;
|
||||
const DEFAULT_MAX_JSON_SIZE = 5 * 1024;
|
||||
const CONFIG_CACHE_TTL_MS = 5000;
|
||||
|
||||
let cachedConfig = null;
|
||||
let cachedConfigTs = 0;
|
||||
|
||||
async function getObservabilityConfig() {
|
||||
if (cachedConfig && (Date.now() - cachedConfigTs) < CONFIG_CACHE_TTL_MS) return cachedConfig;
|
||||
try {
|
||||
const { getSettings } = await import("./settingsRepo.js");
|
||||
const settings = await getSettings();
|
||||
const envEnabled = process.env.OBSERVABILITY_ENABLED !== "false";
|
||||
const enabled = typeof settings.enableObservability === "boolean"
|
||||
? settings.enableObservability
|
||||
: envEnabled;
|
||||
cachedConfig = {
|
||||
enabled,
|
||||
maxRecords: settings.observabilityMaxRecords || parseInt(process.env.OBSERVABILITY_MAX_RECORDS || String(DEFAULT_MAX_RECORDS), 10),
|
||||
batchSize: settings.observabilityBatchSize || parseInt(process.env.OBSERVABILITY_BATCH_SIZE || String(DEFAULT_BATCH_SIZE), 10),
|
||||
flushIntervalMs: settings.observabilityFlushIntervalMs || parseInt(process.env.OBSERVABILITY_FLUSH_INTERVAL_MS || String(DEFAULT_FLUSH_INTERVAL_MS), 10),
|
||||
maxJsonSize: (settings.observabilityMaxJsonSize || parseInt(process.env.OBSERVABILITY_MAX_JSON_SIZE || "5", 10)) * 1024,
|
||||
};
|
||||
} catch {
|
||||
cachedConfig = {
|
||||
enabled: false,
|
||||
maxRecords: DEFAULT_MAX_RECORDS,
|
||||
batchSize: DEFAULT_BATCH_SIZE,
|
||||
flushIntervalMs: DEFAULT_FLUSH_INTERVAL_MS,
|
||||
maxJsonSize: DEFAULT_MAX_JSON_SIZE,
|
||||
};
|
||||
}
|
||||
cachedConfigTs = Date.now();
|
||||
return cachedConfig;
|
||||
}
|
||||
|
||||
let writeBuffer = [];
|
||||
let flushTimer = null;
|
||||
let isFlushing = false;
|
||||
|
||||
function sanitizeHeaders(headers) {
|
||||
if (!headers || typeof headers !== "object") return {};
|
||||
const sensitiveKeys = ["authorization", "x-api-key", "cookie", "token", "api-key"];
|
||||
const sanitized = { ...headers };
|
||||
for (const key of Object.keys(sanitized)) {
|
||||
if (sensitiveKeys.some((s) => key.toLowerCase().includes(s))) delete sanitized[key];
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
function generateDetailId(model) {
|
||||
const timestamp = new Date().toISOString();
|
||||
const random = Math.random().toString(36).substring(2, 8);
|
||||
const modelPart = model ? model.replace(/[^a-zA-Z0-9-]/g, "-") : "unknown";
|
||||
return `${timestamp}-${random}-${modelPart}`;
|
||||
}
|
||||
|
||||
function truncateField(obj, maxSize) {
|
||||
const str = JSON.stringify(obj || {});
|
||||
if (str.length > maxSize) {
|
||||
return { _truncated: true, _originalSize: str.length, _preview: str.substring(0, 200) };
|
||||
}
|
||||
return obj || {};
|
||||
}
|
||||
|
||||
async function flushToDatabase() {
|
||||
if (isFlushing) return;
|
||||
if (writeBuffer.length === 0) return;
|
||||
isFlushing = true;
|
||||
try {
|
||||
// Drain entire buffer (loop in case more pushed during await)
|
||||
while (writeBuffer.length > 0) {
|
||||
const items = writeBuffer.splice(0, writeBuffer.length);
|
||||
const db = await getAdapter();
|
||||
const config = await getObservabilityConfig();
|
||||
|
||||
db.transaction(() => {
|
||||
for (const item of items) {
|
||||
if (!item.id) item.id = generateDetailId(item.model);
|
||||
if (!item.timestamp) item.timestamp = new Date().toISOString();
|
||||
if (item.request?.headers) item.request.headers = sanitizeHeaders(item.request.headers);
|
||||
|
||||
const record = {
|
||||
id: item.id,
|
||||
provider: item.provider || null,
|
||||
model: item.model || null,
|
||||
connectionId: item.connectionId || null,
|
||||
timestamp: item.timestamp,
|
||||
status: item.status || null,
|
||||
latency: item.latency || {},
|
||||
tokens: item.tokens || {},
|
||||
request: truncateField(item.request, config.maxJsonSize),
|
||||
providerRequest: truncateField(item.providerRequest, config.maxJsonSize),
|
||||
providerResponse: truncateField(item.providerResponse, config.maxJsonSize),
|
||||
response: truncateField(item.response, config.maxJsonSize),
|
||||
};
|
||||
|
||||
db.run(
|
||||
`INSERT INTO requestDetails(id, timestamp, provider, model, connectionId, status, data) VALUES(?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET timestamp = excluded.timestamp, provider = excluded.provider, model = excluded.model, connectionId = excluded.connectionId, status = excluded.status, data = excluded.data`,
|
||||
[record.id, record.timestamp, record.provider, record.model, record.connectionId, record.status, stringifyJson(record)]
|
||||
);
|
||||
}
|
||||
|
||||
const cnt = db.get(`SELECT COUNT(*) as c FROM requestDetails`);
|
||||
if (cnt && cnt.c > config.maxRecords) {
|
||||
db.run(
|
||||
`DELETE FROM requestDetails WHERE id IN (SELECT id FROM requestDetails ORDER BY timestamp ASC LIMIT ?)`,
|
||||
[cnt.c - config.maxRecords]
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("[requestDetailsRepo] Batch write failed:", e);
|
||||
} finally {
|
||||
isFlushing = false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveRequestDetail(detail) {
|
||||
const config = await getObservabilityConfig();
|
||||
if (!config.enabled) return;
|
||||
|
||||
writeBuffer.push(detail);
|
||||
|
||||
// Trigger immediate flush if batch threshold reached.
|
||||
// flushToDatabase() drains entire buffer in a loop, so all pushes during await are persisted.
|
||||
if (writeBuffer.length >= config.batchSize) {
|
||||
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
|
||||
flushToDatabase().catch((e) => console.error("[requestDetailsRepo] flush err:", e));
|
||||
} else if (!flushTimer) {
|
||||
flushTimer = setTimeout(() => {
|
||||
flushTimer = null;
|
||||
flushToDatabase().catch(() => {});
|
||||
}, config.flushIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
export async function getRequestDetails(filter = {}) {
|
||||
const db = await getAdapter();
|
||||
const conds = [];
|
||||
const params = [];
|
||||
|
||||
if (filter.provider) { conds.push("provider = ?"); params.push(filter.provider); }
|
||||
if (filter.model) { conds.push("model = ?"); params.push(filter.model); }
|
||||
if (filter.connectionId) { conds.push("connectionId = ?"); params.push(filter.connectionId); }
|
||||
if (filter.status) { conds.push("status = ?"); params.push(filter.status); }
|
||||
if (filter.startDate) { conds.push("timestamp >= ?"); params.push(new Date(filter.startDate).toISOString()); }
|
||||
if (filter.endDate) { conds.push("timestamp <= ?"); params.push(new Date(filter.endDate).toISOString()); }
|
||||
|
||||
const where = conds.length ? `WHERE ${conds.join(" AND ")}` : "";
|
||||
const cntRow = db.get(`SELECT COUNT(*) as c FROM requestDetails ${where}`, params);
|
||||
const totalItems = cntRow ? cntRow.c : 0;
|
||||
|
||||
const page = filter.page || 1;
|
||||
const pageSize = filter.pageSize || 50;
|
||||
const totalPages = Math.ceil(totalItems / pageSize);
|
||||
const offset = (page - 1) * pageSize;
|
||||
|
||||
const rows = db.all(
|
||||
`SELECT data FROM requestDetails ${where} ORDER BY timestamp DESC LIMIT ? OFFSET ?`,
|
||||
[...params, pageSize, offset]
|
||||
);
|
||||
const details = rows.map((r) => parseJson(r.data, {}));
|
||||
|
||||
return {
|
||||
details,
|
||||
pagination: { page, pageSize, totalItems, totalPages, hasNext: page < totalPages, hasPrev: page > 1 },
|
||||
};
|
||||
}
|
||||
|
||||
export async function getRequestDetailById(id) {
|
||||
const db = await getAdapter();
|
||||
const row = db.get(`SELECT data FROM requestDetails WHERE id = ?`, [id]);
|
||||
return row ? parseJson(row.data, null) : null;
|
||||
}
|
||||
|
||||
const _shutdownHandler = async () => {
|
||||
if (flushTimer) { clearTimeout(flushTimer); flushTimer = null; }
|
||||
if (writeBuffer.length > 0) await flushToDatabase();
|
||||
};
|
||||
|
||||
function ensureShutdownHandler() {
|
||||
process.off("beforeExit", _shutdownHandler);
|
||||
process.off("SIGINT", _shutdownHandler);
|
||||
process.off("SIGTERM", _shutdownHandler);
|
||||
process.off("exit", _shutdownHandler);
|
||||
|
||||
process.on("beforeExit", _shutdownHandler);
|
||||
process.on("SIGINT", _shutdownHandler);
|
||||
process.on("SIGTERM", _shutdownHandler);
|
||||
process.on("exit", _shutdownHandler);
|
||||
}
|
||||
|
||||
ensureShutdownHandler();
|
||||
@@ -0,0 +1,98 @@
|
||||
import { getAdapter } from "../driver.js";
|
||||
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
|
||||
|
||||
const DEFAULT_MITM_ROUTER_BASE = "http://127.0.0.1:20128";
|
||||
|
||||
const DEFAULT_SETTINGS = {
|
||||
cloudEnabled: false,
|
||||
tunnelEnabled: false,
|
||||
tunnelUrl: "",
|
||||
tunnelProvider: "cloudflare",
|
||||
tailscaleEnabled: false,
|
||||
tailscaleUrl: "",
|
||||
stickyRoundRobinLimit: 3,
|
||||
providerStrategies: {},
|
||||
comboStrategy: "fallback",
|
||||
comboStickyRoundRobinLimit: 1,
|
||||
comboStrategies: {},
|
||||
requireLogin: true,
|
||||
tunnelDashboardAccess: true,
|
||||
enableObservability: true,
|
||||
observabilityMaxRecords: 1000,
|
||||
observabilityBatchSize: 20,
|
||||
observabilityFlushIntervalMs: 5000,
|
||||
observabilityMaxJsonSize: 5,
|
||||
outboundProxyEnabled: false,
|
||||
outboundProxyUrl: "",
|
||||
outboundNoProxy: "",
|
||||
mitmRouterBaseUrl: DEFAULT_MITM_ROUTER_BASE,
|
||||
dnsToolEnabled: {},
|
||||
rtkEnabled: true,
|
||||
cavemanEnabled: false,
|
||||
cavemanLevel: "full",
|
||||
};
|
||||
|
||||
async function readRaw() {
|
||||
const db = await getAdapter();
|
||||
const row = db.get(`SELECT data FROM settings WHERE id = 1`);
|
||||
return row ? parseJson(row.data, {}) : {};
|
||||
}
|
||||
|
||||
// Merge raw settings with defaults; backward-compat for missing keys
|
||||
function mergeWithDefaults(raw) {
|
||||
const merged = { ...DEFAULT_SETTINGS, ...(raw || {}) };
|
||||
for (const [key, defVal] of Object.entries(DEFAULT_SETTINGS)) {
|
||||
if (merged[key] === undefined) {
|
||||
if (
|
||||
key === "outboundProxyEnabled" &&
|
||||
typeof merged.outboundProxyUrl === "string" &&
|
||||
merged.outboundProxyUrl.trim()
|
||||
) {
|
||||
merged[key] = true;
|
||||
} else {
|
||||
merged[key] = defVal;
|
||||
}
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
export async function getSettings() {
|
||||
const raw = await readRaw();
|
||||
return mergeWithDefaults(raw);
|
||||
}
|
||||
|
||||
// Atomic read-merge-write inside transaction (prevents losing concurrent updates)
|
||||
export async function updateSettings(updates) {
|
||||
const db = await getAdapter();
|
||||
let next;
|
||||
db.transaction(() => {
|
||||
const row = db.get(`SELECT data FROM settings WHERE id = 1`);
|
||||
const current = row ? parseJson(row.data, {}) : {};
|
||||
next = { ...current, ...updates };
|
||||
db.run(
|
||||
`INSERT INTO settings(id, data) VALUES(1, ?) ON CONFLICT(id) DO UPDATE SET data = excluded.data`,
|
||||
[stringifyJson(next)]
|
||||
);
|
||||
});
|
||||
return mergeWithDefaults(next);
|
||||
}
|
||||
|
||||
export async function isCloudEnabled() {
|
||||
const settings = await getSettings();
|
||||
return settings.cloudEnabled === true;
|
||||
}
|
||||
|
||||
export async function getCloudUrl() {
|
||||
const settings = await getSettings();
|
||||
return (
|
||||
settings.cloudUrl ||
|
||||
process.env.CLOUD_URL ||
|
||||
process.env.NEXT_PUBLIC_CLOUD_URL ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
export async function exportSettings() {
|
||||
return await readRaw();
|
||||
}
|
||||
@@ -0,0 +1,698 @@
|
||||
import { EventEmitter } from "events";
|
||||
import { getAdapter } from "../driver.js";
|
||||
import { parseJson, stringifyJson } from "../helpers/jsonCol.js";
|
||||
import { getMeta, setMeta } from "../helpers/metaStore.js";
|
||||
|
||||
const PENDING_TIMEOUT_MS = 60 * 1000;
|
||||
const RING_CAP = 50;
|
||||
const CONN_CACHE_TTL_MS = 30 * 1000;
|
||||
const PERIOD_MS = { "24h": 86400000, "7d": 604800000, "30d": 2592000000, "60d": 5184000000 };
|
||||
|
||||
// In-memory state shared across Next.js modules
|
||||
if (!global._pendingRequests) global._pendingRequests = { byModel: {}, byAccount: {} };
|
||||
if (!global._lastErrorProvider) global._lastErrorProvider = { provider: "", ts: 0 };
|
||||
if (!global._statsEmitter) {
|
||||
global._statsEmitter = new EventEmitter();
|
||||
global._statsEmitter.setMaxListeners(50);
|
||||
}
|
||||
if (!global._pendingTimers) global._pendingTimers = {};
|
||||
if (!global._recentRing) global._recentRing = { items: [], initialized: false };
|
||||
if (!global._connectionMapCache) global._connectionMapCache = { map: {}, ts: 0 };
|
||||
|
||||
const pendingRequests = global._pendingRequests;
|
||||
const lastErrorProvider = global._lastErrorProvider;
|
||||
const pendingTimers = global._pendingTimers;
|
||||
const recentRing = global._recentRing;
|
||||
const connCache = global._connectionMapCache;
|
||||
|
||||
export const statsEmitter = global._statsEmitter;
|
||||
|
||||
function getLocalDateKey(timestamp) {
|
||||
const d = timestamp ? new Date(timestamp) : new Date();
|
||||
return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function addToCounter(target, key, values) {
|
||||
if (!target[key]) target[key] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
|
||||
target[key].requests += values.requests || 1;
|
||||
target[key].promptTokens += values.promptTokens || 0;
|
||||
target[key].completionTokens += values.completionTokens || 0;
|
||||
target[key].cost += values.cost || 0;
|
||||
if (values.meta) Object.assign(target[key], values.meta);
|
||||
}
|
||||
|
||||
function aggregateEntryToDay(day, entry) {
|
||||
const promptTokens = entry.tokens?.prompt_tokens || entry.tokens?.input_tokens || 0;
|
||||
const completionTokens = entry.tokens?.completion_tokens || entry.tokens?.output_tokens || 0;
|
||||
const cost = entry.cost || 0;
|
||||
const vals = { promptTokens, completionTokens, cost };
|
||||
|
||||
day.requests = (day.requests || 0) + 1;
|
||||
day.promptTokens = (day.promptTokens || 0) + promptTokens;
|
||||
day.completionTokens = (day.completionTokens || 0) + completionTokens;
|
||||
day.cost = (day.cost || 0) + cost;
|
||||
|
||||
day.byProvider ||= {};
|
||||
day.byModel ||= {};
|
||||
day.byAccount ||= {};
|
||||
day.byApiKey ||= {};
|
||||
day.byEndpoint ||= {};
|
||||
|
||||
if (entry.provider) addToCounter(day.byProvider, entry.provider, vals);
|
||||
|
||||
const modelKey = entry.provider ? `${entry.model}|${entry.provider}` : entry.model;
|
||||
addToCounter(day.byModel, modelKey, { ...vals, meta: { rawModel: entry.model, provider: entry.provider } });
|
||||
|
||||
if (entry.connectionId) {
|
||||
addToCounter(day.byAccount, entry.connectionId, { ...vals, meta: { rawModel: entry.model, provider: entry.provider } });
|
||||
}
|
||||
|
||||
const apiKeyVal = entry.apiKey && typeof entry.apiKey === "string" ? entry.apiKey : "local-no-key";
|
||||
const akModelKey = `${apiKeyVal}|${entry.model}|${entry.provider || "unknown"}`;
|
||||
addToCounter(day.byApiKey, akModelKey, { ...vals, meta: { rawModel: entry.model, provider: entry.provider, apiKey: entry.apiKey || null } });
|
||||
|
||||
const endpoint = entry.endpoint || "Unknown";
|
||||
const epKey = `${endpoint}|${entry.model}|${entry.provider || "unknown"}`;
|
||||
addToCounter(day.byEndpoint, epKey, { ...vals, meta: { endpoint, rawModel: entry.model, provider: entry.provider } });
|
||||
}
|
||||
|
||||
function pushToRing(entry) {
|
||||
recentRing.items.push(entry);
|
||||
if (recentRing.items.length > RING_CAP) {
|
||||
recentRing.items = recentRing.items.slice(-RING_CAP);
|
||||
}
|
||||
}
|
||||
|
||||
async function getConnectionMapCached() {
|
||||
if (Date.now() - connCache.ts < CONN_CACHE_TTL_MS) return connCache.map;
|
||||
try {
|
||||
const { getProviderConnections } = await import("./connectionsRepo.js");
|
||||
const all = await getProviderConnections();
|
||||
const map = {};
|
||||
for (const c of all) map[c.id] = c.name || c.email || c.id;
|
||||
connCache.map = map;
|
||||
connCache.ts = Date.now();
|
||||
} catch {}
|
||||
return connCache.map;
|
||||
}
|
||||
|
||||
async function ensureRingInitialized() {
|
||||
if (recentRing.initialized) return;
|
||||
recentRing.initialized = true;
|
||||
try {
|
||||
const db = await getAdapter();
|
||||
const rows = db.all(`SELECT timestamp, provider, model, connectionId, apiKey, endpoint, cost, status, tokens FROM usageHistory ORDER BY id DESC LIMIT ?`, [RING_CAP]);
|
||||
recentRing.items = rows.reverse().map((r) => ({
|
||||
timestamp: r.timestamp, provider: r.provider, model: r.model, connectionId: r.connectionId,
|
||||
apiKey: r.apiKey, endpoint: r.endpoint, cost: r.cost, status: r.status,
|
||||
tokens: parseJson(r.tokens, {}),
|
||||
}));
|
||||
} catch {}
|
||||
}
|
||||
|
||||
async function calculateCost(provider, model, tokens) {
|
||||
if (!tokens || !provider || !model) return 0;
|
||||
try {
|
||||
const { getPricingForModel } = await import("./pricingRepo.js");
|
||||
const pricing = await getPricingForModel(provider, model);
|
||||
if (!pricing) return 0;
|
||||
|
||||
let cost = 0;
|
||||
const inputTokens = tokens.prompt_tokens || tokens.input_tokens || 0;
|
||||
const cachedTokens = tokens.cached_tokens || tokens.cache_read_input_tokens || 0;
|
||||
const nonCachedInput = Math.max(0, inputTokens - cachedTokens);
|
||||
cost += nonCachedInput * (pricing.input / 1000000);
|
||||
|
||||
if (cachedTokens > 0) {
|
||||
const cachedRate = pricing.cached || pricing.input;
|
||||
cost += cachedTokens * (cachedRate / 1000000);
|
||||
}
|
||||
|
||||
const outputTokens = tokens.completion_tokens || tokens.output_tokens || 0;
|
||||
cost += outputTokens * (pricing.output / 1000000);
|
||||
|
||||
const reasoningTokens = tokens.reasoning_tokens || 0;
|
||||
if (reasoningTokens > 0) {
|
||||
const rate = pricing.reasoning || pricing.output;
|
||||
cost += reasoningTokens * (rate / 1000000);
|
||||
}
|
||||
|
||||
const cacheCreationTokens = tokens.cache_creation_input_tokens || 0;
|
||||
if (cacheCreationTokens > 0) {
|
||||
const rate = pricing.cache_creation || pricing.input;
|
||||
cost += cacheCreationTokens * (rate / 1000000);
|
||||
}
|
||||
|
||||
return cost;
|
||||
} catch (e) {
|
||||
console.error("Error calculating cost:", e);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
export function trackPendingRequest(model, provider, connectionId, started, error = false) {
|
||||
const modelKey = provider ? `${model} (${provider})` : model;
|
||||
const timerKey = `${connectionId}|${modelKey}`;
|
||||
|
||||
if (!pendingRequests.byModel[modelKey]) pendingRequests.byModel[modelKey] = 0;
|
||||
pendingRequests.byModel[modelKey] = Math.max(0, pendingRequests.byModel[modelKey] + (started ? 1 : -1));
|
||||
if (pendingRequests.byModel[modelKey] === 0) delete pendingRequests.byModel[modelKey];
|
||||
|
||||
if (connectionId) {
|
||||
if (!pendingRequests.byAccount[connectionId]) pendingRequests.byAccount[connectionId] = {};
|
||||
if (!pendingRequests.byAccount[connectionId][modelKey]) pendingRequests.byAccount[connectionId][modelKey] = 0;
|
||||
pendingRequests.byAccount[connectionId][modelKey] = Math.max(0, pendingRequests.byAccount[connectionId][modelKey] + (started ? 1 : -1));
|
||||
if (pendingRequests.byAccount[connectionId][modelKey] === 0) {
|
||||
delete pendingRequests.byAccount[connectionId][modelKey];
|
||||
if (Object.keys(pendingRequests.byAccount[connectionId]).length === 0) {
|
||||
delete pendingRequests.byAccount[connectionId];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (started) {
|
||||
clearTimeout(pendingTimers[timerKey]);
|
||||
pendingTimers[timerKey] = setTimeout(() => {
|
||||
delete pendingTimers[timerKey];
|
||||
if (pendingRequests.byModel[modelKey] > 0) pendingRequests.byModel[modelKey] = 0;
|
||||
if (connectionId && pendingRequests.byAccount[connectionId]?.[modelKey] > 0) {
|
||||
pendingRequests.byAccount[connectionId][modelKey] = 0;
|
||||
}
|
||||
statsEmitter.emit("pending");
|
||||
}, PENDING_TIMEOUT_MS);
|
||||
} else {
|
||||
clearTimeout(pendingTimers[timerKey]);
|
||||
delete pendingTimers[timerKey];
|
||||
}
|
||||
|
||||
if (!started && error && provider) {
|
||||
lastErrorProvider.provider = provider.toLowerCase();
|
||||
lastErrorProvider.ts = Date.now();
|
||||
}
|
||||
|
||||
const t = new Date().toLocaleTimeString("en-US", { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" });
|
||||
console.log(`[${t}] [PENDING] ${started ? "START" : "END"}${error ? " (ERROR)" : ""} | provider=${provider} | model=${model}`);
|
||||
statsEmitter.emit("pending");
|
||||
}
|
||||
|
||||
export async function getActiveRequests() {
|
||||
const activeRequests = [];
|
||||
const connectionMap = await getConnectionMapCached();
|
||||
|
||||
for (const [connectionId, models] of Object.entries(pendingRequests.byAccount)) {
|
||||
for (const [modelKey, count] of Object.entries(models)) {
|
||||
if (count > 0) {
|
||||
const accountName = connectionMap[connectionId] || `Account ${connectionId.slice(0, 8)}...`;
|
||||
const match = modelKey.match(/^(.*) \((.*)\)$/);
|
||||
activeRequests.push({
|
||||
model: match ? match[1] : modelKey,
|
||||
provider: match ? match[2] : "unknown",
|
||||
account: accountName, count,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await ensureRingInitialized();
|
||||
const seen = new Set();
|
||||
const recentRequests = [...recentRing.items]
|
||||
.sort((a, b) => new Date(b.timestamp) - new Date(a.timestamp))
|
||||
.map((e) => {
|
||||
const t = e.tokens || {};
|
||||
return {
|
||||
timestamp: e.timestamp, model: e.model, provider: e.provider || "",
|
||||
promptTokens: t.prompt_tokens || t.input_tokens || 0,
|
||||
completionTokens: t.completion_tokens || t.output_tokens || 0,
|
||||
status: e.status || "ok",
|
||||
};
|
||||
})
|
||||
.filter((e) => {
|
||||
if (e.promptTokens === 0 && e.completionTokens === 0) return false;
|
||||
const minute = e.timestamp ? e.timestamp.slice(0, 16) : "";
|
||||
const key = `${e.model}|${e.provider}|${e.promptTokens}|${e.completionTokens}|${minute}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
})
|
||||
.slice(0, 20);
|
||||
|
||||
const errorProvider = (Date.now() - lastErrorProvider.ts < 10000) ? lastErrorProvider.provider : "";
|
||||
return { activeRequests, recentRequests, errorProvider };
|
||||
}
|
||||
|
||||
export async function saveRequestUsage(entry) {
|
||||
try {
|
||||
const db = await getAdapter();
|
||||
|
||||
if (!entry.timestamp) entry.timestamp = new Date().toISOString();
|
||||
entry.cost = await calculateCost(entry.provider, entry.model, entry.tokens);
|
||||
|
||||
const tokens = entry.tokens || {};
|
||||
const promptTokens = tokens.prompt_tokens || tokens.input_tokens || 0;
|
||||
const completionTokens = tokens.completion_tokens || tokens.output_tokens || 0;
|
||||
|
||||
// All 3 writes (history insert, daily upsert, lifetime counter) in ONE transaction.
|
||||
// better-sqlite3 is sync → no JS yield mid-transaction → no race in same process.
|
||||
db.transaction(() => {
|
||||
db.run(
|
||||
`INSERT INTO usageHistory(timestamp, provider, model, connectionId, apiKey, endpoint, promptTokens, completionTokens, cost, status, tokens, meta) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
entry.timestamp, entry.provider || null, entry.model || null,
|
||||
entry.connectionId || null, entry.apiKey || null, entry.endpoint || null,
|
||||
promptTokens, completionTokens, entry.cost || 0, entry.status || "ok",
|
||||
stringifyJson(tokens), stringifyJson({}),
|
||||
]
|
||||
);
|
||||
|
||||
const dateKey = getLocalDateKey(entry.timestamp);
|
||||
const row = db.get(`SELECT data FROM usageDaily WHERE dateKey = ?`, [dateKey]);
|
||||
const day = row ? parseJson(row.data, {}) : {
|
||||
requests: 0, promptTokens: 0, completionTokens: 0, cost: 0,
|
||||
byProvider: {}, byModel: {}, byAccount: {}, byApiKey: {}, byEndpoint: {},
|
||||
};
|
||||
aggregateEntryToDay(day, entry);
|
||||
db.run(`INSERT INTO usageDaily(dateKey, data) VALUES(?, ?) ON CONFLICT(dateKey) DO UPDATE SET data = excluded.data`, [dateKey, stringifyJson(day)]);
|
||||
|
||||
// Atomic counter increment in same transaction
|
||||
const cur = db.get(`SELECT value FROM _meta WHERE key = 'totalRequestsLifetime'`);
|
||||
const next = (cur ? parseInt(cur.value, 10) : 0) + 1;
|
||||
db.run(`INSERT INTO _meta(key, value) VALUES('totalRequestsLifetime', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, [String(next)]);
|
||||
});
|
||||
|
||||
pushToRing(entry);
|
||||
statsEmitter.emit("update");
|
||||
} catch (e) {
|
||||
console.error("Failed to save usage stats:", e);
|
||||
}
|
||||
}
|
||||
|
||||
export async function getUsageHistory(filter = {}) {
|
||||
const db = await getAdapter();
|
||||
const conds = [];
|
||||
const params = [];
|
||||
|
||||
if (filter.provider) { conds.push("provider = ?"); params.push(filter.provider); }
|
||||
if (filter.model) { conds.push("model = ?"); params.push(filter.model); }
|
||||
if (filter.startDate) { conds.push("timestamp >= ?"); params.push(new Date(filter.startDate).toISOString()); }
|
||||
if (filter.endDate) { conds.push("timestamp <= ?"); params.push(new Date(filter.endDate).toISOString()); }
|
||||
|
||||
const where = conds.length ? `WHERE ${conds.join(" AND ")}` : "";
|
||||
const rows = db.all(`SELECT timestamp, provider, model, connectionId, apiKey, endpoint, cost, status, tokens FROM usageHistory ${where} ORDER BY id ASC`, params);
|
||||
|
||||
return rows.map((r) => ({
|
||||
timestamp: r.timestamp, provider: r.provider, model: r.model,
|
||||
connectionId: r.connectionId, apiKey: r.apiKey, endpoint: r.endpoint,
|
||||
cost: r.cost, status: r.status, tokens: parseJson(r.tokens, {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function loadDaysInRange(adapter, maxDays) {
|
||||
if (maxDays == null) {
|
||||
return adapter.all(`SELECT dateKey, data FROM usageDaily`);
|
||||
}
|
||||
const today = new Date();
|
||||
const cutoff = new Date(today.getFullYear(), today.getMonth(), today.getDate() - maxDays + 1);
|
||||
const cutoffKey = `${cutoff.getFullYear()}-${String(cutoff.getMonth() + 1).padStart(2, "0")}-${String(cutoff.getDate()).padStart(2, "0")}`;
|
||||
return adapter.all(`SELECT dateKey, data FROM usageDaily WHERE dateKey >= ?`, [cutoffKey]);
|
||||
}
|
||||
|
||||
export async function getUsageStats(period = "all") {
|
||||
const db = await getAdapter();
|
||||
|
||||
const [{ getProviderConnections }, { getApiKeys }, { getProviderNodes }] = await Promise.all([
|
||||
import("./connectionsRepo.js"),
|
||||
import("./apiKeysRepo.js"),
|
||||
import("./nodesRepo.js"),
|
||||
]);
|
||||
|
||||
let allConnections = [];
|
||||
try { allConnections = await getProviderConnections(); } catch {}
|
||||
const connectionMap = {};
|
||||
for (const c of allConnections) connectionMap[c.id] = c.name || c.email || c.id;
|
||||
|
||||
const providerNodeNameMap = {};
|
||||
try {
|
||||
const nodes = await getProviderNodes();
|
||||
for (const n of nodes) if (n.id && n.name) providerNodeNameMap[n.id] = n.name;
|
||||
} catch {}
|
||||
|
||||
let allApiKeys = [];
|
||||
try { allApiKeys = await getApiKeys(); } catch {}
|
||||
const apiKeyMap = {};
|
||||
for (const k of allApiKeys) apiKeyMap[k.key] = { name: k.name, id: k.id, createdAt: k.createdAt };
|
||||
|
||||
// recentRequests from live history (last 100 entries enough for 20 deduped)
|
||||
const recentRows = db.all(`SELECT timestamp, provider, model, tokens, status FROM usageHistory ORDER BY id DESC LIMIT 100`);
|
||||
const seen = new Set();
|
||||
const recentRequests = recentRows
|
||||
.map((r) => {
|
||||
const t = parseJson(r.tokens, {}) || {};
|
||||
return {
|
||||
timestamp: r.timestamp, model: r.model, provider: r.provider || "",
|
||||
promptTokens: t.prompt_tokens || t.input_tokens || 0,
|
||||
completionTokens: t.completion_tokens || t.output_tokens || 0,
|
||||
status: r.status || "ok",
|
||||
};
|
||||
})
|
||||
.filter((e) => {
|
||||
if (e.promptTokens === 0 && e.completionTokens === 0) return false;
|
||||
const minute = e.timestamp ? e.timestamp.slice(0, 16) : "";
|
||||
const key = `${e.model}|${e.provider}|${e.promptTokens}|${e.completionTokens}|${minute}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
})
|
||||
.slice(0, 20);
|
||||
|
||||
const stats = {
|
||||
totalRequests: 0,
|
||||
totalPromptTokens: 0, totalCompletionTokens: 0, totalCost: 0,
|
||||
byProvider: {}, byModel: {}, byAccount: {}, byApiKey: {}, byEndpoint: {},
|
||||
last10Minutes: [],
|
||||
pending: pendingRequests,
|
||||
activeRequests: [],
|
||||
recentRequests,
|
||||
errorProvider: (Date.now() - lastErrorProvider.ts < 10000) ? lastErrorProvider.provider : "",
|
||||
};
|
||||
|
||||
// Active requests
|
||||
for (const [connectionId, models] of Object.entries(pendingRequests.byAccount)) {
|
||||
for (const [modelKey, count] of Object.entries(models)) {
|
||||
if (count > 0) {
|
||||
const accountName = connectionMap[connectionId] || `Account ${connectionId.slice(0, 8)}...`;
|
||||
const match = modelKey.match(/^(.*) \((.*)\)$/);
|
||||
stats.activeRequests.push({
|
||||
model: match ? match[1] : modelKey,
|
||||
provider: match ? match[2] : "unknown",
|
||||
account: accountName, count,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// last10Minutes — query 10min window
|
||||
const now = new Date();
|
||||
const currentMinuteStart = new Date(Math.floor(now.getTime() / 60000) * 60000);
|
||||
const tenMinutesAgo = new Date(currentMinuteStart.getTime() - 9 * 60 * 1000);
|
||||
const bucketMap = {};
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const ts = currentMinuteStart.getTime() - (9 - i) * 60 * 1000;
|
||||
bucketMap[ts] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
|
||||
stats.last10Minutes.push(bucketMap[ts]);
|
||||
}
|
||||
const recent10 = db.all(
|
||||
`SELECT timestamp, promptTokens, completionTokens, cost FROM usageHistory WHERE timestamp >= ? AND timestamp <= ?`,
|
||||
[tenMinutesAgo.toISOString(), now.toISOString()]
|
||||
);
|
||||
for (const r of recent10) {
|
||||
const tt = new Date(r.timestamp).getTime();
|
||||
const minuteStart = Math.floor(tt / 60000) * 60000;
|
||||
if (bucketMap[minuteStart]) {
|
||||
bucketMap[minuteStart].requests++;
|
||||
bucketMap[minuteStart].promptTokens += r.promptTokens || 0;
|
||||
bucketMap[minuteStart].completionTokens += r.completionTokens || 0;
|
||||
bucketMap[minuteStart].cost += r.cost || 0;
|
||||
}
|
||||
}
|
||||
|
||||
const useDailySummary = period !== "24h";
|
||||
|
||||
if (useDailySummary) {
|
||||
const periodDays = { "7d": 7, "30d": 30, "60d": 60 };
|
||||
const maxDays = periodDays[period] || null;
|
||||
const dayRows = loadDaysInRange(db, maxDays);
|
||||
|
||||
for (const dr of dayRows) {
|
||||
const dateKey = dr.dateKey;
|
||||
const day = parseJson(dr.data, {});
|
||||
stats.totalPromptTokens += day.promptTokens || 0;
|
||||
stats.totalCompletionTokens += day.completionTokens || 0;
|
||||
stats.totalCost += day.cost || 0;
|
||||
|
||||
for (const [prov, p] of Object.entries(day.byProvider || {})) {
|
||||
if (!stats.byProvider[prov]) stats.byProvider[prov] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
|
||||
stats.byProvider[prov].requests += p.requests || 0;
|
||||
stats.byProvider[prov].promptTokens += p.promptTokens || 0;
|
||||
stats.byProvider[prov].completionTokens += p.completionTokens || 0;
|
||||
stats.byProvider[prov].cost += p.cost || 0;
|
||||
}
|
||||
|
||||
for (const [mk, m] of Object.entries(day.byModel || {})) {
|
||||
const rawModel = m.rawModel || mk.split("|")[0];
|
||||
const provider = m.provider || mk.split("|")[1] || "";
|
||||
const statsKey = provider ? `${rawModel} (${provider})` : rawModel;
|
||||
const providerDisplayName = providerNodeNameMap[provider] || provider;
|
||||
if (!stats.byModel[statsKey]) {
|
||||
stats.byModel[statsKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, lastUsed: dateKey };
|
||||
}
|
||||
stats.byModel[statsKey].requests += m.requests || 0;
|
||||
stats.byModel[statsKey].promptTokens += m.promptTokens || 0;
|
||||
stats.byModel[statsKey].completionTokens += m.completionTokens || 0;
|
||||
stats.byModel[statsKey].cost += m.cost || 0;
|
||||
if (dateKey > (stats.byModel[statsKey].lastUsed || "")) stats.byModel[statsKey].lastUsed = dateKey;
|
||||
}
|
||||
|
||||
for (const [connId, a] of Object.entries(day.byAccount || {})) {
|
||||
const accountName = connectionMap[connId] || `Account ${connId.slice(0, 8)}...`;
|
||||
const rawModel = a.rawModel || "";
|
||||
const provider = a.provider || "";
|
||||
const providerDisplayName = providerNodeNameMap[provider] || provider;
|
||||
const accountKey = `${rawModel} (${provider} - ${accountName})`;
|
||||
if (!stats.byAccount[accountKey]) {
|
||||
stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, connectionId: connId, accountName, lastUsed: dateKey };
|
||||
}
|
||||
stats.byAccount[accountKey].requests += a.requests || 0;
|
||||
stats.byAccount[accountKey].promptTokens += a.promptTokens || 0;
|
||||
stats.byAccount[accountKey].completionTokens += a.completionTokens || 0;
|
||||
stats.byAccount[accountKey].cost += a.cost || 0;
|
||||
if (dateKey > (stats.byAccount[accountKey].lastUsed || "")) stats.byAccount[accountKey].lastUsed = dateKey;
|
||||
}
|
||||
|
||||
for (const [akKey, ak] of Object.entries(day.byApiKey || {})) {
|
||||
const rawModel = ak.rawModel || "";
|
||||
const provider = ak.provider || "";
|
||||
const providerDisplayName = providerNodeNameMap[provider] || provider;
|
||||
const apiKeyVal = ak.apiKey;
|
||||
const keyInfo = apiKeyVal ? apiKeyMap[apiKeyVal] : null;
|
||||
const keyName = keyInfo?.name || (apiKeyVal ? apiKeyVal.slice(0, 8) + "..." : "Local (No API Key)");
|
||||
const apiKeyKey = apiKeyVal || "local-no-key";
|
||||
if (!stats.byApiKey[akKey]) {
|
||||
stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel, provider: providerDisplayName, apiKey: apiKeyVal, keyName, apiKeyKey, lastUsed: dateKey };
|
||||
}
|
||||
stats.byApiKey[akKey].requests += ak.requests || 0;
|
||||
stats.byApiKey[akKey].promptTokens += ak.promptTokens || 0;
|
||||
stats.byApiKey[akKey].completionTokens += ak.completionTokens || 0;
|
||||
stats.byApiKey[akKey].cost += ak.cost || 0;
|
||||
if (dateKey > (stats.byApiKey[akKey].lastUsed || "")) stats.byApiKey[akKey].lastUsed = dateKey;
|
||||
}
|
||||
|
||||
for (const [epKey, ep] of Object.entries(day.byEndpoint || {})) {
|
||||
const endpoint = ep.endpoint || epKey.split("|")[0] || "Unknown";
|
||||
const rawModel = ep.rawModel || "";
|
||||
const provider = ep.provider || "";
|
||||
const providerDisplayName = providerNodeNameMap[provider] || provider;
|
||||
if (!stats.byEndpoint[epKey]) {
|
||||
stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, endpoint, rawModel, provider: providerDisplayName, lastUsed: dateKey };
|
||||
}
|
||||
stats.byEndpoint[epKey].requests += ep.requests || 0;
|
||||
stats.byEndpoint[epKey].promptTokens += ep.promptTokens || 0;
|
||||
stats.byEndpoint[epKey].completionTokens += ep.completionTokens || 0;
|
||||
stats.byEndpoint[epKey].cost += ep.cost || 0;
|
||||
if (dateKey > (stats.byEndpoint[epKey].lastUsed || "")) stats.byEndpoint[epKey].lastUsed = dateKey;
|
||||
}
|
||||
}
|
||||
|
||||
// Overlay precise lastUsed timestamps from history
|
||||
const overlayCutoff = maxDays ? Date.now() - maxDays * 86400000 : 0;
|
||||
const histRows = db.all(
|
||||
`SELECT timestamp, provider, model, connectionId, apiKey, endpoint FROM usageHistory WHERE timestamp >= ?`,
|
||||
[new Date(overlayCutoff).toISOString()]
|
||||
);
|
||||
for (const e of histRows) {
|
||||
const ts = e.timestamp;
|
||||
const modelKey = e.provider ? `${e.model} (${e.provider})` : e.model;
|
||||
if (stats.byModel[modelKey] && new Date(ts) > new Date(stats.byModel[modelKey].lastUsed)) stats.byModel[modelKey].lastUsed = ts;
|
||||
|
||||
if (e.connectionId) {
|
||||
const accountName = connectionMap[e.connectionId] || `Account ${e.connectionId.slice(0, 8)}...`;
|
||||
const accountKey = `${e.model} (${e.provider} - ${accountName})`;
|
||||
if (stats.byAccount[accountKey] && new Date(ts) > new Date(stats.byAccount[accountKey].lastUsed)) stats.byAccount[accountKey].lastUsed = ts;
|
||||
}
|
||||
|
||||
const apiKeyKey = (e.apiKey && typeof e.apiKey === "string")
|
||||
? `${e.apiKey}|${e.model}|${e.provider || "unknown"}`
|
||||
: "local-no-key";
|
||||
if (stats.byApiKey[apiKeyKey] && new Date(ts) > new Date(stats.byApiKey[apiKeyKey].lastUsed)) stats.byApiKey[apiKeyKey].lastUsed = ts;
|
||||
|
||||
const endpoint = e.endpoint || "Unknown";
|
||||
const endpointKey = `${endpoint}|${e.model}|${e.provider || "unknown"}`;
|
||||
if (stats.byEndpoint[endpointKey] && new Date(ts) > new Date(stats.byEndpoint[endpointKey].lastUsed)) stats.byEndpoint[endpointKey].lastUsed = ts;
|
||||
}
|
||||
} else {
|
||||
// 24h: live history
|
||||
const cutoff = new Date(Date.now() - PERIOD_MS["24h"]).toISOString();
|
||||
const filtered = db.all(
|
||||
`SELECT timestamp, provider, model, connectionId, apiKey, endpoint, promptTokens, completionTokens, cost, tokens FROM usageHistory WHERE timestamp >= ?`,
|
||||
[cutoff]
|
||||
);
|
||||
|
||||
for (const r of filtered) {
|
||||
const tokens = parseJson(r.tokens, {}) || {};
|
||||
const promptTokens = tokens.prompt_tokens || 0;
|
||||
const completionTokens = tokens.completion_tokens || 0;
|
||||
const entryCost = r.cost || 0;
|
||||
const providerDisplayName = providerNodeNameMap[r.provider] || r.provider;
|
||||
|
||||
stats.totalPromptTokens += promptTokens;
|
||||
stats.totalCompletionTokens += completionTokens;
|
||||
stats.totalCost += entryCost;
|
||||
|
||||
if (!stats.byProvider[r.provider]) stats.byProvider[r.provider] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0 };
|
||||
stats.byProvider[r.provider].requests++;
|
||||
stats.byProvider[r.provider].promptTokens += promptTokens;
|
||||
stats.byProvider[r.provider].completionTokens += completionTokens;
|
||||
stats.byProvider[r.provider].cost += entryCost;
|
||||
|
||||
const modelKey = r.provider ? `${r.model} (${r.provider})` : r.model;
|
||||
if (!stats.byModel[modelKey]) {
|
||||
stats.byModel[modelKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp };
|
||||
}
|
||||
stats.byModel[modelKey].requests++;
|
||||
stats.byModel[modelKey].promptTokens += promptTokens;
|
||||
stats.byModel[modelKey].completionTokens += completionTokens;
|
||||
stats.byModel[modelKey].cost += entryCost;
|
||||
if (new Date(r.timestamp) > new Date(stats.byModel[modelKey].lastUsed)) stats.byModel[modelKey].lastUsed = r.timestamp;
|
||||
|
||||
if (r.connectionId) {
|
||||
const accountName = connectionMap[r.connectionId] || `Account ${r.connectionId.slice(0, 8)}...`;
|
||||
const accountKey = `${r.model} (${r.provider} - ${accountName})`;
|
||||
if (!stats.byAccount[accountKey]) {
|
||||
stats.byAccount[accountKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, connectionId: r.connectionId, accountName, lastUsed: r.timestamp };
|
||||
}
|
||||
stats.byAccount[accountKey].requests++;
|
||||
stats.byAccount[accountKey].promptTokens += promptTokens;
|
||||
stats.byAccount[accountKey].completionTokens += completionTokens;
|
||||
stats.byAccount[accountKey].cost += entryCost;
|
||||
if (new Date(r.timestamp) > new Date(stats.byAccount[accountKey].lastUsed)) stats.byAccount[accountKey].lastUsed = r.timestamp;
|
||||
}
|
||||
|
||||
if (r.apiKey && typeof r.apiKey === "string") {
|
||||
const keyInfo = apiKeyMap[r.apiKey];
|
||||
const keyName = keyInfo?.name || r.apiKey.slice(0, 8) + "...";
|
||||
const akKey = `${r.apiKey}|${r.model}|${r.provider || "unknown"}`;
|
||||
if (!stats.byApiKey[akKey]) {
|
||||
stats.byApiKey[akKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKey: r.apiKey, keyName, apiKeyKey: r.apiKey, lastUsed: r.timestamp };
|
||||
}
|
||||
const ake = stats.byApiKey[akKey];
|
||||
ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cost += entryCost;
|
||||
if (new Date(r.timestamp) > new Date(ake.lastUsed)) ake.lastUsed = r.timestamp;
|
||||
} else {
|
||||
if (!stats.byApiKey["local-no-key"]) {
|
||||
stats.byApiKey["local-no-key"] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, rawModel: r.model, provider: providerDisplayName, apiKey: null, keyName: "Local (No API Key)", apiKeyKey: "local-no-key", lastUsed: r.timestamp };
|
||||
}
|
||||
const ake = stats.byApiKey["local-no-key"];
|
||||
ake.requests++; ake.promptTokens += promptTokens; ake.completionTokens += completionTokens; ake.cost += entryCost;
|
||||
if (new Date(r.timestamp) > new Date(ake.lastUsed)) ake.lastUsed = r.timestamp;
|
||||
}
|
||||
|
||||
const endpoint = r.endpoint || "Unknown";
|
||||
const epKey = `${endpoint}|${r.model}|${r.provider || "unknown"}`;
|
||||
if (!stats.byEndpoint[epKey]) {
|
||||
stats.byEndpoint[epKey] = { requests: 0, promptTokens: 0, completionTokens: 0, cost: 0, endpoint, rawModel: r.model, provider: providerDisplayName, lastUsed: r.timestamp };
|
||||
}
|
||||
const epe = stats.byEndpoint[epKey];
|
||||
epe.requests++; epe.promptTokens += promptTokens; epe.completionTokens += completionTokens; epe.cost += entryCost;
|
||||
if (new Date(r.timestamp) > new Date(epe.lastUsed)) epe.lastUsed = r.timestamp;
|
||||
}
|
||||
}
|
||||
|
||||
stats.totalRequests = Object.values(stats.byProvider).reduce((sum, p) => sum + (p.requests || 0), 0);
|
||||
return stats;
|
||||
}
|
||||
|
||||
export async function getChartData(period = "7d") {
|
||||
const db = await getAdapter();
|
||||
const now = Date.now();
|
||||
|
||||
if (period === "24h") {
|
||||
const bucketCount = 24;
|
||||
const bucketMs = 3600000;
|
||||
const labelFn = (ts) => new Date(ts).toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit", hour12: false });
|
||||
const startTime = now - bucketCount * bucketMs;
|
||||
const buckets = Array.from({ length: bucketCount }, (_, i) => ({ label: labelFn(startTime + i * bucketMs), tokens: 0, cost: 0 }));
|
||||
|
||||
const rows = db.all(
|
||||
`SELECT timestamp, promptTokens, completionTokens, cost FROM usageHistory WHERE timestamp >= ?`,
|
||||
[new Date(startTime).toISOString()]
|
||||
);
|
||||
for (const r of rows) {
|
||||
const t = new Date(r.timestamp).getTime();
|
||||
if (t < startTime || t > now) continue;
|
||||
const idx = Math.min(Math.floor((t - startTime) / bucketMs), bucketCount - 1);
|
||||
buckets[idx].tokens += (r.promptTokens || 0) + (r.completionTokens || 0);
|
||||
buckets[idx].cost += r.cost || 0;
|
||||
}
|
||||
return buckets;
|
||||
}
|
||||
|
||||
const bucketCount = period === "7d" ? 7 : period === "30d" ? 30 : 60;
|
||||
const today = new Date();
|
||||
const labelFn = (d) => d.toLocaleDateString("en-US", { month: "short", day: "numeric" });
|
||||
|
||||
// Build map of dateKey → day data
|
||||
const dayRows = loadDaysInRange(db, bucketCount);
|
||||
const dayMap = {};
|
||||
for (const r of dayRows) dayMap[r.dateKey] = parseJson(r.data, {});
|
||||
|
||||
return Array.from({ length: bucketCount }, (_, i) => {
|
||||
const d = new Date(today);
|
||||
d.setDate(d.getDate() - (bucketCount - 1 - i));
|
||||
const dateKey = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`;
|
||||
const dayData = dayMap[dateKey];
|
||||
return {
|
||||
label: labelFn(d),
|
||||
tokens: dayData ? (dayData.promptTokens || 0) + (dayData.completionTokens || 0) : 0,
|
||||
cost: dayData ? (dayData.cost || 0) : 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function formatLogDate(date = new Date()) {
|
||||
const pad = (n) => String(n).padStart(2, "0");
|
||||
return `${pad(date.getDate())}-${pad(date.getMonth() + 1)}-${date.getFullYear()} ${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`;
|
||||
}
|
||||
|
||||
// No-op: request log is now derived from usageHistory table on read.
|
||||
export async function appendRequestLog() {}
|
||||
|
||||
export async function getRecentLogs(limit = 200) {
|
||||
try {
|
||||
const db = getAdapter();
|
||||
const rows = db.all(
|
||||
`SELECT timestamp, provider, model, connectionId, promptTokens, completionTokens, status, tokens FROM usageHistory ORDER BY id DESC LIMIT ?`,
|
||||
[limit],
|
||||
);
|
||||
if (!rows.length) return [];
|
||||
|
||||
const connMap = {};
|
||||
try {
|
||||
const { getProviderConnections } = await import("./connectionsRepo.js");
|
||||
const connections = await getProviderConnections();
|
||||
for (const c of connections) connMap[c.id] = c.name || c.email || "";
|
||||
} catch {}
|
||||
|
||||
return rows.map((r) => {
|
||||
const ts = formatLogDate(new Date(r.timestamp));
|
||||
const p = r.provider?.toUpperCase() || "-";
|
||||
const m = r.model || "-";
|
||||
const account = connMap[r.connectionId] || (r.connectionId ? r.connectionId.slice(0, 8) : "-");
|
||||
const tk = r.tokens ? parseJson(r.tokens, {}) : {};
|
||||
const sent = r.promptTokens ?? tk.prompt_tokens ?? "-";
|
||||
const received = r.completionTokens ?? tk.completion_tokens ?? "-";
|
||||
return `${ts} | ${m} | ${p} | ${account} | ${sent} | ${received} | ${r.status || "-"}`;
|
||||
});
|
||||
} catch (e) {
|
||||
console.error("[usageRepo] getRecentLogs failed:", e.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user