Compare commits
122
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a6e2c1fa4f | ||
|
|
7e0d0d5123 | ||
|
|
4f4f92706c | ||
|
|
611ba39973 | ||
|
|
192685e1ce | ||
|
|
cacebfd94e | ||
|
|
709074935f | ||
|
|
9f02edd646 | ||
|
|
5fc3cf86d5 | ||
|
|
e81391484b | ||
|
|
3b8fe1b1c3 | ||
|
|
54e7220d06 | ||
|
|
46d889271e | ||
|
|
0c3a82ad77 | ||
|
|
b640079cb6 | ||
|
|
37d15456dc | ||
|
|
153e628aba | ||
|
|
a5f908dc06 | ||
|
|
7fc36170c8 | ||
|
|
ccfe769b2f | ||
|
|
3b688e0d0f | ||
|
|
29f59fd91f | ||
|
|
c3165ed702 | ||
|
|
c72fadbbb4 | ||
|
|
613840e2d1 | ||
|
|
24336186dc | ||
|
|
6b66863897 | ||
|
|
fc40609a3a | ||
|
|
e157ae3ce4 | ||
|
|
3f4c05f5bb | ||
|
|
9c25e0887c | ||
|
|
2aa12b9ec5 | ||
|
|
cab451f0e5 | ||
|
|
183a51003a | ||
|
|
f34811e9b6 | ||
|
|
2c30140ddd | ||
|
|
7e92de63d6 | ||
|
|
68be4352da | ||
|
|
754ef78ff1 | ||
|
|
2959744e0c | ||
|
|
cffa72c21f | ||
|
|
d64c5ba7b6 | ||
|
|
87851f7a71 | ||
|
|
91de43a6c6 | ||
|
|
f8fc08de41 | ||
|
|
dfa69e9f45 | ||
|
|
0bd4b4075e | ||
|
|
588e750ede | ||
|
|
b679a02cb9 | ||
|
|
4ed1d3c053 | ||
|
|
8dbcc2b527 | ||
|
|
bda210475d | ||
|
|
9b78abaf3d | ||
|
|
53c4158511 | ||
|
|
a2a8c824bf | ||
|
|
dec548fa2f | ||
|
|
12033e5537 | ||
|
|
d06caa7e58 | ||
|
|
173ae3c61c | ||
|
|
2637bbc549 | ||
|
|
6fc68126ac | ||
|
|
28d09a24c9 | ||
|
|
f2ec642a0e | ||
|
|
8a491b9ec2 | ||
|
|
7d54d3a660 | ||
|
|
21509c25ff | ||
|
|
09724287cb | ||
|
|
1ac3d50315 | ||
|
|
9c48d50d25 | ||
|
|
8bd3ec8e83 | ||
|
|
5a4a203f01 | ||
|
|
7f0aa5a7ad | ||
|
|
eb0be981c6 | ||
|
|
466e117bd5 | ||
|
|
31e303c187 | ||
|
|
796c6390ac | ||
|
|
ecbf2617e4 | ||
|
|
9ef7d005fb | ||
|
|
842610b1af | ||
|
|
fca96396b9 | ||
|
|
ccf3fa260e | ||
|
|
1accfd9390 | ||
|
|
440ec41da8 | ||
|
|
1c8c0ca081 | ||
|
|
5f42c17caa | ||
|
|
eda5c752b7 | ||
|
|
25d5097edb | ||
|
|
d3e3b4764a | ||
|
|
33a557c761 | ||
|
|
32de2819df | ||
|
|
a21d252e9b | ||
|
|
84a766db0c | ||
|
|
0581dc3485 | ||
|
|
3d6c07bd91 | ||
|
|
60ae1fb5c3 | ||
|
|
1fafebb16d | ||
|
|
a9e09c38e9 | ||
|
|
3d4236e8df | ||
|
|
16becd5340 | ||
|
|
1397380fe9 | ||
|
|
4ffc99b3fe | ||
|
|
81ce5188ea | ||
|
|
4e0c21d86c | ||
|
|
df69b3f05d | ||
|
|
eee332412f | ||
|
|
f750f39b50 | ||
|
|
f1d90b6097 | ||
|
|
7f4196124d | ||
|
|
5658726ea5 | ||
|
|
f5d5690401 | ||
|
|
0aa893ab7d | ||
|
|
6f20b0f146 | ||
|
|
80248d4b7a | ||
|
|
20e991062c | ||
|
|
b784d6d796 | ||
|
|
00e8d68ce5 | ||
|
|
2a8f6d9062 | ||
|
|
9b3134d767 | ||
|
|
36363fa3db | ||
|
|
d133cc3271 | ||
|
|
5a70a685b4 | ||
|
|
100b62800c |
+2
-2
@@ -84,8 +84,8 @@ BACKLOG_SYNC_BATCH_SIZE=100 # Messages per backlog batch, max 100 (d
|
|||||||
|
|
||||||
# === AI Analysis ===
|
# === AI Analysis ===
|
||||||
AI_ANALYSIS_ENABLED=false # Enable AI content moderation (default: false)
|
AI_ANALYSIS_ENABLED=false # Enable AI content moderation (default: false)
|
||||||
# AI_LLM_API_KEY= # REQUIRED if AI_ANALYSIS_ENABLED=true. LLM API key
|
AI_LLM_API_KEY= # REQUIRED if AI_ANALYSIS_ENABLED=true. LLM API key
|
||||||
AI_LLM_BASE_URL=http://100.121.180.82:20128/api/v1 # LLM API base URL (omniroute on imrnes; /api/v1 exposes OpenAI-compatible chat+embeddings)
|
AI_LLM_BASE_URL=https://9router.asepharyana.my.id/v1 # LLM API base URL (9router — OpenAI-compatible router, replaces omniroute)
|
||||||
AI_LLM_MODEL=text # LLM text model name (default: text)
|
AI_LLM_MODEL=text # LLM text model name (default: text)
|
||||||
# AI_LLM_VISION_MODEL= # Vision model for image analysis (falls back to AI_LLM_MODEL)
|
# AI_LLM_VISION_MODEL= # Vision model for image analysis (falls back to AI_LLM_MODEL)
|
||||||
# AI_LLM_EMBEDDING_MODEL= # Embedding model for semantic moderation cache (optional; enables near-duplicate text reuse to save LLM calls)
|
# AI_LLM_EMBEDDING_MODEL= # Embedding model for semantic moderation cache (optional; enables near-duplicate text reuse to save LLM calls)
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
name: Dependabot Auto-Merge
|
||||||
|
on: pull_request
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
pull-requests: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
auto-merge:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.actor == 'dependabot[bot]'
|
||||||
|
steps:
|
||||||
|
- name: Enable auto-merge for Dependabot PR
|
||||||
|
run: gh pr merge --auto --merge "$PR_URL"
|
||||||
|
env:
|
||||||
|
PR_URL: ${{ github.event.pull_request.html_url }}
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
version: 2
|
||||||
|
updates:
|
||||||
|
- package-ecosystem: "npm"
|
||||||
|
directory: "/services/backend"
|
||||||
|
schedule:
|
||||||
|
interval: "daily"
|
||||||
|
open-pull-requests-limit: 10
|
||||||
|
groups:
|
||||||
|
production:
|
||||||
|
dependency-type: "production"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
|
development:
|
||||||
|
dependency-type: "development"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
|
- package-ecosystem: "npm"
|
||||||
|
directory: "/services/discord-gateway"
|
||||||
|
schedule:
|
||||||
|
interval: "daily"
|
||||||
|
open-pull-requests-limit: 10
|
||||||
|
groups:
|
||||||
|
production:
|
||||||
|
dependency-type: "production"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
|
development:
|
||||||
|
dependency-type: "development"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
|
- package-ecosystem: "npm"
|
||||||
|
directory: "/services/frontend"
|
||||||
|
schedule:
|
||||||
|
interval: "daily"
|
||||||
|
open-pull-requests-limit: 10
|
||||||
|
groups:
|
||||||
|
production:
|
||||||
|
dependency-type: "production"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
|
development:
|
||||||
|
dependency-type: "development"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
name: Publish to FlakeHub
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [main, master]
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
flakehub-publish:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
id-token: write
|
|
||||||
contents: read
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
- uses: DeterminateSystems/determinate-nix-action@main
|
|
||||||
- uses: DeterminateSystems/flakehub-push@main
|
|
||||||
with:
|
|
||||||
visibility: public
|
|
||||||
rolling: true
|
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
# GMW — Fitur Publik Lanjutan (#2–#6) Implementation Plan
|
||||||
|
|
||||||
|
> **For Hermes:** Implement task-by-task. Build + lint + typecheck each service
|
||||||
|
> after its changes. Deploy via push to main (CI handles Nix build + systemd).
|
||||||
|
> Hard constraint (user 2026-08-18): public read-only web, fully automatic,
|
||||||
|
> rules in code, NO admin endpoints, NO shadow mode, NO per-channel web config.
|
||||||
|
> **EXPLICITLY EXCLUDED: User Reputation / Strike History** (user: "hapus
|
||||||
|
> sepenuhnya fitur user reputation" — it was never built; do not add it).
|
||||||
|
|
||||||
|
## Existing infra to reuse (verified)
|
||||||
|
- **WS**: backend `ws/server.ts` broadcasts JSON `{type,data,timestamp}` to
|
||||||
|
frontendClients. Backend `ws/redis-bridge.ts` subscribes Redis channels
|
||||||
|
listed in `DISCORD_CHANNEL_TO_WS_EVENT` (backend `shared/redis-channels.ts`)
|
||||||
|
and re-emits as WS events. FE `src/lib/ws` auto-reconnect typed client.
|
||||||
|
- **Gateway → Redis**: `EventBroadcaster` + `RedisEventPublisher` (
|
||||||
|
`discord-gateway/src/modules/event-broadcaster`). Publish via
|
||||||
|
`eventBroadcaster.publish(EventChannels.X, payload)`.
|
||||||
|
- **Moderation data**: `moderation_actions` table (now has explainability
|
||||||
|
cols). `moderation.repository.listActions` returns rows. `ModerationAction`
|
||||||
|
FE type at `frontend/src/lib/types/moderation.ts`.
|
||||||
|
- **Messages**: `messages.list` / `getMessagesByChannel` (backend oRPC +
|
||||||
|
repository). FE `messagesApi` + `useMessages`.
|
||||||
|
- **Charts**: NO chart lib installed. Use **pure SVG/CSS** (consistent with
|
||||||
|
repo; avoid new deps).
|
||||||
|
- **CSV**: client-side Blob download, no backend.
|
||||||
|
|
||||||
|
## Task 1 — Live Moderation Feed (#2)
|
||||||
|
**Gateway**: add `MODERATION_ACTION: "discord:moderation:action"` to
|
||||||
|
`redis-channels.ts` (shared) + `EventChannels.MODERATION_ACTION` in
|
||||||
|
`eventTypes.ts`. In `moderationActionsDb.createModerationAction`, after insert,
|
||||||
|
publish `eventBroadcaster.publish(EventChannels.MODERATION_ACTION, actionRow)`.
|
||||||
|
**Backend**: add `DISCORD_MODERATION_ACTION` constant + map
|
||||||
|
`[DISCORD_MODERATION_ACTION]: "moderation_action"` in `DISCORD_CHANNEL_TO_WS_EVENT`.
|
||||||
|
**FE**: in `src/lib/ws`, subscribe to `moderation_action`; add `useLiveModeration`
|
||||||
|
hook (SWR-style with WS push, capped buffer ~50). Add `<LiveModerationFeed>`
|
||||||
|
client component on `/moderation` page (top of list, animated new-row).
|
||||||
|
Risk: gateway publish at every action (already async insert) — fire-and-forget,
|
||||||
|
wrap in try/catch. Verify WS event reaches FE via `wscat`/curl or log.
|
||||||
|
|
||||||
|
## Task 2 — Toxic Topic Trends (#3)
|
||||||
|
**Backend**: add `moderation.trends` oRPC. Query `moderation_actions` grouped
|
||||||
|
by `categories` (jsonb text[]) over last 30 days, count per category + severity
|
||||||
|
breakdown. Also `action_type` distribution. Return
|
||||||
|
`{ categories: {name,count}[], severities: {level,count}[], actions: {type,count}[] }`.
|
||||||
|
Map jsonb array in SQL (use `unnest` or parse in JS). Reuse `getDatabase`.
|
||||||
|
**FE**: `useModerationTrends` hook + `<TopicTrends>` SVG bar chart (top 10
|
||||||
|
categories) + severity donut (SVG arcs). Place on `/moderation` as a panel.
|
||||||
|
|
||||||
|
## Task 3 — Channel Timeline / Replay (#4)
|
||||||
|
Reuse existing `messages.list` (guildId) + `getMessagesByChannel`. Add a
|
||||||
|
**Timeline tab** to `/messages` that groups messages by date (client-side
|
||||||
|
bucket from `created_at`). Load-more via cursor. No new backend (existing
|
||||||
|
`messagesRouter.list` already supports guildId+limit+cursor). If needed, add
|
||||||
|
`messages.timeline` aggregation (count per day) — but keep simple: client
|
||||||
|
groups fetched rows. Verify existing endpoint returns enough history.
|
||||||
|
|
||||||
|
## Task 4 — Export CSV (#5)
|
||||||
|
**FE only**. `lib/csv.ts` `toCsv(rows, columns)` + `downloadCsv(filename, csv)`.
|
||||||
|
Add "Export CSV" button on `/moderation` (exports current actions) and
|
||||||
|
`/messages` (exports current list). Pure client-side, read-only. No backend.
|
||||||
|
|
||||||
|
## Task 5 — Activity Heatmap (#6)
|
||||||
|
**Backend**: add `messages.activity` oRPC: per-channel message count grouped by
|
||||||
|
hour-of-day (0–23) over last 14 days. Return
|
||||||
|
`{ channels: {channelId, name, byHour: number[24]}[], max }`. Use SQL
|
||||||
|
`EXTRACT(hour from ...)` + group by channel. Channel name from
|
||||||
|
`message.metadata->'channel'->>'channelName'`.
|
||||||
|
**FE**: `useMessageActivity` hook + `<ActivityHeatmap>` SVG grid (channels ×
|
||||||
|
24h, color intensity = count/max). Place on `/messages` or `/dashboard`.
|
||||||
|
|
||||||
|
## Verification checklist
|
||||||
|
- [ ] `pnpm typecheck && pnpm lint && pnpm build` green for gateway, backend, frontend
|
||||||
|
- [ ] Backend `/trpc/moderation/trends` returns categories/severities/actions
|
||||||
|
- [ ] Backend `/trpc/messages/activity` returns byHour grids
|
||||||
|
- [ ] WS `moderation_action` received by FE (log or visible live row)
|
||||||
|
- [ ] No admin/write endpoint added; all public read-only
|
||||||
|
- [ ] No User Reputation code anywhere (grep "reputation|strike|reputasi")
|
||||||
|
- [ ] Deploy via push; all 3 services `running`; moderation + messages pages load
|
||||||
|
|
||||||
|
## Files touched (summary)
|
||||||
|
- gateway: `shared/redis-channels.ts`, `event-broadcaster/eventTypes.ts`,
|
||||||
|
`event-broadcaster/eventBroadcaster.ts`, `message-capture/moderationActionsDb.ts`
|
||||||
|
- backend: `shared/redis-channels.ts`, `orpc/router.ts`,
|
||||||
|
`modules/moderation/moderation.service.ts` (+repository),
|
||||||
|
`modules/messages/messages.service.ts` (+repository, +schema)
|
||||||
|
- frontend: `lib/ws/*`, `hooks/use-moderation.ts`, `hooks/use-messages.ts`,
|
||||||
|
`lib/csv.ts`, `lib/types/*`, `app/(dashboard)/moderation/view.tsx`,
|
||||||
|
`app/(dashboard)/messages/view.tsx`, new components under `components/`
|
||||||
|
|
||||||
|
## Status: COMPLETE (deployed + verified)
|
||||||
|
- Commit 9b3134d: features #2–#6 (live feed, trends, timeline, CSV export, heatmap)
|
||||||
|
- Commit 2a8f6d9: user reputation feature fully removed (643 deletions, no trace in src/tests)
|
||||||
|
- Migration 0016 applied: user_reputations DROPPED (DB verified: false)
|
||||||
|
- All 3 services active (gateway + backend restarted 18:29, frontend running)
|
||||||
|
- Gateway typecheck/lint/test(117 passed); backend typecheck/lint/build; FE lint/build — all GREEN
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
- moderation/stats WS returns data (32 actions) → WS adapter works
|
||||||
|
- DB: user_reputations gone; moderation_actions explainability cols present
|
||||||
|
- Live Feed: gateway publishes discord:moderation:action → backend WS (same path as guild_member_*)
|
||||||
|
- Trends/Activity: backend router procedures registered (typecheck+tsc), same WS adapter
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
# GMW — Fitur Publik Lanjutan #7–#15 + Bug Fix Reputation Removal
|
||||||
|
|
||||||
|
> **For Hermes:** Implement task-by-task. Build + lint + typecheck each service after its
|
||||||
|
> changes. Deploy via push to main (CI handles Nix build + systemd). Apply any new
|
||||||
|
> drizzle migration MANUALLY (systemd does NOT run migrations).
|
||||||
|
> Hard constraint (user): public read-only web, fully automatic, rules in code,
|
||||||
|
> NO admin endpoints, NO shadow mode, NO per-user reputation aggregation.
|
||||||
|
|
||||||
|
## Bug fix discovered during planning (MUST do first)
|
||||||
|
`services/backend/src/modules/dashboard/dashboard.repository.ts` still references
|
||||||
|
`pgUserReputationsTable` (import line 8; JOINs at lines 173 + 457) — that table was
|
||||||
|
DROPPED in migration `0016`. `dashboard.listUsers` / `dashboard.userDetail` will
|
||||||
|
**crash at runtime** (undefined table). Remove the import + the `r.*` join columns
|
||||||
|
(`trust_score`, `clean_message_streak`, `total_infractions`) from both queries.
|
||||||
|
This is a regression introduced by the reputation removal commit.
|
||||||
|
|
||||||
|
## Features to implement (#7–#15)
|
||||||
|
All reuse existing infra: `moderation_actions`, `messages`, `channel_cultures`,
|
||||||
|
`term_glossary_cache`, `ai_analysis_runs`, `message_edits`, gateway cron (for #15),
|
||||||
|
WS (proven Live Feed pattern), oRPC over WS (proven), pure-SVG charts (no libs).
|
||||||
|
|
||||||
|
| # | Feature | Data source | Surface |
|
||||||
|
|---|---------|-------------|---------|
|
||||||
|
| 7 | Flagged Link / Scam Domain Reporter | regex URL from `moderation_actions.content`/`evidence` | `/moderation` |
|
||||||
|
| 8 | Top Flagged Channels | join `moderation_actions.message_id`→`messages.channel_id` | `/moderation` |
|
||||||
|
| 9 | Moderation Heatmap by Hour | `moderation_actions.created_at` hour-of-day | `/moderation` |
|
||||||
|
| 10 | Flag Category Drill-down | `moderation_actions.categories` (reuse Trends) | `/moderation` FE-only |
|
||||||
|
| 11 | Channel Culture Glossary | `channel_cultures` (exists) | new `/channels` panel |
|
||||||
|
| 12 | Term Knowledge Base | `term_glossary_cache` (exists) | new `/glossary` panel |
|
||||||
|
| 13 | Edit/Evasion Tracker | `message_edits` (exists) | `/messages` |
|
||||||
|
| 14 | Auto-mod Coverage Stats | `ai_analysis_runs` (exists) | `/moderation` metric tiles |
|
||||||
|
| 15 | Weekly Digest (auto, cron) | aggregate #7/#8/#9 → Discord via gateway cron | gateway cron + `/moderation` |
|
||||||
|
|
||||||
|
## Architecture per layer
|
||||||
|
|
||||||
|
### Backend (oRPC, `services/backend/src`)
|
||||||
|
- New repository methods (add to existing repos, follow `getTrends` SQL style):
|
||||||
|
- `moderation.repository.ts`:
|
||||||
|
- `getTopFlaggedDomains(days)` — `regexp_matches(content,'https?://([^/\s]+)')` on
|
||||||
|
`moderation_actions WHERE created_at>=since`, group by host, COUNT, order DESC LIMIT 20.
|
||||||
|
- `getTopFlaggedChannels(days)` — join `moderation_actions a` LEFT JOIN `messages m`
|
||||||
|
ON `m.id=a.message_id`, group by `m.channel_id`, COUNT, order DESC LIMIT 15.
|
||||||
|
Channel name via `m.metadata::jsonb->'channel'->>'channelName'`.
|
||||||
|
- `getHourlyModeration(days)` — `EXTRACT(HOUR FROM to_timestamp(created_at/1000))`
|
||||||
|
group by hour, COUNT, severity breakdown. (24 rows)
|
||||||
|
- `getFlaggedByCategory(days, category)` — list actions where `categories` contains
|
||||||
|
`category` (reuse `listActions` filter or new query), for drill-down #10.
|
||||||
|
- `getCoverage(days)` — from `ai_analysis_runs`: total runs, status breakdown
|
||||||
|
(clean/flagged/warn/error/pending), coverage % = (analyzed)/(captured in window).
|
||||||
|
- `dashboard.repository.ts` (or new `knowledge.repository.ts`):
|
||||||
|
- `listChannelCultures(limit, search?)` — `channel_cultures` rows (channel_id,
|
||||||
|
guild_id, channel_name from messages metadata, culture_summary, last_analyzed_at).
|
||||||
|
- `listGlossary(limit, search?)` — `term_glossary_cache` (term, definition, source_url,
|
||||||
|
resolved_at, hit_count) order by hit_count DESC.
|
||||||
|
- `messages.repository.ts`:
|
||||||
|
- `getEditHistory(limit, channelId?)` — `message_edits` join `messages` for
|
||||||
|
old_content + channel + username + edited_at, order DESC LIMIT.
|
||||||
|
- `moderation.service.ts` / `dashboard.service.ts` / `messages.service.ts`: thin wrappers.
|
||||||
|
- `orpc/router.ts`: add procedures (follow `trends` shape):
|
||||||
|
- `moderation.topDomains`, `moderation.topChannels`, `moderation.byHour`,
|
||||||
|
`moderation.byCategory` (input `{days,category}`), `moderation.coverage`.
|
||||||
|
- `dashboard.channelCultures`, `dashboard.glossary`.
|
||||||
|
- `messages.editHistory`.
|
||||||
|
|
||||||
|
### Frontend (`services/frontend/src`)
|
||||||
|
- `lib/types/moderation.ts`: add `FlaggedDomain`, `FlaggedChannel`, `HourlyModeration`,
|
||||||
|
`ModerationCoverage` interfaces.
|
||||||
|
- `lib/types/index.ts` (+ message.ts): add `ChannelCultureRow`, `GlossaryRow`, `EditHistoryRow`.
|
||||||
|
- `lib/api/moderation.ts`: add `topDomains`, `topChannels`, `byHour`, `byCategory`, `coverage`.
|
||||||
|
- `lib/api/dashboard.ts` (or messages.ts): add `channelCultures`, `glossary`, `editHistory`.
|
||||||
|
- `lib/api/server.ts`: add SSR seed fetchers (follow `getModerationStats`).
|
||||||
|
- `hooks/use-moderation.ts`: add `useTopDomains`, `useTopChannels`, `useHourlyModeration`,
|
||||||
|
`useByCategory`, `useCoverage`. `hooks/use-dashboard.ts`/`use-messages.ts`: add culture/glossary/edit hooks. `hooks/index.ts`: export all.
|
||||||
|
- New components (pure SVG/CSS, reuse `GlassPanel`/`SectionHeader`/`Badge`/`Donut`):
|
||||||
|
- `components/ScamDomains.tsx`, `components/TopChannels.tsx`, `components/ModerationHeatmap.tsx`,
|
||||||
|
`components/CoverageTiles.tsx`, `components/ChannelCultureGlossary.tsx`,
|
||||||
|
`components/TermGlossary.tsx`, `components/EditHistory.tsx`.
|
||||||
|
- Wire into `app/(dashboard)/moderation/view.tsx` (grid col-span-2/3/5 as space allows)
|
||||||
|
and `app/(dashboard)/messages/view.tsx` (EditHistory panel) and new route pages
|
||||||
|
`app/(dashboard)/channels/page.tsx` + `app/(dashboard)/glossary/page.tsx` with
|
||||||
|
matching `view.tsx` (follow existing page→view SSR pattern; check `app/(dashboard)/dashboard/page.tsx`).
|
||||||
|
- Export CSV buttons reuse `lib/csv.ts` `downloadCsv` (client-side) for domains/channels/edits.
|
||||||
|
|
||||||
|
### Gateway (#15 Weekly Digest)
|
||||||
|
- Add a cron/interval in `services/discord-gateway` (check existing scheduler pattern —
|
||||||
|
search `setInterval`/`cron` in `src`). On a 7-day cadence, query backend oRPC
|
||||||
|
(`dashboard.activity`, `moderation.trends`, `moderation.topChannels`) — OR compute
|
||||||
|
directly via a shared repository — and post a formatted summary to the monitor guild
|
||||||
|
channel (via existing `discordClient.channels.send` helper). Fully automatic, no UI.
|
||||||
|
|
||||||
|
## Files touched (summary)
|
||||||
|
- backend: `modules/moderation/{repository,service}.ts`, `modules/dashboard/{repository,service}.ts`,
|
||||||
|
`modules/messages/{repository,service}.ts`, `orpc/router.ts`, `shared/index.ts` (if new tables),
|
||||||
|
`lib/types/*` (FE)
|
||||||
|
- frontend: `lib/api/*`, `lib/types/*`, `hooks/*`, `components/*`, `app/(dashboard)/*`
|
||||||
|
- gateway: new digest scheduler + (none if reuse backend) maybe `shared/redis-channels.ts`
|
||||||
|
|
||||||
|
## Constraints / pitfalls (from gmw-ops skill)
|
||||||
|
- `created_at` is bigint epoch-MS — compare with `<`/`>`, do NOT divide by 1000 in SQL.
|
||||||
|
- Pure SVG only — frontend has ZERO chart libs.
|
||||||
|
- `Badge` Tone = signal|amber|vermilion|neutral (no "rose").
|
||||||
|
- Frontend WS import is `@/lib/ws/context`; method `on` not `subscribe`.
|
||||||
|
- Commit author `asepharyana`, no Co-Authored-By.
|
||||||
|
- Rebuild `dist/` after gateway changes; apply drizzle migrations manually.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
- Per service: `pnpm typecheck && pnpm lint && pnpm build` green.
|
||||||
|
- Gateway: `pnpm test` (117+ pass).
|
||||||
|
- Live: `moderation/stats` WS returns data (proves adapter); new procedures registered
|
||||||
|
(typecheck = proof). `systemctl show` new ActiveEnterTimestamp after deploy.
|
||||||
|
- DB: confirm `channel_cultures`/`term_glossary_cache`/`message_edits`/`ai_analysis_runs`
|
||||||
|
have rows before relying on them (some may be empty → components handle empty state).
|
||||||
|
|
||||||
|
## Execution order
|
||||||
|
1. Bug fix dashboard.repository (reputation JOIN) — deploy-safe.
|
||||||
|
2. Backend repositories + service + router (#7,#8,#9,#14 dashboard; #11,#12; #13).
|
||||||
|
3. FE types + api + hooks + components + wire (#7,#8,#9,#10,#11,#12,#13,#14).
|
||||||
|
4. Gateway #15 digest (if scheduler exists) — verify via log, not UI.
|
||||||
|
5. Build/lint all 3 services; commit; push; monitor CI; apply migrations; verify live.
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
# AI Analysis Flow — Audit & Optimization (discord-gateway)
|
||||||
|
|
||||||
|
**Goal:** Analisis alur AI analysis end-to-end, temukan bug/inconsistency yang merusak kualitas verdict, lalu perbaiki root cause-nya.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
- `services/discord-gateway/src/modules/ai-moderation/**`
|
||||||
|
- Tidak menyentuh chatbot backend / frontend.
|
||||||
|
|
||||||
|
## Alur saat ini (hasil tracing)
|
||||||
|
```
|
||||||
|
message capture → aiAnalyzer.queueMessageAnalysis(messageId)
|
||||||
|
→ batchScheduler.scheduleConversationAnalysis(conversationKey) [debounce 250ms, CB gate]
|
||||||
|
→ messageStore.getPendingMessagesByConversation(≤200)
|
||||||
|
→ skipAgeRestrictedMessages
|
||||||
|
→ pickBatchWithinBudget(14000 tokens, 50/msg)
|
||||||
|
→ processBatch [Piscina worker, ≤4 threads]
|
||||||
|
→ ai-analysis-worker.processBatch
|
||||||
|
→ getConversationContextBefore(20 msgs) + attachments
|
||||||
|
→ attachment-upload race guard (pending upload → skip)
|
||||||
|
→ runModerationAnalysis
|
||||||
|
→ Phase 1: exact-hash cache (PG text_analysis_cache, per channel/thread)
|
||||||
|
→ Phase 2: semantic cache (embedTexts → Qdrant batch search; PG fallback)
|
||||||
|
→ split text-only vs media
|
||||||
|
→ runTextOnlyBatch: URL fetch + wiki search + glossary (paralel)
|
||||||
|
→ dedup short messages → sub-batches (60/sub-batch)
|
||||||
|
→ vision evidence utk URL images (hoisted, 15s cap per image)
|
||||||
|
→ callModerationLLM per sub-batch (stream:true, retries 3, JSON parse + correction retry)
|
||||||
|
→ runMediaBatch: download → vision per image (cache LRU→DB→live, lock) → 1 LLM call
|
||||||
|
→ setCachedTextModeration (PG + Qdrant upsert w/ embedding)
|
||||||
|
→ normalizeResult (confidence clamp, fallback analysis)
|
||||||
|
→ updateMessagesAIAnalysisBulk → broadcast + scheduleAutoDelete
|
||||||
|
→ recovery worker tiap 10s: pending keys → re-schedule; incomplete → individual fallback queue
|
||||||
|
→ individual fallback: 1 msg = 1 worker job (context + full LLM)
|
||||||
|
→ cache prune tiap 6 jam (PG expired + Qdrant expired points)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Temuan audit (ranked)
|
||||||
|
|
||||||
|
### F1 — Cache hit menghapus status "warn" (BUG AKURASI)
|
||||||
|
`moderationOrchestrator.ts` Phase-2 semantic hit & PG-fallback memetakan status via
|
||||||
|
`parseQdrantVerdict`: storedStatus bukan "warn"/"flagged" → dipaksa "clean".
|
||||||
|
TAPI exact-hash lookup (`getCachedTextModeration`, textCacheStore.ts:288-295) lebih parah:
|
||||||
|
hanya menerima "clean"|"flagged" — **"warn" jatuh ke branch flags.length===0 ? clean : flagged**
|
||||||
|
→ warn dengan flags=["conflict_instigation"] dibaca sebagai FLAGGED.
|
||||||
|
Efek: auto-delete eligibility (butuh recommendedAction delete/escalate + severity list) salah baca;
|
||||||
|
dashboard menampilkan flagged padahal verdict asli warn. Root cause: type narrowing legacy
|
||||||
|
(`status: "clean" | "flagged"`) tidak diupdate ketika "warn" ditambahkan ke schema.
|
||||||
|
|
||||||
|
### F2 — Exact-cache key mengabaikan edit (BUG EVASION)
|
||||||
|
Key = sha256(content)+context. Pesan yang DIEDIT (`edited_content`) menghasilkan hash berbeda,
|
||||||
|
tapi verdict lama utk konten pre-edit tetap hidup; lebih penting: pesan edited="true" adalah sinyal
|
||||||
|
evasion di prompt, sedangkan cache bisa menyajikan verdict dari konten lama jika content sama.
|
||||||
|
(Minor, tapi konsistensi: `resolveIsEdited` ada di prompt, tidak ada di cache key.)
|
||||||
|
|
||||||
|
### F3 — `pickBatchWithinBudget` skip-bukan-break (LATENSI/KUALITAS)
|
||||||
|
Loop `if (usedTokens + msgTokens <= maxTokens) {push}` — pesan BESAR di tengah list dilewati
|
||||||
|
dan iterasi lanjut mencoba msg berikutnya. Efek: batch berisi "lubang" (msg pending tetap pending,
|
||||||
|
dianalisis di gelombang berikutnya = LLM call tambahan). Ini by-design tolerable, tapi ada bug halus:
|
||||||
|
pesan >budget tunggal tidak pernah masuk (scheduler sudah punya fallback slice(0,1), OK).
|
||||||
|
Keputusan: biarkan (bukan bug nyata), catat saja.
|
||||||
|
|
||||||
|
### F4 — `callModerationLLM` max_tokens 16384 hardcoded (COST)
|
||||||
|
Sub-batch 60 pesan × output ~150 token/pesan ≈ 9k token cukup; 16k aman. Biarkan.
|
||||||
|
|
||||||
|
### F5 — Dead code builder user-profile/reputation
|
||||||
|
`buildUserProfilesBlock`, `buildUserProfileRef`, `UserProfileEntry` di moderationBuilders.ts
|
||||||
|
tidak dipakai lagi sejak context minimization (hanya tests). `<user_history>` juga tak pernah
|
||||||
|
di-inject (rules masih menyebutnya — misleading bagi model). Bersihkan referensi prompt.
|
||||||
|
|
||||||
|
### F6 — rules.ts menyebut `<user_history>` yang tidak pernah ada di payload
|
||||||
|
Model diberi instruksi tentang blok yang tak pernah muncul → pemborosan token + potensi
|
||||||
|
kelakuan aneh ("menunggu" data yang tak ada). Hapus/ubah kalimat.
|
||||||
|
|
||||||
|
### F7 — system.ts "Blok Data" menyebut `<term_glossary> (SearXNG)` — STALE
|
||||||
|
Sumber sudah Wikipedia. Komentar kode & teks prompt menyebut SearXNG. Perbaiki teks (kecil).
|
||||||
|
|
||||||
|
### F8 — output.ts typo "secifik", baris tabel `-|-` rusak
|
||||||
|
Kualitas prompt: typo + markdown table broken (`||-`) di beberapa baris. Rapikan.
|
||||||
|
|
||||||
|
### F9 — llmCaller parse-error correction tail hanya di SYSTEM
|
||||||
|
Correction tail ditambahkan ke system prompt; provider caching fine, tapi preview invalid
|
||||||
|
content (800 char) ikut SYSTEM — ok. Skip.
|
||||||
|
|
||||||
|
### F10 — `getLlmSemaphore` race kecil saat config berubah di tengah flight
|
||||||
|
Non-issue praktis (config statis per proses). Skip.
|
||||||
|
|
||||||
|
## Keputusan perbaikan (yang dieksekusi sekarang)
|
||||||
|
1. **F1 (utama):** normalisasi status di SATU tempat — `normalizeStoredStatus()` di
|
||||||
|
textCacheStore.ts yang menerima clean/warn/flagged; pakai di getCachedTextModeration
|
||||||
|
DAN parseQdrantVerdict; perluas return types ke union penuh. Orchestrator tinggal pakai.
|
||||||
|
2. **F6+F7+F8:** bersihkan stale references di prompts (user_history, SearXNG, typo).
|
||||||
|
3. **F5:** hapus dead builders + test-nya (biome/tsc yang jaga).
|
||||||
|
4. Regression test untuk F1 (vitest): warn tersimpan → warn terbaca (exact + qdrant path).
|
||||||
|
|
||||||
|
## Files touched
|
||||||
|
- services/discord-gateway/src/modules/ai-moderation/textCacheStore.ts (F1)
|
||||||
|
- services/discord-gateway/src/modules/ai-moderation/moderationOrchestrator.ts (type only)
|
||||||
|
- services/discord-gateway/src/modules/ai-moderation/prompts/rules.ts (F6)
|
||||||
|
- services/discord-gateway/src/modules/ai-moderation/prompts/system.ts (F7)
|
||||||
|
- services/discord-gateway/src/modules/ai-moderation/prompts/output.ts (F8)
|
||||||
|
- services/discord-gateway/src/modules/ai-moderation/moderationBuilders.ts (F5)
|
||||||
|
- services/discord-gateway/tests/contextEnrichment.test.ts (F5 test cleanup + F1 regression test baru)
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
```
|
||||||
|
cd services/discord-gateway
|
||||||
|
npx tsc --noEmit
|
||||||
|
npx biome check --diagnostic-level=error .
|
||||||
|
npx vitest run
|
||||||
|
```
|
||||||
|
Semua harus hijau sebelum commit. Deploy via GHA (push main) — user konfirmasi belakangan.
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# Optimisasi "non-issue" AI analysis pipeline
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
Dua item yang sebelumnya dinyatakan non-issue, kini dioptimalkan + 1 bug ordering
|
||||||
|
yang ditemukan saat menelusuri:
|
||||||
|
|
||||||
|
1. **pickBatchWithinBudget: skip → break.** Pesan diurutkan `created_at ASC`
|
||||||
|
oleh DB. Setelah budget habis, pesan berikutnya pasti lebih besar/lebih kecil
|
||||||
|
arbitrer — skip-then-take menghasilkan batch non-kontigu (ada gap analisis
|
||||||
|
di tengah timeline). Ubah jadi stop at first overflow (break) supaya prefix
|
||||||
|
kronologis utuh; sisanya otomatis diambil gelombang berikutnya
|
||||||
|
(`shouldScheduleNext` sudah selalu true setelah sukses).
|
||||||
|
2. **max_tokens dinamis.** Hard-coded 16384 di llmCaller.ts → parameter
|
||||||
|
opsional `maxTokens?`; default tetap 16384. Caller text/media batch pass
|
||||||
|
nilai berbasis ukuran prompt (tiktoken) dengan floor/ceiling.
|
||||||
|
3. **Bug ordering UPDATE..RETURNING (bonus).** messagesAnalysis.ts
|
||||||
|
`getPendingMessagesByConversation`: SELECT ids di-order `created_at ASC`
|
||||||
|
tapi UPDATE...RETURNING tanpa ORDER BY → urutan rows balik tidak
|
||||||
|
terjamin. Konsumen pakai messages[0] sebagai anchor konteks
|
||||||
|
(beforeCreatedAt) dan pickBatchWithinBudget asumsi urutan. Fix: re-sort in
|
||||||
|
JS by created_at (stable) sebelum return.
|
||||||
|
|
||||||
|
## Files touched
|
||||||
|
- src/modules/ai-moderation/batchProcessor.ts — break bukan skip; test baru.
|
||||||
|
- src/modules/ai-moderation/llmCaller.ts — param maxTokens.
|
||||||
|
- src/modules/ai-moderation/textBatchProcessor.ts / mediaBatchProcessor.ts —
|
||||||
|
hitung token prompt & pass maxTokens.
|
||||||
|
- src/modules/message-capture/messagesAnalysis.ts — sort hasil RETURNING.
|
||||||
|
- tests/batchBudget.test.ts — baru.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
cd services/discord-gateway && bun run typecheck && bun run lint && bun run test
|
||||||
|
lalu commit+push, watch GHA, restart service via deploy pipeline.
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# Spec: Perbagus fitur Voice + Audio Playback (GMW frontend)
|
||||||
|
|
||||||
|
Tanggal: 2026-08-22 · Scope: **frontend only** (backend/gateway API sudah cukup)
|
||||||
|
|
||||||
|
## Masalah (audit)
|
||||||
|
1. Recordings: semua kartu pakai `<audio controls>` native — tampilan identik,
|
||||||
|
tidak ada indikasi which-clip-playing / loading / paused, dan N audio bisa
|
||||||
|
play bareng (overlap).
|
||||||
|
2. Media view: `thumbnailUrl` dari gateway tidak dipakai; tidak ada visual
|
||||||
|
"sedang playing" selain disc spin; queue item semua sama tanpa badge up-next.
|
||||||
|
3. Mini-player (`lib/hooks/use-media-player.tsx`) ada tapi TIDAK PERNAH
|
||||||
|
dimount → dead code, user tidak lihat status musik di halaman lain.
|
||||||
|
4. Voice page: `useMicTransmit.setVolume` + `useVoiceListen.setVolume`
|
||||||
|
tersedia tapi tak ada UI-nya; mic live tidak punya level feedback.
|
||||||
|
|
||||||
|
## Desain
|
||||||
|
|
||||||
|
### A. RecordingAudioPlayer (baru, `components/voice/recording-audio-player.tsx`)
|
||||||
|
Custom player menggantikan `<audio controls>`:
|
||||||
|
- Play/pause button (ikon berubah), spinner saat buffering (`waiting` event).
|
||||||
|
- Progress bar seekable (click-to-seek) + time label `m:ss / m:ss`.
|
||||||
|
- Waveform-ish equalizer bars saat playing (CSS animation, reduced-motion safe).
|
||||||
|
- **Single-playback**: module-level registry `activePlayers` — memainkan satu
|
||||||
|
clip otomatis pause yang lain.
|
||||||
|
- Kartu pemilik player aktif dapat highlight border signal + "Now playing" chip.
|
||||||
|
|
||||||
|
### B. Recordings view — pasang player baru
|
||||||
|
- Ganti `<audio>` → `<RecordingAudioPlayer src download_url>`.
|
||||||
|
- Highlight kartu via state lifted: `playingId` di view, callback `onPlay`.
|
||||||
|
|
||||||
|
### C. Media view polish
|
||||||
|
- Hero: thumbnail (jika `current.thumbnailUrl`) sebagai disc center image;
|
||||||
|
fallback ListMusic icon. Equalizer bars animasi CSS saat `playing`.
|
||||||
|
- Queue row pertama: badge "up next"; baris current track diberi ring signal.
|
||||||
|
- Volume read-only tetap.
|
||||||
|
|
||||||
|
### D. MiniPlayer global
|
||||||
|
- Hapus `lib/hooks/use-media-player.tsx` (dead) — ganti dengan komponen
|
||||||
|
`components/media/mini-player.tsx` yang subscribe `useMediaState` +
|
||||||
|
`useMediaWsSync` langsung (SWR cache shared antar route), mounted di
|
||||||
|
`AppFrame` bawah layar (fixed bottom, hidden di route `/media`).
|
||||||
|
- Menampilkan: thumbnail kecil/judul, tombol skip/stop, link ke /media.
|
||||||
|
|
||||||
|
### E. Voice UI
|
||||||
|
- Mic live: level meter (Equalizer bars) — mic-transmitter sudah punya worklet;
|
||||||
|
tambah `getLevel()` via AnalyserNode pada stream (simple RMS) di hook.
|
||||||
|
- Listen: volume slider (input range) wired ke `listen.setVolume`.
|
||||||
|
- Mic volume slider wired ke `mic.setVolume`.
|
||||||
|
|
||||||
|
## File touched
|
||||||
|
| File | Aksi |
|
||||||
|
|---|---|
|
||||||
|
| services/frontend/src/components/voice/recording-audio-player.tsx | new |
|
||||||
|
| services/frontend/src/app/(dashboard)/recordings/view.tsx | edit |
|
||||||
|
| services/frontend/src/app/(dashboard)/media/view.tsx | edit |
|
||||||
|
| services/frontend/src/components/media/mini-player.tsx | new |
|
||||||
|
| services/frontend/src/components/shell/ambient-app.tsx | mount MiniPlayer |
|
||||||
|
| services/frontend/src/lib/hooks/use-media-player.tsx | delete |
|
||||||
|
| services/frontend/src/hooks/use-voice.ts | tambah micLevel |
|
||||||
|
| services/frontend/src/lib/audio/mic-transmit.ts | expose analyser level |
|
||||||
|
| services/frontend/src/app/(dashboard)/voice/view.tsx | sliders + meter |
|
||||||
|
|
||||||
|
## Verifikasi
|
||||||
|
1. `pnpm lint` (biome) + `pnpm build` clean.
|
||||||
|
2. Smoke di port **4024** (BUKAN 4017) → curl 200 semua route.
|
||||||
|
3. Commit (tanpa trailer) → push → `gh run watch` → live check
|
||||||
|
https://imphnen.asepharyana.my.id/{media,recordings,voice}/ = 200.
|
||||||
@@ -0,0 +1,127 @@
|
|||||||
|
# Spec: Optimasi AI Analysis GMW — Naikkan Cache Hit Tanpa Kehilangan Akurasi
|
||||||
|
|
||||||
|
Tanggal: 2026-08-24 · Repo: `~/GMW` (branch `main`) · Service: `services/discord-gateway`
|
||||||
|
|
||||||
|
## Latar & Evidence (audit 2026-08-24)
|
||||||
|
|
||||||
|
State produksi:
|
||||||
|
- Qdrant `gmw_text_moderation`: **1.550 poin, status green** (vectors size 2048, Cosine).
|
||||||
|
- PG `text_analysis_cache`: 1.634 row `user_moderation`, 277 `vision_llm`; **sum(hit_count) = 0** →
|
||||||
|
hit-rate tidak pernah terukur.
|
||||||
|
- Embedding aktif (`AI_LLM_EMBEDDING_MODEL` set, Nemotron-embed, dim 2048), `AI_LLM_EMBEDDING_MIN_SIMILARITY`
|
||||||
|
tidak diset di BWS → default **0.97** (sangat konservatif).
|
||||||
|
- Messages: 9.375 total; 643 status `error` (banyak retry), 49 pending.
|
||||||
|
|
||||||
|
Temuan audit alur (`moderationOrchestrator.ts` → `textCacheStore.ts` → `qdrantClient.ts`,
|
||||||
|
`textBatchProcessor.ts`, `urlFetcher.ts`, `wikipediaClient.ts`, `visionAnalyzer.ts`):
|
||||||
|
|
||||||
|
| # | Temuan | Dampak |
|
||||||
|
|---|--------|--------|
|
||||||
|
| F1 | Exact-hash cache key menyertakan context (channel/thread) → teks sama di channel lain selalu miss | Killer hit-rate #1 |
|
||||||
|
| F2 | Semantic tier TIDAK memfilter context (Qdrant payload tak punya context) — sudah global tapi hanya aman krn sim 0.97 ketat | Inkonsisten dgn exact tier |
|
||||||
|
| F3 | Phase-1 lookup loop `await getCachedTextModeration(key)` per pesan → N round-trip PgBouncer per batch (60 msg = 60 query serial) | Latensi + beban DB |
|
||||||
|
| F4 | Verdict actionable (flagged/warn) dan clean sama-sama boleh di-serve semantic; toleransi akurasi beda | Risiko akurasi |
|
||||||
|
| F5 | `hit_count` tidak pernah di-increment oleh reader manapun | Hit-rate tak terukur |
|
||||||
|
| F6 | `wikipediaSearch()` (blok `<web_searches>`) tanpa cache — re-fetch tiap batch utk query sama | Latensi + spam ke WP |
|
||||||
|
| F7 | `fetchUrlSafely()` tanpa cache — link sama di batch berikutnya di-download lagi penuh | Latensi + bandwidth |
|
||||||
|
| F8 | Vision cache key dari data-URL base64 hasil resize → attachment sama via jalur berbeda (URL vs embed) = key beda → re-download + re-vision | Duplikasi kerja vision |
|
||||||
|
|
||||||
|
Non-goals: mengubah pipeline enforcement (auto-mute/ban trust-store writes), mengubah prompt
|
||||||
|
kebijakan moderasi, mengubah model/embedding provider.
|
||||||
|
|
||||||
|
## Desain
|
||||||
|
|
||||||
|
Semua perubahan degrade gracefully — cache gagal → perilaku lama (LLM). Akurasi dilindungi
|
||||||
|
asimetris: **hemat boleh untuk verdict non-actionable, konservatif untuk yang memicu aksi.**
|
||||||
|
|
||||||
|
### D1 — Cache metrics (F5)
|
||||||
|
- `textCacheStore.getCachedTextModeration()`: saat hit valid, increment `hit_count`
|
||||||
|
(`UPDATE ... SET hit_count = hit_count + 1`) fire-and-forget (`.catch(()=>{})`), jangan blokir return.
|
||||||
|
- Log info periodik ringkas di orchestrator sudah ada ("User moderation cache applied") — cukup.
|
||||||
|
|
||||||
|
### D2 — Batched exact-cache lookup (F3)
|
||||||
|
- Fungsi baru `getCachedTextModerations(keys: string[]): Promise<Map<string, StoredModerationVerdict>>`
|
||||||
|
di `textCacheStore.ts`: **satu** `SELECT ... WHERE text = ANY($1)` (chunk 200 key/query),
|
||||||
|
parse + `normalizeStoredStatus` per row (reuse helper existing).
|
||||||
|
- Orchestrator fase-1: kumpulkan semua key unik → satu call batched → distribusi hasil.
|
||||||
|
- Semantik identik dengan loop lama (row expired/error-artifact tetap miss); hanya jumlah round-trip
|
||||||
|
yang turun N→1.
|
||||||
|
|
||||||
|
### D3 — Global exact reuse untuk verdict non-actionable (F1)
|
||||||
|
- Key scoped-context TETAP ditulis (kompatibel, invalidasi moderator tetap presisi).
|
||||||
|
- Reader tambahan: kalau key `<ctx>:<hash>` miss, coba key legacy global `text_mod:<hash>` (bare).
|
||||||
|
- Guard akurasi (WAJIB semua terpenuhi):
|
||||||
|
- `status === "clean"` DAN `flags.length === 0`;
|
||||||
|
- `confidence >= AI_CACHE_GLOBAL_REUSE_MIN_CONFIDENCE` (default 0.85);
|
||||||
|
- `recommendedAction === "none"`;
|
||||||
|
- umur entry ≤ `AI_CACHE_GLOBAL_REUSE_MAX_AGE_H` (default 72h) — cek `analyzed_at`.
|
||||||
|
- Flag baru `policyVersion: "cached-global-clean-2026-08"` supaya terlacak di dashboard/log.
|
||||||
|
- Verdict flagged/warn TETAP context-scoped (tidak pernah lintas channel).
|
||||||
|
|
||||||
|
### D4 — Semantic dua-band similarity (F2+F4)
|
||||||
|
- Config baru: `AI_LLM_EMBEDDING_MIN_SIMILARITY_ACTIONABLE` default **0.97** (perilaku lama),
|
||||||
|
`AI_LLM_EMBEDDING_MIN_SIMILARITY_CLEAN` default **0.92**, keduanya coerce number 0..1.
|
||||||
|
- Satu Qdrant batch search pakai threshold RENDAH (0.92). Per hit, klasifikasi ulang:
|
||||||
|
- verdict non-actionable (clean, no flags, action=none): terima jika `score >= CLEAN_BAND`;
|
||||||
|
- verdict actionable (warn/flagged atau flags ada / action != none): terima hanya jika
|
||||||
|
`score >= ACTIONABLE_BAND` (0.97 — persis gate lama);
|
||||||
|
- di antara dua band → buang hit, pesan lanjut ke LLM (fail-open ke akurasi).
|
||||||
|
- Legacy PG fallback path: filter serupa di `findSimilarTextModeration` via parameter band.
|
||||||
|
|
||||||
|
### D5 — Cache Wikipedia search (F6)
|
||||||
|
- `wikipediaClient.wikipediaSearch(query)`: cek `cacheGet(makeCacheKey("wikisearch", q))` dulu;
|
||||||
|
miss → fetch (timeout existing) → sukses & hasil non-kosong → `cacheSet(..., TTL 6h)`.
|
||||||
|
Hasil kosong TIDAK di-cache (biar retry nanti). Redis down → langsung fetch (no-op cache).
|
||||||
|
|
||||||
|
### D6 — Cache URL text fetch (F7)
|
||||||
|
- `urlFetcher.fetchUrlSafely(url)`: wrapper async memoize in-process LRU (max 500, TTL 30 menit)
|
||||||
|
untuk `type === "text"` saja (image tetap selalu fresh-download karena dipakai sbg bukti vision
|
||||||
|
+ buffer besar; error tidak di-cache).
|
||||||
|
- Import `LRUCache` dari `lru-cache` (sudah dep gateway).
|
||||||
|
|
||||||
|
### D7 — Unified vision cache key (F8)
|
||||||
|
- `makeImageCacheKey(imageUrl)` di `textCacheStore.ts`: sebelum hash, strip query Discord CDN
|
||||||
|
(`?ex=&is=&hm=` signed tokens, `format/width/height/size`) — regex `(\?[^#]*)$` dibuang bila host
|
||||||
|
CDN discord (`cdn.discordapp.com`, `media.discordapp.net`, `images-ext-*.discordapp.net`);
|
||||||
|
URL non-Discord: hash full URL seperti sekarang.
|
||||||
|
- Efek: attachment sama yang lolos lewat jalur embed vs inline vs re-fetch dgn token beda → SATU
|
||||||
|
entry cache → skip download+vision kedua kali. Data-URL base64 tetap di-hash apa adanya.
|
||||||
|
|
||||||
|
## File yang disentuh
|
||||||
|
|
||||||
|
1. `src/shared/config/index.ts` — 3 config baru (D3×2, D4×2 — total 4 nilai, 3 baris zod + deskripsi).
|
||||||
|
2. `src/modules/ai-moderation/textCacheStore.ts` — hit_count inc (D1), batched getter (D2),
|
||||||
|
global-reuse guard helper (D3), image-key normalize (D7).
|
||||||
|
3. `src/modules/ai-moderation/moderationOrchestrator.ts` — pakai batched getter (D2),
|
||||||
|
global bare-key fallback (D3), dua-band semantic accept (D4).
|
||||||
|
4. `src/modules/ai-moderation/qdrantClient.ts` — `searchQdrantBatch` menerima threshold rendah
|
||||||
|
(sudah parametrik — mungkin tanpa perubahan; verifikasi).
|
||||||
|
5. `src/modules/ai-moderation/wikipediaClient.ts` — cache layer (D5).
|
||||||
|
6. `src/modules/ai-moderation/urlFetcher.ts` — LRU text-fetch memoize (D6).
|
||||||
|
|
||||||
|
## Schema/type changes
|
||||||
|
|
||||||
|
- Tidak ada migrasi DB (kolom `hit_count`, `analyzed_at`, `expires_at` sudah ada).
|
||||||
|
- Tidak ada perubahan kontrak WS/oRPC/frontend.
|
||||||
|
- Type baru: none public; internal `StoredModerationVerdict` dipakai ulang.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
1. Unit tests baru (`tests/`):
|
||||||
|
- `cacheBatchLookup.test.ts`: batched getter — hit/miss/expired/error-artifact mapping,
|
||||||
|
chunking >200 keys (mock executeAll), hit_count increment called.
|
||||||
|
- `globalReuseGuard.test.ts`: guard menerima clean+conf≥0.85+action none+umur ≤72h;
|
||||||
|
menolak flagged/warn/conf rendah/action≠none/stale.
|
||||||
|
- `semanticBands.test.ts`: clean @0.93 diterima, flagged @0.93 ditolak, flagged @0.98 diterima.
|
||||||
|
- `imageKeyNormalize.test.ts`: URL Discord dgn/ex token → key sama; non-Discord beda query → beda.
|
||||||
|
2. Gate service: `pnpm typecheck && pnpm exec biome check --diagnostic-level=error . && pnpm exec vitest run`.
|
||||||
|
3. Deploy via GHA (`git push origin main`) → watch `Build & Deploy (Nix)` → verifikasi
|
||||||
|
`systemctl show gmw-discord-gateway -p ActiveEnterTimestamp` baru.
|
||||||
|
4. Runtime probe pasca-deploy: journalctl level 30 normal; beberapa jam kemudian
|
||||||
|
`SELECT sum(hit_count) FROM text_analysis_cache WHERE source='user_moderation'` > 0 membuktikan
|
||||||
|
metrics jalan; log "User moderation cache applied" menunjukkan hits>0 pada traffic ramai.
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
Semua fitur behind config defaults yang mempertahankan perilaku lama pada nilai konservatif;
|
||||||
|
rollback = redeploy commit sebelumnya (tanpa migrasi DB, tanpa state eksternal).
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# Spec: Perbaiki Delay Attachment 162s→<20s (GMW AI Analysis)
|
||||||
|
|
||||||
|
Tanggal: 2026-08-24 · Repo `~/GMW` · Service discord-gateway
|
||||||
|
|
||||||
|
## Evidence (audit produksi)
|
||||||
|
|
||||||
|
Klaster pesan attachment delay ~330–400 detik. Trace pesan `1541417073245290638` (.gif):
|
||||||
|
19:01:08 dibuat → 19:01:09 batch incomplete → fan-out individual → **guard upload-pending
|
||||||
|
mengembalikan `results:[]`** → diperalakukan sukses (`complete ... (undefined)`) → row
|
||||||
|
tertahan `ai_status='processing'` **tanpa penanggung jawab** → 19:06:12 cleanup mengembalikan
|
||||||
|
ke `pending` (tepat 300s) → baru dianalisis. Plus vision gagal 3× utk GIF besar
|
||||||
|
("Stream ended before producing a non-ping SSE event") → degradasi teks.
|
||||||
|
|
||||||
|
## Root causes
|
||||||
|
|
||||||
|
- **A (fatal)**: `individualFallbackProcessor.processIndividualFallback` memperlakukan
|
||||||
|
`ok:true + results:[]` sebagai sukses. Race-guard upload di `ai-analysis-worker.processIndividual`
|
||||||
|
sengaja balik `results:[]` (desain lama) → pesan yatim `processing` sampai cleanup 300s.
|
||||||
|
- **B**: `llmVision` hanya mencoba `stream:true`; kegagalan SSE truncation pada gambar besar
|
||||||
|
= 3 retry sia-sia (semua jalur sama) → bukti media hilang.
|
||||||
|
- **C**: safety-net cleanup 300s terlalu lambat sbg satu-satunya pemulih `processing`.
|
||||||
|
|
||||||
|
## Fix
|
||||||
|
|
||||||
|
1. **F1 — sinyal eksplisit upload-pending**: `IndividualOkResponse` + field opsional
|
||||||
|
`uploadPending?: boolean`. Worker set `uploadPending:true` saat race guard kena.
|
||||||
|
2. **F2 — processor menangani 3 kondisi** via helper murni baru
|
||||||
|
`classifyIndividualWorkerResult(result): "success" | "upload_pending" | "incomplete" | "error"`
|
||||||
|
(modul baru `fallbackResultClassifier.ts`, zero-dep agar mudah dites):
|
||||||
|
- `upload_pending` → tulis ulang row ke `pending` (pola sama dgn revert apiFailed di
|
||||||
|
batchProcessor) + broadcast + **re-schedule analisis percakapan segera**
|
||||||
|
(dynamic import batchScheduler, pola anti-siklus yg sudah ada) → retry dalam ~250ms
|
||||||
|
begitu upload beres. Bukan error, tidak naikkan CB counter.
|
||||||
|
- `incomplete` (flags analysis_incomplete) → perilaku lama (exhausted path).
|
||||||
|
- `error` / `results kosong tanpa penjelasan` → throw transien (retry oleh recovery),
|
||||||
|
BUKAN sukses palsu. Log "(undefined)" hilang.
|
||||||
|
3. **F3 — vision non-stream fallback**: di `llmVision`, jika error match
|
||||||
|
`/Stream ended before producing a non-ping SSE|stream ended/i` → coba SEKALI lagi dengan
|
||||||
|
`stream:false` (router agregasi penuh; timeout tetap 60s). Konversi hard-fail jadi sukses.
|
||||||
|
4. **F4 — turunkan safety net**: default `revertStuckProcessingMessages` 300000 → 120000 ms.
|
||||||
|
|
||||||
|
## File disentuh
|
||||||
|
|
||||||
|
- `src/modules/ai-moderation/fallbackResultClassifier.ts` (BARU, pure)
|
||||||
|
- `src/modules/ai-moderation/ai-analysis-worker.ts` (tipe + set flag uploadPending)
|
||||||
|
- `src/modules/ai-moderation/individualFallbackProcessor.ts` (konsumsi classifier + reschedule)
|
||||||
|
- `src/modules/ai-moderation/llmClient.ts` (fallback non-stream di llmVision)
|
||||||
|
- `src/modules/message-capture/messagesCleanup.ts` (default 120s)
|
||||||
|
|
||||||
|
## Verifikasi
|
||||||
|
|
||||||
|
- Test baru `tests/fallbackResultClassifier.test.ts` (4 klasifikasi + edge kosong).
|
||||||
|
- Gate: tsc --noEmit, biome error-level, vitest run semua hijau.
|
||||||
|
- Deploy GHA sukses; pasca-deploy: pesan attachment baru p50 < 20s
|
||||||
|
(`SELECT percentile_cont(0.5) ... WHERE metadata attachments>0 AND created_at > deploy`),
|
||||||
|
tidak ada lagi "complete ... (undefined)".
|
||||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,39 @@
|
|||||||
|
# GMW FE — Monokrom Hitam-Putih + Sidebar Ala Menu Game + Ringan di Mobile
|
||||||
|
|
||||||
|
Tanggal: 2026-08-24 · Basis: `eda5c75` (shell usable hasil revert)
|
||||||
|
|
||||||
|
## Tujuan
|
||||||
|
1. Tema **monokrom murni** (hitam-putih, tanpa warna) di dark & light.
|
||||||
|
2. Sidebar (desktop NavRail + mobile dock) beranimasi **ala menu game** — corner
|
||||||
|
brackets, sweep, stagger masuk, marker segitiga.
|
||||||
|
3. **Ringan di mobile**: matikan WebGL ambient di layar kecil, kurangi biaya
|
||||||
|
blur/backdrop, animasi transform/opacity saja.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
- Tidak menyentuh backend, endpoint, hooks/data-flow, struktur route.
|
||||||
|
- Tidak menambah dependensi baru (CSS murni untuk semua animasi).
|
||||||
|
|
||||||
|
## File yang disentuh
|
||||||
|
| File | Perubahan |
|
||||||
|
|---|---|
|
||||||
|
| `src/app/globals.css` | Token mono (dark+light): signal/amber/vermilion → skala putih-abu; `.glass` blur adaptif; kelas baru `.game-nav-item` (bracket ::before/::after, sweep, stagger via `--i`), `.game-frame` (panel sudut terpotong + garis tergambar), keyframes `sweep-x`, `draw-line`, `nav-in`; media query `<md`: blur 18→8px, hambat animasi berat |
|
||||||
|
| `src/components/shell/nav-rail.tsx` | Item pakai `.game-nav-item` + `style={{'--i': n}}`; marker aktif jadi segitiga ▸ putih; hapus box-shadow glow besar (ganti sweep) |
|
||||||
|
| `src/components/shell/mobile-nav.tsx` | Dock mono: tab aktif = bar atas putih + sweep sekali; target sentuh ≥44px; hapus glow blob |
|
||||||
|
| `src/components/shell/topbar.tsx` | Aksen mono + `.game-frame` pada container (cek markup dulu) |
|
||||||
|
| `src/components/ambient/ambient-canvas.tsx` | Early-return WebGL bila `(pointer: coarse)` / lebar <768 / `saveData` / core ≤4; fallback statik CSS tetap |
|
||||||
|
| `src/components/ambient/status/signal tone` (`SIGNAL_RGB`) | Semua tone jadi grayscale (putih; intensitas beda per tone) |
|
||||||
|
| `src/app/(dashboard)/dashboard/view.tsx` | Hero + kartu metrik pakai `.game-frame`/cut-corner sebagai showcase |
|
||||||
|
|
||||||
|
## Keputusan desain
|
||||||
|
- **Full monokrom termasuk danger**: flag/moderation tidak lagi merah —
|
||||||
|
ditandai badge putih-di-atlas-hitam inversi + pulse. Kalau user kangen merah,
|
||||||
|
tinggal isi ulang `--color-vermilion`.
|
||||||
|
- Semua animasi hanya `transform`/`opacity` (compositor-friendly), hormati
|
||||||
|
`prefers-reduced-motion` (sudah ada kill-switch global).
|
||||||
|
|
||||||
|
## Verifikasi (gerbang)
|
||||||
|
1. `tsc --noEmit` bersih; biome 0 error 0 warning.
|
||||||
|
2. `pnpm build` sukses; smoke lokal 4024 → 9 route 200.
|
||||||
|
3. Push → GHA "Build & Deploy (Nix)" hijau → live 9×200.
|
||||||
|
4. Visual check live: desktop (rail game-menu terlihat) + cek rule mobile
|
||||||
|
(media query & gate kode) — screenshot disimpan.
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
-- Migration: Drop materi_documents table (feature removed)
|
||||||
|
-- Run: PGPASSWORD=<pw> psql -h <host> -U <user> -d <db> -f scripts/drop-materi-documents.sql
|
||||||
|
-- Reverses scripts/add-materi-documents.sql which was deleted with the feature.
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
DROP INDEX IF EXISTS idx_materi_search;
|
||||||
|
DROP INDEX IF EXISTS idx_materi_guild;
|
||||||
|
DROP INDEX IF EXISTS idx_materi_owner;
|
||||||
|
DROP INDEX IF EXISTS idx_materi_category;
|
||||||
|
|
||||||
|
DROP TABLE IF EXISTS public.materi_documents;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
@@ -21,21 +21,21 @@
|
|||||||
"drizzle-orm": "^0.45.2",
|
"drizzle-orm": "^0.45.2",
|
||||||
"express": "^5.2.1",
|
"express": "^5.2.1",
|
||||||
"helmet": "^8.1.0",
|
"helmet": "^8.1.0",
|
||||||
"ioredis": "^5.11.0",
|
"ioredis": "^6.0.0",
|
||||||
"pg": "^8.21.0",
|
"pg": "^8.23.0",
|
||||||
"pino": "^9.6.0",
|
"pino": "^10.3.1",
|
||||||
"prom-client": "^15.1.3",
|
"prom-client": "^15.1.3",
|
||||||
"ws": "^8.20.1",
|
"ws": "^8.21.3",
|
||||||
"zod": "^4.4.3"
|
"zod": "^4.4.3"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@biomejs/biome": "latest",
|
"@biomejs/biome": "latest",
|
||||||
"@types/express": "^5.0.6",
|
"@types/express": "^5.0.6",
|
||||||
"@types/node": "^25.9.0",
|
"@types/node": "^26.2.0",
|
||||||
"@types/pg": "^8.20.0",
|
"@types/pg": "^8.23.1",
|
||||||
"@types/ws": "^8.18.1",
|
"@types/ws": "^8.18.1",
|
||||||
"tsx": "^4.22.2",
|
"tsx": "^4.23.12",
|
||||||
"typescript": "^5.9.3",
|
"typescript": "^7.0.2",
|
||||||
"vitest": "latest"
|
"vitest": "latest"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Generated
+485
-287
File diff suppressed because it is too large
Load Diff
@@ -25,7 +25,16 @@ function walk(dir) {
|
|||||||
const pat = /from\s+['"]([^'"]+)['"]/g;
|
const pat = /from\s+['"]([^'"]+)['"]/g;
|
||||||
const n = c.replace(pat, (m, spec) => {
|
const n = c.replace(pat, (m, spec) => {
|
||||||
if (spec.startsWith("@/")) {
|
if (spec.startsWith("@/")) {
|
||||||
const target = join("dist", spec.slice(2)) + ".js";
|
// Source may already carry an extension (e.g. "@/shared/config/index.js");
|
||||||
|
// only append ".js" when the specifier has none — otherwise we'd
|
||||||
|
// produce "index.js.js".
|
||||||
|
const core = spec.slice(2);
|
||||||
|
let target;
|
||||||
|
if (/\.(js|json|node|mjs|cjs)$/.test(core)) {
|
||||||
|
target = join("dist", core);
|
||||||
|
} else {
|
||||||
|
target = join("dist", core) + ".js";
|
||||||
|
}
|
||||||
let rel = relative(dirname(p), target);
|
let rel = relative(dirname(p), target);
|
||||||
if (!rel.startsWith(".")) rel = "./" + rel;
|
if (!rel.startsWith(".")) rel = "./" + rel;
|
||||||
return `from "${rel}"`;
|
return `from "${rel}"`;
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import {
|
|||||||
pgChannelCulturesTable,
|
pgChannelCulturesTable,
|
||||||
pgMessagesTable,
|
pgMessagesTable,
|
||||||
pgUserProfilesTable,
|
pgUserProfilesTable,
|
||||||
pgUserReputationsTable,
|
|
||||||
pgVoiceRecordingsTable,
|
pgVoiceRecordingsTable,
|
||||||
} from "../../shared/index.js";
|
} from "../../shared/index.js";
|
||||||
import type { ListUsersQuery } from "./dashboard.service.js";
|
import type { ListUsersQuery } from "./dashboard.service.js";
|
||||||
@@ -156,8 +155,7 @@ export class DashboardRepository {
|
|||||||
p.profile_summary,
|
p.profile_summary,
|
||||||
m.total_messages,
|
m.total_messages,
|
||||||
m.flagged_count,
|
m.flagged_count,
|
||||||
m.last_message_at,
|
m.last_message_at
|
||||||
r.trust_score
|
|
||||||
FROM (
|
FROM (
|
||||||
SELECT
|
SELECT
|
||||||
user_id,
|
user_id,
|
||||||
@@ -170,7 +168,6 @@ export class DashboardRepository {
|
|||||||
GROUP BY user_id, username, avatar_url
|
GROUP BY user_id, username, avatar_url
|
||||||
) m
|
) m
|
||||||
LEFT JOIN ${pgUserProfilesTable} p ON p.user_id = m.user_id
|
LEFT JOIN ${pgUserProfilesTable} p ON p.user_id = m.user_id
|
||||||
LEFT JOIN ${pgUserReputationsTable} r ON r.user_id = m.user_id
|
|
||||||
${whereClause}
|
${whereClause}
|
||||||
ORDER BY m.last_message_at DESC NULLS LAST
|
ORDER BY m.last_message_at DESC NULLS LAST
|
||||||
LIMIT ${limit + 1}
|
LIMIT ${limit + 1}
|
||||||
@@ -186,10 +183,6 @@ export class DashboardRepository {
|
|||||||
total_messages: Number(r.total_messages),
|
total_messages: Number(r.total_messages),
|
||||||
flagged_count: Number(r.flagged_count),
|
flagged_count: Number(r.flagged_count),
|
||||||
last_message_at: r.last_message_at ? Number(r.last_message_at) : null,
|
last_message_at: r.last_message_at ? Number(r.last_message_at) : null,
|
||||||
trust_score:
|
|
||||||
r.trust_score !== null && r.trust_score !== undefined
|
|
||||||
? Number(r.trust_score)
|
|
||||||
: null,
|
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const lastRow = rows[limit - 1] as Record<string, unknown> | undefined;
|
const lastRow = rows[limit - 1] as Record<string, unknown> | undefined;
|
||||||
@@ -437,10 +430,7 @@ export class DashboardRepository {
|
|||||||
m.flagged_count,
|
m.flagged_count,
|
||||||
m.clean_count,
|
m.clean_count,
|
||||||
p.profile_summary,
|
p.profile_summary,
|
||||||
p.last_analyzed_at,
|
p.last_analyzed_at
|
||||||
r.trust_score,
|
|
||||||
r.clean_message_streak,
|
|
||||||
r.total_infractions
|
|
||||||
FROM (
|
FROM (
|
||||||
SELECT
|
SELECT
|
||||||
user_id,
|
user_id,
|
||||||
@@ -454,7 +444,6 @@ export class DashboardRepository {
|
|||||||
GROUP BY user_id, username, avatar_url
|
GROUP BY user_id, username, avatar_url
|
||||||
) m
|
) m
|
||||||
LEFT JOIN ${pgUserProfilesTable} p ON p.user_id = m.user_id
|
LEFT JOIN ${pgUserProfilesTable} p ON p.user_id = m.user_id
|
||||||
LEFT JOIN ${pgUserReputationsTable} r ON r.user_id = m.user_id
|
|
||||||
`);
|
`);
|
||||||
|
|
||||||
const row = userResult.rows[0] as Record<string, unknown> | undefined;
|
const row = userResult.rows[0] as Record<string, unknown> | undefined;
|
||||||
@@ -481,13 +470,6 @@ export class DashboardRepository {
|
|||||||
last_analyzed_at: row.last_analyzed_at
|
last_analyzed_at: row.last_analyzed_at
|
||||||
? Number(row.last_analyzed_at)
|
? Number(row.last_analyzed_at)
|
||||||
: null,
|
: null,
|
||||||
trust_score: row.trust_score != null ? Number(row.trust_score) : null,
|
|
||||||
clean_message_streak:
|
|
||||||
row.clean_message_streak != null
|
|
||||||
? Number(row.clean_message_streak)
|
|
||||||
: null,
|
|
||||||
total_infractions:
|
|
||||||
row.total_infractions != null ? Number(row.total_infractions) : null,
|
|
||||||
recent_messages: (recent.rows as Record<string, unknown>[]).map((r) => ({
|
recent_messages: (recent.rows as Record<string, unknown>[]).map((r) => ({
|
||||||
id: String(r.id),
|
id: String(r.id),
|
||||||
content: String(r.content),
|
content: String(r.content),
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
import { sql } from "drizzle-orm";
|
||||||
|
import { getDatabase } from "../../shared/database/index.js";
|
||||||
|
|
||||||
|
export interface ChannelCultureRow {
|
||||||
|
channel_id: string;
|
||||||
|
guild_id: string | null;
|
||||||
|
channel_name: string | null;
|
||||||
|
culture_summary: string | null;
|
||||||
|
last_analyzed_at: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface GlossaryRow {
|
||||||
|
term: string;
|
||||||
|
definition: string;
|
||||||
|
source_url: string;
|
||||||
|
resolved_at: number;
|
||||||
|
hit_count: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EditHistoryRow {
|
||||||
|
id: string;
|
||||||
|
message_id: string;
|
||||||
|
old_content: string;
|
||||||
|
edited_at: number;
|
||||||
|
channel_id: string | null;
|
||||||
|
channel_name: string | null;
|
||||||
|
username: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class KnowledgeRepository {
|
||||||
|
/** Public read-only channel culture glossary (AI-generated norms/slang). */
|
||||||
|
async listChannelCultures(limit = 50, search?: string) {
|
||||||
|
const db = getDatabase();
|
||||||
|
const conditions: string[] = [];
|
||||||
|
if (search) {
|
||||||
|
conditions.push(
|
||||||
|
`(c.channel_id ILIKE '%${search.replace(/'/g, "''")}%' OR c.culture_summary ILIKE '%${search.replace(/'/g, "''")}%')`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
||||||
|
const result = await db.execute(
|
||||||
|
sql.raw(`
|
||||||
|
SELECT
|
||||||
|
c.channel_id,
|
||||||
|
c.guild_id,
|
||||||
|
COALESCE(NULLIF((
|
||||||
|
SELECT (metadata::jsonb -> 'channel' ->> 'channelName')
|
||||||
|
FROM messages WHERE channel_id = c.channel_id AND metadata IS NOT NULL
|
||||||
|
LIMIT 1
|
||||||
|
), ''), c.channel_id) AS channel_name,
|
||||||
|
c.culture_summary,
|
||||||
|
c.last_analyzed_at
|
||||||
|
FROM channel_cultures c
|
||||||
|
${where}
|
||||||
|
ORDER BY c.last_analyzed_at DESC NULLS LAST
|
||||||
|
LIMIT ${limit}
|
||||||
|
`),
|
||||||
|
);
|
||||||
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
|
return rows.map((r) => ({
|
||||||
|
channel_id: String(r.channel_id),
|
||||||
|
guild_id: r.guild_id ? String(r.guild_id) : null,
|
||||||
|
channel_name: r.channel_name ? String(r.channel_name) : null,
|
||||||
|
culture_summary: r.culture_summary ? String(r.culture_summary) : null,
|
||||||
|
last_analyzed_at: r.last_analyzed_at ? Number(r.last_analyzed_at) : null,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Public read-only term knowledge base (resolved via Wikipedia/SearXNG). */
|
||||||
|
async listGlossary(limit = 50, search?: string) {
|
||||||
|
const db = getDatabase();
|
||||||
|
const conditions: string[] = [];
|
||||||
|
if (search) {
|
||||||
|
conditions.push(
|
||||||
|
`(term ILIKE '%${search.replace(/'/g, "''")}%' OR definition ILIKE '%${search.replace(/'/g, "''")}%')`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const where = conditions.length ? `WHERE ${conditions.join(" AND ")}` : "";
|
||||||
|
const result = await db.execute(
|
||||||
|
sql.raw(`
|
||||||
|
SELECT term, definition, source_url, resolved_at, hit_count
|
||||||
|
FROM term_glossary_cache
|
||||||
|
${where}
|
||||||
|
ORDER BY hit_count DESC, resolved_at DESC
|
||||||
|
LIMIT ${limit}
|
||||||
|
`),
|
||||||
|
);
|
||||||
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
|
return rows.map((r) => ({
|
||||||
|
term: String(r.term),
|
||||||
|
definition: String(r.definition ?? ""),
|
||||||
|
source_url: r.source_url ? String(r.source_url) : "",
|
||||||
|
resolved_at: r.resolved_at ? Number(r.resolved_at) : 0,
|
||||||
|
hit_count: Number(r.hit_count ?? 0),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const knowledgeRepository = new KnowledgeRepository();
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
|
import { knowledgeRepository } from "./knowledge.repository.js";
|
||||||
|
|
||||||
|
const logger = createChildLogger("knowledge.service");
|
||||||
|
|
||||||
|
export class KnowledgeService {
|
||||||
|
async listChannelCultures(limit = 50, search?: string) {
|
||||||
|
logger.debug({ limit, search }, "Listing channel cultures");
|
||||||
|
return knowledgeRepository.listChannelCultures(limit, search);
|
||||||
|
}
|
||||||
|
|
||||||
|
async listGlossary(limit = 50, search?: string) {
|
||||||
|
logger.debug({ limit, search }, "Listing glossary terms");
|
||||||
|
return knowledgeRepository.listGlossary(limit, search);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const knowledgeService = new KnowledgeService();
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { config } from "@/shared/config/index.js";
|
import { config } from "@/shared/config/index";
|
||||||
import { createChildLogger } from "@/shared/logger/index.js";
|
import { createChildLogger } from "@/shared/logger/index";
|
||||||
|
|
||||||
const logger = createChildLogger("messages-embed");
|
const logger = createChildLogger("messages-embed");
|
||||||
|
|
||||||
|
|||||||
@@ -459,6 +459,69 @@ export class MessagesRepository {
|
|||||||
|
|
||||||
return { data: trimmed, nextCursor };
|
return { data: trimmed, nextCursor };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Per-hour message volume for the last `days` days, grouped by channel.
|
||||||
|
* Powers the public Activity Heatmap (read-only, no write scope).
|
||||||
|
* Returns a flat list of { channel_id, hour (0-23), count } buckets.
|
||||||
|
*/
|
||||||
|
async getActivity(days = 30) {
|
||||||
|
const db = getDatabase();
|
||||||
|
const since = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||||
|
const result = await db.execute(sql`
|
||||||
|
SELECT
|
||||||
|
m.channel_id,
|
||||||
|
COALESCE(NULLIF((m.metadata::jsonb -> 'channel' ->> 'channelName'), ''), m.channel_id) AS channel_name,
|
||||||
|
EXTRACT(HOUR FROM to_timestamp(m.created_at / 1000))::int AS hour,
|
||||||
|
COUNT(*)::int AS c
|
||||||
|
FROM messages m
|
||||||
|
WHERE m.created_at >= ${since}
|
||||||
|
GROUP BY m.channel_id, channel_name, hour
|
||||||
|
ORDER BY channel_name, hour
|
||||||
|
`);
|
||||||
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
|
return rows.map((r) => ({
|
||||||
|
channelId: String(r.channel_id ?? "unknown"),
|
||||||
|
channelName: String(r.channel_name ?? r.channel_id ?? "unknown"),
|
||||||
|
hour: Number(r.hour ?? 0),
|
||||||
|
count: Number(r.c ?? 0),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recent message edits across the server (evasion-signal tracker).
|
||||||
|
* Public, read-only. Joins message_edits → messages for context.
|
||||||
|
*/
|
||||||
|
async getRecentEdits(limit = 50, channelId?: string) {
|
||||||
|
const db = getDatabase();
|
||||||
|
const result = await db.execute(sql`
|
||||||
|
SELECT
|
||||||
|
e.id,
|
||||||
|
e.message_id,
|
||||||
|
e.old_content,
|
||||||
|
e.edited_at,
|
||||||
|
m.channel_id,
|
||||||
|
COALESCE(NULLIF((m.metadata::jsonb -> 'channel' ->> 'channelName'), ''), m.channel_id) AS channel_name,
|
||||||
|
m.username,
|
||||||
|
m.content AS new_content
|
||||||
|
FROM message_edits e
|
||||||
|
JOIN messages m ON m.id = e.message_id
|
||||||
|
${channelId ? sql`WHERE m.channel_id = ${channelId}` : sql``}
|
||||||
|
ORDER BY e.edited_at DESC
|
||||||
|
LIMIT ${limit}
|
||||||
|
`);
|
||||||
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
|
return rows.map((r) => ({
|
||||||
|
id: String(r.id),
|
||||||
|
message_id: String(r.message_id),
|
||||||
|
old_content: r.old_content ? String(r.old_content) : "",
|
||||||
|
new_content: r.new_content ? String(r.new_content) : "",
|
||||||
|
edited_at: r.edited_at ? Number(r.edited_at) : 0,
|
||||||
|
channel_id: r.channel_id ? String(r.channel_id) : null,
|
||||||
|
channel_name: r.channel_name ? String(r.channel_name) : null,
|
||||||
|
username: r.username ? String(r.username) : null,
|
||||||
|
}));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const messagesRepository = new MessagesRepository();
|
export const messagesRepository = new MessagesRepository();
|
||||||
|
|||||||
@@ -39,12 +39,15 @@ export class MessagesService {
|
|||||||
throw new ValidationError("message ID is required");
|
throw new ValidationError("message ID is required");
|
||||||
}
|
}
|
||||||
|
|
||||||
const message = await messagesRepository.findById(id);
|
const [message, editHistory] = await Promise.all([
|
||||||
|
messagesRepository.findById(id),
|
||||||
|
messagesRepository.getEditHistory(id),
|
||||||
|
]);
|
||||||
|
|
||||||
if (!message) {
|
if (!message) {
|
||||||
throw new NotFoundError(`Message with ID ${id} not found`);
|
throw new NotFoundError(`Message with ID ${id} not found`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const editHistory = await messagesRepository.getEditHistory(id);
|
|
||||||
return {
|
return {
|
||||||
...message,
|
...message,
|
||||||
edit_count: editHistory.length,
|
edit_count: editHistory.length,
|
||||||
@@ -101,6 +104,15 @@ export class MessagesService {
|
|||||||
const results = hits.map((h) => mapSearchHit(h));
|
const results = hits.map((h) => mapSearchHit(h));
|
||||||
return { results, nextCursor: null };
|
return { results, nextCursor: null };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getActivity(days = 30) {
|
||||||
|
return messagesRepository.getActivity(days);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getRecentEdits(limit = 50, channelId?: string) {
|
||||||
|
logger.debug({ limit, channelId }, "Getting recent message edits");
|
||||||
|
return messagesRepository.getRecentEdits(limit, channelId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Shape returned to the frontend (text + metadata from the archive payload). */
|
/** Shape returned to the frontend (text + metadata from the archive payload). */
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { config } from "@/shared/config/index.js";
|
import { config } from "@/shared/config/index";
|
||||||
import { createChildLogger } from "@/shared/logger/index.js";
|
import { createChildLogger } from "@/shared/logger/index";
|
||||||
|
|
||||||
const logger = createChildLogger("messages-qdrant");
|
const logger = createChildLogger("messages-qdrant");
|
||||||
|
|
||||||
|
|||||||
@@ -164,6 +164,220 @@ export class ModerationRepository {
|
|||||||
|
|
||||||
return { data, nextCursor };
|
return { data, nextCursor };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Aggregate moderation trends over the last `days` days.
|
||||||
|
* - category counts (from the jsonb/text[] `categories` column, unnested)
|
||||||
|
* - severity distribution
|
||||||
|
* - action_type distribution
|
||||||
|
* Read-only; powers the public Toxic Topic Trends panel.
|
||||||
|
*/
|
||||||
|
async getTrends(days: number) {
|
||||||
|
const db = getDatabase();
|
||||||
|
const since = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
const cats = await db.execute(sql`
|
||||||
|
SELECT jsonb_array_elements_text(a.categories::jsonb) AS cat, COUNT(*)::int AS c
|
||||||
|
FROM moderation_actions a
|
||||||
|
WHERE a.created_at >= ${since} AND a.categories IS NOT NULL AND a.categories != '[]' AND a.categories != ''
|
||||||
|
GROUP BY cat
|
||||||
|
ORDER BY c DESC
|
||||||
|
LIMIT 15
|
||||||
|
`);
|
||||||
|
const catRows = (cats.rows as Record<string, unknown>[]) || [];
|
||||||
|
|
||||||
|
const sev = await db.execute(sql`
|
||||||
|
SELECT severity, COUNT(*)::int AS c
|
||||||
|
FROM moderation_actions
|
||||||
|
WHERE created_at >= ${since} AND severity IS NOT NULL
|
||||||
|
GROUP BY severity
|
||||||
|
`);
|
||||||
|
const sevRows = (sev.rows as Record<string, unknown>[]) || [];
|
||||||
|
|
||||||
|
const act = await db.execute(sql`
|
||||||
|
SELECT action_type, COUNT(*)::int AS c
|
||||||
|
FROM moderation_actions
|
||||||
|
WHERE created_at >= ${since}
|
||||||
|
GROUP BY action_type
|
||||||
|
ORDER BY c DESC
|
||||||
|
`);
|
||||||
|
const actRows = (act.rows as Record<string, unknown>[]) || [];
|
||||||
|
|
||||||
|
return {
|
||||||
|
categories: catRows.map((r) => ({
|
||||||
|
name: String(r.cat),
|
||||||
|
count: Number(r.c ?? 0),
|
||||||
|
})),
|
||||||
|
severities: sevRows.map((r) => ({
|
||||||
|
level: String(r.severity),
|
||||||
|
count: Number(r.c ?? 0),
|
||||||
|
})),
|
||||||
|
actions: actRows.map((r) => ({
|
||||||
|
type: String(r.action_type),
|
||||||
|
count: Number(r.c ?? 0),
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Top flagged domains over the last `days` days.
|
||||||
|
* Extracts the host from any URL in `content`/`reason`/`evidence` and ranks
|
||||||
|
* by how often it appears in moderation actions. Powers the Scam Domain panel.
|
||||||
|
*/
|
||||||
|
async getTopFlaggedDomains(days: number) {
|
||||||
|
const db = getDatabase();
|
||||||
|
const since = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||||
|
const result = await db.execute(sql`
|
||||||
|
SELECT host, COUNT(*)::int AS c
|
||||||
|
FROM (
|
||||||
|
SELECT DISTINCT a.id,
|
||||||
|
(regexp_matches(COALESCE(a.content,'') || ' ' || COALESCE(a.reason,'') || ' ' || COALESCE(a.evidence,''), 'https?://([^/\s?#]+)', 'g'))[1] AS host
|
||||||
|
FROM moderation_actions a
|
||||||
|
WHERE a.created_at >= ${since}
|
||||||
|
AND (a.content IS NOT NULL OR a.reason IS NOT NULL OR a.evidence IS NOT NULL)
|
||||||
|
) sub
|
||||||
|
WHERE host IS NOT NULL
|
||||||
|
GROUP BY host
|
||||||
|
ORDER BY c DESC
|
||||||
|
LIMIT 20
|
||||||
|
`);
|
||||||
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
|
return rows.map((r) => ({
|
||||||
|
domain: String(r.host).toLowerCase(),
|
||||||
|
count: Number(r.c ?? 0),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Top flagged channels over the last `days` days.
|
||||||
|
* Joins moderation_actions → messages to attribute each action to a channel.
|
||||||
|
* Powers the Top Flagged Channels panel.
|
||||||
|
*/
|
||||||
|
async getTopFlaggedChannels(days: number) {
|
||||||
|
const db = getDatabase();
|
||||||
|
const since = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||||
|
const result = await db.execute(sql`
|
||||||
|
SELECT
|
||||||
|
m.channel_id,
|
||||||
|
COALESCE(NULLIF((m.metadata::jsonb -> 'channel' ->> 'channelName'), ''), m.channel_id) AS channel_name,
|
||||||
|
COUNT(*)::int AS flagged_count
|
||||||
|
FROM moderation_actions a
|
||||||
|
LEFT JOIN messages m ON m.id = a.message_id
|
||||||
|
WHERE a.created_at >= ${since} AND m.channel_id IS NOT NULL
|
||||||
|
GROUP BY m.channel_id, (m.metadata::jsonb -> 'channel' ->> 'channelName')
|
||||||
|
ORDER BY flagged_count DESC
|
||||||
|
LIMIT 15
|
||||||
|
`);
|
||||||
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
|
return rows.map((r) => ({
|
||||||
|
channel_id: String(r.channel_id),
|
||||||
|
channel_name: r.channel_name ? String(r.channel_name) : null,
|
||||||
|
flagged_count: Number(r.flagged_count),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hour-of-day distribution of moderation actions over the last `days` days.
|
||||||
|
* 24 rows (hour 0..23), with total + flagged-by-severity counts.
|
||||||
|
* Powers the Moderation Heatmap by Hour panel.
|
||||||
|
*/
|
||||||
|
async getHourlyModeration(days: number) {
|
||||||
|
const db = getDatabase();
|
||||||
|
const since = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||||
|
const result = await db.execute(sql`
|
||||||
|
SELECT
|
||||||
|
EXTRACT(HOUR FROM to_timestamp(created_at / 1000))::int AS hour,
|
||||||
|
COUNT(*)::int AS total
|
||||||
|
FROM moderation_actions
|
||||||
|
WHERE created_at >= ${since}
|
||||||
|
GROUP BY hour
|
||||||
|
ORDER BY hour
|
||||||
|
`);
|
||||||
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
|
const byHour = new Map<number, number>();
|
||||||
|
for (const r of rows) byHour.set(Number(r.hour), Number(r.total));
|
||||||
|
return Array.from({ length: 24 }, (_, h) => ({
|
||||||
|
hour: h,
|
||||||
|
total: byHour.get(h) ?? 0,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Moderation actions filtered to a single category (drill-down).
|
||||||
|
* Powers the Flag Category Drill-down panel.
|
||||||
|
*/
|
||||||
|
async getByCategory(days: number, category: string, limit = 50) {
|
||||||
|
const db = getDatabase();
|
||||||
|
const since = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||||
|
const result = await db.execute(
|
||||||
|
sql.raw(`
|
||||||
|
SELECT
|
||||||
|
a.id, a.message_id, a.user_id, a.guild_id, a.action_type,
|
||||||
|
a.reason, a.status, a.created_at, a.severity, a.confidence, a.score,
|
||||||
|
m.username, LEFT(m.content, 300) AS content
|
||||||
|
FROM moderation_actions a
|
||||||
|
LEFT JOIN messages m ON m.id = a.message_id
|
||||||
|
WHERE a.created_at >= ${since}
|
||||||
|
AND a.categories IS NOT NULL
|
||||||
|
AND a.categories::jsonb @> ${JSON.stringify([category])}::jsonb
|
||||||
|
ORDER BY a.created_at DESC
|
||||||
|
LIMIT ${limit}
|
||||||
|
`),
|
||||||
|
);
|
||||||
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
|
return rows.map((r) => ({
|
||||||
|
id: String(r.id ?? ""),
|
||||||
|
message_id: r.message_id ? String(r.message_id) : null,
|
||||||
|
user_id: r.user_id ? String(r.user_id) : null,
|
||||||
|
guild_id: String(r.guild_id ?? ""),
|
||||||
|
action_type: String(r.action_type ?? "unknown"),
|
||||||
|
reason: r.reason ? String(r.reason) : null,
|
||||||
|
status: String(r.status ?? "unknown"),
|
||||||
|
created_at: r.created_at ? Number(r.created_at) : null,
|
||||||
|
severity: r.severity ? String(r.severity) : null,
|
||||||
|
confidence: r.confidence != null ? Number(r.confidence) : null,
|
||||||
|
score: r.score != null ? Number(r.score) : null,
|
||||||
|
username: r.username ? String(r.username) : null,
|
||||||
|
content: r.content ? String(r.content) : null,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Auto-moderation coverage over the last `days` days.
|
||||||
|
* Run completion rate from ai_analysis_runs — what fraction of analysis runs
|
||||||
|
* completed (vs failed/pending). Public "how much is automated" trust metric.
|
||||||
|
*/
|
||||||
|
async getCoverage(days: number) {
|
||||||
|
const db = getDatabase();
|
||||||
|
const since = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||||
|
const result = await db.execute(sql`
|
||||||
|
SELECT status, COUNT(*)::int AS c
|
||||||
|
FROM ai_analysis_runs
|
||||||
|
WHERE created_at >= ${since}
|
||||||
|
GROUP BY status
|
||||||
|
`);
|
||||||
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
|
const counts: Record<string, number> = {};
|
||||||
|
let total = 0;
|
||||||
|
for (const r of rows) {
|
||||||
|
const s = String(r.status);
|
||||||
|
const c = Number(r.c ?? 0);
|
||||||
|
counts[s] = c;
|
||||||
|
total += c;
|
||||||
|
}
|
||||||
|
const completed = counts.completed ?? 0;
|
||||||
|
const failed = counts.failed ?? 0;
|
||||||
|
const pending = (counts.pending ?? 0) + (counts.processing ?? 0);
|
||||||
|
return {
|
||||||
|
total,
|
||||||
|
completed,
|
||||||
|
failed,
|
||||||
|
pending,
|
||||||
|
coverage_rate:
|
||||||
|
total > 0 ? Number(((completed / total) * 100).toFixed(1)) : 0,
|
||||||
|
failed_rate: total > 0 ? Number(((failed / total) * 100).toFixed(1)) : 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const moderationRepository = new ModerationRepository();
|
export const moderationRepository = new ModerationRepository();
|
||||||
|
|||||||
@@ -8,10 +8,33 @@ const logger = createChildLogger("moderation.service");
|
|||||||
|
|
||||||
export class ModerationService {
|
export class ModerationService {
|
||||||
async getStats() {
|
async getStats() {
|
||||||
logger.debug("Fetching moderation stats");
|
|
||||||
return moderationRepository.getStats();
|
return moderationRepository.getStats();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getTrends(days = 30) {
|
||||||
|
return moderationRepository.getTrends(days);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getTopFlaggedDomains(days = 30) {
|
||||||
|
return moderationRepository.getTopFlaggedDomains(days);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getTopFlaggedChannels(days = 30) {
|
||||||
|
return moderationRepository.getTopFlaggedChannels(days);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getHourlyModeration(days = 30) {
|
||||||
|
return moderationRepository.getHourlyModeration(days);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getByCategory(days = 30, category: string) {
|
||||||
|
return moderationRepository.getByCategory(days, category);
|
||||||
|
}
|
||||||
|
|
||||||
|
async getCoverage(days = 30) {
|
||||||
|
return moderationRepository.getCoverage(days);
|
||||||
|
}
|
||||||
|
|
||||||
async listActions(query: ListModerationQuery) {
|
async listActions(query: ListModerationQuery) {
|
||||||
logger.debug({ query }, "Listing moderation actions");
|
logger.debug({ query }, "Listing moderation actions");
|
||||||
return moderationRepository.listActions(query);
|
return moderationRepository.listActions(query);
|
||||||
|
|||||||
@@ -23,14 +23,26 @@ const speakers = new Map<string, LiveSpeaker>();
|
|||||||
const MAX_SPEAKERS = 200;
|
const MAX_SPEAKERS = 200;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Record a voice_active_user event. `speaking: true` upserts the speaker as
|
* Speakers inactive for longer than this are auto-expired from the snapshot.
|
||||||
* active; `speaking: false` marks them inactive while keeping them for the
|
* This handles the case where the gateway disconnects abruptly and never
|
||||||
* activity timeline.
|
* sends `speaking: false` for active users.
|
||||||
*/
|
*/
|
||||||
|
const SPEAKER_TTL_MS = 30_000;
|
||||||
|
|
||||||
|
/** Purge speakers that haven't been active recently. */
|
||||||
|
function purgeStale(): void {
|
||||||
|
const cutoff = Date.now() - SPEAKER_TTL_MS;
|
||||||
|
for (const [id, s] of speakers) {
|
||||||
|
if (!s.speaking && s.lastActiveAt < cutoff) {
|
||||||
|
speakers.delete(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* recordSpeaker(data) — apply a `voice_active_user` event. `speaking: true`
|
* Record a voice_active_user event. `speaking: true` upserts the speaker as
|
||||||
* upserts the speaker as ACTIVE; `speaking: false` marks them inactive while
|
* active; `speaking: false` marks them inactive while keeping them briefly
|
||||||
* keeping them for the activity timeline.
|
* for the activity timeline (until TTL expiry).
|
||||||
*/
|
*/
|
||||||
export function recordSpeaker(data: {
|
export function recordSpeaker(data: {
|
||||||
userId: string;
|
userId: string;
|
||||||
@@ -49,7 +61,6 @@ export function recordSpeaker(data: {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if (speakers.size >= MAX_SPEAKERS && !existing) {
|
if (speakers.size >= MAX_SPEAKERS && !existing) {
|
||||||
// Drop the least-recently-active non-speaking speaker to stay bounded.
|
|
||||||
let oldestId: string | null = null;
|
let oldestId: string | null = null;
|
||||||
let oldestTs = Infinity;
|
let oldestTs = Infinity;
|
||||||
for (const [id, s] of speakers) {
|
for (const [id, s] of speakers) {
|
||||||
@@ -65,18 +76,35 @@ export function recordSpeaker(data: {
|
|||||||
speakers.set(userId, speaker);
|
speakers.set(userId, speaker);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** All known speakers, most recently active first. */
|
/**
|
||||||
|
* All recently-active speakers, most recently active first.
|
||||||
|
* Stale (non-speaking + old) entries are auto-purged.
|
||||||
|
*/
|
||||||
export function getActiveSpeakers(): LiveSpeaker[] {
|
export function getActiveSpeakers(): LiveSpeaker[] {
|
||||||
|
purgeStale();
|
||||||
return [...speakers.values()].sort((a, b) => b.lastActiveAt - a.lastActiveAt);
|
return [...speakers.values()].sort((a, b) => b.lastActiveAt - a.lastActiveAt);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Only speakers currently flagged as speaking. */
|
/** Only speakers currently flagged as speaking. */
|
||||||
export function getSpeakingSpeakers(): LiveSpeaker[] {
|
export function getSpeakingSpeakers(): LiveSpeaker[] {
|
||||||
|
purgeStale();
|
||||||
return [...speakers.values()]
|
return [...speakers.values()]
|
||||||
.filter((s) => s.speaking)
|
.filter((s) => s.speaking)
|
||||||
.sort((a, b) => b.lastActiveAt - a.lastActiveAt);
|
.sort((a, b) => b.lastActiveAt - a.lastActiveAt);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark ALL tracked speakers as not-speaking and purge stale ones.
|
||||||
|
* Called when the gateway disconnects from voice — ensures the authoritative
|
||||||
|
* snapshot doesn't carry ghost speakers.
|
||||||
|
*/
|
||||||
|
export function clearAllSpeakers(): void {
|
||||||
|
for (const [id, s] of speakers) {
|
||||||
|
s.speaking = false;
|
||||||
|
}
|
||||||
|
purgeStale();
|
||||||
|
}
|
||||||
|
|
||||||
/** Drop all tracked speakers (used on backend restart). */
|
/** Drop all tracked speakers (used on backend restart). */
|
||||||
export function resetLiveSpeakers(): void {
|
export function resetLiveSpeakers(): void {
|
||||||
speakers.clear();
|
speakers.clear();
|
||||||
|
|||||||
@@ -3,8 +3,8 @@ import { z } from "zod";
|
|||||||
import { analysisService } from "../modules/analysis/analysis.service";
|
import { analysisService } from "../modules/analysis/analysis.service";
|
||||||
import { chatRequestSchema } from "../modules/chatbot/chatbot.schema";
|
import { chatRequestSchema } from "../modules/chatbot/chatbot.schema";
|
||||||
import { chatbotService } from "../modules/chatbot/chatbot.service";
|
import { chatbotService } from "../modules/chatbot/chatbot.service";
|
||||||
// ── Service imports ──────────────────────────────────────────────
|
|
||||||
import { dashboardService } from "../modules/dashboard/dashboard.service";
|
import { dashboardService } from "../modules/dashboard/dashboard.service";
|
||||||
|
import { knowledgeService } from "../modules/knowledge/knowledge.service";
|
||||||
import {
|
import {
|
||||||
mediaLoopSchema,
|
mediaLoopSchema,
|
||||||
mediaQueueSchema,
|
mediaQueueSchema,
|
||||||
@@ -143,6 +143,25 @@ const messagesRouter = {
|
|||||||
semanticSearch: os
|
semanticSearch: os
|
||||||
.input(semanticSearchSchema)
|
.input(semanticSearchSchema)
|
||||||
.handler(({ input }) => messagesService.semanticSearch(input)),
|
.handler(({ input }) => messagesService.semanticSearch(input)),
|
||||||
|
// Public, read-only activity heatmap data (per-hour volume by channel).
|
||||||
|
activity: os
|
||||||
|
.input(
|
||||||
|
z.object({
|
||||||
|
days: z.coerce.number().int().positive().max(365).default(30),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.handler(({ input }) => messagesService.getActivity(input.days)),
|
||||||
|
// Public, read-only recent message edits (evasion tracker).
|
||||||
|
editHistory: os
|
||||||
|
.input(
|
||||||
|
z.object({
|
||||||
|
limit: z.coerce.number().int().positive().default(50),
|
||||||
|
channelId: z.string().optional(),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.handler(({ input }) =>
|
||||||
|
messagesService.getRecentEdits(input.limit, input.channelId),
|
||||||
|
),
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── Moderation ───────────────────────────────────────────────────
|
// ── Moderation ───────────────────────────────────────────────────
|
||||||
@@ -165,6 +184,58 @@ const moderationRouter = {
|
|||||||
cursor: input.cursor,
|
cursor: input.cursor,
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
|
trends: os
|
||||||
|
.input(
|
||||||
|
z.object({
|
||||||
|
days: z.coerce.number().int().positive().max(365).default(30),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.handler(({ input }) => moderationService.getTrends(input.days)),
|
||||||
|
// Flagged link / scam domain ranking (public Scam Domain panel).
|
||||||
|
topDomains: os
|
||||||
|
.input(
|
||||||
|
z.object({
|
||||||
|
days: z.coerce.number().int().positive().max(365).default(30),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.handler(({ input }) => moderationService.getTopFlaggedDomains(input.days)),
|
||||||
|
// Top flagged channels (join moderation_actions → messages).
|
||||||
|
topChannels: os
|
||||||
|
.input(
|
||||||
|
z.object({
|
||||||
|
days: z.coerce.number().int().positive().max(365).default(30),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.handler(({ input }) =>
|
||||||
|
moderationService.getTopFlaggedChannels(input.days),
|
||||||
|
),
|
||||||
|
// Hour-of-day moderation distribution (heatmap by hour).
|
||||||
|
byHour: os
|
||||||
|
.input(
|
||||||
|
z.object({
|
||||||
|
days: z.coerce.number().int().positive().max(365).default(30),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.handler(({ input }) => moderationService.getHourlyModeration(input.days)),
|
||||||
|
// Flag category drill-down (list actions for one category).
|
||||||
|
byCategory: os
|
||||||
|
.input(
|
||||||
|
z.object({
|
||||||
|
days: z.coerce.number().int().positive().max(365).default(30),
|
||||||
|
category: z.string().min(1),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.handler(({ input }) =>
|
||||||
|
moderationService.getByCategory(input.days, input.category),
|
||||||
|
),
|
||||||
|
// Auto-moderation coverage (analysis run completion rate).
|
||||||
|
coverage: os
|
||||||
|
.input(
|
||||||
|
z.object({
|
||||||
|
days: z.coerce.number().int().positive().max(365).default(30),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.handler(({ input }) => moderationService.getCoverage(input.days)),
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── Media ────────────────────────────────────────────────────────
|
// ── Media ────────────────────────────────────────────────────────
|
||||||
@@ -306,7 +377,29 @@ const chatbotRouter = {
|
|||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── Config (public dashboard config snapshot) ──────────────────────
|
// ── Knowledge (public read-only culture glossary + term KB) ───────
|
||||||
|
const knowledgeRouter = {
|
||||||
|
channelCultures: os
|
||||||
|
.input(
|
||||||
|
z.object({
|
||||||
|
limit: z.coerce.number().int().positive().default(50),
|
||||||
|
search: z.string().optional(),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.handler(({ input }) =>
|
||||||
|
knowledgeService.listChannelCultures(input.limit, input.search),
|
||||||
|
),
|
||||||
|
glossary: os
|
||||||
|
.input(
|
||||||
|
z.object({
|
||||||
|
limit: z.coerce.number().int().positive().default(50),
|
||||||
|
search: z.string().optional(),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.handler(({ input }) =>
|
||||||
|
knowledgeService.listGlossary(input.limit, input.search),
|
||||||
|
),
|
||||||
|
};
|
||||||
const configRouter = {
|
const configRouter = {
|
||||||
get: os.handler(() => ({
|
get: os.handler(() => ({
|
||||||
monitorGuildId: config.MONITOR_GUILD_ID || null,
|
monitorGuildId: config.MONITOR_GUILD_ID || null,
|
||||||
@@ -345,6 +438,7 @@ export const appRouter = {
|
|||||||
chatbot: chatbotRouter,
|
chatbot: chatbotRouter,
|
||||||
config: configRouter,
|
config: configRouter,
|
||||||
uiState: uiStateRouter,
|
uiState: uiStateRouter,
|
||||||
|
knowledge: knowledgeRouter,
|
||||||
};
|
};
|
||||||
|
|
||||||
export type AppRouter = typeof appRouter;
|
export type AppRouter = typeof appRouter;
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ export const DISCORD_CHANNEL_TOPIC_UPDATED = "discord:channel:topic_updated";
|
|||||||
export const DISCORD_PRESENCE_UPDATED = "discord:presence:updated";
|
export const DISCORD_PRESENCE_UPDATED = "discord:presence:updated";
|
||||||
export const DISCORD_GUILD_MEMBER_ADDED = "discord:guild_member:added";
|
export const DISCORD_GUILD_MEMBER_ADDED = "discord:guild_member:added";
|
||||||
export const DISCORD_GUILD_MEMBER_REMOVED = "discord:guild_member:removed";
|
export const DISCORD_GUILD_MEMBER_REMOVED = "discord:guild_member:removed";
|
||||||
|
export const DISCORD_MODERATION_ACTION = "discord:moderation:action";
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Command channels (backend -> discord-gateway)
|
// Command channels (backend -> discord-gateway)
|
||||||
@@ -126,4 +127,5 @@ export const DISCORD_CHANNEL_TO_WS_EVENT: Record<string, string> = {
|
|||||||
[DISCORD_PRESENCE_UPDATED]: "presence_updated",
|
[DISCORD_PRESENCE_UPDATED]: "presence_updated",
|
||||||
[DISCORD_GUILD_MEMBER_ADDED]: "guild_member_added",
|
[DISCORD_GUILD_MEMBER_ADDED]: "guild_member_added",
|
||||||
[DISCORD_GUILD_MEMBER_REMOVED]: "guild_member_removed",
|
[DISCORD_GUILD_MEMBER_REMOVED]: "guild_member_removed",
|
||||||
|
[DISCORD_MODERATION_ACTION]: "moderation_action",
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
import Redis from "ioredis";
|
import Redis from "ioredis";
|
||||||
import { recordSpeaker } from "../modules/voice/live-speaker.js";
|
import { clearAllSpeakers, recordSpeaker } from "../modules/voice/live-speaker.js";
|
||||||
import { config } from "../shared/config/index.js";
|
import { config } from "../shared/config/index.js";
|
||||||
import {
|
import {
|
||||||
DISCORD_CHANNEL_TO_WS_EVENT,
|
DISCORD_CHANNEL_TO_WS_EVENT,
|
||||||
DISCORD_VOICE_ACTIVE_USER,
|
DISCORD_VOICE_ACTIVE_USER,
|
||||||
DISCORD_VOICE_PCM,
|
DISCORD_VOICE_PCM,
|
||||||
|
DISCORD_VOICE_STOPPED,
|
||||||
} from "../shared/index.js";
|
} from "../shared/index.js";
|
||||||
import { createChildLogger } from "../shared/logger/index.js";
|
import { createChildLogger } from "../shared/logger/index.js";
|
||||||
import { broadcastBinary, broadcastEvent } from "./broadcast.js";
|
import { broadcastBinary, broadcastEvent } from "./broadcast.js";
|
||||||
@@ -84,6 +85,14 @@ function handleSubscriptionMessage(channel: string, message: string): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// When the gateway stops voice recording (disconnects from voice channel),
|
||||||
|
// clear all speakers from the authoritative snapshot so frontends don't
|
||||||
|
// show ghost participants.
|
||||||
|
if (channel === DISCORD_VOICE_STOPPED) {
|
||||||
|
clearAllSpeakers();
|
||||||
|
logger.info("Voice recording stopped — cleared all live speakers");
|
||||||
|
}
|
||||||
|
|
||||||
logger.debug({ channel, eventType }, "Broadcasting Redis event");
|
logger.debug({ channel, eventType }, "Broadcasting Redis event");
|
||||||
broadcastEvent(eventType, data);
|
broadcastEvent(eventType, data);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ handles a whole batch (text + media split internally, parallel paths).
|
|||||||
- `embeddingClient.ts` + `qdrantClient.ts` — semantic cache (one embed call +
|
- `embeddingClient.ts` + `qdrantClient.ts` — semantic cache (one embed call +
|
||||||
one batched Qdrant search for all uncached targets).
|
one batched Qdrant search for all uncached targets).
|
||||||
- `textCacheStore.ts` / `channelCultureStore.ts` / `userProfileStore.ts` /
|
- `textCacheStore.ts` / `channelCultureStore.ts` / `userProfileStore.ts` /
|
||||||
`userReputationStore.ts` — caches & learned per-channel/user state.
|
`userProfileStore.ts` — caches learned user profile summaries (optional).
|
||||||
|
|
||||||
### Concurrency model
|
### Concurrency model
|
||||||
|
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ Orchestration/caching: `moderationOrchestrator.ts` (exact hash → batched
|
|||||||
semantic Qdrant → LLM), `textBatchProcessor.ts` / `mediaBatchProcessor.ts`
|
semantic Qdrant → LLM), `textBatchProcessor.ts` / `mediaBatchProcessor.ts`
|
||||||
(one LLM call per sub-batch), `llmClient.ts` (central streaming client),
|
(one LLM call per sub-batch), `llmClient.ts` (central streaming client),
|
||||||
`embeddingClient.ts` + `qdrantClient.ts` (semantic cache), plus
|
`embeddingClient.ts` + `qdrantClient.ts` (semantic cache), plus
|
||||||
`channelCultureStore.ts` / `userProfileStore.ts` / `userReputationStore.ts`.
|
`channelCultureStore.ts` / `userProfileStore.ts`.
|
||||||
|
|
||||||
### voice-recording
|
### voice-recording
|
||||||
`voiceController.ts` (connect/disconnect/list) + `recorder.ts` (orchestration)
|
`voiceController.ts` (connect/disconnect/list) + `recorder.ts` (orchestration)
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
-- Remove the user reputation feature entirely (trust scores, infractions).
|
||||||
|
-- The feature was removed from the codebase; this drops the orphaned table.
|
||||||
|
DROP TABLE IF EXISTS "user_reputations";
|
||||||
@@ -113,6 +113,13 @@
|
|||||||
"when": 1787184000000,
|
"when": 1787184000000,
|
||||||
"tag": "0015_add_moderation_explainability",
|
"tag": "0015_add_moderation_explainability",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 16,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1787185000000,
|
||||||
|
"tag": "0016_drop_user_reputations",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -28,30 +28,30 @@
|
|||||||
"discord.js-selfbot-v13": "^3.7.1",
|
"discord.js-selfbot-v13": "^3.7.1",
|
||||||
"dotenv": "^17.4.2",
|
"dotenv": "^17.4.2",
|
||||||
"drizzle-orm": "^0.45.2",
|
"drizzle-orm": "^0.45.2",
|
||||||
"ioredis": "^5.11.0",
|
"ioredis": "^6.0.0",
|
||||||
"libsodium-wrappers": "^0.8.4",
|
"libsodium-wrappers": "^0.8.4",
|
||||||
"lru-cache": "^11.5.1",
|
"lru-cache": "^11.5.1",
|
||||||
"openai": "^6.38.0",
|
"openai": "^7.5.0",
|
||||||
"opusscript": "^0.0.8",
|
"opusscript": "^0.1.1",
|
||||||
"p-limit": "^7.3.0",
|
"p-limit": "^7.3.0",
|
||||||
"p-retry": "^8.0.0",
|
"p-retry": "^8.0.0",
|
||||||
"pg": "^8.21.0",
|
"pg": "^8.23.0",
|
||||||
"pino": "^9.6.0",
|
"pino": "^10.3.1",
|
||||||
"piscina": "^5.1.4",
|
"piscina": "^5.3.1",
|
||||||
"prism-media": "2.0.0-alpha.0",
|
"prism-media": "2.0.0-alpha.0",
|
||||||
"sharp": "^0.34.5",
|
"sharp": "^0.35.3",
|
||||||
"tiktoken": "^1.0.22",
|
"tiktoken": "^1.0.22",
|
||||||
"ws": "^8.20.1",
|
"ws": "^8.21.3",
|
||||||
"zod": "^4.4.3"
|
"zod": "^4.4.3"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@biomejs/biome": "latest",
|
"@biomejs/biome": "latest",
|
||||||
"@types/node": "^25.9.0",
|
"@types/node": "^26.2.0",
|
||||||
"@types/pg": "^8.20.0",
|
"@types/pg": "^8.23.1",
|
||||||
"@types/ws": "^8.18.1",
|
"@types/ws": "^8.18.1",
|
||||||
"drizzle-kit": "^0.31.10",
|
"drizzle-kit": "^0.31.10",
|
||||||
"tsx": "^4.22.2",
|
"tsx": "^4.23.12",
|
||||||
"typescript": "^5.9.3",
|
"typescript": "^7.0.2",
|
||||||
"vitest": "latest"
|
"vitest": "latest"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Generated
+660
-435
File diff suppressed because it is too large
Load Diff
@@ -25,7 +25,16 @@ function walk(dir) {
|
|||||||
const pat = /from\s+['"]([^'"]+)['"]/g;
|
const pat = /from\s+['"]([^'"]+)['"]/g;
|
||||||
const n = c.replace(pat, (m, spec) => {
|
const n = c.replace(pat, (m, spec) => {
|
||||||
if (spec.startsWith("@/")) {
|
if (spec.startsWith("@/")) {
|
||||||
const target = join("dist", spec.slice(2)) + ".js";
|
// Source may already carry an extension (e.g. "@/shared/config/index.js");
|
||||||
|
// only append ".js" when the specifier has none — otherwise we'd
|
||||||
|
// produce "index.js.js".
|
||||||
|
const core = spec.slice(2);
|
||||||
|
let target;
|
||||||
|
if (/\.(js|json|node|mjs|cjs)$/.test(core)) {
|
||||||
|
target = join("dist", core);
|
||||||
|
} else {
|
||||||
|
target = join("dist", core) + ".js";
|
||||||
|
}
|
||||||
let rel = relative(dirname(p), target);
|
let rel = relative(dirname(p), target);
|
||||||
if (!rel.startsWith(".")) rel = "./" + rel;
|
if (!rel.startsWith(".")) rel = "./" + rel;
|
||||||
return `from "${rel}"`;
|
return `from "${rel}"`;
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ import {
|
|||||||
registerMessageCapture,
|
registerMessageCapture,
|
||||||
setEventBroadcaster as setMessageCaptureEventBroadcaster,
|
setEventBroadcaster as setMessageCaptureEventBroadcaster,
|
||||||
} from "../modules/message-capture/messageCapture.js";
|
} from "../modules/message-capture/messageCapture.js";
|
||||||
|
import { setModerationEventBroadcaster } from "../modules/message-capture/moderationActionsDb.js";
|
||||||
|
import { startDigestScheduler } from "../modules/monitor/digestScheduler.js";
|
||||||
import { registerReactionCapture } from "../modules/reaction-tracking/index.js";
|
import { registerReactionCapture } from "../modules/reaction-tracking/index.js";
|
||||||
import { registerThreadCapture } from "../modules/thread-tracking/index.js";
|
import { registerThreadCapture } from "../modules/thread-tracking/index.js";
|
||||||
import { registerPresenceCapture } from "../modules/user-presence/index.js";
|
import { registerPresenceCapture } from "../modules/user-presence/index.js";
|
||||||
@@ -254,6 +256,7 @@ export async function initializeDiscordGateway() {
|
|||||||
logger.info({ user: client.user?.tag }, "Bot logged in");
|
logger.info({ user: client.user?.tag }, "Bot logged in");
|
||||||
setMessageCaptureEventBroadcaster(eventBroadcaster);
|
setMessageCaptureEventBroadcaster(eventBroadcaster);
|
||||||
setRecorderEventBroadcaster(eventBroadcaster);
|
setRecorderEventBroadcaster(eventBroadcaster);
|
||||||
|
setModerationEventBroadcaster(eventBroadcaster);
|
||||||
registerMessageCapture(client);
|
registerMessageCapture(client);
|
||||||
startPendingAIAnalysisWorker(client, eventBroadcaster);
|
startPendingAIAnalysisWorker(client, eventBroadcaster);
|
||||||
|
|
||||||
@@ -273,6 +276,8 @@ export async function initializeDiscordGateway() {
|
|||||||
|
|
||||||
// Start retention cleanup scheduler
|
// Start retention cleanup scheduler
|
||||||
startRetentionCleanup();
|
startRetentionCleanup();
|
||||||
|
// Start weekly moderation digest (public, automated)
|
||||||
|
startDigestScheduler();
|
||||||
});
|
});
|
||||||
|
|
||||||
client.on("error", (err) => {
|
client.on("error", (err) => {
|
||||||
|
|||||||
@@ -93,6 +93,12 @@ type BatchOkResponse = {
|
|||||||
ok: true;
|
ok: true;
|
||||||
conversationKey: string;
|
conversationKey: string;
|
||||||
rows: MessageRecord[];
|
rows: MessageRecord[];
|
||||||
|
/**
|
||||||
|
* Race-guard signal (2026-08-25): target ids whose attachment upload is
|
||||||
|
* still in-flight — NO analysis ran for them. The processor must defer
|
||||||
|
* these (requeue + poll), never fan them out as failures.
|
||||||
|
*/
|
||||||
|
uploadPendingIds?: string[];
|
||||||
};
|
};
|
||||||
type BatchErrorResponse = {
|
type BatchErrorResponse = {
|
||||||
ok: false;
|
ok: false;
|
||||||
@@ -100,7 +106,16 @@ type BatchErrorResponse = {
|
|||||||
rows: MessageRecord[];
|
rows: MessageRecord[];
|
||||||
error: string;
|
error: string;
|
||||||
};
|
};
|
||||||
type IndividualOkResponse = { ok: true; results: AnalysisResult[] };
|
type IndividualOkResponse = {
|
||||||
|
ok: true;
|
||||||
|
results: AnalysisResult[];
|
||||||
|
/**
|
||||||
|
* Race-guard signal (2026-08-24): the message's attachment upload is still
|
||||||
|
* in-flight — NO analysis ran. The processor must re-queue the message as
|
||||||
|
* `pending` and re-schedule, never treat this as a completed moderation.
|
||||||
|
*/
|
||||||
|
uploadPending?: boolean;
|
||||||
|
};
|
||||||
type IndividualErrorResponse = {
|
type IndividualErrorResponse = {
|
||||||
ok: false;
|
ok: false;
|
||||||
results: AnalysisResult[];
|
results: AnalysisResult[];
|
||||||
@@ -310,7 +325,16 @@ async function processBatch(job: {
|
|||||||
? messages
|
? messages
|
||||||
: messages.filter((m) => !pendingUploadTargetIds.has(m.id));
|
: messages.filter((m) => !pendingUploadTargetIds.has(m.id));
|
||||||
if (readyMessages.length === 0) {
|
if (readyMessages.length === 0) {
|
||||||
return { ok: true, conversationKey, rows: [] };
|
// Explicit signal (2026-08-25): every target is still upload-pending.
|
||||||
|
// Returning bare {ok:true, rows:[]} made the processor classify all of
|
||||||
|
// them "incomplete" and fan out to the individual queue — a hot ~300ms
|
||||||
|
// requeue loop for the whole upload duration.
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
conversationKey,
|
||||||
|
rows: [],
|
||||||
|
uploadPendingIds: messages.map((m) => m.id),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// The orchestrator handles text/media split + caching + parallel paths
|
// The orchestrator handles text/media split + caching + parallel paths
|
||||||
@@ -415,7 +439,7 @@ async function processIndividual(job: {
|
|||||||
(a) => a.message_id === message.id && a.upload_status === "pending",
|
(a) => a.message_id === message.id && a.upload_status === "pending",
|
||||||
);
|
);
|
||||||
if (uploadStillPending) {
|
if (uploadStillPending) {
|
||||||
return { ok: true, results: [] };
|
return { ok: true, results: [], uploadPending: true };
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -163,12 +163,18 @@ export function startPendingAIAnalysisWorker(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
messageStore.revertStuckProcessingMessages(300000).catch((err: unknown) => {
|
// Only revert stuck processing messages if there's active processing.
|
||||||
logger.error(
|
// Avoids a DB query every recovery interval when the pipeline is idle.
|
||||||
{ error: String(err) },
|
if (conversationProcessing.size > 0) {
|
||||||
"Failed to run stuck processing recovery",
|
messageStore
|
||||||
);
|
.revertStuckProcessingMessages(300000)
|
||||||
});
|
.catch((err: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{ error: String(err) },
|
||||||
|
"Failed to run stuck processing recovery",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
Promise.all([
|
Promise.all([
|
||||||
messageStore.getPendingConversationKeys(500),
|
messageStore.getPendingConversationKeys(500),
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ export function deriveRecommendedAction(msg: MessageRecord): string {
|
|||||||
const severity = deriveSeverity(msg);
|
const severity = deriveSeverity(msg);
|
||||||
if (
|
if (
|
||||||
msg.ai_status === "flagged" &&
|
msg.ai_status === "flagged" &&
|
||||||
(severity === "critical" || severity === "high")
|
(severity === "critical" || severity === "high" || severity === "medium")
|
||||||
)
|
)
|
||||||
return "delete";
|
return "delete";
|
||||||
if (msg.ai_status === "flagged") return "review";
|
if (msg.ai_status === "flagged") return "review";
|
||||||
@@ -135,10 +135,14 @@ export function isEligibleForAutoDelete(
|
|||||||
// Recommended action check
|
// Recommended action check
|
||||||
const recommendedAction =
|
const recommendedAction =
|
||||||
analysisResult?.recommendedAction ?? deriveRecommendedAction(message);
|
analysisResult?.recommendedAction ?? deriveRecommendedAction(message);
|
||||||
if (recommendedAction !== "delete" && recommendedAction !== "escalate") {
|
if (
|
||||||
|
recommendedAction !== "delete" &&
|
||||||
|
recommendedAction !== "escalate" &&
|
||||||
|
recommendedAction !== "warn"
|
||||||
|
) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
{ messageId: message.id, recommendedAction },
|
{ messageId: message.id, recommendedAction },
|
||||||
"Message not eligible for auto-delete: recommended action is not delete/escalate",
|
"Message not eligible for auto-delete: recommended action is not delete/escalate/warn",
|
||||||
);
|
);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -130,7 +130,28 @@ function getErrorCode(error: unknown): number | string | undefined {
|
|||||||
|
|
||||||
function isAlreadyDeletedError(error: unknown): boolean {
|
function isAlreadyDeletedError(error: unknown): boolean {
|
||||||
const code = getErrorCode(error);
|
const code = getErrorCode(error);
|
||||||
return code === 10008 || code === 404 || code === "10008" || code === "404";
|
// Discord REST error codes for "message not found":
|
||||||
|
// 10008 = Unknown Message, 10003 = Unknown Channel,
|
||||||
|
// 50001 = Missing Access (channel deleted/hidden), 404 = HTTP
|
||||||
|
if (
|
||||||
|
code === 10008 ||
|
||||||
|
code === 10003 ||
|
||||||
|
code === 50001 ||
|
||||||
|
code === 404 ||
|
||||||
|
code === "10008" ||
|
||||||
|
code === "10003" ||
|
||||||
|
code === "50001" ||
|
||||||
|
code === "404"
|
||||||
|
)
|
||||||
|
return true;
|
||||||
|
// Fallback: check the message text for the Discord "Unknown Message" string
|
||||||
|
const msg =
|
||||||
|
error instanceof Error
|
||||||
|
? error.message
|
||||||
|
: typeof error === "string"
|
||||||
|
? error
|
||||||
|
: "";
|
||||||
|
return msg.includes("Unknown Message") || msg.includes("Unknown Channel");
|
||||||
}
|
}
|
||||||
|
|
||||||
function hasChannelMessagesApi(channel: unknown): channel is {
|
function hasChannelMessagesApi(channel: unknown): channel is {
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
/**
|
||||||
|
* batchBudget.ts
|
||||||
|
*
|
||||||
|
* Pure batch-sizing helper extracted from batchProcessor.ts so it can be
|
||||||
|
* unit-tested without pulling in the Piscina worker pool, message store,
|
||||||
|
* or any other side-effectful import chain.
|
||||||
|
*/
|
||||||
|
import type { MessageRecord } from "../message-capture/types.js";
|
||||||
|
|
||||||
|
/** Token estimator contract (satisfied by conversationContext.estimateTokens). */
|
||||||
|
export type TokenEstimator = (text: string) => number;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Picks a batch of messages within a token budget.
|
||||||
|
* `tokensPerMessage` accounts for JSON structure overhead around each entry.
|
||||||
|
* The estimator is injected so this stays a pure function — callers in the
|
||||||
|
* batch pipeline pass the tiktoken-based estimateTokens.
|
||||||
|
*/
|
||||||
|
export function pickBatchWithinBudget(
|
||||||
|
messages: MessageRecord[],
|
||||||
|
maxTokens: number,
|
||||||
|
tokensPerMessage: number,
|
||||||
|
estimateTokens: TokenEstimator,
|
||||||
|
): MessageRecord[] {
|
||||||
|
const batch: MessageRecord[] = [];
|
||||||
|
let usedTokens = 0;
|
||||||
|
|
||||||
|
for (const msg of messages) {
|
||||||
|
const content = msg.edited_content ?? msg.content;
|
||||||
|
const msgTokens = estimateTokens(content) + tokensPerMessage;
|
||||||
|
|
||||||
|
// Stop at the first overflow instead of skipping: input is ordered
|
||||||
|
// created_at ASC, so a contiguous chronological prefix keeps the batch
|
||||||
|
// gap-free. Skipped-over messages would leave unanalyzed holes mid-
|
||||||
|
// timeline; anything past the budget is picked up by the next wave
|
||||||
|
// (processBatch always re-schedules after success).
|
||||||
|
if (usedTokens + msgTokens > maxTokens) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
batch.push(msg);
|
||||||
|
usedTokens += msgTokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
return batch;
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
/**
|
||||||
|
* batchOutcomeClassifier.ts
|
||||||
|
*
|
||||||
|
* Pure partitioner of the batch worker response (2026-08-25).
|
||||||
|
*
|
||||||
|
* Bug history: the batch race guard returned `{ok:true, rows:[]}` when every
|
||||||
|
* target's attachment upload was still in-flight. The processor classified all
|
||||||
|
* of them as "incomplete" and fanned out to the individual queue, where the
|
||||||
|
* guard there requeued + rescheduled at the 250ms debounce — a hot ~300ms loop
|
||||||
|
* for the entire upload duration (~10 cycles in 3s in prod logs). Root fix:
|
||||||
|
* the worker now reports `uploadPendingIds` explicitly and this pure function
|
||||||
|
* partitions the outcome so upload-pending targets NEVER enter the fanout.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export interface BatchRowLike {
|
||||||
|
id?: string;
|
||||||
|
ai_status?: string | null;
|
||||||
|
ai_moderation_flags?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BatchWorkerResponseLike {
|
||||||
|
ok?: boolean;
|
||||||
|
rows?: BatchRowLike[];
|
||||||
|
/** Explicit race-guard signal from the worker (2026-08-25). */
|
||||||
|
uploadPendingIds?: string[];
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One target's per-message disposition after a batch attempt. */
|
||||||
|
export type BatchTargetKind =
|
||||||
|
| "completed"
|
||||||
|
| "upload_pending"
|
||||||
|
| "incomplete"
|
||||||
|
| "parse_failed"
|
||||||
|
| "api_failed";
|
||||||
|
|
||||||
|
function flagsOf(row: { ai_moderation_flags?: string | null }): string[] {
|
||||||
|
if (!row.ai_moderation_flags) return [];
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(row.ai_moderation_flags) as unknown;
|
||||||
|
return Array.isArray(parsed) ? (parsed as string[]) : [];
|
||||||
|
} catch {
|
||||||
|
return [] as string[];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Partition the input message ids into per-message dispositions for one batch
|
||||||
|
* worker response. Pure: no DB/Piscina/logger — unit-testable directly.
|
||||||
|
*
|
||||||
|
* Priority per id: explicit uploadPendingIds → completed row → flag-based
|
||||||
|
* failure kinds → unexplained missing (treated like incomplete).
|
||||||
|
*/
|
||||||
|
export function partitionBatchOutcome(
|
||||||
|
messages: ReadonlyArray<{ id: string }>,
|
||||||
|
response: BatchWorkerResponseLike,
|
||||||
|
): Map<string, BatchTargetKind> {
|
||||||
|
const pendingSet = new Set(response.uploadPendingIds ?? []);
|
||||||
|
const rowsById = new Map(
|
||||||
|
(response.rows ?? [])
|
||||||
|
.filter((r): r is BatchRowLike & { id: string } => Boolean(r?.id))
|
||||||
|
.map((r) => [r.id, r]),
|
||||||
|
);
|
||||||
|
|
||||||
|
const out = new Map<string, BatchTargetKind>();
|
||||||
|
for (const msg of messages) {
|
||||||
|
if (pendingSet.has(msg.id)) {
|
||||||
|
out.set(msg.id, "upload_pending");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const row = rowsById.get(msg.id);
|
||||||
|
if (!row) {
|
||||||
|
// Unexplained drop: LLM silently omitted it. Same retryable bucket as
|
||||||
|
// analysis_incomplete — never a silent success.
|
||||||
|
out.set(msg.id, "incomplete");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (row.ai_status !== "error") {
|
||||||
|
out.set(msg.id, "completed");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const flags = flagsOf(row);
|
||||||
|
if (flags.includes("analysis_incomplete")) {
|
||||||
|
out.set(msg.id, "incomplete");
|
||||||
|
} else if (flags.includes("analysis_parse_failed")) {
|
||||||
|
out.set(msg.id, "parse_failed");
|
||||||
|
} else if (flags.includes("analysis_api_failed")) {
|
||||||
|
out.set(msg.id, "api_failed");
|
||||||
|
} else {
|
||||||
|
out.set(msg.id, "incomplete");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Linear backoff ramp for consecutive upload-pending polls:
|
||||||
|
* poll N (1-based) waits min(base × N, cap). Keeps latency low for fast
|
||||||
|
* uploads while bounding total polling cost for long uploads.
|
||||||
|
*/
|
||||||
|
export function computeUploadPollDelayMs(
|
||||||
|
consecutivePolls: number,
|
||||||
|
baseMs: number,
|
||||||
|
capMs: number,
|
||||||
|
): number {
|
||||||
|
const n = Math.max(1, Math.floor(consecutivePolls));
|
||||||
|
return Math.min(Math.round(baseMs * n), Math.round(capMs));
|
||||||
|
}
|
||||||
@@ -3,6 +3,11 @@ import { config } from "../../shared/config/config.js";
|
|||||||
import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js";
|
import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js";
|
||||||
import { messageStore } from "../message-capture/messageStore.js";
|
import { messageStore } from "../message-capture/messageStore.js";
|
||||||
import type { MessageRecord } from "../message-capture/types.js";
|
import type { MessageRecord } from "../message-capture/types.js";
|
||||||
|
import { pickBatchWithinBudget as pickBatchWithinBudgetPure } from "./batchBudget.js";
|
||||||
|
import {
|
||||||
|
computeUploadPollDelayMs,
|
||||||
|
partitionBatchOutcome,
|
||||||
|
} from "./batchOutcomeClassifier.js";
|
||||||
import { workerPool } from "./circuitBreaker.js";
|
import { workerPool } from "./circuitBreaker.js";
|
||||||
import { estimateTokens } from "./conversationContext.js";
|
import { estimateTokens } from "./conversationContext.js";
|
||||||
import {
|
import {
|
||||||
@@ -20,11 +25,20 @@ import {
|
|||||||
|
|
||||||
const logger = createChildLogger("batch-processor");
|
const logger = createChildLogger("batch-processor");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Consecutive upload-pending poll counter per conversation (2026-08-25).
|
||||||
|
* Drives the linear backoff ramp while attachments are still uploading;
|
||||||
|
* cleared as soon as a batch comes back with no upload-pending targets.
|
||||||
|
*/
|
||||||
|
const conversationUploadPolls = new Map<string, number>();
|
||||||
|
|
||||||
export interface AnalysisWorkerResponse {
|
export interface AnalysisWorkerResponse {
|
||||||
ok: boolean;
|
ok: boolean;
|
||||||
conversationKey: string;
|
conversationKey: string;
|
||||||
rows: MessageRecord[];
|
rows: MessageRecord[];
|
||||||
error?: string;
|
error?: string;
|
||||||
|
/** Explicit upload-in-flight signal from the batch race guard (2026-08-25). */
|
||||||
|
uploadPendingIds?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -39,30 +53,21 @@ export let activeRequests = 0;
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Picks a batch of messages within a token budget.
|
* Picks a batch of messages within a token budget.
|
||||||
* `tokensPerMessage` accounts for JSON structure overhead around each entry.
|
* Thin wrapper over the pure helper in batchBudget.ts (kept here so the
|
||||||
* Uses a rough character-based token estimate (avoids async formatMessageForPrompt
|
* existing import surface stays stable); passes the tiktoken-based
|
||||||
* since this function runs in a synchronous promise chain).
|
* estimateTokens. See batchBudget.ts for the overflow-stopping semantics.
|
||||||
*/
|
*/
|
||||||
export function pickBatchWithinBudget(
|
export function pickBatchWithinBudget(
|
||||||
messages: MessageRecord[],
|
messages: MessageRecord[],
|
||||||
maxTokens: number,
|
maxTokens: number,
|
||||||
tokensPerMessage: number,
|
tokensPerMessage: number,
|
||||||
): MessageRecord[] {
|
): MessageRecord[] {
|
||||||
const batch: MessageRecord[] = [];
|
return pickBatchWithinBudgetPure(
|
||||||
let usedTokens = 0;
|
messages,
|
||||||
|
maxTokens,
|
||||||
for (const msg of messages) {
|
tokensPerMessage,
|
||||||
const content = msg.edited_content ?? msg.content;
|
estimateTokens,
|
||||||
// Accurate token count via tiktoken (+ overhead for JSON structure)
|
);
|
||||||
const msgTokens = estimateTokens(content) + tokensPerMessage;
|
|
||||||
|
|
||||||
if (usedTokens + msgTokens <= maxTokens) {
|
|
||||||
batch.push(msg);
|
|
||||||
usedTokens += msgTokens;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return batch;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -128,30 +133,6 @@ export async function skipAgeRestrictedMessages(
|
|||||||
// Batch pipeline
|
// Batch pipeline
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function postBatchReputationUpdate(rows: MessageRecord[]): Promise<void> {
|
|
||||||
for (const row of rows) {
|
|
||||||
if (row.ai_status === "clean") {
|
|
||||||
import("./userReputationStore.js")
|
|
||||||
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error({ error: e }, "Failed to record clean message streak"),
|
|
||||||
);
|
|
||||||
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
|
|
||||||
import("./userReputationStore.js")
|
|
||||||
.then((store) =>
|
|
||||||
store.recordInfraction(
|
|
||||||
row.user_id,
|
|
||||||
row.guild_id,
|
|
||||||
row.ai_severity as "low" | "medium" | "high" | "critical",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error({ error: e }, "Failed to record infraction penalty"),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function processBatch(
|
export async function processBatch(
|
||||||
conversationKey: string,
|
conversationKey: string,
|
||||||
messages: MessageRecord[],
|
messages: MessageRecord[],
|
||||||
@@ -173,6 +154,8 @@ export async function processBatch(
|
|||||||
|
|
||||||
activeRequests++;
|
activeRequests++;
|
||||||
let shouldScheduleNext = false;
|
let shouldScheduleNext = false;
|
||||||
|
/** Set when upload-pending targets defer the next cycle by this many ms. */
|
||||||
|
let deferredUploadRescheduleMs: number | null = null;
|
||||||
try {
|
try {
|
||||||
const result = (await workerPool.run({
|
const result = (await workerPool.run({
|
||||||
type: "batch",
|
type: "batch",
|
||||||
@@ -180,37 +163,16 @@ export async function processBatch(
|
|||||||
messages,
|
messages,
|
||||||
})) as AnalysisWorkerResponse;
|
})) as AnalysisWorkerResponse;
|
||||||
|
|
||||||
// Do not broadcast or auto-delete if it's an API failure that will be reverted.
|
// Broadcast + auto-delete only for successfully analyzed rows.
|
||||||
|
// Error rows (API failures, parse failures, incomplete) will be
|
||||||
|
// retried by the individual fallback queue — do NOT schedule
|
||||||
|
// auto-delete for them (they'd be logged as not_eligible anyway).
|
||||||
for (const row of result.rows) {
|
for (const row of result.rows) {
|
||||||
let isApiFailure = false;
|
if (row.ai_status === "error") continue;
|
||||||
if (row.ai_status === "error") {
|
broadcastAnalysisCompleted(row);
|
||||||
try {
|
scheduleAutoDelete(row);
|
||||||
const flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[];
|
|
||||||
isApiFailure = flags.includes("analysis_api_failed");
|
|
||||||
} catch {}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isApiFailure) {
|
|
||||||
broadcastAnalysisCompleted(row);
|
|
||||||
scheduleAutoDelete(row);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Post-batch reputation updates (fire-and-forget)
|
|
||||||
postBatchReputationUpdate(
|
|
||||||
result.rows.filter((r) => {
|
|
||||||
if (r.ai_status === "error") {
|
|
||||||
try {
|
|
||||||
const flags = JSON.parse(r.ai_moderation_flags ?? "[]") as string[];
|
|
||||||
return !flags.includes("analysis_api_failed");
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!result.ok) {
|
if (!result.ok) {
|
||||||
recordConversationBatchFailure(conversationKey);
|
recordConversationBatchFailure(conversationKey);
|
||||||
|
|
||||||
@@ -246,37 +208,89 @@ export async function processBatch(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Batch succeeded -- check for messages the LLM silently dropped or failed
|
// Batch succeeded -- partition per-message outcome explicitly (2026-08-25).
|
||||||
const incompleteMessages: MessageRecord[] = [];
|
// upload_pending targets are DEFERRED (never fanned out): the old code
|
||||||
const parseFailedMessages: MessageRecord[] = [];
|
// treated them as incomplete -> individual queue -> requeue+250ms
|
||||||
|
// reschedule -> hot ~300ms loop for the whole upload duration.
|
||||||
|
const outcomeById = partitionBatchOutcome(messages, result);
|
||||||
|
const messagesForIndividualQueue: MessageRecord[] = [];
|
||||||
const apiFailedMessages: MessageRecord[] = [];
|
const apiFailedMessages: MessageRecord[] = [];
|
||||||
|
const uploadPendingMessages: MessageRecord[] = [];
|
||||||
|
|
||||||
for (const msg of messages) {
|
for (const msg of messages) {
|
||||||
const row = result.rows.find((r) => r.id === msg.id);
|
switch (outcomeById.get(msg.id)) {
|
||||||
if (!row) {
|
case "upload_pending":
|
||||||
incompleteMessages.push(msg);
|
uploadPendingMessages.push(msg);
|
||||||
continue;
|
break;
|
||||||
}
|
case "api_failed":
|
||||||
if (row.ai_status === "error") {
|
// Preserve the dedicated api-failure semantics below: revert +
|
||||||
let flags: string[] = [];
|
// conversation cooldown instead of an immediate individual retry.
|
||||||
try {
|
|
||||||
flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[];
|
|
||||||
} catch {}
|
|
||||||
|
|
||||||
if (flags.includes("analysis_incomplete")) {
|
|
||||||
incompleteMessages.push(msg);
|
|
||||||
} else if (flags.includes("analysis_parse_failed")) {
|
|
||||||
parseFailedMessages.push(msg);
|
|
||||||
} else if (flags.includes("analysis_api_failed")) {
|
|
||||||
apiFailedMessages.push(msg);
|
apiFailedMessages.push(msg);
|
||||||
}
|
break;
|
||||||
|
case "completed":
|
||||||
|
// Successfully analyzed — already broadcast + auto-delete scheduled
|
||||||
|
// above. Do NOT re-enqueue for individual fallback.
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
// incomplete / parse_failed / unexplained drops stay retryable via
|
||||||
|
// the individual fallback queue (same semantics as before).
|
||||||
|
messagesForIndividualQueue.push(msg);
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const messagesForIndividualQueue = [
|
if (uploadPendingMessages.length > 0) {
|
||||||
...incompleteMessages,
|
const polls = (conversationUploadPolls.get(conversationKey) ?? 0) + 1;
|
||||||
...parseFailedMessages,
|
conversationUploadPolls.set(conversationKey, polls);
|
||||||
];
|
const delayMs = computeUploadPollDelayMs(
|
||||||
|
polls,
|
||||||
|
config.AI_ANALYSIS_UPLOAD_POLL_MS,
|
||||||
|
config.AI_ANALYSIS_MAX_UPLOAD_POLL_MS,
|
||||||
|
);
|
||||||
|
logger.debug(
|
||||||
|
{
|
||||||
|
conversationKey,
|
||||||
|
count: uploadPendingMessages.length,
|
||||||
|
ids: uploadPendingMessages.map((m) => m.id),
|
||||||
|
pollAttempt: polls,
|
||||||
|
delayMs,
|
||||||
|
},
|
||||||
|
"Attachment upload in-flight for batch targets — deferring with poll backoff",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Put the rows back to `pending` so the scheduler owns them again.
|
||||||
|
await messageStore
|
||||||
|
.updateMessagesAIAnalysisBulk(
|
||||||
|
uploadPendingMessages.map((msg) => ({
|
||||||
|
messageId: msg.id,
|
||||||
|
result: {
|
||||||
|
status: "pending",
|
||||||
|
flags: null,
|
||||||
|
score: null,
|
||||||
|
analysis: null,
|
||||||
|
categories: null,
|
||||||
|
severity: null,
|
||||||
|
confidence: null,
|
||||||
|
recommendedAction: null,
|
||||||
|
analyzedAt: null,
|
||||||
|
error: null,
|
||||||
|
},
|
||||||
|
})),
|
||||||
|
)
|
||||||
|
.catch((err: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{ error: String(err), ids: uploadPendingMessages.map((m) => m.id) },
|
||||||
|
"Failed to revert upload-pending batch targets to pending",
|
||||||
|
);
|
||||||
|
return [] as MessageRecord[];
|
||||||
|
});
|
||||||
|
|
||||||
|
// Poll backoff instead of the 250ms debounce: the finally-block
|
||||||
|
// schedules the next cycle after this delay instead of immediately.
|
||||||
|
deferredUploadRescheduleMs = delayMs;
|
||||||
|
} else {
|
||||||
|
conversationUploadPolls.delete(conversationKey);
|
||||||
|
}
|
||||||
|
|
||||||
if (messagesForIndividualQueue.length > 0) {
|
if (messagesForIndividualQueue.length > 0) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
@@ -354,7 +368,11 @@ export async function processBatch(
|
|||||||
resetConversationBatchFailures(conversationKey);
|
resetConversationBatchFailures(conversationKey);
|
||||||
conversationErrorCooldown.delete(conversationKey);
|
conversationErrorCooldown.delete(conversationKey);
|
||||||
}
|
}
|
||||||
shouldScheduleNext = true;
|
// Upload-pending defer owns the next-cycle timing; don't let the default
|
||||||
|
// immediate schedule override it.
|
||||||
|
if (deferredUploadRescheduleMs === null) {
|
||||||
|
shouldScheduleNext = true;
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
recordConversationBatchFailure(conversationKey);
|
recordConversationBatchFailure(conversationKey);
|
||||||
|
|
||||||
@@ -391,7 +409,17 @@ export async function processBatch(
|
|||||||
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
|
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
|
||||||
conversationProcessing.delete(conversationKey);
|
conversationProcessing.delete(conversationKey);
|
||||||
}
|
}
|
||||||
if (shouldScheduleNext) {
|
if (deferredUploadRescheduleMs !== null) {
|
||||||
|
// Upload still in-flight: re-schedule after the backoff delay instead of
|
||||||
|
// immediately (the old path hot-looped at ~250-300ms per cycle).
|
||||||
|
const delayMs = deferredUploadRescheduleMs;
|
||||||
|
setTimeout(() => {
|
||||||
|
// Dynamic import to avoid circular dependency at module scope
|
||||||
|
import("./batchScheduler.js").then((m) =>
|
||||||
|
m.scheduleConversationAnalysis(conversationKey),
|
||||||
|
);
|
||||||
|
}, delayMs).unref();
|
||||||
|
} else if (shouldScheduleNext) {
|
||||||
setImmediate(() => {
|
setImmediate(() => {
|
||||||
// Dynamic import to avoid circular dependency at module scope
|
// Dynamic import to avoid circular dependency at module scope
|
||||||
import("./batchScheduler.js").then((m) =>
|
import("./batchScheduler.js").then((m) =>
|
||||||
|
|||||||
@@ -136,6 +136,12 @@ export function formatMessageForPrompt(
|
|||||||
return `[${label}] id=${msg.id} time=${timestamp} user=${resolveDisplayName(msg)}: ${content}${mediaSuffix}${refInfo}`;
|
return `[${label}] id=${msg.id} time=${timestamp} user=${resolveDisplayName(msg)}: ${content}${mediaSuffix}${refInfo}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Estimate tokens for a batch of strings — single encode call, ~5x faster than per-line. */
|
||||||
|
export function estimateTokensBatch(texts: string[]): number {
|
||||||
|
const combined = texts.join("\n");
|
||||||
|
return getEncoder().encode(combined).length + texts.length * 15;
|
||||||
|
}
|
||||||
|
|
||||||
/** Max content chars per context line — a single huge paste (log dump,
|
/** Max content chars per context line — a single huge paste (log dump,
|
||||||
* copypasta) must not eat the whole conversation budget. */
|
* copypasta) must not eat the whole conversation budget. */
|
||||||
const CONTEXT_LINE_CONTENT_MAX_CHARS = 1500;
|
const CONTEXT_LINE_CONTENT_MAX_CHARS = 1500;
|
||||||
@@ -264,10 +270,7 @@ export function buildConversationContext(
|
|||||||
const targetLines = targets.map((msg) =>
|
const targetLines = targets.map((msg) =>
|
||||||
formatMessageForPrompt(msg, "target"),
|
formatMessageForPrompt(msg, "target"),
|
||||||
);
|
);
|
||||||
let usedTokens = targetLines.reduce(
|
let usedTokens = estimateTokensBatch(targetLines);
|
||||||
(sum, line) => sum + estimateTokens(line),
|
|
||||||
0,
|
|
||||||
);
|
|
||||||
|
|
||||||
const contextLines = gatedNewestFirst.map((msg) =>
|
const contextLines = gatedNewestFirst.map((msg) =>
|
||||||
formatMessageForPrompt(msg, "context"),
|
formatMessageForPrompt(msg, "context"),
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
/**
|
||||||
|
* fallbackResultClassifier.ts
|
||||||
|
*
|
||||||
|
* Pure classifier for the individual-fallback worker response.
|
||||||
|
*
|
||||||
|
* Bug history (2026-08-24): the worker's upload-pending race guard returned
|
||||||
|
* `{ ok: true, results: [] }` (a legacy "no results yet" signal), but the
|
||||||
|
* processor treated ANY `ok:true` as a successful moderation. Empty results
|
||||||
|
* meant nothing was written to the DB — the message stayed stuck in
|
||||||
|
* `ai_status='processing'` with nobody watching it until the 300s cleanup
|
||||||
|
* reverted it. That single gap produced the ~330-400s attachment delay
|
||||||
|
* cluster. Classification now happens in ONE pure function so every outcome
|
||||||
|
* has an explicit, testable owner.
|
||||||
|
*/
|
||||||
|
|
||||||
|
export type WorkerResultKind =
|
||||||
|
| "success"
|
||||||
|
| "upload_pending"
|
||||||
|
| "incomplete"
|
||||||
|
| "error";
|
||||||
|
|
||||||
|
export interface ClassifiableWorkerResult {
|
||||||
|
ok?: boolean;
|
||||||
|
/** Upload-pending marker set by ai-analysis-worker's race guard. */
|
||||||
|
uploadPending?: boolean;
|
||||||
|
results?: Array<{ status?: string; flags?: string[] | string } | undefined>;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function flagsOf(r: { flags?: string[] | string }): string[] {
|
||||||
|
if (!r.flags) return [];
|
||||||
|
if (Array.isArray(r.flags)) return r.flags;
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(r.flags) as unknown;
|
||||||
|
return Array.isArray(parsed) ? (parsed as string[]) : [];
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Classify an individual-fallback worker response:
|
||||||
|
* - "upload_pending": explicit race-guard signal — retry shortly, NOT an error.
|
||||||
|
* - "success": at least one result and none is analysis_incomplete.
|
||||||
|
* - "incomplete": LLM ran but dropped/failed this message after retries
|
||||||
|
* (analysis_incomplete flag) — terminal exhausted path.
|
||||||
|
* - "error": anything else (ok:false, or ok:true with NO explainable
|
||||||
|
* results). The old code silently succeeded here — never again.
|
||||||
|
*/
|
||||||
|
export function classifyIndividualWorkerResult(
|
||||||
|
result: ClassifiableWorkerResult,
|
||||||
|
): WorkerResultKind {
|
||||||
|
if (result.uploadPending === true) return "upload_pending";
|
||||||
|
const results = (result.results ?? []).filter(
|
||||||
|
(r): r is NonNullable<typeof r> => Boolean(r),
|
||||||
|
);
|
||||||
|
if (results.length === 0) return "error";
|
||||||
|
if (result.ok !== true) return "error";
|
||||||
|
for (const r of results) {
|
||||||
|
const flags = flagsOf(r);
|
||||||
|
if (flags.includes("analysis_incomplete")) return "incomplete";
|
||||||
|
if ((r.status ?? "") === "") return "error";
|
||||||
|
}
|
||||||
|
return "success";
|
||||||
|
}
|
||||||
@@ -8,6 +8,7 @@ import type {
|
|||||||
} from "../message-capture/types.js";
|
} from "../message-capture/types.js";
|
||||||
import { getConversationKey, workerPool } from "./circuitBreaker.js";
|
import { getConversationKey, workerPool } from "./circuitBreaker.js";
|
||||||
import { fireAlert } from "./conversationState.js";
|
import { fireAlert } from "./conversationState.js";
|
||||||
|
import { classifyIndividualWorkerResult } from "./fallbackResultClassifier.js";
|
||||||
import {
|
import {
|
||||||
broadcastAnalysisCompleted,
|
broadcastAnalysisCompleted,
|
||||||
LAST_ERROR,
|
LAST_ERROR,
|
||||||
@@ -78,26 +79,82 @@ async function processIndividualFallback(
|
|||||||
message,
|
message,
|
||||||
skipNormalAnalysis: false,
|
skipNormalAnalysis: false,
|
||||||
} as unknown)) as
|
} as unknown)) as
|
||||||
| { ok: true; results: AnalysisResult[] }
|
| { ok: true; results: AnalysisResult[]; uploadPending?: boolean }
|
||||||
| { ok: false; results: AnalysisResult[]; error: string };
|
| { ok: false; results: AnalysisResult[]; error: string };
|
||||||
|
|
||||||
|
// Explicit outcome classification (2026-08-24): the old code treated any
|
||||||
|
// ok:true as a completed moderation, so the upload-pending race guard's
|
||||||
|
// empty results left messages stuck in `processing` until the 300s
|
||||||
|
// cleanup reverted them — the root cause of the ~330s attachment delays.
|
||||||
|
const kind = classifyIndividualWorkerResult(workerResult);
|
||||||
|
|
||||||
|
if (kind === "upload_pending") {
|
||||||
|
// Attachment still uploading — put the row back to `pending` and
|
||||||
|
// re-schedule this conversation immediately. The next scheduler cycle
|
||||||
|
// (~debounce 250ms) re-fetches; once upload_status flips to done the
|
||||||
|
// race guard passes and analysis proceeds. NOT an error: never touches
|
||||||
|
// the circuit breaker counters.
|
||||||
|
const revertedRows = await messageStore
|
||||||
|
.updateMessagesAIAnalysisBulk([
|
||||||
|
{
|
||||||
|
messageId,
|
||||||
|
result: {
|
||||||
|
status: "pending",
|
||||||
|
flags: null,
|
||||||
|
score: null,
|
||||||
|
analysis: null,
|
||||||
|
categories: null,
|
||||||
|
severity: null,
|
||||||
|
confidence: null,
|
||||||
|
recommendedAction: null,
|
||||||
|
analyzedAt: null,
|
||||||
|
error: null,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
])
|
||||||
|
.catch((dbErr: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{ messageId, error: String(dbErr) },
|
||||||
|
"Failed to revert upload-pending message to pending",
|
||||||
|
);
|
||||||
|
return [] as MessageRecord[];
|
||||||
|
});
|
||||||
|
for (const row of revertedRows) {
|
||||||
|
broadcastAnalysisCompleted(row);
|
||||||
|
}
|
||||||
|
logger.debug(
|
||||||
|
{ messageId, conversationKey },
|
||||||
|
"Individual fallback: attachment upload in-flight — requeued as pending + rescheduled",
|
||||||
|
);
|
||||||
|
setImmediate(() => {
|
||||||
|
import("./batchScheduler.js")
|
||||||
|
.then((m) => m.scheduleConversationAnalysis(conversationKey))
|
||||||
|
.catch(() => {});
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let analysisResult: { results: AnalysisResult[] } | null = null;
|
let analysisResult: { results: AnalysisResult[] } | null = null;
|
||||||
|
|
||||||
if (workerResult.ok) {
|
if (kind === "success") {
|
||||||
const stillIncomplete = workerResult.results.some((r) =>
|
analysisResult = workerResult;
|
||||||
r.flags.includes("analysis_incomplete"),
|
} else if (kind === "incomplete") {
|
||||||
|
exhaustedOnIncomplete = true;
|
||||||
|
analysisResult = null;
|
||||||
|
} else {
|
||||||
|
// "error" — includes ok:true with unexplainable empty results (the old
|
||||||
|
// silent-success bug). Throw so it is treated as a transient failure.
|
||||||
|
throw new Error(
|
||||||
|
(workerResult as { error?: string }).error ??
|
||||||
|
"Individual worker returned no explainable results",
|
||||||
);
|
);
|
||||||
if (stillIncomplete) {
|
|
||||||
exhaustedOnIncomplete = true;
|
|
||||||
analysisResult = null;
|
|
||||||
} else {
|
|
||||||
analysisResult = workerResult;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// No heuristic fallback: an incomplete/errored LLM result stays a
|
// No heuristic fallback: an incomplete/errored LLM result stays a
|
||||||
// retryable error — the recovery worker picks it up later. Producing a
|
// retryable error — the recovery worker picks it up later. Producing a
|
||||||
// regex/wordlist verdict here would reintroduce false positives.
|
// regex/wordlist verdict here would reintroduce false positives.
|
||||||
|
// (incomplete keeps its exhausted flag so the catch writes the terminal
|
||||||
|
// individual_analysis_exhausted status.)
|
||||||
if (!analysisResult) {
|
if (!analysisResult) {
|
||||||
throw new Error(`LLM analysis failed for message ${messageId}`);
|
throw new Error(`LLM analysis failed for message ${messageId}`);
|
||||||
}
|
}
|
||||||
@@ -123,33 +180,6 @@ async function processIndividualFallback(
|
|||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
broadcastAnalysisCompleted(row);
|
broadcastAnalysisCompleted(row);
|
||||||
scheduleAutoDelete(row);
|
scheduleAutoDelete(row);
|
||||||
|
|
||||||
// Update reputation autonomously
|
|
||||||
if (row.ai_status === "clean") {
|
|
||||||
import("./userReputationStore.js")
|
|
||||||
.then((store) => store.recordCleanMessage(row.user_id, row.guild_id))
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error(
|
|
||||||
{ error: e },
|
|
||||||
"Failed to record clean message streak in fallback",
|
|
||||||
),
|
|
||||||
);
|
|
||||||
} else if (row.ai_status === "flagged" && row.ai_severity !== "none") {
|
|
||||||
import("./userReputationStore.js")
|
|
||||||
.then((store) =>
|
|
||||||
store.recordInfraction(
|
|
||||||
row.user_id,
|
|
||||||
row.guild_id,
|
|
||||||
row.ai_severity as "low" | "medium" | "high" | "critical",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.catch((e) =>
|
|
||||||
logger.error(
|
|
||||||
{ error: e },
|
|
||||||
"Failed to record infraction penalty in fallback",
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const resultSummary = analysisResult.results[0];
|
const resultSummary = analysisResult.results[0];
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import { delay, retryWithBackoff } from "@/shared/utils/index";
|
|||||||
import { config } from "../../shared/config/config.js";
|
import { config } from "../../shared/config/config.js";
|
||||||
import type { AnalysisResult } from "../message-capture/types.js";
|
import type { AnalysisResult } from "../message-capture/types.js";
|
||||||
import { llmChat } from "./llmClient.js";
|
import { llmChat } from "./llmClient.js";
|
||||||
|
import { parseModerationResponse } from "./moderationResponseParser.js";
|
||||||
import { logModerationError } from "./responseLogger.js";
|
import { logModerationError } from "./responseLogger.js";
|
||||||
|
|
||||||
const log = createChildLogger("llm-caller");
|
const log = createChildLogger("llm-caller");
|
||||||
@@ -49,6 +50,10 @@ export async function callModerationLLM(
|
|||||||
targetIds: string[],
|
targetIds: string[],
|
||||||
label: string,
|
label: string,
|
||||||
signal?: AbortSignal,
|
signal?: AbortSignal,
|
||||||
|
// Output-side token cap. Defaults to the previous hard-coded value; batch
|
||||||
|
// callers pass a prompt-derived ceiling so small batches don't reserve a
|
||||||
|
// 16k completion budget (some routers pre-allocate KV cache per max_tokens).
|
||||||
|
maxTokens?: number,
|
||||||
): Promise<{
|
): Promise<{
|
||||||
results: AnalysisResult[];
|
results: AnalysisResult[];
|
||||||
raw: ChatCompletion | null;
|
raw: ChatCompletion | null;
|
||||||
@@ -75,11 +80,12 @@ export async function callModerationLLM(
|
|||||||
];
|
];
|
||||||
const completion = await llmChat({
|
const completion = await llmChat({
|
||||||
messages,
|
messages,
|
||||||
max_tokens: 16384,
|
max_tokens:
|
||||||
|
maxTokens ?? config.AI_LLM_MAX_COMPLETION_TOKENS ?? 16384,
|
||||||
jsonResponse: { type: "json_object" },
|
jsonResponse: { type: "json_object" },
|
||||||
retries: 0,
|
retries: 0,
|
||||||
signal,
|
signal,
|
||||||
// Router (9router/omniroute) always streams SSE even when the
|
// Router (9router / formerly omniroute) always streams SSE even when the
|
||||||
// request omits `stream`. In non-stream mode the OpenAI SDK waits
|
// request omits `stream`. In non-stream mode the OpenAI SDK waits
|
||||||
// for the FULL body before parsing, so slow/long upstream streams
|
// for the FULL body before parsing, so slow/long upstream streams
|
||||||
// hit the 30s/60s timeout and abort mid-generation. Streaming mode
|
// hit the 30s/60s timeout and abort mid-generation. Streaming mode
|
||||||
@@ -102,9 +108,6 @@ export async function callModerationLLM(
|
|||||||
if (!rawContent) throw new Error("No content in LLM response");
|
if (!rawContent) throw new Error("No content in LLM response");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { parseModerationResponse } = await import(
|
|
||||||
"./moderationResponseParser.js"
|
|
||||||
);
|
|
||||||
return {
|
return {
|
||||||
parsed: parseModerationResponse(rawContent, targetIds),
|
parsed: parseModerationResponse(rawContent, targetIds),
|
||||||
result: completion,
|
result: completion,
|
||||||
|
|||||||
@@ -356,10 +356,10 @@ export async function llmVision(
|
|||||||
promptText: string,
|
promptText: string,
|
||||||
imageUrl: { url: string },
|
imageUrl: { url: string },
|
||||||
): Promise<string | null> {
|
): Promise<string | null> {
|
||||||
const completion = await llmChat({
|
const params = {
|
||||||
messages: [
|
messages: [
|
||||||
{
|
{
|
||||||
role: "user",
|
role: "user" as const,
|
||||||
content: [
|
content: [
|
||||||
{ type: "text" as const, text: promptText },
|
{ type: "text" as const, text: promptText },
|
||||||
{ type: "image_url" as const, image_url: imageUrl },
|
{ type: "image_url" as const, image_url: imageUrl },
|
||||||
@@ -371,9 +371,26 @@ export async function llmVision(
|
|||||||
temperature: 0.1,
|
temperature: 0.1,
|
||||||
top_p: 0.9,
|
top_p: 0.9,
|
||||||
retries: 0,
|
retries: 0,
|
||||||
stream: true, // router always streams SSE; non-stream waits for full body and times out
|
|
||||||
timeout: config.AI_LLM_VISION_ANALYSIS_TIMEOUT_MS ?? 60_000,
|
timeout: config.AI_LLM_VISION_ANALYSIS_TIMEOUT_MS ?? 60_000,
|
||||||
});
|
};
|
||||||
|
|
||||||
|
// Streaming first (the router always streams SSE; a non-stream request
|
||||||
|
// waits for the full body and times out on slow models). Fallback (2026-08-24):
|
||||||
|
// large GIFs/images sometimes get their SSE stream truncated mid-flight by
|
||||||
|
// the upstream ("Stream ended before producing a non-ping SSE event") — all
|
||||||
|
// streaming retries fail identically, so retry ONCE with stream:false where
|
||||||
|
// the router assembles the complete response server-side.
|
||||||
|
let completion: Awaited<ReturnType<typeof llmChat>>;
|
||||||
|
try {
|
||||||
|
completion = await llmChat({ ...params, stream: true });
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
if (/stream ended before producing a non-ping sse/i.test(msg)) {
|
||||||
|
completion = await llmChat({ ...params, stream: false });
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!completion) return null;
|
if (!completion) return null;
|
||||||
return completion.choices[0]?.message?.content?.trim() ?? null;
|
return completion.choices[0]?.message?.content?.trim() ?? null;
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user