Compare commits
104
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a3a91aa2ce | ||
|
|
75ff9274b3 | ||
|
|
e3016a858a | ||
|
|
f0874d8634 | ||
|
|
8d6b48fb4c | ||
|
|
9c9cd8917e | ||
|
|
4b58a159d0 | ||
|
|
f53a2d4f4a | ||
|
|
76daa7026c | ||
|
|
430e3d137c | ||
|
|
de8b521bb4 | ||
|
|
8bd7f5a98f | ||
|
|
ffbe9959ab | ||
|
|
14bd20f072 | ||
|
|
4d0bbed596 | ||
|
|
ae41f64e46 | ||
|
|
e7b0f02620 | ||
|
|
120aa95818 | ||
|
|
870f7546fb | ||
|
|
565b3da086 | ||
|
|
a54c34c6b3 | ||
|
|
7375e2ed52 | ||
|
|
5196cb0221 | ||
|
|
631b5e1027 | ||
|
|
8049b70738 | ||
|
|
ea23c405fa | ||
|
|
4991164591 | ||
|
|
1590479f58 | ||
|
|
0de393625f | ||
|
|
cccfd89266 | ||
|
|
17aad2cc39 | ||
|
|
a6e2c1fa4f | ||
|
|
7e0d0d5123 | ||
|
|
4f4f92706c | ||
|
|
611ba39973 | ||
|
|
192685e1ce | ||
|
|
cacebfd94e | ||
|
|
709074935f | ||
|
|
9f02edd646 | ||
|
|
5fc3cf86d5 | ||
|
|
e81391484b | ||
|
|
3b8fe1b1c3 | ||
|
|
54e7220d06 | ||
|
|
46d889271e | ||
|
|
0c3a82ad77 | ||
|
|
b640079cb6 | ||
|
|
37d15456dc | ||
|
|
153e628aba | ||
|
|
a5f908dc06 | ||
|
|
7fc36170c8 | ||
|
|
ccfe769b2f | ||
|
|
3b688e0d0f | ||
|
|
29f59fd91f | ||
|
|
c3165ed702 | ||
|
|
c72fadbbb4 | ||
|
|
613840e2d1 | ||
|
|
24336186dc | ||
|
|
6b66863897 | ||
|
|
fc40609a3a | ||
|
|
e157ae3ce4 | ||
|
|
3f4c05f5bb | ||
|
|
9c25e0887c | ||
|
|
2aa12b9ec5 | ||
|
|
cab451f0e5 | ||
|
|
183a51003a | ||
|
|
f34811e9b6 | ||
|
|
2c30140ddd | ||
|
|
7e92de63d6 | ||
|
|
68be4352da | ||
|
|
754ef78ff1 | ||
|
|
2959744e0c | ||
|
|
cffa72c21f | ||
|
|
d64c5ba7b6 | ||
|
|
87851f7a71 | ||
|
|
91de43a6c6 | ||
|
|
f8fc08de41 | ||
|
|
dfa69e9f45 | ||
|
|
0bd4b4075e | ||
|
|
b679a02cb9 | ||
|
|
4ed1d3c053 | ||
|
|
8dbcc2b527 | ||
|
|
bda210475d | ||
|
|
9b78abaf3d | ||
|
|
53c4158511 | ||
|
|
a2a8c824bf | ||
|
|
dec548fa2f | ||
|
|
12033e5537 | ||
|
|
d06caa7e58 | ||
|
|
173ae3c61c | ||
|
|
2637bbc549 | ||
|
|
6fc68126ac | ||
|
|
28d09a24c9 | ||
|
|
f2ec642a0e | ||
|
|
8a491b9ec2 | ||
|
|
7d54d3a660 | ||
|
|
21509c25ff | ||
|
|
09724287cb | ||
|
|
1ac3d50315 | ||
|
|
9c48d50d25 | ||
|
|
8bd3ec8e83 | ||
|
|
5a4a203f01 | ||
|
|
7f0aa5a7ad | ||
|
|
eb0be981c6 | ||
|
|
466e117bd5 |
No files matched your search
+2
-2
@@ -85,8 +85,8 @@ BACKLOG_SYNC_BATCH_SIZE=100 # Messages per backlog batch, max 100 (d
|
|||||||
# === AI Analysis ===
|
# === AI Analysis ===
|
||||||
AI_ANALYSIS_ENABLED=false # Enable AI content moderation (default: false)
|
AI_ANALYSIS_ENABLED=false # Enable AI content moderation (default: false)
|
||||||
AI_LLM_API_KEY= # REQUIRED if AI_ANALYSIS_ENABLED=true. LLM API key
|
AI_LLM_API_KEY= # REQUIRED if AI_ANALYSIS_ENABLED=true. LLM API key
|
||||||
AI_LLM_BASE_URL=https://9router.asepharyana.my.id/v1 # LLM API base URL (9router — OpenAI-compatible router, replaces omniroute)
|
AI_LLM_BASE_URL=http://100.121.180.82:20128/api/v1 # LLM API base URL (omniroute — OpenAI-compatible router on imrnes, Tailscale 100.121.180.82)
|
||||||
AI_LLM_MODEL=claude-opus-5 # LLM text model name (default: claude-opus-5)
|
AI_LLM_MODEL=text # LLM text model name (default: text)
|
||||||
# AI_LLM_VISION_MODEL= # Vision model for image analysis (falls back to AI_LLM_MODEL)
|
# AI_LLM_VISION_MODEL= # Vision model for image analysis (falls back to AI_LLM_MODEL)
|
||||||
# AI_LLM_EMBEDDING_MODEL= # Embedding model for semantic moderation cache (optional; enables near-duplicate text reuse to save LLM calls)
|
# AI_LLM_EMBEDDING_MODEL= # Embedding model for semantic moderation cache (optional; enables near-duplicate text reuse to save LLM calls)
|
||||||
# AI_LLM_EMBEDDING_MIN_SIMILARITY=0.97 # Min cosine similarity to reuse a cached verdict (default: 0.97)
|
# AI_LLM_EMBEDDING_MIN_SIMILARITY=0.97 # Min cosine similarity to reuse a cached verdict (default: 0.97)
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
name: Dependabot Auto-Merge
|
||||||
|
on: pull_request
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
pull-requests: write
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
auto-merge:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
if: github.actor == 'dependabot[bot]'
|
||||||
|
steps:
|
||||||
|
- name: Enable auto-merge for Dependabot PR
|
||||||
|
run: gh pr merge --auto --merge "$PR_URL"
|
||||||
|
env:
|
||||||
|
PR_URL: ${{ github.event.pull_request.html_url }}
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
version: 2
|
||||||
|
updates:
|
||||||
|
- package-ecosystem: "npm"
|
||||||
|
directory: "/services/backend"
|
||||||
|
schedule:
|
||||||
|
interval: "daily"
|
||||||
|
open-pull-requests-limit: 10
|
||||||
|
groups:
|
||||||
|
production:
|
||||||
|
dependency-type: "production"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
|
development:
|
||||||
|
dependency-type: "development"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
|
- package-ecosystem: "npm"
|
||||||
|
directory: "/services/discord-gateway"
|
||||||
|
schedule:
|
||||||
|
interval: "daily"
|
||||||
|
open-pull-requests-limit: 10
|
||||||
|
groups:
|
||||||
|
production:
|
||||||
|
dependency-type: "production"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
|
development:
|
||||||
|
dependency-type: "development"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
|
- package-ecosystem: "npm"
|
||||||
|
directory: "/services/frontend"
|
||||||
|
schedule:
|
||||||
|
interval: "daily"
|
||||||
|
open-pull-requests-limit: 10
|
||||||
|
groups:
|
||||||
|
production:
|
||||||
|
dependency-type: "production"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
|
development:
|
||||||
|
dependency-type: "development"
|
||||||
|
update-types: ["minor", "patch"]
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
name: Publish to FlakeHub
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [main, master]
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
flakehub-publish:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
permissions:
|
|
||||||
id-token: write
|
|
||||||
contents: read
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v6
|
|
||||||
- uses: DeterminateSystems/determinate-nix-action@main
|
|
||||||
- uses: DeterminateSystems/flakehub-push@main
|
|
||||||
with:
|
|
||||||
visibility: public
|
|
||||||
rolling: true
|
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
# Spec: Recordings — filter per user + export WAV (Audacity)
|
||||||
|
|
||||||
|
## Konteks / Gejala
|
||||||
|
Halaman `services/frontend/src/app/(dashboard)/recordings` menampilkan semua
|
||||||
|
rekaman voice (deck). User ingin:
|
||||||
|
1. **Filter per orang** (tampil rekaman satu user saja).
|
||||||
|
2. **Export ke format untuk Audacity** (buka & edit rekaman di Audacity).
|
||||||
|
|
||||||
|
## Fakta saat ini (verified)
|
||||||
|
- Backend `recordings.list` (services/backend/src/orpc/router.ts:291) SUDAH
|
||||||
|
menerima `userId`/`channelId` filter → `RecordingsService.getRecent`.
|
||||||
|
- Frontend `recordingsApi.list(limit, channelId, userId, cursor)` (lib/api/recordings.ts)
|
||||||
|
sudah meneruskan `userId`. `useLoadMoreRecordings` juga sudah bawa userId.
|
||||||
|
- Tapi UI `RecordingsView` (app/(dashboard)/recordings/view.tsx) TIDAK punya
|
||||||
|
filter UI, dan `useRecordingsPage` dipanggil tanpa userId → semua tampil.
|
||||||
|
- Setiap rekaman punya `download_url` (MP3 di TeleUploader), `user_id`, `username`.
|
||||||
|
- Audacity membuka MP3/OGG tapi editing paling bersih dari WAV (uncompressed)
|
||||||
|
/ FLAC (lossless). Backend TIDAK punya ffmpeg & Nix flake backend tak include
|
||||||
|
ffmpeg → transcode server-side bukan pilihan. Browser punya codec MP3 → export
|
||||||
|
WAV via Web Audio API (client-side) adalah solusi self-contained terbaik.
|
||||||
|
|
||||||
|
## Keputusan desain
|
||||||
|
1. **Filter per user**: UI dropdown (Semua User + per user) di header halaman.
|
||||||
|
Memilih user → re-fetch `recordingsApi.list(50, undefined, userId)` (server
|
||||||
|
filter, benar untuk dataset besar + pagination). Dropdown dibangun dari
|
||||||
|
distinct `user_id`/`username` pada items yang sedang tampil.
|
||||||
|
2. **Export WAV (Audacity)**: client-side via Web Audio API.
|
||||||
|
- Per kartu: tombol "WAV" → decode `download_url` → WAV 16-bit PCM → download.
|
||||||
|
- Header: tombol "EXPORT WAV (N)" → gabung (concat) semua rekaman yang
|
||||||
|
sedang tampil (ter-filter) jadi 1 file WAV → download. Ideal untuk analisis
|
||||||
|
/ mixdown per orang.
|
||||||
|
- Implementasi di `lib/audio/wav.ts` (decode + encode + concat), tanpa dep baru.
|
||||||
|
|
||||||
|
## Perubahan
|
||||||
|
|
||||||
|
### Frontend
|
||||||
|
- **`src/lib/audio/wav.ts`** (baru):
|
||||||
|
- `decodeAudio(url: string): Promise<AudioBuffer>` — fetch arrayBuffer →
|
||||||
|
`new AudioContext().decodeAudioData`.
|
||||||
|
- `audioBufferToWav(buf: AudioBuffer, sampleRate=48000): Blob` — PCM 16-bit
|
||||||
|
interleaved, mono→stereo handling, RIFF/WAVE writer. Audacity-importable.
|
||||||
|
- `concatBuffers(buffers: AudioBuffer[]): AudioBuffer` — gabung di channel 0
|
||||||
|
(mono) dengan sample-rate max; untuk export gabungan.
|
||||||
|
- `downloadWav(blob: Blob, filename: string): void` — obj URL + <a download>.
|
||||||
|
- **`src/app/(dashboard)/recordings/view.tsx`**:
|
||||||
|
- Toolbar filter: dropdown user (built from distinct items) + tombol reset.
|
||||||
|
- State `filterUserId`; saat berubah → `recordingsApi.list(50, undefined, id)`
|
||||||
|
→ set ke SWR (key includes filter), reset pagination.
|
||||||
|
- Tombol "WAV" per kartu (disabled jika `!r.download_url`).
|
||||||
|
- Tombol "EXPORT WAV (N)" di header (disabled jika 0 item punya download_url);
|
||||||
|
concat semua items ter-filter yang punya download_url.
|
||||||
|
- Status loading saat export (spinner/disable).
|
||||||
|
- **`src/hooks/use-recordings.ts`**: `useRecordingsPage` menerima `userId?` dan
|
||||||
|
memasukkan ke key + call, supaya filter re-fetch bersih (per-user cache key).
|
||||||
|
`useRecordings`/`useLoadMoreRecordings` propagate `userId`.
|
||||||
|
- **`src/lib/types/recording.ts`**: tidak berubah (userId dari items).
|
||||||
|
|
||||||
|
### Backend / gateway
|
||||||
|
- Tidak ada perubahan. Filter & export sepenuhnya frontend.
|
||||||
|
|
||||||
|
## File yang disentuh (frontend only)
|
||||||
|
- `src/lib/audio/wav.ts` (baru)
|
||||||
|
- `src/app/(dashboard)/recordings/view.tsx`
|
||||||
|
- `src/hooks/use-recordings.ts`
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
1. `cd services/frontend && pnpm typecheck` (tsc --noEmit) — 0 error.
|
||||||
|
2. `pnpm lint` (biome check src/) — exit 0.
|
||||||
|
3. `pnpm build` (next build) — hijau.
|
||||||
|
4. Manual (user): buka /recordings; pilih user di dropdown → hanya rekaman user
|
||||||
|
itu; klik WAV di kartu → file .wav ter-download & terbuka di Audacity; klik
|
||||||
|
EXPORT WAV (filtered) → satu .wav gabungan.
|
||||||
|
5. Push → CI `Build & Deploy (Nix)` (frontend job) hijau → deploy landing.
|
||||||
|
|
||||||
|
## Risiko / Trade-off
|
||||||
|
- Web Audio decode MP3 di client: butuh CORS pada download_url (TeleUploader
|
||||||
|
asepharyana.my.id — sudah same-serve/proxied, CORS ikut origin). Jika 403/CORS
|
||||||
|
gagal, error toaster + fallback manual (RAW MP3 tetap ada).
|
||||||
|
- concat gabungan = mono 48k; Audacity bisa edit per-channel nanti. Acceptable.
|
||||||
|
- Filter client (dropdown dari items yang dimuat) hanya menawarkan user yang
|
||||||
|
sudah tampil; dataset besar bisa pakai search nanti. Server filter benar untuk
|
||||||
|
yang dipilih.
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
# Spec: Recordings v2 — transcription, search, filters, leaderboard, sessions
|
||||||
|
|
||||||
|
## Konteks
|
||||||
|
4 fitur lanjutan untuk halaman /recordings (dipilih user):
|
||||||
|
1. Tampilkan transkripsi + search by kata kunci
|
||||||
|
2. Filter lanjutan: by channel + rentang tanggal
|
||||||
|
3. Kelompokkan klip jadi "sesi rapat" + autoplay berurutan + export satu sesi
|
||||||
|
4. Leaderboard bicara per user + ringkasan
|
||||||
|
|
||||||
|
## Fakta terverifikasi (2026-08-30)
|
||||||
|
- `voice_recordings` kolom: id, user_id, username, avatar_url, guild_id,
|
||||||
|
channel_id, channel_name, filename, size_bytes, download_url, upload_status,
|
||||||
|
upload_error, created_at, uploaded_at, **transcription** (schema
|
||||||
|
`shared/database/schema.ts:246`). Index user_id/channel_id/created_at.
|
||||||
|
- **Transcription 0/10.640** terisi prod: `AI_VOICE_TRANSCRIPTION_ENABLED`
|
||||||
|
default **false** (config/index.ts:333) & tidak diset di BWS env →
|
||||||
|
`transcribeRecording` (voiceTranscriber.ts) langsung return null.
|
||||||
|
`AI_LLM_BASE_URL` + `AI_LLM_API_KEY` SUDAH dikonfig (Whisper via router GMW).
|
||||||
|
- Transcriber hardcode `language: "en"` (voiceTranscriber.ts:34) — salah utk
|
||||||
|
ucapan campur id/en. Utk auto-detect: hapus param `language` (Whisper
|
||||||
|
auto-detect jika tidak diberikan).
|
||||||
|
- Backend `RecordingsService.getRecent` (recordings.service.ts) TIDAK select
|
||||||
|
`transcription`; SUDAH dukung filter `channelId`+`userId`+`cursor`; belum
|
||||||
|
dukung date-range & keyword search. `RecordingRow` interface juga tak punya
|
||||||
|
`transcription`.
|
||||||
|
- Tidak ada kolom `session_id` / `duration_ms` → sesi grouping = heuristik
|
||||||
|
(channel sama + gap created_at), durasi leaderboard = estimasi dari
|
||||||
|
size_bytes (MP3 128kbps: durasi_s ≈ size_bytes*8/128000).
|
||||||
|
|
||||||
|
## Keputusan desain
|
||||||
|
1. **Aktifkan transkripsi (fondasi)**: transcriber auto-detect (hapus
|
||||||
|
`language:"en"`), set secret BWS `AI_VOICE_TRANSCRIPTION_ENABLED=true`
|
||||||
|
(dibaca runtime oleh bws-exec saat service start). Rekaman BARU dapat
|
||||||
|
transkripsi. Backfill rekaman lama TIDAK dilakukan (pilih user: fokus baru;
|
||||||
|
file OGG lama kemungkinan besar sudah tidak dipakai).
|
||||||
|
2. **Backend** — perluas `getRecent`:
|
||||||
|
- select `transcription` (+ interface RecordingRow + FE type)
|
||||||
|
- filter baru: `q` (ILIKE on transcription + username), `startDate`/`endDate`
|
||||||
|
(created_at range, bigint ms)
|
||||||
|
- endpoint baru `recordings.summary`: agregasi per user → {user_id,
|
||||||
|
username, avatar_url, clips, est_duration_s, words, last_at}. `words`
|
||||||
|
dihitung dari transcription (tokenisasi spasi). Return sorted by clips.
|
||||||
|
3. **Frontend**:
|
||||||
|
- Kartu: tampilkan transkripsi (collapse/expand line-clamp) + durasi estimasi.
|
||||||
|
- Toolbar: search box (q), Select channel, date range (start/end), speaker
|
||||||
|
(sudah ada), reset filter.
|
||||||
|
- Tab/segment "Tape Deck" vs "Leaderboard": leaderboard render summary per
|
||||||
|
user + klik → filter deck by user itu.
|
||||||
|
- Sesi grouping (deck view): klip di-group jadi sesi bila channel sama &
|
||||||
|
gap antar klip < SESSION_GAP_MS (default 120s). Header sesi (channel,
|
||||||
|
waktu mulai, jumlah klip, total durasi). Autoplay tombol "Play session" &
|
||||||
|
"Export session WAV" (concat klip sesi via lib/audio/wav.ts yg sudah ada).
|
||||||
|
|
||||||
|
## Perubahan file
|
||||||
|
|
||||||
|
### Gateway (Tahap 1)
|
||||||
|
- `voiceTranscriber.ts`: hapus baris `language: "en"`.
|
||||||
|
- (secret) set `AI_VOICE_TRANSCRIPTION_ENABLED=true` via bws.
|
||||||
|
|
||||||
|
### Backend (Tahap 2)
|
||||||
|
- `recordings.service.ts`: interface + select + getRecent tambah transcription;
|
||||||
|
tambah filter q/startDate/endDate; method getSummary() untuk leaderboard.
|
||||||
|
- `orpc/router.ts`: procedur `recordings.list` schema tambah fields; prosedur
|
||||||
|
baru `recordings.summary`.
|
||||||
|
|
||||||
|
### Frontend (Tahap 3-5)
|
||||||
|
- `lib/types/recording.ts`: tambah transcription, est_duration_s opsional,
|
||||||
|
Summary type.
|
||||||
|
- `lib/api/recordings.ts`: list tambah q/startDate/endDate; + summary().
|
||||||
|
- `hooks/use-recordings.ts`: propagate filter baru ke key+call; hook
|
||||||
|
useRecordingsSummary.
|
||||||
|
- `app/(dashboard)/recordings/view.tsx`: toolbar search+channel+date, kartu
|
||||||
|
transkripsi, tab leaderboard, grouping sesi + autoplay + export sesi.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
- tiap tahap: gateway `pnpm build` + `biome check src/`; backend `pnpm build`
|
||||||
|
+ `biome check src/ tests/`; FE `pnpm build` + `biome check src/`.
|
||||||
|
- CI Build & Deploy hijau tiap tahap; deploy landing dicek via
|
||||||
|
`systemctl show gmw-<svc>.service --property=ActiveEnterTimestamp`.
|
||||||
|
- Tahap 1c: setelah deploy + rekaman baru, cek
|
||||||
|
`SELECT COUNT(*) FROM voice_recordings WHERE transcription IS NOT NULL`.
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
# Spec: Perbaiki alur voice → recording (miss & terpotong)
|
||||||
|
|
||||||
|
## Konteks & Gejala
|
||||||
|
User melaporkan alur voice sampai recording **banyak miss** (audio tidak tercatat)
|
||||||
|
dan **terpotong** (satu alur bicara kebelah jadi beberapa segmen / audio putus di
|
||||||
|
tengah). Ini domain `services/discord-gateway/src/modules/voice-recording/`.
|
||||||
|
|
||||||
|
Pipeline per user yang mulai bicara (speaking "start"):
|
||||||
|
```
|
||||||
|
receiver.speaking "start" → speakingHandler(userId)
|
||||||
|
├─ await collectUserMetadata(...) ← roundtrip API, subscribe tertunda
|
||||||
|
├─ receiver.subscribe(userId, {end: AfterSilence, duration: 3000ms}) → audioStream
|
||||||
|
├─ attach data/end/error handlers → audioStream.pipe(PacketFilter) → oggPacketStream
|
||||||
|
├─ SegmentManager.open() → OggLogicalBitstream → file .ogg
|
||||||
|
├─ data: SegmentManager.rotateIfNeeded (rotasi 5s) + decoder.write (web PCM tho
|
||||||
|
└─ end: SegmentManager.close() → segmen finish → finalizeSegment upload + transkrip
|
||||||
|
```
|
||||||
|
|
||||||
|
## Root cause (dari pembacaan kode — justifikasi di bawah)
|
||||||
|
|
||||||
|
### A. MISS bagian awal bicara — subscribe tertunda (utama)
|
||||||
|
`speakingHandler.ts:53` melakukan `await collectUserMetadata(...)` SEBELUM
|
||||||
|
`receiver.subscribe`. `collectUserMetadata` (metadata.ts:30) pada cold path
|
||||||
|
(cache miss) melakukan `client.users.fetch` + `guild.members.fetch` roundtrip
|
||||||
|
Discord API (ratusan ms–detik). Selama await, seluruh opus awal bocor → awal
|
||||||
|
kalimat hilang. Cache menghilangkan ini untuk user yang pernah ter-record, tapi
|
||||||
|
user baru/evict (cache max 200) kena setiap kali.
|
||||||
|
|
||||||
|
### B. Double-subscribe race
|
||||||
|
Guard `receiver.subscriptions.has(userId)` di `speakingHandler.ts:63` diletakkan
|
||||||
|
SETELAH `await collectUserMetadata`. Dua event "start" cepat keduanya melewati
|
||||||
|
guard (belum subscribe) → dua subscription → audio terbelah/ganda per user.
|
||||||
|
|
||||||
|
### C. TERPOTONG di jeda bicara — AfterSilence 3000ms
|
||||||
|
`AUDIO_STREAM_SILENCE_DURATION_MS=3000`. Setelah 3s diam, stream auto-`end` →
|
||||||
|
`SegmentManager.close` → segmen baru saat bicara lagi. Jeda normal (berpikir,
|
||||||
|
interupsi) memecah 1 alur bicara jadi beberapa segmen/file. Ini source "terpotong".
|
||||||
|
Segmen pendek hasil jeda <1s juga DIBUANG di `finalizeSegment` (MIN_DURATION_MS=1000)
|
||||||
|
→ miss kata singkat ("ya", "siap").
|
||||||
|
|
||||||
|
### D. Rotasi segmen 5s di tengah bicara
|
||||||
|
`RECORDING_SEGMENT_MS=5000`: `rotateIfNeeded` menutup bitstream & membuka baru
|
||||||
|
setiap 5s walau bicara kontinu. Pipenya di-re-wire di dalam handler data →
|
||||||
|
window drop kecil + continuity file pecah (bukan masalah besar, tapi berkontribusi).
|
||||||
|
|
||||||
|
### E. Tidak ada sinkronisasi "stop" speaking & stream "end" flaky
|
||||||
|
Handler hanya listen "start"; mengandalkan `AfterSilence` untuk emit "end".
|
||||||
|
Bug @discordjs/voice yang dikenal: `AfterSilence` bisa TIDAK emit "end" saat
|
||||||
|
koneksi gagal/teardown → segmen menggantung & tidak pernah finalize/upload
|
||||||
|
(recording "hilang"). Tidak ada watchdog.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
Hanya `services/discord-gateway/src/modules/voice-recording/` (+ config index bila
|
||||||
|
perlu default baru). Tidak menyentuh playback (player.ts), transmitter (voice dari
|
||||||
|
browser → Discord, arah berlawanan), muxer (konsolidasi akhir), atau screen-share
|
||||||
|
video (hanya audio SSRC via `hookScreenShareAudio` sudah ada & dibiarkan).
|
||||||
|
|
||||||
|
## Perubahan
|
||||||
|
|
||||||
|
### 1. Speak-before-metadata: subscribe LEBIH DULU, metadata paralel
|
||||||
|
`recorder/speakingHandler.ts`:
|
||||||
|
- Pindahkan `receiver.subscribe` + pipeline setup ke ATAS, SEGERA di handler,
|
||||||
|
sebelum `collectUserMetadata`.
|
||||||
|
- Jalankan `collectUserMetadata` secara paralel non-blocking; gunakan metadata
|
||||||
|
cache untuk registrasi segmen saat finalize.
|
||||||
|
- Pertahankan guard skip bot/user (bot bisa dicek dari `client.users.cache` /
|
||||||
|
`client.user.id` tanpa await) SEBELUM subscribe — jangan tunggu fetch user.
|
||||||
|
|
||||||
|
Detail konkret:
|
||||||
|
```
|
||||||
|
async handler(userId):
|
||||||
|
if (userId === client.user?.id) return;
|
||||||
|
if (receiver.subscriptions.has(userId)) return; // guard kini di DEPAN, tanpa await
|
||||||
|
// (belum tahu bot? gunakan cache user; subscribe dulu biar nggak miss)
|
||||||
|
clone = subscribe(userId, {AfterSilence, duration}) // TANPA await metadata
|
||||||
|
setup pipeline (data/end/error, pipe, open segment)
|
||||||
|
collectUserMetadata(...).then(meta => {
|
||||||
|
if (meta.bot) { drain & close subscription (jangan simpan) }
|
||||||
|
else { registrasi ulang metadata utk segmen aktif }
|
||||||
|
})
|
||||||
|
```
|
||||||
|
Karena listener `start` dibuang untuk bot, harus tutup subscription bot tanpa
|
||||||
|
menyimpan segmen (buang hasil). Pakai `receiver.subscriptions.get(userId)?.destroy()`.
|
||||||
|
|
||||||
|
### 2. Selesaikan race double-subscribe (bagian dari #1)
|
||||||
|
Guard `receiver.subscriptions.has(userId)` diletakkan SINCRON di awal (sebelum
|
||||||
|
await). Karena `subscribe` sinkron dan `subscriptions` terisi sinkron saat
|
||||||
|
dipanggil, event "start" kedua yang tiba setelah subscribe akan melihat
|
||||||
|
subscription aktif → di-skip. Tidak ada await antara guard & subscribe.
|
||||||
|
|
||||||
|
### 3. Naikkan AfterSilence + tail-length → kurangi "terpotong"
|
||||||
|
`shared/config/index.ts`:
|
||||||
|
- `AUDIO_STREAM_SILENCE_DURATION_MS` default 3000 → 4000 (beri ruang jeda
|
||||||
|
alami; Discord packet 20ms, 4s masih wajar, tidak membengkak file).
|
||||||
|
Opsional via env override di production (tidak wajib komit env).
|
||||||
|
|
||||||
|
### 4. Segmen berorientasi "burst bicara" daripada rotasi jam
|
||||||
|
`recorder/segment.ts` + `recorder/speakingHandler.ts`:
|
||||||
|
- Hapus/lepas rotasi SEGMEN berbasis waktu (RECORDING_SEGMENT_MS). Alih-alih,
|
||||||
|
satu segmen = satu burst bicara (buka di "start", tutup di "end"/AfterSilence
|
||||||
|
end). Ini menghilangkan pemecahan di tengah kalimat.
|
||||||
|
- `RECORDING_SEGMENT_MS` tetap dipakai untuk rotasi decoder web-PCM (broadcast
|
||||||
|
live), di mana segmen besar bisa menunda frame — biarkan seperti ada.
|
||||||
|
JADI: `SegmentManager.rotateIfNeeded` TIDAK lagi dipanggil pada jalur OGG
|
||||||
|
recording; decoder rotate tetap dijalankan.
|
||||||
|
|
||||||
|
Catatan: dengan satu segmen per burst, ukuran file ~ durasi bicara. File panjang
|
||||||
|
dibutuhkan transkrip & transcode; tidak ada batas keras yang perlu di-override.
|
||||||
|
Watchdog di #5 membatasi durasi menggantung.
|
||||||
|
|
||||||
|
### 5. Watchdog end-of-burst & teardown recovery
|
||||||
|
`recorder/speakingHandler.ts`:
|
||||||
|
- Setelah subscribe, arm timer watchdog (mis. `config twin`/hitung) yang menutup
|
||||||
|
segmen jika `AfterSilence` tidak emit "end" dalam X detik setelah "stop"
|
||||||
|
speaking — atau, lebih sederhana & robust: dengarkan BOTH stream "end" DAN
|
||||||
|
timer dari `receiver.speaking` "stop" (hingga @discordjs/voice meng-klaim
|
||||||
|
AfterSilence). Bila "stop" fire, mulai countdown kecil (mis. 500ms) lalu
|
||||||
|
`segmentManager.close` + `decoder.destroy` + destroy subscription jika stream
|
||||||
|
belum "end".
|
||||||
|
- Ini menutup A: segmen menggantung → jadi pasti finalize & upload.
|
||||||
|
|
||||||
|
Implementasi: subscriptionStream (audioStream) + track milik per-user di
|
||||||
|
Map<userId, {audioStream, segmentManager, decoder, timer}>; handler "stop"
|
||||||
|
menjadwalkan finalize.
|
||||||
|
|
||||||
|
### 6. Naikkan/ambil MIN segmen duration lebih rendah
|
||||||
|
`recorder/segmentFinalizer.ts`: MIN_DURATION_MS 1000 → 300ms. Kata pendek
|
||||||
|
("ya", "siap") tetap tersimpan. GUI biarkan.
|
||||||
|
|
||||||
|
## File yang disentuh
|
||||||
|
- `src/modules/voice-recording/recorder/speakingHandler.ts` (utama: subscribe
|
||||||
|
first, guard depan, watchdog stop, hapus rotasi segmen dari jalur OGG)
|
||||||
|
- `src/modules/voice-recording/recorder/segment.ts` (opsional: API close/open,
|
||||||
|
pertahankan rotate untuk decoder tapi tak dipakai jalur OGG)
|
||||||
|
- `src/modules/voice-recording/recorder/streamSetup.ts` (kecil: backfill subscribe
|
||||||
|
supaya return subscription utk cleanup/destroy bot)
|
||||||
|
- `src/modules/voice-recording/recorder/segmentFinalizer.ts` (MIN_DURATION)
|
||||||
|
- `src/shared/config/index.ts` (default AfterSilence 4000)
|
||||||
|
|
||||||
|
## Yang TIDAK disentuh
|
||||||
|
- `transmitter.ts` (arah browser→Discord, bukan recording)
|
||||||
|
- `player.ts`, `mediaSource.ts`, `screenShareAudio.ts` (hook SSRC sudah benar)
|
||||||
|
- `muxer.ts` (konsolidasi akhir tetap jalan)
|
||||||
|
- Flake/deps/bundle
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
1. `cd services/discord-gateway && pnpm typecheck` (tsc --noEmit) — 0 error.
|
||||||
|
2. `pnpm lint` (biome check src/) — exit 0.
|
||||||
|
3. `pnpm build` (tsc → dist/).
|
||||||
|
4. Unit test baru (vitest, kalau infra tes ada):
|
||||||
|
- subscribe terjadi tanpa await metadata (spy urutan panggilan)
|
||||||
|
- double-start skip lewat guard sinkron
|
||||||
|
- "stop" → watchdog finalize segmen walau stream tidak "end"
|
||||||
|
- MIN_DURATION 300ms menyimpan kata pendek
|
||||||
|
5. Smoke/CI: `nix flake check` (eval). Push → CI `Build & Deploy (Nix)` hijau →
|
||||||
|
deploy landing (`systemctl show gmw-discord-gateway.service --property=ActiveEnterTimestamp`).
|
||||||
|
6. Runtime manual (user): join voice, bicara dengan jeda >3s, pastikan 1 alur
|
||||||
|
kontinu = 1 segmen utuh (bukan 3), dan awal kata tidak hilang.
|
||||||
|
|
||||||
|
## Risiko / Trade-off
|
||||||
|
- Subscribe-before-metadata: burst bot akan dikumpulkan sesaat lalu dibuang
|
||||||
|
(cost kecil: buang segmen). Lebih baik miss bot daripada miss user.
|
||||||
|
- AfterSilence naik: file lebih panjang sedikit saat jeda; upload/transkrip
|
||||||
|
timeout (transcodeToMp3 30s) tetap aman.
|
||||||
|
- Satu segmen per burst: tidak ada rotasi paksa → durasi segmen = durasi bicara
|
||||||
|
(bisa menit). Transkrip & transcode tetap ok. Watchdog batasi menggantung.
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
# GMW — Fix mobile navbar "tidak bisa pindah halaman" (root cause)
|
||||||
|
|
||||||
|
## Context / symptom
|
||||||
|
- User (phone, mobile viewport <md): pressing the bottom mobile nav items does NOT
|
||||||
|
navigate ("Masih sama, walau sudah ditekan tidak pindah halaman").
|
||||||
|
- Previous fix (commits ae41f64e, 4d0bbed5) repositioned the chatbot FAB above the
|
||||||
|
nav and expanded nav to all 7 items — but the bug persisted.
|
||||||
|
|
||||||
|
## Root cause (verified live 2026-08-28)
|
||||||
|
Reproduced on the LIVE site (`imphnen.asepharyana.my.id`) with Playwright @375px:
|
||||||
|
1. **Next `<Link>` client-side navigation is dead app-wide.** Clicking a mobile nav
|
||||||
|
`<Link>` fires the click (event reaches document, `inLink=true`,
|
||||||
|
`defaultPrevented=false`) but **the URL never changes**. `window.next.router.push('/voice')`
|
||||||
|
also does nothing. The desktop NavRail navigates only because it uses a **plain
|
||||||
|
`<a href>`** (full browser navigation bypasses the broken router).
|
||||||
|
2. **A React hydration failure (#418 "server rendered text didn't match the client")**
|
||||||
|
is thrown on live pages — the only console error. Cause: relative-time text
|
||||||
|
(`formatRelativeTime(e.edited_at)` / `formatRelativeTime(m.created_at)` in the
|
||||||
|
SSR-seeded messages/edit-history feed uses `Date.now()`; server and client
|
||||||
|
render slightly different text → hydration mismatch → React re-renders, and the
|
||||||
|
Next client router ends up non-functional.
|
||||||
|
|
||||||
|
## Why desktop "worked" / mobile didn't
|
||||||
|
- `NavRail` = plain `<a href>` → **hard** navigation (works).
|
||||||
|
- `MobileNav` = Next `<Link>` → **client** navigation → dead router.
|
||||||
|
User's instruction "ikuti cara kerja sidebar" = make mobile nav behave like the
|
||||||
|
sidebar (plain anchors).
|
||||||
|
|
||||||
|
## Changes
|
||||||
|
### 1. `src/components/shell/mobile-nav.tsx`
|
||||||
|
Replace Next `<Link>` with a **plain `<a href>`** (mirrors NavRail). Keep:
|
||||||
|
`usePathname` for active-state styling + `scrollIntoView` for snap-to-active.
|
||||||
|
Drop the now-unused `Link` import (biome import-order — run biome check --write).
|
||||||
|
|
||||||
|
### 2. Hydration root cause — `formatRelativeTime` in SSR-seeded feeds
|
||||||
|
Add `suppressHydrationWarning` to the timestamp elements in the SSR-seeded
|
||||||
|
components that render live-relative text so server/client text drift no longer
|
||||||
|
throws #418:
|
||||||
|
- `src/components/EditHistory.tsx` (line ~111 `edited {formatRelativeTime(...)}`)
|
||||||
|
- `src/app/(dashboard)/messages/view.tsx` MessageRow channel/time (line ~580) +
|
||||||
|
semantic row (line ~364)
|
||||||
|
- `src/components/LiveModerationFeed.tsx` (line ~139)
|
||||||
|
- (recordings/view.tsx, TermGlossary, ChannelCultureGlossary, CategoryDrilldown,
|
||||||
|
chatbot — chatbot is client-after-mount; add where SSR-seeded.)
|
||||||
|
|
||||||
|
NOTE: `suppressHydrationWarning` is safe for these single-text spans; the values
|
||||||
|
are cosmetic and self-correct on the next interval/render.
|
||||||
|
|
||||||
|
## Verification (non-prod, NEVER 4017)
|
||||||
|
- `pnpm format` + `pnpm lint` (biome) + `pnpm build` clean.
|
||||||
|
- Smoke on :4024 (fresh `next dev`): mobile click on nav item actually navigates
|
||||||
|
(URL changes) AND no React #418 in console.
|
||||||
|
- After push: `gh run watch`, then live `imphnen.asepharyana.my.id` @375px: nav tap
|
||||||
|
navigates, console free of #418.
|
||||||
|
|
||||||
|
## Files
|
||||||
|
- services/frontend/src/components/shell/mobile-nav.tsx
|
||||||
|
- services/frontend/src/components/EditHistory.tsx
|
||||||
|
- services/frontend/src/app/(dashboard)/messages/view.tsx
|
||||||
|
- services/frontend/src/components/LiveModerationFeed.tsx
|
||||||
|
- (others only if #418 persists)
|
||||||
@@ -82,6 +82,17 @@
|
|||||||
[ "$d" = "node_modules" ] && continue; \
|
[ "$d" = "node_modules" ] && continue; \
|
||||||
grep -qF ".pnpm/$d" $TMPDIR/prod-pnms.txt || rm -rf "$d"; \
|
grep -qF ".pnpm/$d" $TMPDIR/prod-pnms.txt || rm -rf "$d"; \
|
||||||
done ) || true
|
done ) || true
|
||||||
|
# Drop runtime-dead packages that still land in the prod graph:
|
||||||
|
# - `@types/*` (pure TypeScript declarations) get pulled in as
|
||||||
|
# REAL dependencies by type-aware deps (discord-api-types ->
|
||||||
|
# @types/node, pg-protocol -> @types/pg, ...) even though nothing
|
||||||
|
# ever `require`s them at runtime. Safe to strip.
|
||||||
|
# - `opusscript` is only a pure-JS fallback Opus engine that
|
||||||
|
# prism-media's loader uses IF `@discordjs/opus` (native, always
|
||||||
|
# present/prebuilt) fails to load. Since the native engine loads,
|
||||||
|
# opusscript is never executed — dead weight pulled in via
|
||||||
|
# discord.js-selfbot-v13's dependency. Strip it too.
|
||||||
|
( cd node_modules/.pnpm && rm -rf @types+* opusscript@* 2>/dev/null ) || true
|
||||||
# Drop symlinks whose .pnpm target was pruned (top-level, scoped dirs,
|
# Drop symlinks whose .pnpm target was pruned (top-level, scoped dirs,
|
||||||
# hoist, .bin — any depth). Mirrors stdenv's noBrokenSymlinks check,
|
# hoist, .bin — any depth). Mirrors stdenv's noBrokenSymlinks check,
|
||||||
# which would otherwise fail the fixupPhase.
|
# which would otherwise fail the fixupPhase.
|
||||||
@@ -132,14 +143,17 @@ WRAPPER
|
|||||||
|
|
||||||
src = ./services/discord-gateway;
|
src = ./services/discord-gateway;
|
||||||
|
|
||||||
|
# cmake + rust/cargo were inherited for node-datachannel-style native
|
||||||
|
# deps — that's 9router/omniroute, NOT GMW. GMW's only native deps
|
||||||
|
# are @discordjs/opus + sharp, both PREBUILT (node-pre-gyp / @img
|
||||||
|
# libvips download), so no cmake or rust toolchain is needed.
|
||||||
|
# python3/gnumake/gcc stay as node-gyp fallback for @discordjs/opus.
|
||||||
nativeBuildInputs = [
|
nativeBuildInputs = [
|
||||||
nodejs pnpm
|
nodejs pnpm
|
||||||
pkgs.python3 pkgs.gnumake pkgs.gcc pkgs.cmake
|
pkgs.python3 pkgs.gnumake pkgs.gcc
|
||||||
pkgs.rustc pkgs.cargo
|
|
||||||
pkgs.pkg-config
|
pkgs.pkg-config
|
||||||
pkgs.openssl
|
pkgs.openssl
|
||||||
pkgs.openssl.dev
|
pkgs.openssl.dev
|
||||||
pkgs.git # for any FetchContent-based deps during native builds
|
|
||||||
pkgs.cacert
|
pkgs.cacert
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|||||||
@@ -21,21 +21,21 @@
|
|||||||
"drizzle-orm": "^0.45.2",
|
"drizzle-orm": "^0.45.2",
|
||||||
"express": "^5.2.1",
|
"express": "^5.2.1",
|
||||||
"helmet": "^8.1.0",
|
"helmet": "^8.1.0",
|
||||||
"ioredis": "^5.11.0",
|
"ioredis": "^6.0.0",
|
||||||
"pg": "^8.21.0",
|
"pg": "^8.23.0",
|
||||||
"pino": "^9.6.0",
|
"pino": "^10.3.1",
|
||||||
"prom-client": "^15.1.3",
|
"prom-client": "^15.1.3",
|
||||||
"ws": "^8.20.1",
|
"ws": "^8.21.3",
|
||||||
"zod": "^4.4.3"
|
"zod": "^4.4.3"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@biomejs/biome": "latest",
|
"@biomejs/biome": "latest",
|
||||||
"@types/express": "^5.0.6",
|
"@types/express": "^5.0.6",
|
||||||
"@types/node": "^25.9.0",
|
"@types/node": "^26.3.0",
|
||||||
"@types/pg": "^8.20.0",
|
"@types/pg": "^8.23.1",
|
||||||
"@types/ws": "^8.18.1",
|
"@types/ws": "^8.18.1",
|
||||||
"tsx": "^4.22.2",
|
"tsx": "^4.23.12",
|
||||||
"typescript": "^5.9.3",
|
"typescript": "^7.0.2",
|
||||||
"vitest": "latest"
|
"vitest": "latest"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Generated
+486
-287
File diff suppressed because it is too large.
Load diff
@@ -158,7 +158,7 @@ Gaya ngobrol:
|
|||||||
tool_choice: "auto",
|
tool_choice: "auto",
|
||||||
max_tokens: 600,
|
max_tokens: 600,
|
||||||
temperature: 0.4,
|
temperature: 0.4,
|
||||||
// Non-streaming: request a single complete response. 9router may
|
// Non-streaming: request a single complete response. omniroute may
|
||||||
// still emit SSE even with stream:false, so the parser below
|
// still emit SSE even with stream:false, so the parser below
|
||||||
// handle both raw-JSON and SSE bodies.
|
// handle both raw-JSON and SSE bodies.
|
||||||
stream: false,
|
stream: false,
|
||||||
@@ -176,7 +176,7 @@ Gaya ngobrol:
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
// Parse the body into content + tool_calls. 9router may return either
|
// Parse the body into content + tool_calls. omniroute may return either
|
||||||
// a single JSON object (stream:false honored) or SSE text (stream
|
// a single JSON object (stream:false honored) or SSE text (stream
|
||||||
// implied) — parseResponse handles both.
|
// implied) — parseResponse handles both.
|
||||||
const { content, toolCalls } = this.parseResponse(
|
const { content, toolCalls } = this.parseResponse(
|
||||||
@@ -252,7 +252,7 @@ Gaya ngobrol:
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Parse an LLM HTTP body into content + tool_calls. Handles both shapes
|
* Parse an LLM HTTP body into content + tool_calls. Handles both shapes
|
||||||
* 9router can return: a single JSON object (stream:false honored) or SSE
|
* omniroute can return: a single JSON object (stream:false honored) or SSE
|
||||||
* text (stream implied). For SSE we delegate to parseSse.
|
* text (stream implied). For SSE we delegate to parseSse.
|
||||||
*/
|
*/
|
||||||
private parseResponse(body: string): {
|
private parseResponse(body: string): {
|
||||||
@@ -306,7 +306,7 @@ Gaya ngobrol:
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Parse an SSE stream body into accumulated content + any tool_calls.
|
* Parse an SSE stream body into accumulated content + any tool_calls.
|
||||||
* 9router (and most OpenAI-compatible routers) emit `data: {json}` lines
|
* omniroute (and most OpenAI-compatible routers) emit `data: {json}` lines
|
||||||
* even when stream is only implied; we must collect deltas manually.
|
* even when stream is only implied; we must collect deltas manually.
|
||||||
*/
|
*/
|
||||||
private parseSse(body: string): {
|
private parseSse(body: string): {
|
||||||
|
|||||||
@@ -469,17 +469,20 @@ export class MessagesRepository {
|
|||||||
const db = getDatabase();
|
const db = getDatabase();
|
||||||
const since = Date.now() - days * 24 * 60 * 60 * 1000;
|
const since = Date.now() - days * 24 * 60 * 60 * 1000;
|
||||||
const result = await db.execute(sql`
|
const result = await db.execute(sql`
|
||||||
SELECT channel_id,
|
SELECT
|
||||||
EXTRACT(HOUR FROM to_timestamp(created_at / 1000))::int AS hour,
|
m.channel_id,
|
||||||
COUNT(*)::int AS c
|
COALESCE(NULLIF((m.metadata::jsonb -> 'channel' ->> 'channelName'), ''), m.channel_id) AS channel_name,
|
||||||
FROM messages
|
EXTRACT(HOUR FROM to_timestamp(m.created_at / 1000))::int AS hour,
|
||||||
WHERE created_at >= ${since}
|
COUNT(*)::int AS c
|
||||||
GROUP BY channel_id, hour
|
FROM messages m
|
||||||
ORDER BY channel_id, hour
|
WHERE m.created_at >= ${since}
|
||||||
|
GROUP BY m.channel_id, channel_name, hour
|
||||||
|
ORDER BY channel_name, hour
|
||||||
`);
|
`);
|
||||||
const rows = (result.rows as Record<string, unknown>[]) || [];
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
return rows.map((r) => ({
|
return rows.map((r) => ({
|
||||||
channelId: String(r.channel_id ?? "unknown"),
|
channelId: String(r.channel_id ?? "unknown"),
|
||||||
|
channelName: String(r.channel_name ?? r.channel_id ?? "unknown"),
|
||||||
hour: Number(r.hour ?? 0),
|
hour: Number(r.hour ?? 0),
|
||||||
count: Number(r.c ?? 0),
|
count: Number(r.c ?? 0),
|
||||||
}));
|
}));
|
||||||
@@ -491,31 +494,28 @@ export class MessagesRepository {
|
|||||||
*/
|
*/
|
||||||
async getRecentEdits(limit = 50, channelId?: string) {
|
async getRecentEdits(limit = 50, channelId?: string) {
|
||||||
const db = getDatabase();
|
const db = getDatabase();
|
||||||
const where = channelId
|
const result = await db.execute(sql`
|
||||||
? `WHERE m.channel_id = '${channelId.replace(/'/g, "''")}'`
|
SELECT
|
||||||
: "";
|
e.id,
|
||||||
const result = await db.execute(
|
e.message_id,
|
||||||
sql.raw(`
|
e.old_content,
|
||||||
SELECT
|
e.edited_at,
|
||||||
e.id,
|
m.channel_id,
|
||||||
e.message_id,
|
COALESCE(NULLIF((m.metadata::jsonb -> 'channel' ->> 'channelName'), ''), m.channel_id) AS channel_name,
|
||||||
e.old_content,
|
m.username,
|
||||||
e.edited_at,
|
COALESCE(m.edited_content, m.content) AS new_content
|
||||||
m.channel_id,
|
FROM message_edits e
|
||||||
COALESCE(NULLIF((m.metadata::jsonb -> 'channel' ->> 'channelName'), ''), m.channel_id) AS channel_name,
|
JOIN messages m ON m.id = e.message_id
|
||||||
m.username
|
${channelId ? sql`WHERE m.channel_id = ${channelId}` : sql``}
|
||||||
FROM message_edits e
|
ORDER BY e.edited_at DESC
|
||||||
JOIN messages m ON m.id = e.message_id
|
LIMIT ${limit}
|
||||||
${where}
|
`);
|
||||||
ORDER BY e.edited_at DESC
|
|
||||||
LIMIT ${limit}
|
|
||||||
`),
|
|
||||||
);
|
|
||||||
const rows = (result.rows as Record<string, unknown>[]) || [];
|
const rows = (result.rows as Record<string, unknown>[]) || [];
|
||||||
return rows.map((r) => ({
|
return rows.map((r) => ({
|
||||||
id: String(r.id),
|
id: String(r.id),
|
||||||
message_id: String(r.message_id),
|
message_id: String(r.message_id),
|
||||||
old_content: r.old_content ? String(r.old_content) : "",
|
old_content: r.old_content ? String(r.old_content) : "",
|
||||||
|
new_content: r.new_content ? String(r.new_content) : "",
|
||||||
edited_at: r.edited_at ? Number(r.edited_at) : 0,
|
edited_at: r.edited_at ? Number(r.edited_at) : 0,
|
||||||
channel_id: r.channel_id ? String(r.channel_id) : null,
|
channel_id: r.channel_id ? String(r.channel_id) : null,
|
||||||
channel_name: r.channel_name ? String(r.channel_name) : null,
|
channel_name: r.channel_name ? String(r.channel_name) : null,
|
||||||
|
|||||||
@@ -39,12 +39,15 @@ export class MessagesService {
|
|||||||
throw new ValidationError("message ID is required");
|
throw new ValidationError("message ID is required");
|
||||||
}
|
}
|
||||||
|
|
||||||
const message = await messagesRepository.findById(id);
|
const [message, editHistory] = await Promise.all([
|
||||||
|
messagesRepository.findById(id),
|
||||||
|
messagesRepository.getEditHistory(id),
|
||||||
|
]);
|
||||||
|
|
||||||
if (!message) {
|
if (!message) {
|
||||||
throw new NotFoundError(`Message with ID ${id} not found`);
|
throw new NotFoundError(`Message with ID ${id} not found`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const editHistory = await messagesRepository.getEditHistory(id);
|
|
||||||
return {
|
return {
|
||||||
...message,
|
...message,
|
||||||
edit_count: editHistory.length,
|
edit_count: editHistory.length,
|
||||||
|
|||||||
@@ -124,7 +124,8 @@ export class ModerationRepository {
|
|||||||
a.score,
|
a.score,
|
||||||
a.evidence,
|
a.evidence,
|
||||||
a.policy_version,
|
a.policy_version,
|
||||||
m.username,
|
a.username,
|
||||||
|
a.server_nick,
|
||||||
LEFT(m.content, 300) AS content
|
LEFT(m.content, 300) AS content
|
||||||
FROM moderation_actions a
|
FROM moderation_actions a
|
||||||
LEFT JOIN messages m ON m.id = a.message_id
|
LEFT JOIN messages m ON m.id = a.message_id
|
||||||
@@ -155,6 +156,7 @@ export class ModerationRepository {
|
|||||||
evidence: parseJsonArray(r.evidence),
|
evidence: parseJsonArray(r.evidence),
|
||||||
policy_version: r.policy_version ? String(r.policy_version) : null,
|
policy_version: r.policy_version ? String(r.policy_version) : null,
|
||||||
username: r.username ? String(r.username) : null,
|
username: r.username ? String(r.username) : null,
|
||||||
|
server_nick: r.server_nick ? String(r.server_nick) : null,
|
||||||
content: r.content ? String(r.content) : null,
|
content: r.content ? String(r.content) : null,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -314,7 +316,7 @@ export class ModerationRepository {
|
|||||||
SELECT
|
SELECT
|
||||||
a.id, a.message_id, a.user_id, a.guild_id, a.action_type,
|
a.id, a.message_id, a.user_id, a.guild_id, a.action_type,
|
||||||
a.reason, a.status, a.created_at, a.severity, a.confidence, a.score,
|
a.reason, a.status, a.created_at, a.severity, a.confidence, a.score,
|
||||||
m.username, LEFT(m.content, 300) AS content
|
a.username, LEFT(m.content, 300) AS content
|
||||||
FROM moderation_actions a
|
FROM moderation_actions a
|
||||||
LEFT JOIN messages m ON m.id = a.message_id
|
LEFT JOIN messages m ON m.id = a.message_id
|
||||||
WHERE a.created_at >= ${since}
|
WHERE a.created_at >= ${since}
|
||||||
@@ -338,6 +340,7 @@ export class ModerationRepository {
|
|||||||
confidence: r.confidence != null ? Number(r.confidence) : null,
|
confidence: r.confidence != null ? Number(r.confidence) : null,
|
||||||
score: r.score != null ? Number(r.score) : null,
|
score: r.score != null ? Number(r.score) : null,
|
||||||
username: r.username ? String(r.username) : null,
|
username: r.username ? String(r.username) : null,
|
||||||
|
server_nick: r.server_nick ? String(r.server_nick) : null,
|
||||||
content: r.content ? String(r.content) : null,
|
content: r.content ? String(r.content) : null,
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,14 +23,26 @@ const speakers = new Map<string, LiveSpeaker>();
|
|||||||
const MAX_SPEAKERS = 200;
|
const MAX_SPEAKERS = 200;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Record a voice_active_user event. `speaking: true` upserts the speaker as
|
* Speakers inactive for longer than this are auto-expired from the snapshot.
|
||||||
* active; `speaking: false` marks them inactive while keeping them for the
|
* This handles the case where the gateway disconnects abruptly and never
|
||||||
* activity timeline.
|
* sends `speaking: false` for active users.
|
||||||
*/
|
*/
|
||||||
|
const SPEAKER_TTL_MS = 30_000;
|
||||||
|
|
||||||
|
/** Purge speakers that haven't been active recently. */
|
||||||
|
function purgeStale(): void {
|
||||||
|
const cutoff = Date.now() - SPEAKER_TTL_MS;
|
||||||
|
for (const [id, s] of speakers) {
|
||||||
|
if (!s.speaking && s.lastActiveAt < cutoff) {
|
||||||
|
speakers.delete(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* recordSpeaker(data) — apply a `voice_active_user` event. `speaking: true`
|
* Record a voice_active_user event. `speaking: true` upserts the speaker as
|
||||||
* upserts the speaker as ACTIVE; `speaking: false` marks them inactive while
|
* active; `speaking: false` marks them inactive while keeping them briefly
|
||||||
* keeping them for the activity timeline.
|
* for the activity timeline (until TTL expiry).
|
||||||
*/
|
*/
|
||||||
export function recordSpeaker(data: {
|
export function recordSpeaker(data: {
|
||||||
userId: string;
|
userId: string;
|
||||||
@@ -49,7 +61,6 @@ export function recordSpeaker(data: {
|
|||||||
};
|
};
|
||||||
|
|
||||||
if (speakers.size >= MAX_SPEAKERS && !existing) {
|
if (speakers.size >= MAX_SPEAKERS && !existing) {
|
||||||
// Drop the least-recently-active non-speaking speaker to stay bounded.
|
|
||||||
let oldestId: string | null = null;
|
let oldestId: string | null = null;
|
||||||
let oldestTs = Infinity;
|
let oldestTs = Infinity;
|
||||||
for (const [id, s] of speakers) {
|
for (const [id, s] of speakers) {
|
||||||
@@ -65,18 +76,35 @@ export function recordSpeaker(data: {
|
|||||||
speakers.set(userId, speaker);
|
speakers.set(userId, speaker);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** All known speakers, most recently active first. */
|
/**
|
||||||
|
* All recently-active speakers, most recently active first.
|
||||||
|
* Stale (non-speaking + old) entries are auto-purged.
|
||||||
|
*/
|
||||||
export function getActiveSpeakers(): LiveSpeaker[] {
|
export function getActiveSpeakers(): LiveSpeaker[] {
|
||||||
|
purgeStale();
|
||||||
return [...speakers.values()].sort((a, b) => b.lastActiveAt - a.lastActiveAt);
|
return [...speakers.values()].sort((a, b) => b.lastActiveAt - a.lastActiveAt);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Only speakers currently flagged as speaking. */
|
/** Only speakers currently flagged as speaking. */
|
||||||
export function getSpeakingSpeakers(): LiveSpeaker[] {
|
export function getSpeakingSpeakers(): LiveSpeaker[] {
|
||||||
|
purgeStale();
|
||||||
return [...speakers.values()]
|
return [...speakers.values()]
|
||||||
.filter((s) => s.speaking)
|
.filter((s) => s.speaking)
|
||||||
.sort((a, b) => b.lastActiveAt - a.lastActiveAt);
|
.sort((a, b) => b.lastActiveAt - a.lastActiveAt);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark ALL tracked speakers as not-speaking and purge stale ones.
|
||||||
|
* Called when the gateway disconnects from voice — ensures the authoritative
|
||||||
|
* snapshot doesn't carry ghost speakers.
|
||||||
|
*/
|
||||||
|
export function clearAllSpeakers(): void {
|
||||||
|
for (const [id, s] of speakers) {
|
||||||
|
s.speaking = false;
|
||||||
|
}
|
||||||
|
purgeStale();
|
||||||
|
}
|
||||||
|
|
||||||
/** Drop all tracked speakers (used on backend restart). */
|
/** Drop all tracked speakers (used on backend restart). */
|
||||||
export function resetLiveSpeakers(): void {
|
export function resetLiveSpeakers(): void {
|
||||||
speakers.clear();
|
speakers.clear();
|
||||||
|
|||||||
@@ -131,7 +131,7 @@ export const configSchema = z
|
|||||||
AI_LLM_BASE_URL: z
|
AI_LLM_BASE_URL: z
|
||||||
.string()
|
.string()
|
||||||
.url()
|
.url()
|
||||||
.default("https://9router.asepharyana.my.id/v1"),
|
.default("http://100.121.180.82:20128/api/v1"),
|
||||||
AI_LLM_MODEL: z.string().default("text"),
|
AI_LLM_MODEL: z.string().default("text"),
|
||||||
AI_LLM_VISION_MODEL: z.string().optional(),
|
AI_LLM_VISION_MODEL: z.string().optional(),
|
||||||
AI_LLM_EMBEDDING_MODEL: z.string().optional(),
|
AI_LLM_EMBEDDING_MODEL: z.string().optional(),
|
||||||
|
|||||||
@@ -1,5 +1,22 @@
|
|||||||
// Shared message row mapper for backend repository modules
|
// Shared message row mapper for backend repository modules
|
||||||
|
|
||||||
|
function resolveServerNick(row: Record<string, unknown>): string | null {
|
||||||
|
const metadata = row.metadata;
|
||||||
|
if (!metadata || typeof metadata !== "string") {
|
||||||
|
return (row.username as string | null) ?? null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(metadata) as {
|
||||||
|
member?: { displayName?: string | null } | null;
|
||||||
|
};
|
||||||
|
return (
|
||||||
|
parsed?.member?.displayName ?? (row.username as string | null) ?? null
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
return (row.username as string | null) ?? null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export interface MappedMessage {
|
export interface MappedMessage {
|
||||||
id: string;
|
id: string;
|
||||||
guild_id: string;
|
guild_id: string;
|
||||||
@@ -7,6 +24,8 @@ export interface MappedMessage {
|
|||||||
thread_id: string | null;
|
thread_id: string | null;
|
||||||
user_id: string;
|
user_id: string;
|
||||||
username: string;
|
username: string;
|
||||||
|
/** Member's server-specific display name (nickname), from metadata.member.displayName. Falls back to username when absent. */
|
||||||
|
server_nick: string | null;
|
||||||
avatar_url: string | null;
|
avatar_url: string | null;
|
||||||
content: string;
|
content: string;
|
||||||
edited_content: string | null;
|
edited_content: string | null;
|
||||||
@@ -42,6 +61,7 @@ export function mapMessageRow(row: Record<string, unknown>): MappedMessage {
|
|||||||
thread_id: (row.thread_id as string | null) ?? null,
|
thread_id: (row.thread_id as string | null) ?? null,
|
||||||
user_id: String(row.user_id ?? ""),
|
user_id: String(row.user_id ?? ""),
|
||||||
username: String(row.username ?? ""),
|
username: String(row.username ?? ""),
|
||||||
|
server_nick: resolveServerNick(row),
|
||||||
avatar_url: (row.avatar_url as string | null) ?? null,
|
avatar_url: (row.avatar_url as string | null) ?? null,
|
||||||
content: String(row.content ?? ""),
|
content: String(row.content ?? ""),
|
||||||
edited_content: (row.edited_content as string | null) ?? null,
|
edited_content: (row.edited_content as string | null) ?? null,
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
import Redis from "ioredis";
|
import Redis from "ioredis";
|
||||||
import { recordSpeaker } from "../modules/voice/live-speaker.js";
|
import {
|
||||||
|
clearAllSpeakers,
|
||||||
|
recordSpeaker,
|
||||||
|
} from "../modules/voice/live-speaker.js";
|
||||||
import { config } from "../shared/config/index.js";
|
import { config } from "../shared/config/index.js";
|
||||||
import {
|
import {
|
||||||
DISCORD_CHANNEL_TO_WS_EVENT,
|
DISCORD_CHANNEL_TO_WS_EVENT,
|
||||||
DISCORD_VOICE_ACTIVE_USER,
|
DISCORD_VOICE_ACTIVE_USER,
|
||||||
DISCORD_VOICE_PCM,
|
DISCORD_VOICE_PCM,
|
||||||
|
DISCORD_VOICE_STOPPED,
|
||||||
} from "../shared/index.js";
|
} from "../shared/index.js";
|
||||||
import { createChildLogger } from "../shared/logger/index.js";
|
import { createChildLogger } from "../shared/logger/index.js";
|
||||||
import { broadcastBinary, broadcastEvent } from "./broadcast.js";
|
import { broadcastBinary, broadcastEvent } from "./broadcast.js";
|
||||||
@@ -84,6 +88,14 @@ function handleSubscriptionMessage(channel: string, message: string): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// When the gateway stops voice recording (disconnects from voice channel),
|
||||||
|
// clear all speakers from the authoritative snapshot so frontends don't
|
||||||
|
// show ghost participants.
|
||||||
|
if (channel === DISCORD_VOICE_STOPPED) {
|
||||||
|
clearAllSpeakers();
|
||||||
|
logger.info("Voice recording stopped — cleared all live speakers");
|
||||||
|
}
|
||||||
|
|
||||||
logger.debug({ channel, eventType }, "Broadcasting Redis event");
|
logger.debug({ channel, eventType }, "Broadcasting Redis event");
|
||||||
broadcastEvent(eventType, data);
|
broadcastEvent(eventType, data);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -139,5 +139,5 @@ pipeline gauges — `ai_analysis_queued_conversations`,
|
|||||||
numeric snowflake IDs never trigger false positives.
|
numeric snowflake IDs never trigger false positives.
|
||||||
- **Semantic cache is batched** (one embed call + one Qdrant batch search),
|
- **Semantic cache is batched** (one embed call + one Qdrant batch search),
|
||||||
not N sequential round-trips. `ensureQdrantCollection` is memoized.
|
not N sequential round-trips. `ensureQdrantCollection` is memoized.
|
||||||
- **Streaming is mandatory** against the 9router base URL (non-stream waits for
|
- **Streaming is mandatory** against the omniroute base URL (non-stream waits for
|
||||||
the full body and times out). `llmClient` aggregates SSE chunks.
|
the full body and times out). `llmClient` aggregates SSE chunks.
|
||||||
+9
@@ -0,0 +1,9 @@
|
|||||||
|
ALTER TABLE "moderation_actions" ADD COLUMN IF NOT EXISTS "username" text;
|
||||||
|
--> statement-breakpoint
|
||||||
|
-- Backfill username from messages table for existing actions
|
||||||
|
UPDATE "moderation_actions" a
|
||||||
|
SET "username" = m."username"
|
||||||
|
FROM "messages" m
|
||||||
|
WHERE a."message_id" = m."id"
|
||||||
|
AND a."username" IS NULL
|
||||||
|
AND m."username" IS NOT NULL;
|
||||||
+1
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE "moderation_actions" ADD COLUMN IF NOT EXISTS "server_name" text;
|
||||||
+26
@@ -0,0 +1,26 @@
|
|||||||
|
-- Rename server_name (guild name) -> server_nick (member's server nickname)
|
||||||
|
-- server_name was added in 0018 but never populated (all NULL), so renaming
|
||||||
|
-- the empty column is safe and keeps retention semantics correct.
|
||||||
|
-- Idempotent across all possible prior states:
|
||||||
|
-- • server_name present, server_nick absent -> rename
|
||||||
|
-- • both present (backfill created server_nick) -> drop the stale server_name
|
||||||
|
-- • server_nick present only -> already done, no-op
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
IF EXISTS (
|
||||||
|
SELECT 1 FROM information_schema.columns
|
||||||
|
WHERE table_name = 'moderation_actions' AND column_name = 'server_name'
|
||||||
|
) AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM information_schema.columns
|
||||||
|
WHERE table_name = 'moderation_actions' AND column_name = 'server_nick'
|
||||||
|
) THEN
|
||||||
|
ALTER TABLE "moderation_actions" RENAME COLUMN "server_name" TO "server_nick";
|
||||||
|
ELSIF EXISTS (
|
||||||
|
SELECT 1 FROM information_schema.columns
|
||||||
|
WHERE table_name = 'moderation_actions' AND column_name = 'server_name'
|
||||||
|
) THEN
|
||||||
|
-- server_nick already exists (e.g. backfilled out-of-band) — drop the
|
||||||
|
-- now-redundant server_name column instead of renaming into a conflict.
|
||||||
|
ALTER TABLE "moderation_actions" DROP COLUMN "server_name";
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
@@ -120,6 +120,27 @@
|
|||||||
"when": 1787185000000,
|
"when": 1787185000000,
|
||||||
"tag": "0016_drop_user_reputations",
|
"tag": "0016_drop_user_reputations",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 17,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1787810000000,
|
||||||
|
"tag": "0017_add_username_to_moderation_actions",
|
||||||
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 18,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1787820000000,
|
||||||
|
"tag": "0018_add_server_name_to_moderation_actions",
|
||||||
|
"breakpoints": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"idx": 19,
|
||||||
|
"version": "7",
|
||||||
|
"when": 1787830000000,
|
||||||
|
"tag": "0019_rename_server_name_to_server_nick",
|
||||||
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
@@ -28,30 +28,30 @@
|
|||||||
"discord.js-selfbot-v13": "^3.7.1",
|
"discord.js-selfbot-v13": "^3.7.1",
|
||||||
"dotenv": "^17.4.2",
|
"dotenv": "^17.4.2",
|
||||||
"drizzle-orm": "^0.45.2",
|
"drizzle-orm": "^0.45.2",
|
||||||
"ioredis": "^5.11.0",
|
"ioredis": "^6.0.0",
|
||||||
"libsodium-wrappers": "^0.8.4",
|
"libsodium-wrappers": "^0.8.4",
|
||||||
"lru-cache": "^11.5.1",
|
"lru-cache": "^11.5.1",
|
||||||
"openai": "^6.38.0",
|
"openai": "^7.5.0",
|
||||||
"opusscript": "^0.0.8",
|
"opusscript": "^0.1.1",
|
||||||
"p-limit": "^7.3.0",
|
"p-limit": "^7.3.0",
|
||||||
"p-retry": "^8.0.0",
|
"p-retry": "^8.0.0",
|
||||||
"pg": "^8.21.0",
|
"pg": "^8.23.0",
|
||||||
"pino": "^9.6.0",
|
"pino": "^10.3.1",
|
||||||
"piscina": "^5.1.4",
|
"piscina": "^5.3.1",
|
||||||
"prism-media": "2.0.0-alpha.0",
|
"prism-media": "2.0.0-alpha.0",
|
||||||
"sharp": "^0.34.5",
|
"sharp": "^0.35.3",
|
||||||
"tiktoken": "^1.0.22",
|
"tiktoken": "^1.0.22",
|
||||||
"ws": "^8.20.1",
|
"ws": "^8.21.3",
|
||||||
"zod": "^4.4.3"
|
"zod": "^4.4.3"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@biomejs/biome": "latest",
|
"@biomejs/biome": "latest",
|
||||||
"@types/node": "^25.9.0",
|
"@types/node": "^26.3.0",
|
||||||
"@types/pg": "^8.20.0",
|
"@types/pg": "^8.23.1",
|
||||||
"@types/ws": "^8.18.1",
|
"@types/ws": "^8.18.1",
|
||||||
"drizzle-kit": "^0.31.10",
|
"drizzle-kit": "^0.31.10",
|
||||||
"tsx": "^4.22.2",
|
"tsx": "^4.23.12",
|
||||||
"typescript": "^5.9.3",
|
"typescript": "^7.0.2",
|
||||||
"vitest": "latest"
|
"vitest": "latest"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Generated
+660
-435
File diff suppressed because it is too large.
Load diff
@@ -93,12 +93,6 @@ type BatchOkResponse = {
|
|||||||
ok: true;
|
ok: true;
|
||||||
conversationKey: string;
|
conversationKey: string;
|
||||||
rows: MessageRecord[];
|
rows: MessageRecord[];
|
||||||
/**
|
|
||||||
* Race-guard signal (2026-08-25): target ids whose attachment upload is
|
|
||||||
* still in-flight — NO analysis ran for them. The processor must defer
|
|
||||||
* these (requeue + poll), never fan them out as failures.
|
|
||||||
*/
|
|
||||||
uploadPendingIds?: string[];
|
|
||||||
};
|
};
|
||||||
type BatchErrorResponse = {
|
type BatchErrorResponse = {
|
||||||
ok: false;
|
ok: false;
|
||||||
@@ -109,12 +103,6 @@ type BatchErrorResponse = {
|
|||||||
type IndividualOkResponse = {
|
type IndividualOkResponse = {
|
||||||
ok: true;
|
ok: true;
|
||||||
results: AnalysisResult[];
|
results: AnalysisResult[];
|
||||||
/**
|
|
||||||
* Race-guard signal (2026-08-24): the message's attachment upload is still
|
|
||||||
* in-flight — NO analysis ran. The processor must re-queue the message as
|
|
||||||
* `pending` and re-schedule, never treat this as a completed moderation.
|
|
||||||
*/
|
|
||||||
uploadPending?: boolean;
|
|
||||||
};
|
};
|
||||||
type IndividualErrorResponse = {
|
type IndividualErrorResponse = {
|
||||||
ok: false;
|
ok: false;
|
||||||
@@ -309,40 +297,9 @@ async function processBatch(job: {
|
|||||||
...contextIds,
|
...contextIds,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
// Attachment-upload race guard: a message whose attachment is still being
|
|
||||||
// uploaded (upload_status='pending') must not be analyzed yet. Its
|
|
||||||
// uploaded_url is not ready, and falling back to the Discord CDN link often
|
|
||||||
// 404s (expired/purged) — which used to silently produce a text-only
|
|
||||||
// verdict ("lampiran yang gagal terbaca"). Leave those targets pending; the
|
|
||||||
// next worker cycle picks them up after the upload lands.
|
|
||||||
const pendingUploadTargetIds = new Set(
|
|
||||||
(attachments ?? [])
|
|
||||||
.filter((a) => a.upload_status === "pending")
|
|
||||||
.map((a) => a.message_id),
|
|
||||||
);
|
|
||||||
const readyMessages =
|
|
||||||
pendingUploadTargetIds.size === 0
|
|
||||||
? messages
|
|
||||||
: messages.filter((m) => !pendingUploadTargetIds.has(m.id));
|
|
||||||
if (readyMessages.length === 0) {
|
|
||||||
// Explicit signal (2026-08-25): every target is still upload-pending.
|
|
||||||
// Returning bare {ok:true, rows:[]} made the processor classify all of
|
|
||||||
// them "incomplete" and fan out to the individual queue — a hot ~300ms
|
|
||||||
// requeue loop for the whole upload duration.
|
|
||||||
return {
|
|
||||||
ok: true,
|
|
||||||
conversationKey,
|
|
||||||
rows: [],
|
|
||||||
uploadPendingIds: messages.map((m) => m.id),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// The orchestrator handles text/media split + caching + parallel paths
|
|
||||||
// internally, so a 20-message batch = 1 text LLM call (+1 media call
|
|
||||||
// when media is present), not N per-message calls.
|
|
||||||
const analysisStart = Date.now();
|
const analysisStart = Date.now();
|
||||||
const moderationResult = await runModerationAnalysis({
|
const moderationResult = await runModerationAnalysis({
|
||||||
targets: readyMessages,
|
targets: messages,
|
||||||
contextBlock,
|
contextBlock,
|
||||||
attachments,
|
attachments,
|
||||||
});
|
});
|
||||||
@@ -351,7 +308,7 @@ async function processBatch(job: {
|
|||||||
const results = moderationResult.results.map((r) =>
|
const results = moderationResult.results.map((r) =>
|
||||||
normalizeResult(
|
normalizeResult(
|
||||||
r as unknown as AnalysisResult,
|
r as unknown as AnalysisResult,
|
||||||
readyMessages.find((m) => m.id === r.messageId),
|
messages.find((m) => m.id === r.messageId),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -380,10 +337,9 @@ async function processBatch(job: {
|
|||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
total: readyMessages.length,
|
total: messages.length,
|
||||||
saved: allRows.length,
|
saved: allRows.length,
|
||||||
conversationKey,
|
conversationKey,
|
||||||
skippedPendingUpload: messages.length - readyMessages.length,
|
|
||||||
},
|
},
|
||||||
"LLM batch analysis complete",
|
"LLM batch analysis complete",
|
||||||
);
|
);
|
||||||
@@ -431,16 +387,9 @@ async function processIndividual(job: {
|
|||||||
...contextIds,
|
...contextIds,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
// Same attachment-upload race guard as the batch path: while the upload is
|
// Analysis uses the Discord CDN URL directly (archive-only uploader).
|
||||||
// still in-flight the uploaded_url is not ready and the Discord CDN fallback
|
// No upload-pending race guard: analysis proceeds regardless of upload
|
||||||
// often 404s — analyzing now would silently produce a text-only verdict.
|
// status, since discord_url is available immediately at capture time.
|
||||||
// Return no results so the message stays pending for the next cycle.
|
|
||||||
const uploadStillPending = (attachments ?? []).some(
|
|
||||||
(a) => a.message_id === message.id && a.upload_status === "pending",
|
|
||||||
);
|
|
||||||
if (uploadStillPending) {
|
|
||||||
return { ok: true, results: [], uploadPending: true };
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const moderationResult = await runModerationAnalysis({
|
const moderationResult = await runModerationAnalysis({
|
||||||
|
|||||||
@@ -163,12 +163,18 @@ export function startPendingAIAnalysisWorker(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
messageStore.revertStuckProcessingMessages(300000).catch((err: unknown) => {
|
// Only revert stuck processing messages if there's active processing.
|
||||||
logger.error(
|
// Avoids a DB query every recovery interval when the pipeline is idle.
|
||||||
{ error: String(err) },
|
if (conversationProcessing.size > 0) {
|
||||||
"Failed to run stuck processing recovery",
|
messageStore
|
||||||
);
|
.revertStuckProcessingMessages(300000)
|
||||||
});
|
.catch((err: unknown) => {
|
||||||
|
logger.error(
|
||||||
|
{ error: String(err) },
|
||||||
|
"Failed to run stuck processing recovery",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
Promise.all([
|
Promise.all([
|
||||||
messageStore.getPendingConversationKeys(500),
|
messageStore.getPendingConversationKeys(500),
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ export function deriveRecommendedAction(msg: MessageRecord): string {
|
|||||||
const severity = deriveSeverity(msg);
|
const severity = deriveSeverity(msg);
|
||||||
if (
|
if (
|
||||||
msg.ai_status === "flagged" &&
|
msg.ai_status === "flagged" &&
|
||||||
(severity === "critical" || severity === "high")
|
(severity === "critical" || severity === "high" || severity === "medium")
|
||||||
)
|
)
|
||||||
return "delete";
|
return "delete";
|
||||||
if (msg.ai_status === "flagged") return "review";
|
if (msg.ai_status === "flagged") return "review";
|
||||||
@@ -135,10 +135,14 @@ export function isEligibleForAutoDelete(
|
|||||||
// Recommended action check
|
// Recommended action check
|
||||||
const recommendedAction =
|
const recommendedAction =
|
||||||
analysisResult?.recommendedAction ?? deriveRecommendedAction(message);
|
analysisResult?.recommendedAction ?? deriveRecommendedAction(message);
|
||||||
if (recommendedAction !== "delete" && recommendedAction !== "escalate") {
|
if (
|
||||||
|
recommendedAction !== "delete" &&
|
||||||
|
recommendedAction !== "escalate" &&
|
||||||
|
recommendedAction !== "warn"
|
||||||
|
) {
|
||||||
logger.debug(
|
logger.debug(
|
||||||
{ messageId: message.id, recommendedAction },
|
{ messageId: message.id, recommendedAction },
|
||||||
"Message not eligible for auto-delete: recommended action is not delete/escalate",
|
"Message not eligible for auto-delete: recommended action is not delete/escalate/warn",
|
||||||
);
|
);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import type { Client, PermissionString } from "discord.js-selfbot-v13";
|
|||||||
import { LRUCache } from "lru-cache";
|
import { LRUCache } from "lru-cache";
|
||||||
import { createChildLogger } from "@/shared/logger/index";
|
import { createChildLogger } from "@/shared/logger/index";
|
||||||
import { config } from "../../shared/config/config.js";
|
import { config } from "../../shared/config/config.js";
|
||||||
|
import { parseRichMessageMetadata } from "../message-capture/messageMetadata.js";
|
||||||
import { messageStore } from "../message-capture/messageStore.js";
|
import { messageStore } from "../message-capture/messageStore.js";
|
||||||
import type { MessageRecord } from "../message-capture/types.js";
|
import type { MessageRecord } from "../message-capture/types.js";
|
||||||
import {
|
import {
|
||||||
@@ -10,6 +11,7 @@ import {
|
|||||||
} from "./autoDeleteEligibility.js";
|
} from "./autoDeleteEligibility.js";
|
||||||
import { logDeletionToChannel } from "./autoDeleteLogger.js";
|
import { logDeletionToChannel } from "./autoDeleteLogger.js";
|
||||||
import { sendDeletionNotification } from "./autoDeleteNotify.js";
|
import { sendDeletionNotification } from "./autoDeleteNotify.js";
|
||||||
|
import { llmChat } from "./llmClient.js";
|
||||||
import { verdictToActionFields } from "./verdictToActionFields.js";
|
import { verdictToActionFields } from "./verdictToActionFields.js";
|
||||||
|
|
||||||
const logger = createChildLogger("auto-delete-manager");
|
const logger = createChildLogger("auto-delete-manager");
|
||||||
@@ -94,6 +96,26 @@ export async function resetOffensiveNickname(
|
|||||||
}
|
}
|
||||||
// setNickname(null) = remove nickname → Discord shows global username
|
// setNickname(null) = remove nickname → Discord shows global username
|
||||||
await member.setNickname(null, "[auto] nickname melanggar aturan server");
|
await member.setNickname(null, "[auto] nickname melanggar aturan server");
|
||||||
|
|
||||||
|
// ── Check if the global username is ALSO offensive ──────────────
|
||||||
|
// After reset, the member shows their global username. If that is
|
||||||
|
// also offensive (gambling/scam keywords), replace it with a random
|
||||||
|
// generated username so the user can't circumvent the filter by
|
||||||
|
// having an offensive global username.
|
||||||
|
const refreshedMember = await guild.members.fetch(userId);
|
||||||
|
const globalUsername = refreshedMember.user?.username ?? "";
|
||||||
|
if (await isGlobalUsernameOffensiveAI(globalUsername)) {
|
||||||
|
const randomName = generateRandomUsername();
|
||||||
|
await refreshedMember.setNickname(
|
||||||
|
randomName,
|
||||||
|
"[auto] global username juga melanggar, diganti random",
|
||||||
|
);
|
||||||
|
logger.info(
|
||||||
|
{ messageId, guildId, userId, globalUsername, randomName },
|
||||||
|
"Global username also offensive — replaced with random username",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
recentNicknameResets.set(cooldownKey, Date.now());
|
recentNicknameResets.set(cooldownKey, Date.now());
|
||||||
logger.info(
|
logger.info(
|
||||||
{ messageId, guildId, userId },
|
{ messageId, guildId, userId },
|
||||||
@@ -119,6 +141,69 @@ export async function resetOffensiveNickname(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ─── AI-Based Global Username Check ──────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ask the AI moderation LLM whether a global username is offensive
|
||||||
|
* (gambling, scam, spam, NSFW, etc.). Returns `true` if the LLM
|
||||||
|
* judges the username as violating server rules.
|
||||||
|
*
|
||||||
|
* Fail-open: if the LLM call fails or times out, returns `false`
|
||||||
|
* so the nickname reset still completes — we don't want a broken LLM
|
||||||
|
* to block enforcement.
|
||||||
|
*/
|
||||||
|
async function isGlobalUsernameOffensiveAI(username: string): Promise<boolean> {
|
||||||
|
try {
|
||||||
|
const completion = await llmChat({
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: "system",
|
||||||
|
content:
|
||||||
|
'Kamu adalah moderator Discord. Tentukan apakah username berikut melanggar aturan server (judi, togel, scam, spam, NSFW, SARA, atau ofensif). Jawab HANYA dengan JSON: {"offensive": true} atau {"offensive": false}. Jangan penjelasan tambahan.',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: `Username: "${username}"`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
max_tokens: 50,
|
||||||
|
temperature: 0,
|
||||||
|
stream: false,
|
||||||
|
timeout: 10_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
const content = completion?.choices?.[0]?.message?.content?.trim() ?? "";
|
||||||
|
// Parse JSON response — handle both strict JSON and markdown-wrapped
|
||||||
|
const jsonMatch = content.match(
|
||||||
|
/\{[^}]*"offensive"\s*:\s*(true|false)[^}]*\}/,
|
||||||
|
);
|
||||||
|
if (jsonMatch) {
|
||||||
|
const parsed = JSON.parse(jsonMatch[0]) as { offensive: boolean };
|
||||||
|
return parsed.offensive === true;
|
||||||
|
}
|
||||||
|
// Fallback: if response contains "true" anywhere, treat as offensive
|
||||||
|
return content.toLowerCase().includes("true");
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
username,
|
||||||
|
error: error instanceof Error ? error.message : String(error),
|
||||||
|
},
|
||||||
|
"AI username check failed — failing open (not offensive)",
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Generate a random fallback username: "User" + 5-digit number.
|
||||||
|
* Guaranteed unique per call (random suffix).
|
||||||
|
*/
|
||||||
|
function generateRandomUsername(): string {
|
||||||
|
const suffix = Math.floor(10000 + Math.random() * 90000);
|
||||||
|
return `User${suffix}`;
|
||||||
|
}
|
||||||
|
|
||||||
// ─── Error Handling Utilities ────────────────────────────────────────
|
// ─── Error Handling Utilities ────────────────────────────────────────
|
||||||
|
|
||||||
function getErrorCode(error: unknown): number | string | undefined {
|
function getErrorCode(error: unknown): number | string | undefined {
|
||||||
@@ -130,7 +215,28 @@ function getErrorCode(error: unknown): number | string | undefined {
|
|||||||
|
|
||||||
function isAlreadyDeletedError(error: unknown): boolean {
|
function isAlreadyDeletedError(error: unknown): boolean {
|
||||||
const code = getErrorCode(error);
|
const code = getErrorCode(error);
|
||||||
return code === 10008 || code === 404 || code === "10008" || code === "404";
|
// Discord REST error codes for "message not found":
|
||||||
|
// 10008 = Unknown Message, 10003 = Unknown Channel,
|
||||||
|
// 50001 = Missing Access (channel deleted/hidden), 404 = HTTP
|
||||||
|
if (
|
||||||
|
code === 10008 ||
|
||||||
|
code === 10003 ||
|
||||||
|
code === 50001 ||
|
||||||
|
code === 404 ||
|
||||||
|
code === "10008" ||
|
||||||
|
code === "10003" ||
|
||||||
|
code === "50001" ||
|
||||||
|
code === "404"
|
||||||
|
)
|
||||||
|
return true;
|
||||||
|
// Fallback: check the message text for the Discord "Unknown Message" string
|
||||||
|
const msg =
|
||||||
|
error instanceof Error
|
||||||
|
? error.message
|
||||||
|
: typeof error === "string"
|
||||||
|
? error
|
||||||
|
: "";
|
||||||
|
return msg.includes("Unknown Message") || msg.includes("Unknown Channel");
|
||||||
}
|
}
|
||||||
|
|
||||||
function hasChannelMessagesApi(channel: unknown): channel is {
|
function hasChannelMessagesApi(channel: unknown): channel is {
|
||||||
@@ -164,6 +270,21 @@ function hasPermissionApi(channel: unknown): channel is {
|
|||||||
|
|
||||||
// ─── Database Action Log ─────────────────────────────────────────────
|
// ─── Database Action Log ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract the member's server-specific display name (nickname) from a
|
||||||
|
* captured message's metadata. Falls back to the global username when no
|
||||||
|
* server nickname exists (member.displayName defaults to the guild nickname
|
||||||
|
* when set, otherwise the global username).
|
||||||
|
*/
|
||||||
|
function resolveServerNick(message: MessageRecord): string | null {
|
||||||
|
try {
|
||||||
|
const parsed = parseRichMessageMetadata(message.metadata);
|
||||||
|
return parsed?.member?.displayName ?? message.username ?? null;
|
||||||
|
} catch {
|
||||||
|
return message.username ?? null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function logAutoDeleteAttempt(
|
async function logAutoDeleteAttempt(
|
||||||
message: MessageRecord,
|
message: MessageRecord,
|
||||||
result: AutoDeleteResult,
|
result: AutoDeleteResult,
|
||||||
@@ -176,6 +297,8 @@ async function logAutoDeleteAttempt(
|
|||||||
action_type: "delete_message",
|
action_type: "delete_message",
|
||||||
reason: result.reason,
|
reason: result.reason,
|
||||||
...verdictToActionFields(message),
|
...verdictToActionFields(message),
|
||||||
|
username: message.username,
|
||||||
|
server_nick: resolveServerNick(message),
|
||||||
executed_by: "auto-delete-manager",
|
executed_by: "auto-delete-manager",
|
||||||
status: result.deleted
|
status: result.deleted
|
||||||
? "executed"
|
? "executed"
|
||||||
@@ -241,6 +364,8 @@ export async function attemptAutoDeleteFlaggedMessage(
|
|||||||
reason:
|
reason:
|
||||||
"nickname melanggar aturan server (offensive_username); pesan dibiarkan",
|
"nickname melanggar aturan server (offensive_username); pesan dibiarkan",
|
||||||
...verdictToActionFields(message),
|
...verdictToActionFields(message),
|
||||||
|
username: message.username,
|
||||||
|
server_nick: resolveServerNick(message),
|
||||||
executed_by: "auto-delete-manager",
|
executed_by: "auto-delete-manager",
|
||||||
status: resetOk ? "executed" : "failed",
|
status: resetOk ? "executed" : "failed",
|
||||||
error: resetOk ? null : "nickname_reset_failed",
|
error: resetOk ? null : "nickname_reset_failed",
|
||||||
|
|||||||
@@ -1,15 +1,7 @@
|
|||||||
/**
|
/**
|
||||||
* batchOutcomeClassifier.ts
|
* batchOutcomeClassifier.ts
|
||||||
*
|
*
|
||||||
* Pure partitioner of the batch worker response (2026-08-25).
|
* Pure partitioner of the batch worker response.
|
||||||
*
|
|
||||||
* Bug history: the batch race guard returned `{ok:true, rows:[]}` when every
|
|
||||||
* target's attachment upload was still in-flight. The processor classified all
|
|
||||||
* of them as "incomplete" and fanned out to the individual queue, where the
|
|
||||||
* guard there requeued + rescheduled at the 250ms debounce — a hot ~300ms loop
|
|
||||||
* for the entire upload duration (~10 cycles in 3s in prod logs). Root fix:
|
|
||||||
* the worker now reports `uploadPendingIds` explicitly and this pure function
|
|
||||||
* partitions the outcome so upload-pending targets NEVER enter the fanout.
|
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export interface BatchRowLike {
|
export interface BatchRowLike {
|
||||||
@@ -21,15 +13,12 @@ export interface BatchRowLike {
|
|||||||
export interface BatchWorkerResponseLike {
|
export interface BatchWorkerResponseLike {
|
||||||
ok?: boolean;
|
ok?: boolean;
|
||||||
rows?: BatchRowLike[];
|
rows?: BatchRowLike[];
|
||||||
/** Explicit race-guard signal from the worker (2026-08-25). */
|
|
||||||
uploadPendingIds?: string[];
|
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** One target's per-message disposition after a batch attempt. */
|
/** One target's per-message disposition after a batch attempt. */
|
||||||
export type BatchTargetKind =
|
export type BatchTargetKind =
|
||||||
| "completed"
|
| "completed"
|
||||||
| "upload_pending"
|
|
||||||
| "incomplete"
|
| "incomplete"
|
||||||
| "parse_failed"
|
| "parse_failed"
|
||||||
| "api_failed";
|
| "api_failed";
|
||||||
@@ -40,7 +29,7 @@ function flagsOf(row: { ai_moderation_flags?: string | null }): string[] {
|
|||||||
const parsed = JSON.parse(row.ai_moderation_flags) as unknown;
|
const parsed = JSON.parse(row.ai_moderation_flags) as unknown;
|
||||||
return Array.isArray(parsed) ? (parsed as string[]) : [];
|
return Array.isArray(parsed) ? (parsed as string[]) : [];
|
||||||
} catch {
|
} catch {
|
||||||
return [] as string[];
|
return [];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -48,14 +37,13 @@ function flagsOf(row: { ai_moderation_flags?: string | null }): string[] {
|
|||||||
* Partition the input message ids into per-message dispositions for one batch
|
* Partition the input message ids into per-message dispositions for one batch
|
||||||
* worker response. Pure: no DB/Piscina/logger — unit-testable directly.
|
* worker response. Pure: no DB/Piscina/logger — unit-testable directly.
|
||||||
*
|
*
|
||||||
* Priority per id: explicit uploadPendingIds → completed row → flag-based
|
* Priority per id: completed row → flag-based failure kinds → unexplained
|
||||||
* failure kinds → unexplained missing (treated like incomplete).
|
* missing (treated like incomplete).
|
||||||
*/
|
*/
|
||||||
export function partitionBatchOutcome(
|
export function partitionBatchOutcome(
|
||||||
messages: ReadonlyArray<{ id: string }>,
|
messages: ReadonlyArray<{ id: string }>,
|
||||||
response: BatchWorkerResponseLike,
|
response: BatchWorkerResponseLike,
|
||||||
): Map<string, BatchTargetKind> {
|
): Map<string, BatchTargetKind> {
|
||||||
const pendingSet = new Set(response.uploadPendingIds ?? []);
|
|
||||||
const rowsById = new Map(
|
const rowsById = new Map(
|
||||||
(response.rows ?? [])
|
(response.rows ?? [])
|
||||||
.filter((r): r is BatchRowLike & { id: string } => Boolean(r?.id))
|
.filter((r): r is BatchRowLike & { id: string } => Boolean(r?.id))
|
||||||
@@ -64,10 +52,6 @@ export function partitionBatchOutcome(
|
|||||||
|
|
||||||
const out = new Map<string, BatchTargetKind>();
|
const out = new Map<string, BatchTargetKind>();
|
||||||
for (const msg of messages) {
|
for (const msg of messages) {
|
||||||
if (pendingSet.has(msg.id)) {
|
|
||||||
out.set(msg.id, "upload_pending");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const row = rowsById.get(msg.id);
|
const row = rowsById.get(msg.id);
|
||||||
if (!row) {
|
if (!row) {
|
||||||
// Unexplained drop: LLM silently omitted it. Same retryable bucket as
|
// Unexplained drop: LLM silently omitted it. Same retryable bucket as
|
||||||
@@ -92,17 +76,3 @@ export function partitionBatchOutcome(
|
|||||||
}
|
}
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Linear backoff ramp for consecutive upload-pending polls:
|
|
||||||
* poll N (1-based) waits min(base × N, cap). Keeps latency low for fast
|
|
||||||
* uploads while bounding total polling cost for long uploads.
|
|
||||||
*/
|
|
||||||
export function computeUploadPollDelayMs(
|
|
||||||
consecutivePolls: number,
|
|
||||||
baseMs: number,
|
|
||||||
capMs: number,
|
|
||||||
): number {
|
|
||||||
const n = Math.max(1, Math.floor(consecutivePolls));
|
|
||||||
return Math.min(Math.round(baseMs * n), Math.round(capMs));
|
|
||||||
}
|
|
||||||
@@ -4,10 +4,7 @@ import { isAgeRestrictedMetadata } from "../message-capture/messageMetadata.js";
|
|||||||
import { messageStore } from "../message-capture/messageStore.js";
|
import { messageStore } from "../message-capture/messageStore.js";
|
||||||
import type { MessageRecord } from "../message-capture/types.js";
|
import type { MessageRecord } from "../message-capture/types.js";
|
||||||
import { pickBatchWithinBudget as pickBatchWithinBudgetPure } from "./batchBudget.js";
|
import { pickBatchWithinBudget as pickBatchWithinBudgetPure } from "./batchBudget.js";
|
||||||
import {
|
import { partitionBatchOutcome } from "./batchOutcomeClassifier.js";
|
||||||
computeUploadPollDelayMs,
|
|
||||||
partitionBatchOutcome,
|
|
||||||
} from "./batchOutcomeClassifier.js";
|
|
||||||
import { workerPool } from "./circuitBreaker.js";
|
import { workerPool } from "./circuitBreaker.js";
|
||||||
import { estimateTokens } from "./conversationContext.js";
|
import { estimateTokens } from "./conversationContext.js";
|
||||||
import {
|
import {
|
||||||
@@ -25,20 +22,11 @@ import {
|
|||||||
|
|
||||||
const logger = createChildLogger("batch-processor");
|
const logger = createChildLogger("batch-processor");
|
||||||
|
|
||||||
/**
|
|
||||||
* Consecutive upload-pending poll counter per conversation (2026-08-25).
|
|
||||||
* Drives the linear backoff ramp while attachments are still uploading;
|
|
||||||
* cleared as soon as a batch comes back with no upload-pending targets.
|
|
||||||
*/
|
|
||||||
const conversationUploadPolls = new Map<string, number>();
|
|
||||||
|
|
||||||
export interface AnalysisWorkerResponse {
|
export interface AnalysisWorkerResponse {
|
||||||
ok: boolean;
|
ok: boolean;
|
||||||
conversationKey: string;
|
conversationKey: string;
|
||||||
rows: MessageRecord[];
|
rows: MessageRecord[];
|
||||||
error?: string;
|
error?: string;
|
||||||
/** Explicit upload-in-flight signal from the batch race guard (2026-08-25). */
|
|
||||||
uploadPendingIds?: string[];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -154,8 +142,6 @@ export async function processBatch(
|
|||||||
|
|
||||||
activeRequests++;
|
activeRequests++;
|
||||||
let shouldScheduleNext = false;
|
let shouldScheduleNext = false;
|
||||||
/** Set when upload-pending targets defer the next cycle by this many ms. */
|
|
||||||
let deferredUploadRescheduleMs: number | null = null;
|
|
||||||
try {
|
try {
|
||||||
const result = (await workerPool.run({
|
const result = (await workerPool.run({
|
||||||
type: "batch",
|
type: "batch",
|
||||||
@@ -163,20 +149,14 @@ export async function processBatch(
|
|||||||
messages,
|
messages,
|
||||||
})) as AnalysisWorkerResponse;
|
})) as AnalysisWorkerResponse;
|
||||||
|
|
||||||
// Do not broadcast or auto-delete if it's an API failure that will be reverted.
|
// Broadcast + auto-delete only for successfully analyzed rows.
|
||||||
|
// Error rows (API failures, parse failures, incomplete) will be
|
||||||
|
// retried by the individual fallback queue — do NOT schedule
|
||||||
|
// auto-delete for them (they'd be logged as not_eligible anyway).
|
||||||
for (const row of result.rows) {
|
for (const row of result.rows) {
|
||||||
let isApiFailure = false;
|
if (row.ai_status === "error") continue;
|
||||||
if (row.ai_status === "error") {
|
broadcastAnalysisCompleted(row);
|
||||||
try {
|
scheduleAutoDelete(row);
|
||||||
const flags = JSON.parse(row.ai_moderation_flags ?? "[]") as string[];
|
|
||||||
isApiFailure = flags.includes("analysis_api_failed");
|
|
||||||
} catch {}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isApiFailure) {
|
|
||||||
broadcastAnalysisCompleted(row);
|
|
||||||
scheduleAutoDelete(row);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!result.ok) {
|
if (!result.ok) {
|
||||||
@@ -214,19 +194,16 @@ export async function processBatch(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Batch succeeded -- partition per-message outcome explicitly (2026-08-25).
|
// Batch succeeded -- partition per-message outcome (2026-08-25).
|
||||||
// upload_pending targets are DEFERRED (never fanned out): the old code
|
|
||||||
// treated them as incomplete -> individual queue -> requeue+250ms
|
|
||||||
// reschedule -> hot ~300ms loop for the whole upload duration.
|
|
||||||
const outcomeById = partitionBatchOutcome(messages, result);
|
const outcomeById = partitionBatchOutcome(messages, result);
|
||||||
const messagesForIndividualQueue: MessageRecord[] = [];
|
const messagesForIndividualQueue: MessageRecord[] = [];
|
||||||
const apiFailedMessages: MessageRecord[] = [];
|
const apiFailedMessages: MessageRecord[] = [];
|
||||||
const uploadPendingMessages: MessageRecord[] = [];
|
|
||||||
|
|
||||||
for (const msg of messages) {
|
for (const msg of messages) {
|
||||||
switch (outcomeById.get(msg.id)) {
|
switch (outcomeById.get(msg.id)) {
|
||||||
case "upload_pending":
|
case "completed":
|
||||||
uploadPendingMessages.push(msg);
|
// Successfully analyzed — already broadcast + auto-delete scheduled
|
||||||
|
// above. Do NOT re-enqueue for individual fallback.
|
||||||
break;
|
break;
|
||||||
case "api_failed":
|
case "api_failed":
|
||||||
// Preserve the dedicated api-failure semantics below: revert +
|
// Preserve the dedicated api-failure semantics below: revert +
|
||||||
@@ -241,59 +218,6 @@ export async function processBatch(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (uploadPendingMessages.length > 0) {
|
|
||||||
const polls = (conversationUploadPolls.get(conversationKey) ?? 0) + 1;
|
|
||||||
conversationUploadPolls.set(conversationKey, polls);
|
|
||||||
const delayMs = computeUploadPollDelayMs(
|
|
||||||
polls,
|
|
||||||
config.AI_ANALYSIS_UPLOAD_POLL_MS,
|
|
||||||
config.AI_ANALYSIS_MAX_UPLOAD_POLL_MS,
|
|
||||||
);
|
|
||||||
logger.debug(
|
|
||||||
{
|
|
||||||
conversationKey,
|
|
||||||
count: uploadPendingMessages.length,
|
|
||||||
ids: uploadPendingMessages.map((m) => m.id),
|
|
||||||
pollAttempt: polls,
|
|
||||||
delayMs,
|
|
||||||
},
|
|
||||||
"Attachment upload in-flight for batch targets — deferring with poll backoff",
|
|
||||||
);
|
|
||||||
|
|
||||||
// Put the rows back to `pending` so the scheduler owns them again.
|
|
||||||
await messageStore
|
|
||||||
.updateMessagesAIAnalysisBulk(
|
|
||||||
uploadPendingMessages.map((msg) => ({
|
|
||||||
messageId: msg.id,
|
|
||||||
result: {
|
|
||||||
status: "pending",
|
|
||||||
flags: null,
|
|
||||||
score: null,
|
|
||||||
analysis: null,
|
|
||||||
categories: null,
|
|
||||||
severity: null,
|
|
||||||
confidence: null,
|
|
||||||
recommendedAction: null,
|
|
||||||
analyzedAt: null,
|
|
||||||
error: null,
|
|
||||||
},
|
|
||||||
})),
|
|
||||||
)
|
|
||||||
.catch((err: unknown) => {
|
|
||||||
logger.error(
|
|
||||||
{ error: String(err), ids: uploadPendingMessages.map((m) => m.id) },
|
|
||||||
"Failed to revert upload-pending batch targets to pending",
|
|
||||||
);
|
|
||||||
return [] as MessageRecord[];
|
|
||||||
});
|
|
||||||
|
|
||||||
// Poll backoff instead of the 250ms debounce: the finally-block
|
|
||||||
// schedules the next cycle after this delay instead of immediately.
|
|
||||||
deferredUploadRescheduleMs = delayMs;
|
|
||||||
} else {
|
|
||||||
conversationUploadPolls.delete(conversationKey);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (messagesForIndividualQueue.length > 0) {
|
if (messagesForIndividualQueue.length > 0) {
|
||||||
logger.warn(
|
logger.warn(
|
||||||
{
|
{
|
||||||
@@ -370,11 +294,7 @@ export async function processBatch(
|
|||||||
resetConversationBatchFailures(conversationKey);
|
resetConversationBatchFailures(conversationKey);
|
||||||
conversationErrorCooldown.delete(conversationKey);
|
conversationErrorCooldown.delete(conversationKey);
|
||||||
}
|
}
|
||||||
// Upload-pending defer owns the next-cycle timing; don't let the default
|
shouldScheduleNext = true;
|
||||||
// immediate schedule override it.
|
|
||||||
if (deferredUploadRescheduleMs === null) {
|
|
||||||
shouldScheduleNext = true;
|
|
||||||
}
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
recordConversationBatchFailure(conversationKey);
|
recordConversationBatchFailure(conversationKey);
|
||||||
|
|
||||||
@@ -411,17 +331,7 @@ export async function processBatch(
|
|||||||
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
|
if (conversationProcessing.get(conversationKey) === processingStartedAt) {
|
||||||
conversationProcessing.delete(conversationKey);
|
conversationProcessing.delete(conversationKey);
|
||||||
}
|
}
|
||||||
if (deferredUploadRescheduleMs !== null) {
|
if (shouldScheduleNext) {
|
||||||
// Upload still in-flight: re-schedule after the backoff delay instead of
|
|
||||||
// immediately (the old path hot-looped at ~250-300ms per cycle).
|
|
||||||
const delayMs = deferredUploadRescheduleMs;
|
|
||||||
setTimeout(() => {
|
|
||||||
// Dynamic import to avoid circular dependency at module scope
|
|
||||||
import("./batchScheduler.js").then((m) =>
|
|
||||||
m.scheduleConversationAnalysis(conversationKey),
|
|
||||||
);
|
|
||||||
}, delayMs).unref();
|
|
||||||
} else if (shouldScheduleNext) {
|
|
||||||
setImmediate(() => {
|
setImmediate(() => {
|
||||||
// Dynamic import to avoid circular dependency at module scope
|
// Dynamic import to avoid circular dependency at module scope
|
||||||
import("./batchScheduler.js").then((m) =>
|
import("./batchScheduler.js").then((m) =>
|
||||||
|
|||||||
@@ -136,6 +136,12 @@ export function formatMessageForPrompt(
|
|||||||
return `[${label}] id=${msg.id} time=${timestamp} user=${resolveDisplayName(msg)}: ${content}${mediaSuffix}${refInfo}`;
|
return `[${label}] id=${msg.id} time=${timestamp} user=${resolveDisplayName(msg)}: ${content}${mediaSuffix}${refInfo}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Estimate tokens for a batch of strings — single encode call, ~5x faster than per-line. */
|
||||||
|
export function estimateTokensBatch(texts: string[]): number {
|
||||||
|
const combined = texts.join("\n");
|
||||||
|
return getEncoder().encode(combined).length + texts.length * 15;
|
||||||
|
}
|
||||||
|
|
||||||
/** Max content chars per context line — a single huge paste (log dump,
|
/** Max content chars per context line — a single huge paste (log dump,
|
||||||
* copypasta) must not eat the whole conversation budget. */
|
* copypasta) must not eat the whole conversation budget. */
|
||||||
const CONTEXT_LINE_CONTENT_MAX_CHARS = 1500;
|
const CONTEXT_LINE_CONTENT_MAX_CHARS = 1500;
|
||||||
@@ -264,10 +270,7 @@ export function buildConversationContext(
|
|||||||
const targetLines = targets.map((msg) =>
|
const targetLines = targets.map((msg) =>
|
||||||
formatMessageForPrompt(msg, "target"),
|
formatMessageForPrompt(msg, "target"),
|
||||||
);
|
);
|
||||||
let usedTokens = targetLines.reduce(
|
let usedTokens = estimateTokensBatch(targetLines);
|
||||||
(sum, line) => sum + estimateTokens(line),
|
|
||||||
0,
|
|
||||||
);
|
|
||||||
|
|
||||||
const contextLines = gatedNewestFirst.map((msg) =>
|
const contextLines = gatedNewestFirst.map((msg) =>
|
||||||
formatMessageForPrompt(msg, "context"),
|
formatMessageForPrompt(msg, "context"),
|
||||||
|
|||||||
@@ -13,16 +13,10 @@
|
|||||||
* has an explicit, testable owner.
|
* has an explicit, testable owner.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export type WorkerResultKind =
|
export type WorkerResultKind = "success" | "incomplete" | "error";
|
||||||
| "success"
|
|
||||||
| "upload_pending"
|
|
||||||
| "incomplete"
|
|
||||||
| "error";
|
|
||||||
|
|
||||||
export interface ClassifiableWorkerResult {
|
export interface ClassifiableWorkerResult {
|
||||||
ok?: boolean;
|
ok?: boolean;
|
||||||
/** Upload-pending marker set by ai-analysis-worker's race guard. */
|
|
||||||
uploadPending?: boolean;
|
|
||||||
results?: Array<{ status?: string; flags?: string[] | string } | undefined>;
|
results?: Array<{ status?: string; flags?: string[] | string } | undefined>;
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
@@ -40,7 +34,6 @@ function flagsOf(r: { flags?: string[] | string }): string[] {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Classify an individual-fallback worker response:
|
* Classify an individual-fallback worker response:
|
||||||
* - "upload_pending": explicit race-guard signal — retry shortly, NOT an error.
|
|
||||||
* - "success": at least one result and none is analysis_incomplete.
|
* - "success": at least one result and none is analysis_incomplete.
|
||||||
* - "incomplete": LLM ran but dropped/failed this message after retries
|
* - "incomplete": LLM ran but dropped/failed this message after retries
|
||||||
* (analysis_incomplete flag) — terminal exhausted path.
|
* (analysis_incomplete flag) — terminal exhausted path.
|
||||||
@@ -50,7 +43,6 @@ function flagsOf(r: { flags?: string[] | string }): string[] {
|
|||||||
export function classifyIndividualWorkerResult(
|
export function classifyIndividualWorkerResult(
|
||||||
result: ClassifiableWorkerResult,
|
result: ClassifiableWorkerResult,
|
||||||
): WorkerResultKind {
|
): WorkerResultKind {
|
||||||
if (result.uploadPending === true) return "upload_pending";
|
|
||||||
const results = (result.results ?? []).filter(
|
const results = (result.results ?? []).filter(
|
||||||
(r): r is NonNullable<typeof r> => Boolean(r),
|
(r): r is NonNullable<typeof r> => Boolean(r),
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -79,75 +79,21 @@ async function processIndividualFallback(
|
|||||||
message,
|
message,
|
||||||
skipNormalAnalysis: false,
|
skipNormalAnalysis: false,
|
||||||
} as unknown)) as
|
} as unknown)) as
|
||||||
| { ok: true; results: AnalysisResult[]; uploadPending?: boolean }
|
| { ok: true; results: AnalysisResult[] }
|
||||||
| { ok: false; results: AnalysisResult[]; error: string };
|
| { ok: false; results: AnalysisResult[]; error: string };
|
||||||
|
|
||||||
// Explicit outcome classification (2026-08-24): the old code treated any
|
// Explicit outcome classification: the old code treated any
|
||||||
// ok:true as a completed moderation, so the upload-pending race guard's
|
// ok:true as a completed moderation, so empty or unexplainable
|
||||||
// empty results left messages stuck in `processing` until the 300s
|
// results left messages stuck in `processing` until the 300s
|
||||||
// cleanup reverted them — the root cause of the ~330s attachment delays.
|
// cleanup reverted them.
|
||||||
const kind = classifyIndividualWorkerResult(workerResult);
|
const kind = classifyIndividualWorkerResult(workerResult);
|
||||||
|
|
||||||
if (kind === "upload_pending") {
|
|
||||||
// Attachment still uploading — put the row back to `pending` and
|
|
||||||
// re-schedule this conversation immediately. The next scheduler cycle
|
|
||||||
// (~debounce 250ms) re-fetches; once upload_status flips to done the
|
|
||||||
// race guard passes and analysis proceeds. NOT an error: never touches
|
|
||||||
// the circuit breaker counters.
|
|
||||||
const revertedRows = await messageStore
|
|
||||||
.updateMessagesAIAnalysisBulk([
|
|
||||||
{
|
|
||||||
messageId,
|
|
||||||
result: {
|
|
||||||
status: "pending",
|
|
||||||
flags: null,
|
|
||||||
score: null,
|
|
||||||
analysis: null,
|
|
||||||
categories: null,
|
|
||||||
severity: null,
|
|
||||||
confidence: null,
|
|
||||||
recommendedAction: null,
|
|
||||||
analyzedAt: null,
|
|
||||||
error: null,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
])
|
|
||||||
.catch((dbErr: unknown) => {
|
|
||||||
logger.error(
|
|
||||||
{ messageId, error: String(dbErr) },
|
|
||||||
"Failed to revert upload-pending message to pending",
|
|
||||||
);
|
|
||||||
return [] as MessageRecord[];
|
|
||||||
});
|
|
||||||
for (const row of revertedRows) {
|
|
||||||
broadcastAnalysisCompleted(row);
|
|
||||||
}
|
|
||||||
logger.debug(
|
|
||||||
{ messageId, conversationKey },
|
|
||||||
"Individual fallback: attachment upload in-flight — requeued as pending + rescheduled",
|
|
||||||
);
|
|
||||||
setImmediate(() => {
|
|
||||||
import("./batchScheduler.js")
|
|
||||||
.then((m) => m.scheduleConversationAnalysis(conversationKey))
|
|
||||||
.catch(() => {});
|
|
||||||
});
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let analysisResult: { results: AnalysisResult[] } | null = null;
|
let analysisResult: { results: AnalysisResult[] } | null = null;
|
||||||
|
|
||||||
if (kind === "success") {
|
if (kind === "success") {
|
||||||
analysisResult = workerResult;
|
analysisResult = workerResult;
|
||||||
} else if (kind === "incomplete") {
|
|
||||||
exhaustedOnIncomplete = true;
|
|
||||||
analysisResult = null;
|
|
||||||
} else {
|
} else {
|
||||||
// "error" — includes ok:true with unexplainable empty results (the old
|
exhaustedOnIncomplete = kind === "incomplete";
|
||||||
// silent-success bug). Throw so it is treated as a transient failure.
|
analysisResult = null;
|
||||||
throw new Error(
|
|
||||||
(workerResult as { error?: string }).error ??
|
|
||||||
"Individual worker returned no explainable results",
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// No heuristic fallback: an incomplete/errored LLM result stays a
|
// No heuristic fallback: an incomplete/errored LLM result stays a
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import { delay, retryWithBackoff } from "@/shared/utils/index";
|
|||||||
import { config } from "../../shared/config/config.js";
|
import { config } from "../../shared/config/config.js";
|
||||||
import type { AnalysisResult } from "../message-capture/types.js";
|
import type { AnalysisResult } from "../message-capture/types.js";
|
||||||
import { llmChat } from "./llmClient.js";
|
import { llmChat } from "./llmClient.js";
|
||||||
|
import { parseModerationResponse } from "./moderationResponseParser.js";
|
||||||
import { logModerationError } from "./responseLogger.js";
|
import { logModerationError } from "./responseLogger.js";
|
||||||
|
|
||||||
const log = createChildLogger("llm-caller");
|
const log = createChildLogger("llm-caller");
|
||||||
@@ -79,11 +80,12 @@ export async function callModerationLLM(
|
|||||||
];
|
];
|
||||||
const completion = await llmChat({
|
const completion = await llmChat({
|
||||||
messages,
|
messages,
|
||||||
max_tokens: maxTokens ?? 16384,
|
max_tokens:
|
||||||
|
maxTokens ?? config.AI_LLM_MAX_COMPLETION_TOKENS ?? 16384,
|
||||||
jsonResponse: { type: "json_object" },
|
jsonResponse: { type: "json_object" },
|
||||||
retries: 0,
|
retries: 0,
|
||||||
signal,
|
signal,
|
||||||
// Router (9router / formerly omniroute) always streams SSE even when the
|
// Router (omniroute) always streams SSE even when the
|
||||||
// request omits `stream`. In non-stream mode the OpenAI SDK waits
|
// request omits `stream`. In non-stream mode the OpenAI SDK waits
|
||||||
// for the FULL body before parsing, so slow/long upstream streams
|
// for the FULL body before parsing, so slow/long upstream streams
|
||||||
// hit the 30s/60s timeout and abort mid-generation. Streaming mode
|
// hit the 30s/60s timeout and abort mid-generation. Streaming mode
|
||||||
@@ -106,9 +108,6 @@ export async function callModerationLLM(
|
|||||||
if (!rawContent) throw new Error("No content in LLM response");
|
if (!rawContent) throw new Error("No content in LLM response");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const { parseModerationResponse } = await import(
|
|
||||||
"./moderationResponseParser.js"
|
|
||||||
);
|
|
||||||
return {
|
return {
|
||||||
parsed: parseModerationResponse(rawContent, targetIds),
|
parsed: parseModerationResponse(rawContent, targetIds),
|
||||||
result: completion,
|
result: completion,
|
||||||
|
|||||||
@@ -94,7 +94,7 @@ type LLMResponseChunk = {
|
|||||||
* `delta.content`; falls back to reasoning fields so reasoning-only models
|
* `delta.content`; falls back to reasoning fields so reasoning-only models
|
||||||
* still produce usable aggregated text. Providers differ in the field name:
|
* still produce usable aggregated text. Providers differ in the field name:
|
||||||
* - DeepSeek-style / Cloudflare gemma → `delta.reasoning_content`
|
* - DeepSeek-style / Cloudflare gemma → `delta.reasoning_content`
|
||||||
* - mimo (via 9router) streams reasoning in `delta.reasoning` +
|
* - mimo (via omniroute) streams reasoning in `delta.reasoning` +
|
||||||
* `delta.reasoning_details[].text` (content:"") — without these fallbacks
|
* `delta.reasoning_details[].text` (content:"") — without these fallbacks
|
||||||
* vision aggregation came back empty ("Vision API null response").
|
* vision aggregation came back empty ("Vision API null response").
|
||||||
* Exported for unit tests.
|
* Exported for unit tests.
|
||||||
@@ -225,7 +225,7 @@ export function buildLlmParams(
|
|||||||
reasoning: { enabled: false },
|
reasoning: { enabled: false },
|
||||||
// vLLM / Qwen / litellm
|
// vLLM / Qwen / litellm
|
||||||
chat_template_kwargs: { enable_thinking: false },
|
chat_template_kwargs: { enable_thinking: false },
|
||||||
// Anthropic / Claude-format (9router exposes thinkingFormat
|
// Anthropic / Claude-format (omniroute exposes thinkingFormat
|
||||||
// "claude-adaptive" / "claude-budget" on its reasoning models)
|
// "claude-adaptive" / "claude-budget" on its reasoning models)
|
||||||
thinking: { type: "disabled" },
|
thinking: { type: "disabled" },
|
||||||
} as Record<string, unknown>);
|
} as Record<string, unknown>);
|
||||||
|
|||||||
@@ -296,14 +296,13 @@ export async function downloadAndExtractFrame(
|
|||||||
imageMap: Map<string, MessageImagePart[]>,
|
imageMap: Map<string, MessageImagePart[]>,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const log = createChildLogger("mediaAnalysis");
|
const log = createChildLogger("mediaAnalysis");
|
||||||
// Prefer the upload proxy (uploaded_url); the Discord CDN link can expire
|
// Analysis now uses the Discord CDN URL directly (uploaded_url is archive-only).
|
||||||
// or be purged (404), and a non-OK response used to silently drop the image
|
// Try discord_url first; fall back to uploaded_url (Tele proxy) if the CDN
|
||||||
// from vision analysis (no log, empty image map → text-only verdict). Try
|
// link returns a non-OK response (expired/purged).
|
||||||
// each candidate URL in order and surface failures.
|
|
||||||
const urlCandidates = [
|
const urlCandidates = [
|
||||||
att.uploaded_url,
|
att.discord_url,
|
||||||
att.discord_url && att.discord_url !== att.uploaded_url
|
att.uploaded_url && att.uploaded_url !== att.discord_url
|
||||||
? att.discord_url
|
? att.uploaded_url
|
||||||
: null,
|
: null,
|
||||||
].filter((u): u is string => Boolean(u));
|
].filter((u): u is string => Boolean(u));
|
||||||
if (urlCandidates.length === 0) return;
|
if (urlCandidates.length === 0) return;
|
||||||
|
|||||||
@@ -461,7 +461,9 @@ export async function runModerationAnalysis(
|
|||||||
cacheKey,
|
cacheKey,
|
||||||
stored,
|
stored,
|
||||||
embeddingsByKey.get(cacheKey),
|
embeddingsByKey.get(cacheKey),
|
||||||
).catch(() => {});
|
).catch((err: unknown) => {
|
||||||
|
log.warn({ cacheKey, error: String(err) }, "Cache write failed");
|
||||||
|
});
|
||||||
|
|
||||||
// Dual-key write-back (2026-08-24): the FIRST analysis of a message runs
|
// Dual-key write-back (2026-08-24): the FIRST analysis of a message runs
|
||||||
// WITH conversation context (accurate), but its verdict is also stored
|
// WITH conversation context (accurate), but its verdict is also stored
|
||||||
|
|||||||
@@ -69,6 +69,26 @@ export async function buildCorrectedFewShotExamples(): Promise<string> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Few-shot correction cache (refreshes hourly)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
let _correctedExamplesCache: string | null = null;
|
||||||
|
let _correctedExamplesCacheAt = 0;
|
||||||
|
const CORRECTED_CACHE_TTL_MS = 60 * 60 * 1000;
|
||||||
|
|
||||||
|
async function getCachedCorrectedExamples(): Promise<string> {
|
||||||
|
const now = Date.now();
|
||||||
|
if (
|
||||||
|
_correctedExamplesCache !== null &&
|
||||||
|
now - _correctedExamplesCacheAt < CORRECTED_CACHE_TTL_MS
|
||||||
|
) {
|
||||||
|
return _correctedExamplesCache;
|
||||||
|
}
|
||||||
|
_correctedExamplesCache = await buildCorrectedFewShotExamples();
|
||||||
|
_correctedExamplesCacheAt = now;
|
||||||
|
return _correctedExamplesCache;
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Text-only batch
|
// Text-only batch
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -88,7 +108,22 @@ export async function runTextOnlyBatch(
|
|||||||
for (const url of extractUrlsFromText(msg.edited_content ?? msg.content))
|
for (const url of extractUrlsFromText(msg.edited_content ?? msg.content))
|
||||||
allUrls.add(url);
|
allUrls.add(url);
|
||||||
}
|
}
|
||||||
const urlArr = Array.from(allUrls).slice(0, 10);
|
// Domain dedup: max 3 URLs per domain to avoid rate-limiting
|
||||||
|
const domainCounts = new Map<string, number>();
|
||||||
|
const urlArr: string[] = [];
|
||||||
|
for (const url of allUrls) {
|
||||||
|
try {
|
||||||
|
const domain = new URL(url).hostname;
|
||||||
|
const count = domainCounts.get(domain) ?? 0;
|
||||||
|
if (count >= 3) continue;
|
||||||
|
domainCounts.set(domain, count + 1);
|
||||||
|
} catch {
|
||||||
|
/* invalid URL, skip */
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
urlArr.push(url);
|
||||||
|
if (urlArr.length >= 10) break;
|
||||||
|
}
|
||||||
if (urlArr.length === 0) {
|
if (urlArr.length === 0) {
|
||||||
return {
|
return {
|
||||||
text: new Map<string, string>(),
|
text: new Map<string, string>(),
|
||||||
@@ -192,7 +227,7 @@ export async function runTextOnlyBatch(
|
|||||||
// Corrected false-positive examples are static per batch — fetch ONCE
|
// Corrected false-positive examples are static per batch — fetch ONCE
|
||||||
// here instead of inside the per-sub-batch retry closure (which would
|
// here instead of inside the per-sub-batch retry closure (which would
|
||||||
// re-query the DB on every sub-batch and every parse-error retry).
|
// re-query the DB on every sub-batch and every parse-error retry).
|
||||||
const correctedExamples = await buildCorrectedFewShotExamples();
|
const correctedExamples = await getCachedCorrectedExamples();
|
||||||
|
|
||||||
// ── URL images → multimodal vision evidence (hoisted out of the sub-batch
|
// ── URL images → multimodal vision evidence (hoisted out of the sub-batch
|
||||||
// loop) ───────────────────────────────────────────────────────────
|
// loop) ───────────────────────────────────────────────────────────
|
||||||
@@ -368,9 +403,13 @@ export async function runTextOnlyBatch(
|
|||||||
);
|
);
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
if (err.name === "AbortError" || abortController.signal.aborted) {
|
if (err.name === "AbortError" || abortController.signal.aborted) {
|
||||||
throw new Error(
|
// Sub-batch timed out — log but DO NOT throw. Previous sub-batches'
|
||||||
`Text-only batch sub-batch ${i + 1} timed out for messages ${targetIds.join(", ")}`,
|
// results are already in allResults; throwing would discard them.
|
||||||
|
log.warn(
|
||||||
|
{ subBatch: i + 1, targetIds, timeoutMs },
|
||||||
|
"Sub-batch timed out — preserving partial results from prior sub-batches",
|
||||||
);
|
);
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
throw err;
|
throw err;
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -103,6 +103,8 @@ export class ModerationHandler {
|
|||||||
| "kick_user"
|
| "kick_user"
|
||||||
| "ban_user",
|
| "ban_user",
|
||||||
reason: payload.reason ?? null,
|
reason: payload.reason ?? null,
|
||||||
|
username: null,
|
||||||
|
server_nick: null,
|
||||||
executed_by: payload.executed_by ?? "command-handler",
|
executed_by: payload.executed_by ?? "command-handler",
|
||||||
status: "executed",
|
status: "executed",
|
||||||
error: null,
|
error: null,
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import type { Client } from "discord.js-selfbot-v13";
|
import type { Client } from "discord.js-selfbot-v13";
|
||||||
|
import Redis from "ioredis";
|
||||||
import { config } from "../../shared/config/config.js";
|
import { config } from "../../shared/config/config.js";
|
||||||
import type { CommandMessage, CommandReply } from "../../shared/index.js";
|
import type { CommandMessage, CommandReply } from "../../shared/index.js";
|
||||||
import { createChildLogger } from "../../shared/logger/index.js";
|
import { createChildLogger } from "../../shared/logger/index.js";
|
||||||
@@ -152,13 +153,28 @@ export class VoiceHandler {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Double-check: verify the voice controller also reports connected
|
||||||
|
if (this.voiceController) {
|
||||||
|
const vcStatus = this.voiceController.getStatus();
|
||||||
|
if (!vcStatus.connected) {
|
||||||
|
this.logger.warn(
|
||||||
|
"Player reports connected but voice controller says disconnected — stale state",
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
id: cmd.id,
|
||||||
|
success: false,
|
||||||
|
data: null,
|
||||||
|
error: "Voice channel connection is stale — reconnect first",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// IMPORTANT: transmitter needs its OWN Redis client because it calls
|
// IMPORTANT: transmitter needs its OWN Redis client because it calls
|
||||||
// .subscribe() which converts the connection to subscriber mode. Reusing
|
// .subscribe() which converts the connection to subscriber mode. Reusing
|
||||||
// the publish connection from CommandHandler would corrupt it and break
|
// the publish connection from CommandHandler would corrupt it and break
|
||||||
// every command reply + status update.
|
// every command reply + status update.
|
||||||
const { default: IORedis } = await import("ioredis");
|
const transmitRedis = new Redis(config.REDIS_URL);
|
||||||
const transmitRedis = new IORedis(config.REDIS_URL);
|
|
||||||
await voiceTransmitter.start(transmitRedis);
|
await voiceTransmitter.start(transmitRedis);
|
||||||
|
|
||||||
const status = voiceTransmitter.getStatus();
|
const status = voiceTransmitter.getStatus();
|
||||||
|
|||||||
@@ -50,6 +50,22 @@ export function finalizeSegment(input: SegmentFinalizerInput): void {
|
|||||||
} = input;
|
} = input;
|
||||||
|
|
||||||
const endTime = currentSegment.endTime ?? Date.now();
|
const endTime = currentSegment.endTime ?? Date.now();
|
||||||
|
const durationMs = endTime - currentSegment.startTime;
|
||||||
|
|
||||||
|
// Discard only very short segments — a blip of ambient noise or a mic
|
||||||
|
// click. At 300ms, brief acknowledgements ("ya", "siap", "ok") are still
|
||||||
|
// kept while true artifacts are dropped.
|
||||||
|
const MIN_DURATION_MS = 300;
|
||||||
|
if (durationMs < MIN_DURATION_MS) {
|
||||||
|
logger.debug(
|
||||||
|
{ filename: currentSegment.filename, durationMs },
|
||||||
|
"Segment too short, discarding",
|
||||||
|
);
|
||||||
|
// Clean up the OGG file
|
||||||
|
fsPromises.unlink(currentSegment.filename).catch(() => {});
|
||||||
|
fsPromises.unlink(currentSegment.jsonFilename).catch(() => {});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (config.VERBOSE) {
|
if (config.VERBOSE) {
|
||||||
logger.info({ filename: currentSegment.filename }, "Segment saved");
|
logger.info({ filename: currentSegment.filename }, "Segment saved");
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ import type { VoiceConnection } from "@discordjs/voice";
|
|||||||
import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
|
import type { Client, VoiceChannel } from "discord.js-selfbot-v13";
|
||||||
import { createChildLogger } from "@/shared/logger/index";
|
import { createChildLogger } from "@/shared/logger/index";
|
||||||
import type { EventBroadcaster } from "../../event-broadcaster/eventBroadcaster.js";
|
import type { EventBroadcaster } from "../../event-broadcaster/eventBroadcaster.js";
|
||||||
|
import type {
|
||||||
|
SegmentState,
|
||||||
|
UserMetadata,
|
||||||
|
} from "../../message-capture/types.js";
|
||||||
import { collectUserMetadata } from "./metadata.js";
|
import { collectUserMetadata } from "./metadata.js";
|
||||||
import { finalizeSegment } from "./segmentFinalizer.js";
|
import { finalizeSegment } from "./segmentFinalizer.js";
|
||||||
import type { RecordingSession } from "./sessionRecording.js";
|
import type { RecordingSession } from "./sessionRecording.js";
|
||||||
@@ -25,13 +29,21 @@ export interface SpeakingHandlerContext {
|
|||||||
/**
|
/**
|
||||||
* Creates the event handler for `receiver.speaking.on("start", handler)`.
|
* Creates the event handler for `receiver.speaking.on("start", handler)`.
|
||||||
*
|
*
|
||||||
* The returned handler manages the full lifecycle for a user who starts speaking:
|
* # Why this is structured the way it is (recording-completeness)
|
||||||
* 1. Validates the user (skip bot self, skip already-subscribed)
|
*
|
||||||
* 2. Collects user metadata and notifies the event broadcaster
|
* `receiver.speaking"start"` fires from `onUdpMessage` the moment the FIRST
|
||||||
* 3. Sets up the audio stream, decoder, packet filter, and segment manager
|
* Opus packet for a user arrives, and the opus bytes are forwarded to the
|
||||||
* 4. Attaches stream event handlers (data, end, error) BEFORE piping
|
* subscription stream **only if a subscription already exists** — otherwise
|
||||||
* 5. Pipes audio through the packet filter for OGG recording
|
* they are dropped (`subscriptions.get(userId)` is undefined → `return`).
|
||||||
* 6. Handles segment completion (metadata write, upload trigger)
|
*
|
||||||
|
* That means the subscription MUST be created synchronously (no `await` before
|
||||||
|
* it): every frame between "start" and the subscribe call is otherwise silently
|
||||||
|
* discarded → the START of a burst is MISSING (the user's core complaint).
|
||||||
|
* The previous code `await`ed `collectUserMetadata` (a Discord REST roundtrip,
|
||||||
|
* slow on cache miss) BEFORE subscribing — that was the main source of misses.
|
||||||
|
*
|
||||||
|
* Metadata is now fetched in the background. If the speaker turns out to be a
|
||||||
|
* bot, the just-started burst is discarded (unlinked, never uploaded).
|
||||||
*/
|
*/
|
||||||
export function createSpeakingHandler(
|
export function createSpeakingHandler(
|
||||||
ctx: SpeakingHandlerContext,
|
ctx: SpeakingHandlerContext,
|
||||||
@@ -47,110 +59,176 @@ export function createSpeakingHandler(
|
|||||||
} = ctx;
|
} = ctx;
|
||||||
|
|
||||||
return async (userId: string) => {
|
return async (userId: string) => {
|
||||||
// Skip the bot's own audio
|
// Skip the bot's own audio.
|
||||||
if (userId === client.user?.id) return;
|
if (userId === client.user?.id) return;
|
||||||
|
|
||||||
const userMetadata = await collectUserMetadata(client, userId, channel);
|
// Synchronous guard BEFORE any await. Two "start" events can race during a
|
||||||
if (userMetadata.bot) return;
|
// subscription; guard here so we never create a second subscription.
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
{ userId, username: userMetadata.username },
|
|
||||||
"Voice activity detected",
|
|
||||||
);
|
|
||||||
|
|
||||||
// Skip if user already has an active stream subscription
|
|
||||||
// (check BEFORE broadcast to avoid false positive events)
|
|
||||||
if (receiver.subscriptions.has(userId)) return;
|
if (receiver.subscriptions.has(userId)) return;
|
||||||
|
|
||||||
// Notify webserver / WebSocket clients
|
logger.debug({ userId }, "Voice activity detected");
|
||||||
|
|
||||||
|
// Optimistic speaking:true (real metadata arrives async).
|
||||||
eventBroadcaster?.voiceActiveUser(userId, {
|
eventBroadcaster?.voiceActiveUser(userId, {
|
||||||
username: userMetadata.username,
|
username: userId,
|
||||||
avatar: userMetadata.avatarUrl,
|
avatar: "",
|
||||||
speaking: true,
|
speaking: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
// Ensure per-user recording directory
|
// Subscribe IMMEDIATELY (synchronously — guard is already done above, no
|
||||||
|
// `await` since then), so the subscription exists before any Opus frame
|
||||||
|
// arrives. Frames are then buffered by the AudioReceiveStream while we
|
||||||
|
// finish setup below (mkdir is non-network and fast).
|
||||||
const userDir = path.join(recordingsDir, userId);
|
const userDir = path.join(recordingsDir, userId);
|
||||||
|
const { audioStream, packetFilter, segmentManager, decoder } =
|
||||||
|
setupUserStream({
|
||||||
|
userId,
|
||||||
|
receiver,
|
||||||
|
userDir,
|
||||||
|
onPcmData: (pcm) => {
|
||||||
|
if (pcmSender) {
|
||||||
|
pcmSender(pcm, userId);
|
||||||
|
} else {
|
||||||
|
eventBroadcaster?.voicePcmData(pcm, userId);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Ensure per-user recording directory (subscription is already live, so
|
||||||
|
// this await does NOT drop audio — frames buffer in the subscription).
|
||||||
await fsPromises.mkdir(userDir, { recursive: true }).catch(() => {
|
await fsPromises.mkdir(userDir, { recursive: true }).catch(() => {
|
||||||
// Directory already exists, ignore
|
// Directory already exists, ignore.
|
||||||
});
|
});
|
||||||
|
|
||||||
try {
|
const activeSession = activeSessions.get(channel.guild.id);
|
||||||
// Step 1: Set up stream components (subscribe, decoder, filter, segment
|
|
||||||
// manager). NOTE: pipe() is NOT called here — we attach event handlers
|
// Mutable per-burst state.
|
||||||
// first to prevent data loss from race conditions.
|
let userMetadata: UserMetadata | null = null;
|
||||||
const { audioStream, packetFilter, segmentManager, decoder } =
|
let accepted = true; // false once we learn it's a bot → discard burst
|
||||||
setupUserStream({
|
let finalized = false;
|
||||||
userId,
|
|
||||||
receiver,
|
/** Unlink the given segment's files (bot/error discard path). */
|
||||||
userDir,
|
const discardSegmentFiles = (seg: SegmentState | null): void => {
|
||||||
onPcmData: (pcm) => {
|
if (!seg) return;
|
||||||
if (pcmSender) {
|
fsPromises.unlink(seg.filename).catch(() => {});
|
||||||
pcmSender(pcm, userId);
|
fsPromises.unlink(seg.jsonFilename).catch(() => {});
|
||||||
} else {
|
};
|
||||||
eventBroadcaster?.voicePcmData(pcm, userId);
|
|
||||||
|
const emitSpeakingFalse = (): void => {
|
||||||
|
eventBroadcaster?.voiceActiveUser(userId, {
|
||||||
|
username: userMetadata?.username ?? userId,
|
||||||
|
avatar: userMetadata?.avatarUrl ?? "",
|
||||||
|
speaking: false,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Close the current (per-burst) segment and — once the underlying file has
|
||||||
|
* finished flushing to disk — finalize it (or discard it if bot/error).
|
||||||
|
* Safe to call multiple times (guarded by `finalized`).
|
||||||
|
*/
|
||||||
|
const finishBurst = (): void => {
|
||||||
|
if (finalized) return;
|
||||||
|
finalized = true;
|
||||||
|
const seg = segmentManager.close(oggPacketStream);
|
||||||
|
decoder.destroy();
|
||||||
|
|
||||||
|
const doFinalize = (): void => {
|
||||||
|
Promise.resolve(userMetadata).then((meta) => {
|
||||||
|
if (meta && accepted && seg) {
|
||||||
|
try {
|
||||||
|
finalizeSegment({
|
||||||
|
currentSegment: seg,
|
||||||
|
userMetadata: meta,
|
||||||
|
activeSession,
|
||||||
|
guildId: channel.guild.id,
|
||||||
|
channelId: channel.id,
|
||||||
|
channelName: channel.name,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
logger.error({ userId, error: msg }, "Segment finalize failed");
|
||||||
}
|
}
|
||||||
},
|
} else {
|
||||||
|
// Bot audio, metadata failure, or no segment opened — discard.
|
||||||
|
discardSegmentFiles(seg);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
};
|
||||||
|
|
||||||
// Step 2: Attach all audioStream event handlers BEFORE pipe()
|
// finalizeSegment reads the OGG file, so wait for the write stream to
|
||||||
audioStream.on("data", (chunk: Buffer) => {
|
// flush before touching it.
|
||||||
if (chunk.length < 8) return;
|
if (seg?.out.writableFinished) {
|
||||||
segmentManager.rotateIfNeeded(packetFilter);
|
doFinalize();
|
||||||
decoder.rotateIfNeeded();
|
} else {
|
||||||
decoder.write(chunk);
|
seg?.out.once("finish", doFinalize);
|
||||||
});
|
}
|
||||||
|
};
|
||||||
|
|
||||||
audioStream.on("end", () => {
|
// Attach audioStream handlers BEFORE pipe() (prevents data loss).
|
||||||
segmentManager.close(packetFilter);
|
audioStream.on("data", (chunk: Buffer) => {
|
||||||
decoder.destroy();
|
if (chunk.length < 8) return;
|
||||||
|
// One segment per burst — no time-based rotation here. Only rotate the
|
||||||
|
// web-PCM broadcast decoder to bound its memory.
|
||||||
|
decoder.rotateIfNeeded();
|
||||||
|
decoder.write(chunk);
|
||||||
|
});
|
||||||
|
|
||||||
|
audioStream.on("end", () => {
|
||||||
|
finishBurst();
|
||||||
|
emitSpeakingFalse();
|
||||||
|
});
|
||||||
|
|
||||||
|
audioStream.on("error", (error: Error) => {
|
||||||
|
logger.error({ userId, error: error.message }, "Audio stream error");
|
||||||
|
finishBurst();
|
||||||
|
emitSpeakingFalse();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Pipe for OGG recording (handlers already attached).
|
||||||
|
const oggPacketStream = audioStream.pipe(packetFilter);
|
||||||
|
|
||||||
|
// Open the (single, per-burst) segment.
|
||||||
|
const currentSegment = segmentManager.open(oggPacketStream);
|
||||||
|
|
||||||
|
// Handle file-write errors on the underlying write stream.
|
||||||
|
currentSegment.out.on("error", (err: unknown) => {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
logger.error({ userId, error: msg }, "File write error");
|
||||||
|
});
|
||||||
|
|
||||||
|
// Handle packet-filter errors (rare) by closing the burst.
|
||||||
|
packetFilter.on("error", (err: Error) => {
|
||||||
|
logger.error({ userId, error: err.message }, "PacketFilter error");
|
||||||
|
finishBurst();
|
||||||
|
emitSpeakingFalse();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Fetch user metadata in the background; reject bots by discarding burst.
|
||||||
|
collectUserMetadata(client, userId, channel)
|
||||||
|
.then((meta) => {
|
||||||
|
if (meta.bot) {
|
||||||
|
accepted = false;
|
||||||
|
finishBurst();
|
||||||
|
emitSpeakingFalse();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
userMetadata = meta;
|
||||||
eventBroadcaster?.voiceActiveUser(userId, {
|
eventBroadcaster?.voiceActiveUser(userId, {
|
||||||
username: userMetadata.username,
|
username: meta.username,
|
||||||
avatar: userMetadata.avatarUrl,
|
avatar: meta.avatarUrl,
|
||||||
speaking: false,
|
speaking: true,
|
||||||
});
|
});
|
||||||
});
|
})
|
||||||
|
.catch((err: unknown) => {
|
||||||
audioStream.on("error", (error: Error) => {
|
|
||||||
segmentManager.close(packetFilter);
|
|
||||||
decoder.destroy();
|
|
||||||
logger.error({ userId, error: error.message }, "Audio stream error");
|
|
||||||
});
|
|
||||||
|
|
||||||
// Step 3: Now pipe for OGG recording (safe — event handlers attached)
|
|
||||||
const oggPacketStream = audioStream.pipe(packetFilter);
|
|
||||||
|
|
||||||
// Step 4: Open the first segment
|
|
||||||
const activeSession = activeSessions.get(channel.guild.id);
|
|
||||||
const currentSegment = segmentManager.open(oggPacketStream);
|
|
||||||
|
|
||||||
// Step 5: Handle segment file completion
|
|
||||||
currentSegment.out.on("finish", () => {
|
|
||||||
finalizeSegment({
|
|
||||||
currentSegment,
|
|
||||||
userMetadata,
|
|
||||||
activeSession,
|
|
||||||
guildId: channel.guild.id,
|
|
||||||
channelId: channel.id,
|
|
||||||
channelName: channel.name,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
currentSegment.out.on("error", (err: unknown) => {
|
|
||||||
const msg = err instanceof Error ? err.message : String(err);
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
logger.error({ userId, error: msg }, "File write error");
|
logger.warn(
|
||||||
|
{ userId, error: msg },
|
||||||
|
"Metadata fetch failed, dropping burst",
|
||||||
|
);
|
||||||
|
accepted = false;
|
||||||
|
finishBurst();
|
||||||
|
emitSpeakingFalse();
|
||||||
});
|
});
|
||||||
|
|
||||||
// Step 6: Handle packet filter errors
|
|
||||||
packetFilter.on("error", (err) => {
|
|
||||||
segmentManager.close(oggPacketStream);
|
|
||||||
logger.error({ userId, error: err.message }, "PacketFilter error");
|
|
||||||
});
|
|
||||||
} catch (e) {
|
|
||||||
logger.error(
|
|
||||||
{ userId, error: e instanceof Error ? e.message : String(e) },
|
|
||||||
"Failed to create stream",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -23,10 +23,18 @@ export class VoiceTransmitter {
|
|||||||
private readonly TRANSMIT_CHANNEL = BACKEND_VOICE_TRANSMIT;
|
private readonly TRANSMIT_CHANNEL = BACKEND_VOICE_TRANSMIT;
|
||||||
/** Queue for PCM chunks when backpressure is active */
|
/** Queue for PCM chunks when backpressure is active */
|
||||||
private backpressureQueue: Buffer[] = [];
|
private backpressureQueue: Buffer[] = [];
|
||||||
|
/** Max queued chunks before dropping oldest (prevents unbounded memory growth) */
|
||||||
|
private static readonly MAX_QUEUE = 500;
|
||||||
/** Serialise start/stop to prevent races between rapid toggle commands */
|
/** Serialise start/stop to prevent races between rapid toggle commands */
|
||||||
private gate = Promise.resolve();
|
private gate = Promise.resolve();
|
||||||
/** Set true before sending SIGTERM so exit handler knows it's intentional */
|
/** Set true before sending SIGTERM so exit handler knows it's intentional */
|
||||||
private _expectedExit = false;
|
private _expectedExit = false;
|
||||||
|
/** Auto-stop timer: if no PCM received for this long, stop transmitter */
|
||||||
|
private _activityTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
|
/** How long to wait without PCM before auto-stopping (10s) */
|
||||||
|
private static readonly ACTIVITY_TIMEOUT_MS = 10_000;
|
||||||
|
/** Max stderr bytes to retain for error reporting */
|
||||||
|
private static readonly MAX_STDERR_BYTES = 4096;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Start listening for PCM audio data from Redis and stream to Discord
|
* Start listening for PCM audio data from Redis and stream to Discord
|
||||||
@@ -48,7 +56,6 @@ export class VoiceTransmitter {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.redisSub = redis;
|
|
||||||
this.isActive = true;
|
this.isActive = true;
|
||||||
|
|
||||||
// Create PCM input stream
|
// Create PCM input stream
|
||||||
@@ -115,10 +122,14 @@ export class VoiceTransmitter {
|
|||||||
this.pcmStream.pipe(this.ffmpegProcess.stdin);
|
this.pcmStream.pipe(this.ffmpegProcess.stdin);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log FFmpeg stderr for debugging
|
// Log FFmpeg stderr for debugging (cap to prevent unbounded growth)
|
||||||
const stderrChunks: Buffer[] = [];
|
const stderrChunks: Buffer[] = [];
|
||||||
|
let stderrLen = 0;
|
||||||
this.ffmpegProcess.stderr?.on("data", (chunk: Buffer) => {
|
this.ffmpegProcess.stderr?.on("data", (chunk: Buffer) => {
|
||||||
stderrChunks.push(chunk);
|
stderrLen += chunk.length;
|
||||||
|
if (stderrLen <= VoiceTransmitter.MAX_STDERR_BYTES) {
|
||||||
|
stderrChunks.push(chunk);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
this.ffmpegProcess.on("error", (err) => {
|
this.ffmpegProcess.on("error", (err) => {
|
||||||
@@ -135,8 +146,11 @@ export class VoiceTransmitter {
|
|||||||
const stderr = Buffer.concat(stderrChunks).toString();
|
const stderr = Buffer.concat(stderrChunks).toString();
|
||||||
logger.error(
|
logger.error(
|
||||||
{ code, stderr: stderr.slice(-500) },
|
{ code, stderr: stderr.slice(-500) },
|
||||||
"FFmpeg exited with error",
|
"FFmpeg exited with error — auto-stopping transmitter",
|
||||||
);
|
);
|
||||||
|
// FFmpeg crashed — auto-stop to prevent silent audio loss.
|
||||||
|
// Fire-and-forget; stop() serialises with the gate.
|
||||||
|
void this.stop();
|
||||||
} else {
|
} else {
|
||||||
logger.debug({ code }, "FFmpeg process exited");
|
logger.debug({ code }, "FFmpeg process exited");
|
||||||
}
|
}
|
||||||
@@ -155,11 +169,15 @@ export class VoiceTransmitter {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Subscribe to Redis channel for PCM data
|
// Subscribe to Redis channel for PCM data
|
||||||
await this.redisSub.subscribe(this.TRANSMIT_CHANNEL);
|
await redis.subscribe(this.TRANSMIT_CHANNEL);
|
||||||
|
// Assign AFTER subscription succeeds — prevents race with stop()
|
||||||
|
this.redisSub = redis;
|
||||||
logger.info(
|
logger.info(
|
||||||
{ channel: this.TRANSMIT_CHANNEL },
|
{ channel: this.TRANSMIT_CHANNEL },
|
||||||
"Subscribed to transmit channel",
|
"Subscribed to transmit channel",
|
||||||
);
|
);
|
||||||
|
// Start activity timer — will auto-stop if no PCM arrives within timeout
|
||||||
|
this.resetActivityTimer();
|
||||||
|
|
||||||
this.redisSub.on("message", (channel, message) => {
|
this.redisSub.on("message", (channel, message) => {
|
||||||
if (
|
if (
|
||||||
@@ -175,8 +193,21 @@ export class VoiceTransmitter {
|
|||||||
const pcmBuffer = Buffer.from(data.buffer, "base64");
|
const pcmBuffer = Buffer.from(data.buffer, "base64");
|
||||||
const stream = this.pcmStream;
|
const stream = this.pcmStream;
|
||||||
const canContinue = stream.write(pcmBuffer);
|
const canContinue = stream.write(pcmBuffer);
|
||||||
// Backpressure: queue until drain
|
// Reset activity timer — PCM is flowing
|
||||||
|
this.resetActivityTimer();
|
||||||
|
// Backpressure: queue until drain (cap to prevent memory leak)
|
||||||
if (!canContinue) {
|
if (!canContinue) {
|
||||||
|
if (this.backpressureQueue.length >= VoiceTransmitter.MAX_QUEUE) {
|
||||||
|
// Drop oldest chunks to free memory — real-time audio,
|
||||||
|
// stale data is useless
|
||||||
|
const dropCount = Math.floor(VoiceTransmitter.MAX_QUEUE * 0.25);
|
||||||
|
this.backpressureQueue.splice(0, dropCount);
|
||||||
|
logger.debug(
|
||||||
|
{ dropped: dropCount },
|
||||||
|
"Backpressure overflow — dropping oldest PCM chunks",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
this.backpressureQueue.push(pcmBuffer);
|
||||||
stream.once("drain", () => {
|
stream.once("drain", () => {
|
||||||
const currentStream = this.pcmStream;
|
const currentStream = this.pcmStream;
|
||||||
if (!currentStream || !this.isActive) return;
|
if (!currentStream || !this.isActive) return;
|
||||||
@@ -210,6 +241,30 @@ export class VoiceTransmitter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reset the voice activity timer. Called on every PCM chunk received.
|
||||||
|
* If no chunks arrive for ACTIVITY_TIMEOUT_MS, auto-stop the transmitter
|
||||||
|
* to prevent dead air and wasted resources.
|
||||||
|
*/
|
||||||
|
private resetActivityTimer(): void {
|
||||||
|
this.clearActivityTimer();
|
||||||
|
this._activityTimer = setTimeout(() => {
|
||||||
|
if (!this.isActive) return;
|
||||||
|
logger.warn(
|
||||||
|
{ timeoutMs: VoiceTransmitter.ACTIVITY_TIMEOUT_MS },
|
||||||
|
"Voice activity timeout — no PCM received, auto-stopping transmitter",
|
||||||
|
);
|
||||||
|
void this.stop();
|
||||||
|
}, VoiceTransmitter.ACTIVITY_TIMEOUT_MS);
|
||||||
|
}
|
||||||
|
|
||||||
|
private clearActivityTimer(): void {
|
||||||
|
if (this._activityTimer !== null) {
|
||||||
|
clearTimeout(this._activityTimer);
|
||||||
|
this._activityTimer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Stop transmitting and clean up resources
|
* Stop transmitting and clean up resources
|
||||||
*/
|
*/
|
||||||
@@ -232,6 +287,7 @@ export class VoiceTransmitter {
|
|||||||
|
|
||||||
this.isActive = false;
|
this.isActive = false;
|
||||||
|
|
||||||
|
this.clearActivityTimer();
|
||||||
this.backpressureQueue = [];
|
this.backpressureQueue = [];
|
||||||
|
|
||||||
if (this.pcmStream) {
|
if (this.pcmStream) {
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ export async function transcribeRecording(
|
|||||||
const response = await openai.audio.transcriptions.create({
|
const response = await openai.audio.transcriptions.create({
|
||||||
file: createReadStream(oggPath),
|
file: createReadStream(oggPath),
|
||||||
model: "whisper-1",
|
model: "whisper-1",
|
||||||
language: "en",
|
// No `language` param → Whisper auto-detects (handles id/en mixed).
|
||||||
response_format: "text",
|
response_format: "text",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -53,10 +53,15 @@ export const configSchema = z
|
|||||||
DECODER_COOLDOWN_MS: z.coerce.number().positive().default(30000),
|
DECODER_COOLDOWN_MS: z.coerce.number().positive().default(30000),
|
||||||
|
|
||||||
// ── Audio ────────────────────────────────────────────────────────────
|
// ── Audio ────────────────────────────────────────────────────────────
|
||||||
|
// AfterSilence: how long a voice burst may stay silent before the receive
|
||||||
|
// stream ends the segment. Raised 3000→4000 so natural pauses in speech
|
||||||
|
// (thinking gaps, interruptions) don't split one utterance into multiple
|
||||||
|
// segments ("terpotong"). Tunable via env; larger = fewer splits but a
|
||||||
|
// longer silent tail on each recording.
|
||||||
AUDIO_STREAM_SILENCE_DURATION_MS: z.coerce
|
AUDIO_STREAM_SILENCE_DURATION_MS: z.coerce
|
||||||
.number()
|
.number()
|
||||||
.positive()
|
.positive()
|
||||||
.default(3000),
|
.default(4000),
|
||||||
PACKET_FILTER_MIN_SIZE: z.coerce.number().positive().default(8),
|
PACKET_FILTER_MIN_SIZE: z.coerce.number().positive().default(8),
|
||||||
OPUS_FRAME_SIZE: z.coerce.number().positive().default(960),
|
OPUS_FRAME_SIZE: z.coerce.number().positive().default(960),
|
||||||
AUDIO_SAMPLE_RATE: z.coerce.number().positive().default(48000),
|
AUDIO_SAMPLE_RATE: z.coerce.number().positive().default(48000),
|
||||||
@@ -149,8 +154,8 @@ export const configSchema = z
|
|||||||
AI_LLM_BASE_URL: z
|
AI_LLM_BASE_URL: z
|
||||||
.string()
|
.string()
|
||||||
.url()
|
.url()
|
||||||
.default("https://9router.asepharyana.my.id/v1"),
|
.default("http://100.121.180.82:20128/api/v1"),
|
||||||
AI_LLM_MODEL: z.string().default("claude-opus-5"),
|
AI_LLM_MODEL: z.string().default("text"),
|
||||||
// Vision uses the SAME router/base URL as text moderation
|
// Vision uses the SAME router/base URL as text moderation
|
||||||
// (AI_LLM_BASE_URL) but a different model alias. The dedicated NVIDIA
|
// (AI_LLM_BASE_URL) but a different model alias. The dedicated NVIDIA
|
||||||
// multimodal endpoint was removed.
|
// multimodal endpoint was removed.
|
||||||
@@ -197,6 +202,11 @@ export const configSchema = z
|
|||||||
.positive()
|
.positive()
|
||||||
.default(1024),
|
.default(1024),
|
||||||
AI_LLM_TEXT_BATCH_SIZE: z.coerce.number().int().positive().default(60),
|
AI_LLM_TEXT_BATCH_SIZE: z.coerce.number().int().positive().default(60),
|
||||||
|
AI_LLM_MAX_COMPLETION_TOKENS: z.coerce
|
||||||
|
.number()
|
||||||
|
.int()
|
||||||
|
.positive()
|
||||||
|
.default(16384),
|
||||||
AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS: z.coerce
|
AI_LLM_MEDIA_ANALYSIS_TIMEOUT_MS: z.coerce
|
||||||
.number()
|
.number()
|
||||||
.int()
|
.int()
|
||||||
|
|||||||
@@ -56,25 +56,56 @@ async function getFirstMigrationTag(): Promise<string> {
|
|||||||
return first.tag;
|
return first.tag;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read the Drizzle migration journal and return the `when` (folderMillis)
|
||||||
|
* timestamp of the LAST migration entry. Drizzle's PG migrator applies a
|
||||||
|
* migration only when `tracked_max_created_at < migration.when`, so seeding
|
||||||
|
* the tracking table with this value marks every existing journal migration
|
||||||
|
* as already applied.
|
||||||
|
*/
|
||||||
|
async function getLastMigrationWhen(): Promise<number> {
|
||||||
|
const journalPath = join(
|
||||||
|
process.cwd(),
|
||||||
|
"drizzle/migrations/meta/_journal.json",
|
||||||
|
);
|
||||||
|
const raw = await readFile(journalPath, "utf-8");
|
||||||
|
const journal: MigrationJournal = JSON.parse(raw);
|
||||||
|
|
||||||
|
if (!journal.entries || journal.entries.length === 0) {
|
||||||
|
throw new Error(
|
||||||
|
"Migration journal is empty — cannot determine last migration timestamp",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const last = journal.entries.reduce((newest, entry) =>
|
||||||
|
entry.when > newest.when ? entry : newest,
|
||||||
|
);
|
||||||
|
return last.when;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Seed Drizzle's __drizzle_migrations tracking table for pre-existing databases
|
* Seed Drizzle's __drizzle_migrations tracking table for pre-existing databases
|
||||||
* that were created manually or by an earlier migration system (e.g., the old
|
* that were created manually or by an earlier migration system (e.g., the old
|
||||||
* checkSchemaExists short-circuit). Without this, Drizzle attempts to re-create
|
* checkSchemaExists short-circuit). Without this, Drizzle attempts to re-create
|
||||||
* all tables from 0000 and fails with "relation already exists".
|
* all tables from 0000 and fails with "relation already exists".
|
||||||
|
*
|
||||||
|
* The Drizzle PG migrator (drizzle-orm 0.45.x) only reads the LAST tracked
|
||||||
|
* migration (`SELECT ... ORDER BY created_at DESC LIMIT 1` from
|
||||||
|
* <migrationsSchema>.__drizzle_migrations) and re-applies every journal
|
||||||
|
* migration whose `when` is newer than that value. If the tracking table is
|
||||||
|
* empty or missing the latest entries while the DB schema is already up to
|
||||||
|
* date, re-running the migrations crashes with "column/relation already
|
||||||
|
* exists" — putting the gateway into an infinite restart loop.
|
||||||
|
*
|
||||||
|
* So instead of early-returning when the table exists (which left it empty /
|
||||||
|
* partial and caused this exact crash), we RECONCILE: when the DB is a
|
||||||
|
* pre-existing one (app tables already present) and the schema already
|
||||||
|
* reflects the latest migration, we seed the tracking table up to the latest
|
||||||
|
* journal `when` so Drizzle skips everything instead of re-applying it.
|
||||||
*/
|
*/
|
||||||
async function seedDrizzleHistory(client: PoolClient): Promise<void> {
|
async function seedDrizzleHistory(client: PoolClient): Promise<void> {
|
||||||
const exists = await client.query(`
|
// Check whether the app tables pre-exist (old ./migrations/ SQL or manual
|
||||||
SELECT EXISTS (
|
// creation). If not, this is a brand-new database — let Drizzle handle it.
|
||||||
SELECT FROM information_schema.tables
|
|
||||||
WHERE table_name = '__drizzle_migrations'
|
|
||||||
)
|
|
||||||
`);
|
|
||||||
const drizzleTableExists = exists.rows[0]?.exists === true;
|
|
||||||
if (drizzleTableExists) {
|
|
||||||
return; // already seeded, nothing to do
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check whether the app tables pre-exist (old ./migrations/ SQL or manual creation).
|
|
||||||
const hasTextCache = await client.query(`
|
const hasTextCache = await client.query(`
|
||||||
SELECT EXISTS (
|
SELECT EXISTS (
|
||||||
SELECT FROM information_schema.columns
|
SELECT FROM information_schema.columns
|
||||||
@@ -85,12 +116,19 @@ async function seedDrizzleHistory(client: PoolClient): Promise<void> {
|
|||||||
return; // brand-new database, let Drizzle handle everything
|
return; // brand-new database, let Drizzle handle everything
|
||||||
}
|
}
|
||||||
|
|
||||||
const firstMigrationTag = await getFirstMigrationTag();
|
// Final schema "at latest migration" sentinel: the moderation_actions
|
||||||
|
// table at 0019 has a server_nick column. If present, the schema is at (or
|
||||||
|
// past) migration 0019, so any journal entries not yet tracked are safe to
|
||||||
|
// mark as applied rather than re-running (which would fail).
|
||||||
|
const atLatest = await client.query(`
|
||||||
|
SELECT EXISTS (
|
||||||
|
SELECT FROM information_schema.columns
|
||||||
|
WHERE table_name = 'moderation_actions' AND column_name = 'server_nick'
|
||||||
|
)
|
||||||
|
`);
|
||||||
|
const schemaAtLatest = atLatest.rows[0]?.exists === true;
|
||||||
|
|
||||||
logger.info(
|
const lastMigrationWhen = await getLastMigrationWhen();
|
||||||
{ firstMigrationTag },
|
|
||||||
"Seeding Drizzle migration history — marking first migration as already applied on this pre-existing database",
|
|
||||||
);
|
|
||||||
|
|
||||||
// Create the Drizzle tracking table if it does not exist.
|
// Create the Drizzle tracking table if it does not exist.
|
||||||
// PG15+ locks down CREATE on the public schema for non-owner roles,
|
// PG15+ locks down CREATE on the public schema for non-owner roles,
|
||||||
@@ -128,21 +166,43 @@ async function seedDrizzleHistory(client: PoolClient): Promise<void> {
|
|||||||
throw createErr;
|
throw createErr;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Drizzle's __drizzle_migrations table has no UNIQUE(hash)
|
|
||||||
// constraint, so check manually before inserting.
|
// Current max tracked created_at. Drizzle only re-applies migrations newer
|
||||||
const alreadySeeded = await client.query(
|
// than this, so if it already reaches the journal's last `when`, nothing to do.
|
||||||
`SELECT 1 FROM "__drizzle_migrations" WHERE hash = $1 LIMIT 1`,
|
const tracked = await client.query(`
|
||||||
[firstMigrationTag],
|
SELECT COALESCE(MAX(created_at), 0) AS max_created FROM "__drizzle_migrations"
|
||||||
);
|
`);
|
||||||
if (alreadySeeded.rows.length === 0) {
|
const trackedMax = Number(tracked.rows[0]?.max_created ?? 0);
|
||||||
await client.query(
|
|
||||||
`INSERT INTO "__drizzle_migrations" (hash, created_at) VALUES ($1, $2)`,
|
if (trackedMax >= lastMigrationWhen) {
|
||||||
[firstMigrationTag, Date.now()],
|
return; // already fully tracked — Drizzle will apply only genuinely-pending ones
|
||||||
);
|
|
||||||
}
|
}
|
||||||
logger.info(
|
|
||||||
{ firstMigrationTag },
|
if (!schemaAtLatest) {
|
||||||
"Drizzle history seeded — first migration marked applied",
|
// Schema is NOT yet at the latest journal migration — this is a genuinely
|
||||||
|
// old DB that needs the pending migrations to run for real. Let Drizzle
|
||||||
|
// apply them. (Idempotent migration SQL guards legacy partial states.)
|
||||||
|
logger.info(
|
||||||
|
{ trackedMax, lastMigrationWhen },
|
||||||
|
"DB schema behind latest journal migration — leaving pending migrations to run",
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Schema already reflects the latest migration but the tracking table is
|
||||||
|
// behind (e.g. migrations applied manually/out-of-band without being
|
||||||
|
// recorded). Seed up to the last journal `when` so Drizzle skips them
|
||||||
|
// instead of re-running already-applied DDL and crashing. This is the
|
||||||
|
// recovery that broke the gateway's infinite-restart loop.
|
||||||
|
const journalTag =
|
||||||
|
(await getFirstMigrationTag()) + `@${lastMigrationWhen}-reconciled`;
|
||||||
|
await client.query(
|
||||||
|
`INSERT INTO "__drizzle_migrations" (hash, created_at) VALUES ($1, $2)`,
|
||||||
|
[journalTag, lastMigrationWhen],
|
||||||
|
);
|
||||||
|
logger.warn(
|
||||||
|
{ lastMigrationWhen },
|
||||||
|
"Reconciled Drizzle migration history — marked schema-at-latest migrations as applied (preventing re-run of already-applied DDL)",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -642,6 +642,8 @@ export const pgModerationActionsTable = pgTable(
|
|||||||
score: pgReal("score"), // 0..1 raw model score
|
score: pgReal("score"), // 0..1 raw model score
|
||||||
evidence: pgText("evidence"), // JSON array of short quoted snippets
|
evidence: pgText("evidence"), // JSON array of short quoted snippets
|
||||||
policy_version: pgText("policy_version"), // rules.ts policy version string
|
policy_version: pgText("policy_version"), // rules.ts policy version string
|
||||||
|
username: pgText("username"), // denormalized from messages table for retention safety
|
||||||
|
server_nick: pgText("server_nick"), // member's server-specific display name (nickname), from metadata.member.displayName
|
||||||
},
|
},
|
||||||
(table) => ({
|
(table) => ({
|
||||||
messageIdIdx: pgIndex("idx_moderation_actions_message_id").on(
|
messageIdIdx: pgIndex("idx_moderation_actions_message_id").on(
|
||||||
|
|||||||
@@ -59,6 +59,8 @@ export const pgModerationActionsTable = pgTable(
|
|||||||
score: pgReal("score"), // 0..1 raw model score
|
score: pgReal("score"), // 0..1 raw model score
|
||||||
evidence: pgText("evidence"), // JSON array of short quoted snippets
|
evidence: pgText("evidence"), // JSON array of short quoted snippets
|
||||||
policy_version: pgText("policy_version"), // rules.ts policy version string
|
policy_version: pgText("policy_version"), // rules.ts policy version string
|
||||||
|
username: pgText("username"), // denormalized from messages table for retention safety
|
||||||
|
server_nick: pgText("server_nick"), // member's server-specific display name (nickname), from metadata.member.displayName
|
||||||
},
|
},
|
||||||
(table) => ({
|
(table) => ({
|
||||||
messageIdIdx: pgIndex("idx_moderation_actions_message_id").on(
|
messageIdIdx: pgIndex("idx_moderation_actions_message_id").on(
|
||||||
|
|||||||
Loaded 50 of 131 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user