MythEclipse/GMW
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(analytics): improve topic word filtering with Indonesian stop words and stricter noise reduction

Browse Source
This commit is contained in:
MythEclipse
2026-06-03 20:08:03 +07:00
parent cbee35c348
commit 8918dd09e2
1 changed files with 18 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/backend/src/modules/analytics/analytics.repository.ts
+18 -8
View File
@@ -325,13 +325,13 @@ export class AnalyticsRepository {
FROM messages, FROM messages,
LATERAL UNNEST(STRING_TO_ARRAY(content, ' ')) AS word LATERAL UNNEST(STRING_TO_ARRAY(content, ' ')) AS word
${filter.where} ${filter.where}
AND content IS NOT NULL AND content IS NOT NULL AND content != ''
AND content != '' AND LENGTH(TRIM(BOTH '.,!?;:\"()[]{}' FROM word)) >= 4
AND LENGTH(TRIM(BOTH '.,!?;:\"()[]{}' FROM word)) >= 3 AND word !~ '^<.+:\d+>$'
AND word !~ '^<.+:\\d+>$' AND word !~ '^\['
AND word !~ '^https?://' AND word !~ '^https?://'
AND word !~ '^discord\.(gg|app|com)' AND word !~ '^discord\.'
AND word !~ '^cdn\.discord' AND word !~ '^cdn\.'
) )
SELECT SELECT
word AS topic, word AS topic,
@@ -349,10 +349,20 @@ export class AnalyticsRepository {
'the','and','for','are','but','not','you','all','can','has', 'the','and','for','are','but','not','you','all','can','has',
'was','were','been','like','just','that','this','with','your', 'was','were','been','like','just','that','this','with','your',
'from','they','have','what','when','where','which','their', 'from','they','have','what','when','where','which','their',
'about','would','could','should','very','also','than','then' 'about','would','could','should','very','also','than','then',
'mau','lagi','jadi','aja','nya','apa','orang',
'lihat','kak','bro','bang','mas','pack','sih','dong','kok',
'nih','deh','kali','loh','lho','doang',
'gue','lo','lu','gua','elo','ane','wkwk','wkwkwk',
'wkwkwkwk','wkwkwkwkwk','haha','hahaha','hehe','wk','wkwk',
'kalo','buat','udah','jir','kan','tuh','pake','dulu',
'banget','kayak','kya','kyk','klo','karna','soalnya',
'bikin','bilang','makan','minum','tidur','main','pergi','pulang',
'mana','cuma','kah','udh','gitu','gini','gtu','gni',
'mending','wkakak','wakak'
) )
GROUP BY word GROUP BY word
ORDER BY count DESC ORDER BY COUNT(*) DESC
LIMIT 10 LIMIT 10
`, `,
filter.params, filter.params,