feat(archive): rich metadata in semantic search — username/channel/guild context + guild filter
Archive payload now stores username, channel_id, guild_id, thread_id and the real message created_at (not embed time). Backend searchArray accepts an optional guildId and applies a Qdrant payload filter so results can be scoped to the guild being viewed. API/frontend expose the new fields and the semantic results card shows who said it, in which channel, and when — turning bare text blobs into contextual results. Old points fall back to analyzed_at and omit the new fields gracefully.
This commit is contained in:
1 parent
c704fbf7a5
commit
26a690943d
8 files changed
+84
-8
No files matched your search
@@ -105,6 +105,7 @@ export class MessagesService {
|
|||||||
vector,
|
vector,
|
||||||
input.limit,
|
input.limit,
|
||||||
config.AI_LLM_EMBEDDING_ARCHIVE_MIN_SIMILARITY,
|
config.AI_LLM_EMBEDDING_ARCHIVE_MIN_SIMILARITY,
|
||||||
|
input.guildId,
|
||||||
);
|
);
|
||||||
const results = hits.map((h) => mapSearchHit(h));
|
const results = hits.map((h) => mapSearchHit(h));
|
||||||
return { results, nextCursor: null };
|
return { results, nextCursor: null };
|
||||||
@@ -120,16 +121,31 @@ export class MessagesService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Shape returned to the frontend (text + metadata from the archive payload). */
|
/** Shape returned to the frontend (text + rich metadata from the archive payload). */
|
||||||
function mapSearchHit(hit: {
|
function mapSearchHit(hit: {
|
||||||
score: number;
|
score: number;
|
||||||
payload: { text: string; content_hash?: string; analyzed_at: number };
|
payload: {
|
||||||
|
text: string;
|
||||||
|
content_hash?: string;
|
||||||
|
analyzed_at: number;
|
||||||
|
username?: string;
|
||||||
|
channel_id?: string;
|
||||||
|
guild_id?: string;
|
||||||
|
thread_id?: string | null;
|
||||||
|
created_at?: number;
|
||||||
|
};
|
||||||
}) {
|
}) {
|
||||||
return {
|
return {
|
||||||
message_id: hit.payload.content_hash ?? null,
|
message_id: hit.payload.content_hash ?? null,
|
||||||
content: hit.payload.text,
|
content: hit.payload.text,
|
||||||
score: hit.score,
|
score: hit.score,
|
||||||
created_at: hit.payload.analyzed_at,
|
// Prefer the real message timestamp; fall back to embed time for old
|
||||||
|
// points that predate rich metadata.
|
||||||
|
created_at: hit.payload.created_at ?? hit.payload.analyzed_at,
|
||||||
|
username: hit.payload.username ?? null,
|
||||||
|
channel_id: hit.payload.channel_id ?? null,
|
||||||
|
guild_id: hit.payload.guild_id ?? null,
|
||||||
|
thread_id: hit.payload.thread_id ?? null,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,11 @@ export interface ArchiveHit {
|
|||||||
content_hash?: string;
|
content_hash?: string;
|
||||||
analyzed_at: number;
|
analyzed_at: number;
|
||||||
expires_at: number;
|
expires_at: number;
|
||||||
|
username?: string;
|
||||||
|
channel_id?: string;
|
||||||
|
guild_id?: string;
|
||||||
|
thread_id?: string | null;
|
||||||
|
created_at?: number;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -61,6 +66,7 @@ export async function searchArchive(
|
|||||||
vector: number[],
|
vector: number[],
|
||||||
limit: number,
|
limit: number,
|
||||||
scoreThreshold: number,
|
scoreThreshold: number,
|
||||||
|
guildId?: string,
|
||||||
): Promise<ArchiveHit[]> {
|
): Promise<ArchiveHit[]> {
|
||||||
if (!config.QDRANT_URL) return [];
|
if (!config.QDRANT_URL) return [];
|
||||||
try {
|
try {
|
||||||
@@ -72,6 +78,17 @@ export async function searchArchive(
|
|||||||
limit,
|
limit,
|
||||||
score_threshold: scoreThreshold,
|
score_threshold: scoreThreshold,
|
||||||
with_payload: true,
|
with_payload: true,
|
||||||
|
// Optional scope: only return vectors from a specific guild's archive.
|
||||||
|
// Old points (embedded before rich metadata) have no guild_id payload —
|
||||||
|
// the `must` match simply excludes them, which is the correct behavior
|
||||||
|
// for a guild-scoped search.
|
||||||
|
...(guildId
|
||||||
|
? {
|
||||||
|
filter: {
|
||||||
|
must: [{ key: "guild_id", match: { value: guildId } }],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
},
|
},
|
||||||
)) as {
|
)) as {
|
||||||
result?: Array<{
|
result?: Array<{
|
||||||
|
|||||||
@@ -37,6 +37,14 @@ export interface QdrantVerdictPayload {
|
|||||||
/** Bare content hash (16 hex chars) — enables content-based invalidation
|
/** Bare content hash (16 hex chars) — enables content-based invalidation
|
||||||
* regardless of the (context-scoped) point id. */
|
* regardless of the (context-scoped) point id. */
|
||||||
content_hash?: string;
|
content_hash?: string;
|
||||||
|
// Archive metadata (archiveEmbedder writes these; moderation cache doesn't).
|
||||||
|
// Optional so the same payload shape serves both the moderation cache
|
||||||
|
// collection and the public message archive collection.
|
||||||
|
username?: string;
|
||||||
|
channel_id?: string;
|
||||||
|
guild_id?: string;
|
||||||
|
thread_id?: string | null;
|
||||||
|
created_at?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
function baseUrl(): string {
|
function baseUrl(): string {
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ export interface ArchiveMessage {
|
|||||||
username: string;
|
username: string;
|
||||||
channel_id: string;
|
channel_id: string;
|
||||||
guild_id: string;
|
guild_id: string;
|
||||||
|
thread_id: string | null;
|
||||||
created_at: number;
|
created_at: number;
|
||||||
/** True when the message came from an age-restricted (NSFW) channel. NSFW
|
/** True when the message came from an age-restricted (NSFW) channel. NSFW
|
||||||
* content is deliberately NOT embedded into the public archive so it can't
|
* content is deliberately NOT embedded into the public archive so it can't
|
||||||
@@ -58,6 +59,13 @@ export function archiveMessageEmbedded(message: ArchiveMessage): void {
|
|||||||
{
|
{
|
||||||
text: normalized.slice(0, 4000),
|
text: normalized.slice(0, 4000),
|
||||||
flags: "",
|
flags: "",
|
||||||
|
// Rich metadata so public semantic search results can be shown in
|
||||||
|
// context (who said it, where, when) instead of a bare text blob.
|
||||||
|
username: message.username ?? "",
|
||||||
|
channel_id: message.channel_id ?? "",
|
||||||
|
guild_id: message.guild_id ?? "",
|
||||||
|
thread_id: message.thread_id ?? null,
|
||||||
|
created_at: message.created_at ?? Date.now(),
|
||||||
analyzed_at: Date.now(),
|
analyzed_at: Date.now(),
|
||||||
// 5-year persistent window (archive is NOT a TTL cache).
|
// 5-year persistent window (archive is NOT a TTL cache).
|
||||||
expires_at: Date.now() + 1000 * 60 * 60 * 24 * 365 * 5,
|
expires_at: Date.now() + 1000 * 60 * 60 * 24 * 365 * 5,
|
||||||
|
|||||||
@@ -129,6 +129,7 @@ export function MessagesView({
|
|||||||
const semantic = useSemanticSearch(
|
const semantic = useSemanticSearch(
|
||||||
query,
|
query,
|
||||||
query.trim().length >= 2 && semanticMode,
|
query.trim().length >= 2 && semanticMode,
|
||||||
|
guildId,
|
||||||
);
|
);
|
||||||
const activity = useMessageActivity(30);
|
const activity = useMessageActivity(30);
|
||||||
const edits = useRecentEdits(50, undefined, initialEdits);
|
const edits = useRecentEdits(50, undefined, initialEdits);
|
||||||
@@ -360,6 +361,16 @@ export function MessagesView({
|
|||||||
<span className="font-mono text-[10px] font-semibold text-signal">
|
<span className="font-mono text-[10px] font-semibold text-signal">
|
||||||
{(r.score * 100).toFixed(0)}% RELEVANCE
|
{(r.score * 100).toFixed(0)}% RELEVANCE
|
||||||
</span>
|
</span>
|
||||||
|
{r.username && (
|
||||||
|
<span className="font-mono text-[10px] font-medium text-ink">
|
||||||
|
{r.username}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{r.channel_id && (
|
||||||
|
<span className="font-mono text-[10px] text-ink-muted">
|
||||||
|
#{r.channel_id}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
<span
|
<span
|
||||||
className="ml-auto font-mono text-[10px] text-ink-muted"
|
className="ml-auto font-mono text-[10px] text-ink-muted"
|
||||||
suppressHydrationWarning
|
suppressHydrationWarning
|
||||||
|
|||||||
@@ -212,13 +212,21 @@ export function useMessageSearch(query: string, enabled: boolean) {
|
|||||||
|
|
||||||
// ── Semantic Search (public archive, Qdrant) ──────
|
// ── Semantic Search (public archive, Qdrant) ──────
|
||||||
|
|
||||||
export function useSemanticSearch(query: string, enabled: boolean) {
|
export function useSemanticSearch(
|
||||||
|
query: string,
|
||||||
|
enabled: boolean,
|
||||||
|
guildId?: string | null,
|
||||||
|
) {
|
||||||
return useSWR<SemanticSearchResult[]>(
|
return useSWR<SemanticSearchResult[]>(
|
||||||
enabled && query.trim().length >= 2
|
enabled && query.trim().length >= 2
|
||||||
? ["semantic-search", query.trim()]
|
? ["semantic-search", query.trim(), guildId ?? ""]
|
||||||
: null,
|
: null,
|
||||||
async () => {
|
async () => {
|
||||||
const res = await messagesApi.semanticSearch(query.trim(), 10);
|
const res = await messagesApi.semanticSearch(
|
||||||
|
query.trim(),
|
||||||
|
10,
|
||||||
|
guildId ?? undefined,
|
||||||
|
);
|
||||||
return res.results;
|
return res.results;
|
||||||
},
|
},
|
||||||
{ keepPreviousData: true },
|
{ keepPreviousData: true },
|
||||||
|
|||||||
@@ -70,8 +70,12 @@ export const messagesApi = {
|
|||||||
}>,
|
}>,
|
||||||
|
|
||||||
// Public semantic search over the persistent message archive (Qdrant).
|
// Public semantic search over the persistent message archive (Qdrant).
|
||||||
semanticSearch: (query: string, limit?: number) =>
|
semanticSearch: (query: string, limit?: number, guildId?: string) =>
|
||||||
orpc.messages.semanticSearch({ query, limit }) as unknown as Promise<{
|
orpc.messages.semanticSearch({
|
||||||
|
query,
|
||||||
|
limit,
|
||||||
|
...(guildId ? { guildId } : {}),
|
||||||
|
}) as unknown as Promise<{
|
||||||
results: SemanticSearchResult[];
|
results: SemanticSearchResult[];
|
||||||
nextCursor: null;
|
nextCursor: null;
|
||||||
}>,
|
}>,
|
||||||
|
|||||||
@@ -173,6 +173,10 @@ export interface SemanticSearchResult {
|
|||||||
content: string;
|
content: string;
|
||||||
score: number;
|
score: number;
|
||||||
created_at: number;
|
created_at: number;
|
||||||
|
username: string | null;
|
||||||
|
channel_id: string | null;
|
||||||
|
guild_id: string | null;
|
||||||
|
thread_id: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface MessageActivityBucket {
|
export interface MessageActivityBucket {
|
||||||
|
|||||||
Reference in new issue
Block a user