Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Enhance context retrieval with author information and metadata
Ylber Gashi0d7a90283646
diff --git a/apps/server/src/lib/assistant.ts b/apps/server/src/lib/assistant.tsindex cdc6f3c8..d3585302 100644--- a/apps/server/src/lib/assistant.ts+++ b/apps/server/src/lib/assistant.ts@@ -177,6 +177,7 @@ async function fetchWorkspaceDetails(workspaceId: string) {async function generateResponse(state: AssistantChainState) {const workspace = await fetchWorkspaceDetails(state.workspaceId);+const formattedChatHistory = state.chatHistory.map((doc) => {const timestamp = doc.metadata.createdAt@@ -186,12 +187,13 @@ async function generateResponse(state: AssistantChainState) {return `- [${timestamp}] ${authorName}: ${doc.pageContent}`;}).join('\n');+const formattedContext = state.topContext.map((doc) => {const timestamp = doc.metadata?.createdAt? new Date(doc.metadata.createdAt).toLocaleString(): 'Unknown time';- const authorName = doc.metadata?.author || 'Unknown';+ const authorName = doc.metadata?.author?.name || 'Unknown';let contextHeader = `Source ID: ${doc.metadata.id}\nAuthor: ${authorName}\nTime: ${timestamp}`;if (doc.metadata?.type === 'node' &&@@ -214,6 +216,7 @@ async function generateResponse(state: AssistantChainState) {formattedDocuments: formattedContext,question: state.userInput,});+return { finalAnswer: result.answer, citations: result.citations };}@@ -256,6 +259,7 @@ async function fetchDatabaseContext(state: AssistantChainState) {};}));+return { databaseContext };}@@ -283,6 +287,7 @@ function selectTopContext(if (reranked.length === 0) return [];const maxScore = Math.max(...reranked.map((item) => item.score));const threshold = maxScore * 0.5;+return reranked.filter((item) => item.score >= threshold && item.score > 0).sort((a, b) => b.score - a.score)diff --git a/apps/server/src/services/document-retrieval-service.ts b/apps/server/src/services/document-retrieval-service.tsindex f6f50bcc..a7737f51 100644--- a/apps/server/src/services/document-retrieval-service.ts+++ b/apps/server/src/services/document-retrieval-service.ts@@ -18,19 +18,24 @@ export class DocumentRetrievalService {limit = configuration.ai.retrieval.hybridSearch.maxResults): Promise<Document[]> {const embedding = await this.embeddings.embedQuery(query);- if (!embedding) return [];+ if (!embedding) {+ return [];+ }+const semanticResults = await this.semanticSearch(embedding,workspaceId,userId,limit);+const keywordResults = await this.keywordSearch(query,workspaceId,userId,limit);+return this.combineSearchResults(semanticResults, keywordResults);}@@ -54,6 +59,7 @@ export class DocumentRetrievalService {'document_embeddings.document_id as id','document_embeddings.text','documents.created_at',+ 'documents.created_by','document_embeddings.chunk as chunk_index',sql<number>`${sql.raw(`'[${embedding}]'::vector`)} <=> document_embeddings.embedding_vector`.as('similarity'@@ -64,6 +70,7 @@ export class DocumentRetrievalService {'document_embeddings.document_id','document_embeddings.text','documents.created_at',+ 'documents.created_by','document_embeddings.chunk',]).orderBy('similarity', 'asc')@@ -76,6 +83,7 @@ export class DocumentRetrievalService {score: result.similarity,type: 'semantic',createdAt: result.created_at,+ createdBy: result.created_by,chunkIndex: result.chunk_index,}));}@@ -100,6 +108,7 @@ export class DocumentRetrievalService {'document_embeddings.document_id as id','document_embeddings.text','documents.created_at',+ 'documents.created_by','document_embeddings.chunk as chunk_index',sql<number>`ts_rank(document_embeddings.search_vector, websearch_to_tsquery('english', ${query}))`.as('rank'@@ -114,6 +123,7 @@ export class DocumentRetrievalService {'document_embeddings.document_id','document_embeddings.text','documents.created_at',+ 'documents.created_by','document_embeddings.chunk',]).orderBy('rank', 'desc')@@ -126,14 +136,15 @@ export class DocumentRetrievalService {score: result.rank,type: 'keyword',createdAt: result.created_at,+ createdBy: result.created_by,chunkIndex: result.chunk_index,}));}- private combineSearchResults(+ private async combineSearchResults(semanticResults: SearchResult[],keywordResults: SearchResult[]- ): Document[] {+ ): Promise<Document[]> {const { semanticSearchWeight, keywordSearchWeight } =configuration.ai.retrieval.hybridSearch;const maxSemanticScore = Math.max(@@ -141,9 +152,11 @@ export class DocumentRetrievalService {1);const maxKeywordScore = Math.max(...keywordResults.map((r) => r.score), 1);+const combined = new Map<string, SearchResult & { finalScore: number }>();const createKey = (result: SearchResult) =>`${result.id}-${result.chunkIndex}`;+const calculateRecencyBoost = (createdAt: Date | undefined | null): number => {@@ -182,21 +195,50 @@ export class DocumentRetrievalService {}});+ // Fetch all unique author IDs from the results+ const authorIds = Array.from(+ new Set(+ Array.from(combined.values())+ .map((r) => r.createdBy)+ .filter((id): id is string => id !== undefined && id !== null)+ )+ );++ // Bulk fetch author information+ const authors =+ authorIds.length > 0+ ? await database+ .selectFrom('users')+ .select(['id', 'name'])+ .where('id', 'in', authorIds)+ .execute()+ : [];++ const authorMap = new Map(authors.map((author) => [author.id, author]));+return Array.from(combined.values()).sort((a, b) => b.finalScore - a.finalScore)- .map(- (result) =>- new Document({- pageContent: result.text,- metadata: {- id: result.id,- score: result.finalScore,- createdAt: result.createdAt,- type: 'document',- chunkIndex: result.chunkIndex,- },- })- );+ .map((result) => {+ const author = result.createdBy+ ? authorMap.get(result.createdBy)+ : null;+ return new Document({+ pageContent: result.text,+ metadata: {+ id: result.id,+ score: result.finalScore,+ createdAt: result.createdAt,+ type: 'document',+ chunkIndex: result.chunkIndex,+ author: author+ ? {+ id: author.id,+ name: author.name || 'Anonymous',+ }+ : null,+ },+ });+ });}}diff --git a/apps/server/src/services/node-retrieval-service.ts b/apps/server/src/services/node-retrieval-service.tsindex 84d1f740..c52b85be 100644--- a/apps/server/src/services/node-retrieval-service.ts+++ b/apps/server/src/services/node-retrieval-service.ts@@ -19,19 +19,24 @@ export class NodeRetrievalService {limit = configuration.ai.retrieval.hybridSearch.maxResults): Promise<Document[]> {const embedding = await this.embeddings.embedQuery(query);- if (!embedding) return [];+ if (!embedding) {+ return [];+ }+const semanticResults = await this.semanticSearch(embedding,workspaceId,userId,limit);+const keywordResults = await this.keywordSearch(query,workspaceId,userId,limit);+return this.combineSearchResults(semanticResults, keywordResults);}@@ -54,6 +59,7 @@ export class NodeRetrievalService {'node_embeddings.node_id as id','node_embeddings.text','nodes.created_at',+ 'nodes.created_by','node_embeddings.chunk as chunk_index',sql<number>`${sql.raw(`'[${embedding}]'::vector`)} <=> node_embeddings.embedding_vector`.as('similarity'@@ -64,6 +70,7 @@ export class NodeRetrievalService {'node_embeddings.node_id','node_embeddings.text','nodes.created_at',+ 'nodes.created_by','node_embeddings.chunk',]).orderBy('similarity', 'asc')@@ -76,6 +83,7 @@ export class NodeRetrievalService {score: result.similarity,type: 'semantic',createdAt: result.created_at,+ createdBy: result.created_by,chunkIndex: result.chunk_index,}));}@@ -99,6 +107,7 @@ export class NodeRetrievalService {'node_embeddings.node_id as id','node_embeddings.text','nodes.created_at',+ 'nodes.created_by','node_embeddings.chunk as chunk_index',sql<number>`ts_rank(node_embeddings.search_vector, websearch_to_tsquery('english', ${query}))`.as('rank'@@ -113,6 +122,7 @@ export class NodeRetrievalService {'node_embeddings.node_id','node_embeddings.text','nodes.created_at',+ 'nodes.created_by','node_embeddings.chunk',]).orderBy('rank', 'desc')@@ -125,14 +135,15 @@ export class NodeRetrievalService {score: result.rank,type: 'keyword',createdAt: result.created_at,+ createdBy: result.created_by,chunkIndex: result.chunk_index,}));}- private combineSearchResults(+ private async combineSearchResults(semanticResults: SearchResult[],keywordResults: SearchResult[]- ): Document[] {+ ): Promise<Document[]> {const { semanticSearchWeight, keywordSearchWeight } =configuration.ai.retrieval.hybridSearch;const maxSemanticScore = Math.max(@@ -140,9 +151,11 @@ export class NodeRetrievalService {1);const maxKeywordScore = Math.max(...keywordResults.map((r) => r.score), 1);+const combined = new Map<string, SearchResult & { finalScore: number }>();const createKey = (result: SearchResult) =>`${result.id}-${result.chunkIndex}`;+const calculateRecencyBoost = (createdAt: Date | undefined | null): number => {@@ -181,21 +194,50 @@ export class NodeRetrievalService {}});+ // Fetch all unique author IDs from the results+ const authorIds = Array.from(+ new Set(+ Array.from(combined.values())+ .map((r) => r.createdBy)+ .filter((id): id is string => id !== undefined && id !== null)+ )+ );++ // Bulk fetch author information+ const authors =+ authorIds.length > 0+ ? await database+ .selectFrom('users')+ .select(['id', 'name'])+ .where('id', 'in', authorIds)+ .execute()+ : [];++ const authorMap = new Map(authors.map((author) => [author.id, author]));+return Array.from(combined.values()).sort((a, b) => b.finalScore - a.finalScore)- .map(- (result) =>- new Document({- pageContent: result.text,- metadata: {- id: result.id,- score: result.finalScore,- createdAt: result.createdAt,- type: 'node',- chunkIndex: result.chunkIndex,- },- })- );+ .map((result) => {+ const author = result.createdBy+ ? authorMap.get(result.createdBy)+ : null;+ return new Document({+ pageContent: result.text,+ metadata: {+ id: result.id,+ score: result.finalScore,+ createdAt: result.createdAt,+ type: 'node',+ chunkIndex: result.chunkIndex,+ author: author+ ? {+ id: author.id,+ name: author.name || 'Anonymous',+ }+ : null,+ },+ });+ });}}diff --git a/apps/server/src/types/retrieval.ts b/apps/server/src/types/retrieval.tsindex b88452dc..4eb992ec 100644--- a/apps/server/src/types/retrieval.ts+++ b/apps/server/src/types/retrieval.ts@@ -4,5 +4,6 @@ export type SearchResult = {score: number;type: 'semantic' | 'keyword';createdAt?: Date;+ createdBy?: string;chunkIndex: number;};