Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Enhance context retrieval with author information and metadata

Ylber Gashi0d7a90283646
diff --git a/apps/server/src/lib/assistant.ts b/apps/server/src/lib/assistant.ts
index cdc6f3c8..d3585302 100644
--- a/apps/server/src/lib/assistant.ts
+++ b/apps/server/src/lib/assistant.ts
@@ -177,6 +177,7 @@ async function fetchWorkspaceDetails(workspaceId: string) {
async function generateResponse(state: AssistantChainState) {
const workspace = await fetchWorkspaceDetails(state.workspaceId);
+
const formattedChatHistory = state.chatHistory
.map((doc) => {
const timestamp = doc.metadata.createdAt
@@ -186,12 +187,13 @@ async function generateResponse(state: AssistantChainState) {
return `- [${timestamp}] ${authorName}: ${doc.pageContent}`;
})
.join('\n');
+
const formattedContext = state.topContext
.map((doc) => {
const timestamp = doc.metadata?.createdAt
? new Date(doc.metadata.createdAt).toLocaleString()
: 'Unknown time';
- const authorName = doc.metadata?.author || 'Unknown';
+ const authorName = doc.metadata?.author?.name || 'Unknown';
let contextHeader = `Source ID: ${doc.metadata.id}\nAuthor: ${authorName}\nTime: ${timestamp}`;
if (
doc.metadata?.type === 'node' &&
@@ -214,6 +216,7 @@ async function generateResponse(state: AssistantChainState) {
formattedDocuments: formattedContext,
question: state.userInput,
});
+
return { finalAnswer: result.answer, citations: result.citations };
}
@@ -256,6 +259,7 @@ async function fetchDatabaseContext(state: AssistantChainState) {
};
})
);
+
return { databaseContext };
}
@@ -283,6 +287,7 @@ function selectTopContext(
if (reranked.length === 0) return [];
const maxScore = Math.max(...reranked.map((item) => item.score));
const threshold = maxScore * 0.5;
+
return reranked
.filter((item) => item.score >= threshold && item.score > 0)
.sort((a, b) => b.score - a.score)
diff --git a/apps/server/src/services/document-retrieval-service.ts b/apps/server/src/services/document-retrieval-service.ts
index f6f50bcc..a7737f51 100644
--- a/apps/server/src/services/document-retrieval-service.ts
+++ b/apps/server/src/services/document-retrieval-service.ts
@@ -18,19 +18,24 @@ export class DocumentRetrievalService {
limit = configuration.ai.retrieval.hybridSearch.maxResults
): Promise<Document[]> {
const embedding = await this.embeddings.embedQuery(query);
- if (!embedding) return [];
+ if (!embedding) {
+ return [];
+ }
+
const semanticResults = await this.semanticSearch(
embedding,
workspaceId,
userId,
limit
);
+
const keywordResults = await this.keywordSearch(
query,
workspaceId,
userId,
limit
);
+
return this.combineSearchResults(semanticResults, keywordResults);
}
@@ -54,6 +59,7 @@ export class DocumentRetrievalService {
'document_embeddings.document_id as id',
'document_embeddings.text',
'documents.created_at',
+ 'documents.created_by',
'document_embeddings.chunk as chunk_index',
sql<number>`${sql.raw(`'[${embedding}]'::vector`)} <=> document_embeddings.embedding_vector`.as(
'similarity'
@@ -64,6 +70,7 @@ export class DocumentRetrievalService {
'document_embeddings.document_id',
'document_embeddings.text',
'documents.created_at',
+ 'documents.created_by',
'document_embeddings.chunk',
])
.orderBy('similarity', 'asc')
@@ -76,6 +83,7 @@ export class DocumentRetrievalService {
score: result.similarity,
type: 'semantic',
createdAt: result.created_at,
+ createdBy: result.created_by,
chunkIndex: result.chunk_index,
}));
}
@@ -100,6 +108,7 @@ export class DocumentRetrievalService {
'document_embeddings.document_id as id',
'document_embeddings.text',
'documents.created_at',
+ 'documents.created_by',
'document_embeddings.chunk as chunk_index',
sql<number>`ts_rank(document_embeddings.search_vector, websearch_to_tsquery('english', ${query}))`.as(
'rank'
@@ -114,6 +123,7 @@ export class DocumentRetrievalService {
'document_embeddings.document_id',
'document_embeddings.text',
'documents.created_at',
+ 'documents.created_by',
'document_embeddings.chunk',
])
.orderBy('rank', 'desc')
@@ -126,14 +136,15 @@ export class DocumentRetrievalService {
score: result.rank,
type: 'keyword',
createdAt: result.created_at,
+ createdBy: result.created_by,
chunkIndex: result.chunk_index,
}));
}
- private combineSearchResults(
+ private async combineSearchResults(
semanticResults: SearchResult[],
keywordResults: SearchResult[]
- ): Document[] {
+ ): Promise<Document[]> {
const { semanticSearchWeight, keywordSearchWeight } =
configuration.ai.retrieval.hybridSearch;
const maxSemanticScore = Math.max(
@@ -141,9 +152,11 @@ export class DocumentRetrievalService {
1
);
const maxKeywordScore = Math.max(...keywordResults.map((r) => r.score), 1);
+
const combined = new Map<string, SearchResult & { finalScore: number }>();
const createKey = (result: SearchResult) =>
`${result.id}-${result.chunkIndex}`;
+
const calculateRecencyBoost = (
createdAt: Date | undefined | null
): number => {
@@ -182,21 +195,50 @@ export class DocumentRetrievalService {
}
});
+ // Fetch all unique author IDs from the results
+ const authorIds = Array.from(
+ new Set(
+ Array.from(combined.values())
+ .map((r) => r.createdBy)
+ .filter((id): id is string => id !== undefined && id !== null)
+ )
+ );
+
+ // Bulk fetch author information
+ const authors =
+ authorIds.length > 0
+ ? await database
+ .selectFrom('users')
+ .select(['id', 'name'])
+ .where('id', 'in', authorIds)
+ .execute()
+ : [];
+
+ const authorMap = new Map(authors.map((author) => [author.id, author]));
+
return Array.from(combined.values())
.sort((a, b) => b.finalScore - a.finalScore)
- .map(
- (result) =>
- new Document({
- pageContent: result.text,
- metadata: {
- id: result.id,
- score: result.finalScore,
- createdAt: result.createdAt,
- type: 'document',
- chunkIndex: result.chunkIndex,
- },
- })
- );
+ .map((result) => {
+ const author = result.createdBy
+ ? authorMap.get(result.createdBy)
+ : null;
+ return new Document({
+ pageContent: result.text,
+ metadata: {
+ id: result.id,
+ score: result.finalScore,
+ createdAt: result.createdAt,
+ type: 'document',
+ chunkIndex: result.chunkIndex,
+ author: author
+ ? {
+ id: author.id,
+ name: author.name || 'Anonymous',
+ }
+ : null,
+ },
+ });
+ });
}
}
diff --git a/apps/server/src/services/node-retrieval-service.ts b/apps/server/src/services/node-retrieval-service.ts
index 84d1f740..c52b85be 100644
--- a/apps/server/src/services/node-retrieval-service.ts
+++ b/apps/server/src/services/node-retrieval-service.ts
@@ -19,19 +19,24 @@ export class NodeRetrievalService {
limit = configuration.ai.retrieval.hybridSearch.maxResults
): Promise<Document[]> {
const embedding = await this.embeddings.embedQuery(query);
- if (!embedding) return [];
+ if (!embedding) {
+ return [];
+ }
+
const semanticResults = await this.semanticSearch(
embedding,
workspaceId,
userId,
limit
);
+
const keywordResults = await this.keywordSearch(
query,
workspaceId,
userId,
limit
);
+
return this.combineSearchResults(semanticResults, keywordResults);
}
@@ -54,6 +59,7 @@ export class NodeRetrievalService {
'node_embeddings.node_id as id',
'node_embeddings.text',
'nodes.created_at',
+ 'nodes.created_by',
'node_embeddings.chunk as chunk_index',
sql<number>`${sql.raw(`'[${embedding}]'::vector`)} <=> node_embeddings.embedding_vector`.as(
'similarity'
@@ -64,6 +70,7 @@ export class NodeRetrievalService {
'node_embeddings.node_id',
'node_embeddings.text',
'nodes.created_at',
+ 'nodes.created_by',
'node_embeddings.chunk',
])
.orderBy('similarity', 'asc')
@@ -76,6 +83,7 @@ export class NodeRetrievalService {
score: result.similarity,
type: 'semantic',
createdAt: result.created_at,
+ createdBy: result.created_by,
chunkIndex: result.chunk_index,
}));
}
@@ -99,6 +107,7 @@ export class NodeRetrievalService {
'node_embeddings.node_id as id',
'node_embeddings.text',
'nodes.created_at',
+ 'nodes.created_by',
'node_embeddings.chunk as chunk_index',
sql<number>`ts_rank(node_embeddings.search_vector, websearch_to_tsquery('english', ${query}))`.as(
'rank'
@@ -113,6 +122,7 @@ export class NodeRetrievalService {
'node_embeddings.node_id',
'node_embeddings.text',
'nodes.created_at',
+ 'nodes.created_by',
'node_embeddings.chunk',
])
.orderBy('rank', 'desc')
@@ -125,14 +135,15 @@ export class NodeRetrievalService {
score: result.rank,
type: 'keyword',
createdAt: result.created_at,
+ createdBy: result.created_by,
chunkIndex: result.chunk_index,
}));
}
- private combineSearchResults(
+ private async combineSearchResults(
semanticResults: SearchResult[],
keywordResults: SearchResult[]
- ): Document[] {
+ ): Promise<Document[]> {
const { semanticSearchWeight, keywordSearchWeight } =
configuration.ai.retrieval.hybridSearch;
const maxSemanticScore = Math.max(
@@ -140,9 +151,11 @@ export class NodeRetrievalService {
1
);
const maxKeywordScore = Math.max(...keywordResults.map((r) => r.score), 1);
+
const combined = new Map<string, SearchResult & { finalScore: number }>();
const createKey = (result: SearchResult) =>
`${result.id}-${result.chunkIndex}`;
+
const calculateRecencyBoost = (
createdAt: Date | undefined | null
): number => {
@@ -181,21 +194,50 @@ export class NodeRetrievalService {
}
});
+ // Fetch all unique author IDs from the results
+ const authorIds = Array.from(
+ new Set(
+ Array.from(combined.values())
+ .map((r) => r.createdBy)
+ .filter((id): id is string => id !== undefined && id !== null)
+ )
+ );
+
+ // Bulk fetch author information
+ const authors =
+ authorIds.length > 0
+ ? await database
+ .selectFrom('users')
+ .select(['id', 'name'])
+ .where('id', 'in', authorIds)
+ .execute()
+ : [];
+
+ const authorMap = new Map(authors.map((author) => [author.id, author]));
+
return Array.from(combined.values())
.sort((a, b) => b.finalScore - a.finalScore)
- .map(
- (result) =>
- new Document({
- pageContent: result.text,
- metadata: {
- id: result.id,
- score: result.finalScore,
- createdAt: result.createdAt,
- type: 'node',
- chunkIndex: result.chunkIndex,
- },
- })
- );
+ .map((result) => {
+ const author = result.createdBy
+ ? authorMap.get(result.createdBy)
+ : null;
+ return new Document({
+ pageContent: result.text,
+ metadata: {
+ id: result.id,
+ score: result.finalScore,
+ createdAt: result.createdAt,
+ type: 'node',
+ chunkIndex: result.chunkIndex,
+ author: author
+ ? {
+ id: author.id,
+ name: author.name || 'Anonymous',
+ }
+ : null,
+ },
+ });
+ });
}
}
diff --git a/apps/server/src/types/retrieval.ts b/apps/server/src/types/retrieval.ts
index b88452dc..4eb992ec 100644
--- a/apps/server/src/types/retrieval.ts
+++ b/apps/server/src/types/retrieval.ts
@@ -4,5 +4,6 @@ export type SearchResult = {
score: number;
type: 'semantic' | 'keyword';
createdAt?: Date;
+ createdBy?: string;
chunkIndex: number;
};