Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Refactor assistant response generation and context retrieval
Ylber Gashi16b78a0e02a8
diff --git a/apps/server/src/jobs/assistant-response.ts b/apps/server/src/jobs/assistant-response.tsindex c5bb998c..52439757 100644--- a/apps/server/src/jobs/assistant-response.ts+++ b/apps/server/src/jobs/assistant-response.ts@@ -26,12 +26,8 @@ import {} from '@/services/llm-service';import { CallbackHandler } from 'langfuse-langchain';import { fetchNode } from '@/lib/nodes';-import { sql } from 'kysely';import { recordsRetrievalService } from '@/services/records-retrieval-service';-// ----------------------------------------------------------------------// Job Input & Type Definitions-// ---------------------------------------------------------------------export type AssistantResponseInput = {type: 'assistant_response';messageId: string;@@ -84,41 +80,41 @@ const ResponseState = Annotation.Root({shouldFilter: boolean;filters: Array<{databaseId: string;- filters: any[]; // DatabaseViewFilterAttributes[]+ filters: any[];}>;}>(),});// ----------------------------------------------------------------------// Chain Node Functions with Improved Naming+// Chain Node Functions// ----------------------------------------------------------------------async function generateRewrittenQuery(state: typeof ResponseState.State) {+const generateRewrittenQuery = async (state: typeof ResponseState.State) => {const rewritten = await rewriteQuery(state.userInput);return { rewrittenQuery: rewritten };-}+};-async function assessIntent(state: typeof ResponseState.State) {+const assessIntent = async (state: typeof ResponseState.State) => {const formattedChatHistory = state.chatHistory.map((doc) => {const timestamp = doc.metadata.createdAt? new Date(doc.metadata.createdAt).toLocaleString(): 'Unknown time';- const authorName = doc.metadata.authorName || 'Unknown User';+ const authorName = doc.metadata.authorName || 'User';return `- [${timestamp}] ${authorName}: ${doc.pageContent}`;}).join('\n');const intent = await assessUserIntent(state.userInput, formattedChatHistory);return { intent };-}+};-async function generateNoContextResponse(state: typeof ResponseState.State) {+const generateNoContextResponse = async (state: typeof ResponseState.State) => {const formattedChatHistory = state.chatHistory.map((doc) => {const timestamp = doc.metadata.createdAt? new Date(doc.metadata.createdAt).toLocaleString(): 'Unknown time';- const authorName = doc.metadata.authorName || 'Unknown User';+ const authorName = doc.metadata.authorName || 'User';return `- [${timestamp}] ${authorName}: ${doc.pageContent}`;}).join('\n');@@ -128,9 +124,9 @@ async function generateNoContextResponse(state: typeof ResponseState.State) {formattedChatHistory);return { finalAnswer: answer };-}+};-async function fetchContextDocuments(state: typeof ResponseState.State) {+const fetchContextDocuments = async (state: typeof ResponseState.State) => {const [nodeResults, documentResults] = await Promise.all([nodeRetrievalService.retrieve(state.rewrittenQuery,@@ -146,7 +142,6 @@ async function fetchContextDocuments(state: typeof ResponseState.State) {let databaseResults: Document[] = [];- // If we have database filters, fetch the filtered recordsif (state.databaseFilters.shouldFilter) {const filteredRecords = await Promise.all(state.databaseFilters.filters.map(async (filter) => {@@ -158,17 +153,12 @@ async function fetchContextDocuments(state: typeof ResponseState.State) {filters: filter.filters,sorts: [],page: 1,- count: 10, // Fetch top 10 matching records+ count: 10,});-- // Get the database node to access its nameconst dbNode = await fetchNode(filter.databaseId);if (!dbNode || dbNode.type !== 'database') return [];-const dbAttributes = dbNode.attributes as DatabaseAttributes;-- // Convert records to Documentsreturn records.map((record) => {const recordNode = record as unknown as RecordNode;return new Document({@@ -188,80 +178,78 @@ async function fetchContextDocuments(state: typeof ResponseState.State) {});}));-- // Flatten the array of arrays into a single arraydatabaseResults = filteredRecords.flat();}return {contextDocuments: [...nodeResults, ...documentResults, ...databaseResults],};-}+};-async function fetchChatHistory(state: typeof ResponseState.State) {+const fetchChatHistory = async (state: typeof ResponseState.State) => {const messages = await database.selectFrom('nodes')- .innerJoin('collaborations', (join) =>- join- .onRef('collaborations.node_id', '=', 'nodes.root_id')- .on('collaborations.collaborator_id', '=', sql.lit(state.userId))- .on('collaborations.deleted_at', 'is', null)- )- .where('nodes.parent_id', '=', state.parentMessageId)- .where('nodes.type', '=', 'message')- .where('nodes.id', '!=', state.currentMessageId)- .orderBy('nodes.created_at', 'asc')+ .where('parent_id', '=', state.parentMessageId)+ .where('type', '=', 'message')+ .where('id', '!=', state.currentMessageId)+ .where('workspace_id', '=', state.workspaceId)+ .orderBy('created_at', 'asc').selectAll().execute();return {- chatHistory: messages.map(- (message) =>- new Document({- pageContent:- getNodeModel(message.attributes.type)?.extractNodeText(- message.id,- message.attributes- )?.attributes || '',- metadata: {- id: message.id,- type: 'message',- createdAt: message.created_at,- author: message.created_by,- authorName: message.created_by === 'colanode_ai' ? 'You' : 'User',- },- })- ),+ chatHistory: messages.map((message) => {+ const isAI = message.created_by === 'colanode_ai';+ // Extract text using the node model's extractor for clean output+ const extracted = getNodeModel(message.attributes.type)?.extractNodeText(+ message.id,+ message.attributes+ );+ const text = extracted ? extracted.attributes : '';+ return new Document({+ pageContent: text || '',+ metadata: {+ id: message.id,+ type: 'message',+ createdAt: message.created_at,+ author: message.created_by,+ authorName: isAI ? 'Colanode AI' : 'User',+ },+ });+ }),};-}+};-async function rerankContextDocuments(state: typeof ResponseState.State) {- const reranked = await rerankContext(- state.contextDocuments,+const rerankContextDocuments = async (state: typeof ResponseState.State) => {+ const reranked = await rerankDocuments(+ state.contextDocuments.map((doc) => ({+ content: doc.pageContent,+ type: doc.metadata.type,+ sourceId: doc.metadata.id,+ })),state.rewrittenQuery);return { rerankedContext: reranked };-}+};-async function selectRelevantDocuments(state: typeof ResponseState.State) {+const selectRelevantDocuments = async (state: typeof ResponseState.State) => {const topDocs = selectTopContext(state.rerankedContext,5,state.contextDocuments);return { topContext: topDocs };-}+};-// Add new helper function to fetch workspace details-async function fetchWorkspaceDetails(workspaceId: string) {+const fetchWorkspaceDetails = async (workspaceId: string) => {return database.selectFrom('workspaces').where('id', '=', workspaceId).select(['name', 'id']).executeTakeFirst();-}+};-async function generateResponse(state: typeof ResponseState.State) {+const generateResponse = async (state: typeof ResponseState.State) => {const workspace = await fetchWorkspaceDetails(state.workspaceId);const formattedChatHistory = state.chatHistory@@ -269,29 +257,26 @@ async function generateResponse(state: typeof ResponseState.State) {const timestamp = doc.metadata.createdAt? new Date(doc.metadata.createdAt).toLocaleString(): 'Unknown time';- const authorName = doc.metadata.authorName || 'Unknown User';+ const authorName = doc.metadata.authorName || 'User';return `- [${timestamp}] ${authorName}: ${doc.pageContent}`;}).join('\n');- const formattedMessages = state.topContext- .filter((doc) => doc.metadata.type === 'message')- .map((doc) => {- const timestamp = doc.metadata.createdAt- ? new Date(doc.metadata.createdAt).toLocaleString()- : 'Unknown time';- const authorName = doc.metadata.authorName || 'Unknown User';- return `- [Source ${doc.metadata.id}]\n- Author: ${authorName}\n- Time: ${timestamp}\n- Content: ${doc.pageContent}\n`;- })- .join('\n');-- const formattedDocuments = state.topContext- .filter((doc) => doc.metadata.type === 'document')+ const formattedContext = state.topContext.map((doc) => {- const timestamp = doc.metadata.createdAt+ const timestamp = doc.metadata?.createdAt? new Date(doc.metadata.createdAt).toLocaleString(): 'Unknown time';- return `- [Source ${doc.metadata.id}]\n- Time: ${timestamp}\n- Content: ${doc.pageContent}\n`;+ const authorName = doc.metadata?.author || 'Unknown';+ let contextHeader = `Source ID: ${doc.metadata.id}\nAuthor: ${authorName}\nTime: ${timestamp}`;+ if (+ doc.metadata?.type === 'node' &&+ doc.metadata.nodeType === 'record' &&+ doc.metadata.parentContext?.name+ ) {+ contextHeader += `\nDatabase: ${doc.metadata.parentContext.name}`;+ }+ return `${contextHeader}\nContent: "${doc.pageContent}"\n`;}).join('\n');@@ -301,8 +286,8 @@ async function generateResponse(state: typeof ResponseState.State) {userName: state.userDetails.name,userEmail: state.userDetails.email,formattedChatHistory,- formattedMessages,- formattedDocuments,+ formattedMessages: '',+ formattedDocuments: formattedContext,question: state.userInput,});@@ -310,10 +295,9 @@ async function generateResponse(state: typeof ResponseState.State) {finalAnswer: result.answer,citations: result.citations,};-}+};-async function fetchDatabaseContext(state: typeof ResponseState.State) {- // Fetch all databases the user has access to+const fetchDatabaseContext = async (state: typeof ResponseState.State) => {const databases = await database.selectFrom('nodes as n').innerJoin('collaborations as c', 'c.node_id', 'n.root_id')@@ -324,11 +308,9 @@ async function fetchDatabaseContext(state: typeof ResponseState.State) {.selectAll().execute();- // For each database, fetch schema and sample recordsconst databaseContext = await Promise.all(databases.map(async (db) => {const dbNode = db as unknown as DatabaseNode;- // Get sample recordsconst sampleRecords = await recordsRetrievalService.retrieveByFilters(db.id,state.workspaceId,@@ -337,11 +319,9 @@ async function fetchDatabaseContext(state: typeof ResponseState.State) {filters: [],sorts: [],page: 1,- count: 5, // Fetch 5 sample records+ count: 5,});-- // Extract field information from database attributesconst fields = dbNode.attributes.fields || {};const formattedFields = Object.entries(fields).reduce((acc, [id, field]) => ({@@ -353,7 +333,6 @@ async function fetchDatabaseContext(state: typeof ResponseState.State) {}),{});-return {id: db.id,name: dbNode.attributes.name || 'Untitled Database',@@ -364,11 +343,11 @@ async function fetchDatabaseContext(state: typeof ResponseState.State) {);return { databaseContext };-}+};-async function generateDatabaseFilterAttributes(+const generateDatabaseFilterAttributes = async (state: typeof ResponseState.State-) {+) => {if (state.intent === 'no_context' || !state.databaseContext.length) {return {databaseFilters: {@@ -384,10 +363,46 @@ async function generateDatabaseFilterAttributes(});return { databaseFilters: filters };-}+};++const selectTopContext = (+ reranked: {+ index: number;+ score: number;+ type: string;+ sourceId: string;+ }[],+ max: number,+ contextDocuments: Document[]+): Document[] => {+ if (reranked.length === 0) return [];+ const maxScore = Math.max(...reranked.map((item) => item.score));+ const threshold = maxScore * 0.5;+ return reranked+ .filter((item) => item.score >= threshold && item.score > 0)+ .sort((a, b) => b.score - a.score)+ .slice(0, max)+ .map((item) => {+ if (item.index >= 0 && item.index < contextDocuments.length) {+ const doc = contextDocuments[item.index];+ if (!doc) {+ return undefined;+ }++ if (+ doc.metadata.id === item.sourceId &&+ doc.metadata.type === item.type+ ) {+ return doc;+ }+ }+ return undefined;+ })+ .filter((doc): doc is Document => doc !== undefined);+};// ----------------------------------------------------------------------// Build the Response Chain Graph+// Response Chain Graph// ---------------------------------------------------------------------const assistantResponseChain = new StateGraph(ResponseState).addNode('generateRewrittenQuery', generateRewrittenQuery)@@ -405,10 +420,8 @@ const assistantResponseChain = new StateGraph(ResponseState).addConditionalEdges('assessIntent', (state) => {return state.intent === 'no_context'? 'generateNoContextResponse'- : 'fetchDatabaseContext';+ : 'generateRewrittenQuery';})- .addEdge('fetchDatabaseContext', 'generateDatabaseFilterAttributes')- .addEdge('generateDatabaseFilterAttributes', 'generateRewrittenQuery').addEdge('generateRewrittenQuery', 'fetchContextDocuments').addEdge('fetchContextDocuments', 'rerankContextDocuments').addEdge('rerankContextDocuments', 'selectRelevantDocuments')@@ -417,9 +430,6 @@ const assistantResponseChain = new StateGraph(ResponseState).addEdge('generateNoContextResponse', '__end__').compile();-// ----------------------------------------------------------------------// Initialize Langfuse Callback Handler-// ---------------------------------------------------------------------const langfuseCallback = new CallbackHandler({publicKey: configuration.ai.langfuse.publicKey,secretKey: configuration.ai.langfuse.secretKey,@@ -479,13 +489,14 @@ export const assistantResponseHandler: JobHandler<workspaceId,userId: user.id,userDetails: {- name: user.name || 'Unknown User',+ name: user.name || 'User',email: user.email || '[email protected]',},parentMessageId: message.parent_id || message.id,currentMessageId: message.id,originalMessage: message,intent: 'retrieve',+ databaseFilters: { shouldFilter: false, filters: [] },},{ callbacks: [langfuseCallback] });@@ -508,22 +519,19 @@ export const assistantResponseHandler: JobHandler<}};-// ----------------------------------------------------------------------// Helper Functions-// ----------------------------------------------------------------------async function createAndPublishResponse(+const createAndPublishResponse = async (response: string,citations: Array<{ sourceId: string; quote: string }> | undefined,originalMessage: any,workspaceId: string-) {+) => {const id = generateId(IdType.Message);const blockId = generateId(IdType.Block);const messageAttributes: NodeAttributes = {type: 'message',subtype: 'answer',- parentId: originalMessage.parent_id || originalMessage.id,+ parentId: originalMessage.parent_id,content: {[blockId]: {id: blockId,@@ -575,49 +583,4 @@ async function createAndPublishResponse(rootId: createdMessage.root_id,workspaceId: createdMessage.workspace_id,});-}--async function rerankContext(context: Document[], query: string) {- const documentsToRerank = context.map((doc) => ({- content: doc.pageContent,- type: doc.metadata.type,- sourceId: doc.metadata.id,- }));-- return await rerankDocuments(documentsToRerank, query);-}--function selectTopContext(- reranked: {- index: number;- score: number;- type: string;- sourceId: string;- }[],- max: number,- contextDocuments: Document[]-): Document[] {- if (reranked.length === 0) return [];-- const maxScore = Math.max(...reranked.map((item) => item.score));- const threshold = maxScore * 0.5;-- return reranked- .filter((item) => item.score >= threshold && item.score > 0)- .sort((a, b) => b.score - a.score)- .slice(0, max)- .map((item) => {- if (item.index >= 0 && item.index < contextDocuments.length) {- const doc = contextDocuments[item.index];- if (!doc) return undefined;- if (- doc.metadata.id === item.sourceId &&- doc.metadata.type === item.type- ) {- return doc;- }- }- return undefined;- })- .filter((doc): doc is Document => doc !== undefined);-}+};diff --git a/apps/server/src/jobs/embed-node.ts b/apps/server/src/jobs/embed-node.tsindex d751c2de..cfa4e406 100644--- a/apps/server/src/jobs/embed-node.ts+++ b/apps/server/src/jobs/embed-node.ts@@ -6,6 +6,7 @@ import { CreateNodeEmbedding, SelectNode } from '@/data/schema';import { sql } from 'kysely';import { fetchNode } from '@/lib/nodes';import { getNodeModel } from '@colanode/core';+import { FieldValue, DatabaseAttributes } from '@colanode/core';export type EmbedNodeInput = {type: 'embed_node';@@ -20,30 +21,74 @@ declare module '@/types/jobs' {}}-const extractNodeText = (node: SelectNode): string => {+const extractNodeText = async (+ node: SelectNode+): Promise<string | null | undefined> => {if (!node) {- return '';+ return;}const nodeModel = getNodeModel(node.attributes.type);if (!nodeModel) {- return '';+ return;}const nodeText = nodeModel.extractNodeText(node.id, node.attributes);if (!nodeText) {- return '';+ return;}- const textParts: string[] = [];- if (nodeText.name) {- textParts.push(nodeText.name);- }- if (nodeText.attributes) {- textParts.push(nodeText.attributes);+ if (node.attributes.type === 'record') {+ const fields = node.attributes.fields;+ if (!fields) {+ return;+ }++ const databaseNode = await database+ .selectFrom('nodes')+ .selectAll()+ .where('id', '=', node.attributes.databaseId)+ .executeTakeFirst();++ if (!databaseNode) {+ return;+ }++ const databaseAttrs = databaseNode.attributes as DatabaseAttributes;+ const fieldTexts = Object.entries(fields)+ .map(([fieldId, field]) => {+ const typedField = field as FieldValue;+ const fieldInfo = databaseAttrs.fields[fieldId];+ if (!fieldInfo) {+ return null;+ }++ let valueText = '';+ switch (typedField.type) {+ case 'string':+ case 'text':+ valueText = typedField.value.toString();+ break;+ case 'number':+ valueText = typedField.value.toString();+ break;+ case 'boolean':+ valueText = typedField.value ? 'Yes' : 'No';+ break;+ case 'string_array':+ valueText = typedField.value.join(', ');+ break;+ default:+ break;+ }+ return `${fieldInfo.name}: ${valueText}`;+ })+ .filter(Boolean);++ return `Name: ${nodeText.name || 'Untitled Record'}\n${fieldTexts.join('\n')}`;}- return textParts.join('\n').trim();+ return nodeText.attributes;};export const embedNodeHandler = async (input: {@@ -56,17 +101,18 @@ export const embedNodeHandler = async (input: {const { nodeId } = input;const node = await fetchNode(nodeId);+if (!node) {return;}const nodeModel = getNodeModel(node.attributes.type);- // Skip nodes that are handled by embed documents job- if (!nodeModel || nodeModel.documentSchema) {+ // Skip page nodes since they store content in documents+ if (!nodeModel || node.attributes.type === 'page') {return;}- const text = extractNodeText(node);+ const text = await extractNodeText(node);if (!text || text.trim() === '') {await database.deleteFrom('node_embeddings')diff --git a/apps/server/src/services/chunking-service.ts b/apps/server/src/services/chunking-service.tsindex 0682d8bf..a223618b 100644--- a/apps/server/src/services/chunking-service.ts+++ b/apps/server/src/services/chunking-service.ts@@ -2,7 +2,7 @@ import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';import { configuration } from '@/lib/configuration';import { database } from '@/data/database';import { addContextToChunk } from '@/services/llm-service';-import { getNodeModel, NodeType } from '@colanode/core';+import { getNodeModel, NodeType, FieldAttributes } from '@colanode/core';import type { SelectNode, SelectDocument } from '@/data/schema';type BaseMetadata = {@@ -27,7 +27,7 @@ type BaseMetadata = {export type NodeMetadata = BaseMetadata & {type: 'node';nodeType: NodeType;- fields?: Record<string, unknown> | null;+ fieldInfo?: Record<string, { type: string; name: string }>;};export type DocumentMetadata = BaseMetadata & {@@ -122,10 +122,33 @@ export class ChunkingService {}}+ // For record nodes, fetch the database schema to provide field context+ let fieldInfo: Record<string, { type: string; name: string }> | undefined;+ if (node.attributes.type === 'record') {+ const databaseNode = await database+ .selectFrom('nodes')+ .selectAll()+ .where('id', '=', node.attributes.databaseId)+ .executeTakeFirst();++ if (databaseNode?.attributes.type === 'database') {+ fieldInfo = Object.entries(databaseNode.attributes.fields).reduce(+ (acc, [fieldId, field]) => ({+ ...acc,+ [fieldId]: {+ type: (field as FieldAttributes).type,+ name: (field as FieldAttributes).name,+ },+ }),+ {} as Record<string, { type: string; name: string }>+ );+ }+ }+return {type: 'node',nodeType: node.attributes.type,- fields: 'fields' in node.attributes ? node.attributes.fields : null,+ fieldInfo,...baseMetadata,};}@@ -175,7 +198,7 @@ export class ChunkingService {}const nodeText = nodeModel.extractNodeText(node.id, node.attributes);- if (!nodeText || !nodeText.name) {+ if (!nodeText) {return undefined;}@@ -201,7 +224,7 @@ export class ChunkingService {return {id: node.id,- name: nodeText.name,+ name: nodeText.name ?? node.attributes.type,createdAt: node.created_at,createdBy: node.created_by,updatedAt: node.updated_at,@@ -235,19 +258,19 @@ export class ChunkingService {parentNode.attributes);- // Get the full path by traversing up the tree+ // Get the full path by traversing up the tree, ordered from root to leafconst pathNodes = await database.selectFrom('node_paths').innerJoin('nodes', 'nodes.id', 'node_paths.ancestor_id').select(['nodes.id', 'nodes.attributes']).where('node_paths.descendant_id', '=', node.id)- .orderBy('node_paths.level', 'asc')+ .orderBy('node_paths.level', 'desc').execute();const path = pathNodes.map((n) => {const model = getNodeModel(n.attributes.type);- return model?.extractNodeText(n.id, n.attributes)?.name ?? 'Untitled';+ return model?.extractNodeText(n.id, n.attributes)?.name ?? '';}).join(' / ');diff --git a/apps/server/src/services/llm-service.ts b/apps/server/src/services/llm-service.tsindex 75cfb208..05954030 100644--- a/apps/server/src/services/llm-service.ts+++ b/apps/server/src/services/llm-service.ts@@ -53,9 +53,9 @@ const databaseFilterSchema = z.object({type DatabaseFilterResult = z.infer<typeof databaseFilterSchema>;-export function getChatModel(+export const getChatModel = (task: keyof typeof configuration.ai.models-): ChatOpenAI | ChatGoogleGenerativeAI {+): ChatOpenAI | ChatGoogleGenerativeAI => {const modelConfig = configuration.ai.models[task];if (!configuration.ai.enabled) {throw new Error('AI is disabled.');@@ -80,7 +80,7 @@ export function getChatModel(default:throw new Error(`Unsupported AI provider: ${modelConfig.provider}`);}-}+};// Updated prompt templates using type-safe node types and contextconst queryRewritePrompt = PromptTemplate.fromTemplate(@@ -124,25 +124,31 @@ Return an array of rankings in JSON format.`const answerPrompt = ChatPromptTemplate.fromTemplate(`You are Colanode's AI assistant.-+CURRENT TIME: {currentTimestamp}WORKSPACE: {workspaceName}USER: {userName} ({userEmail})-+CONVERSATION HISTORY:{formattedChatHistory}--RELATED CONTEXT:-Messages:-{formattedMessages}-Documents:++RELEVANT CONTEXT:{formattedDocuments}-+USER QUERY:{question}--Provide a clear, professional answer. Then, in a separate citations array, list exact quotes (with source IDs) used to form your answer.-Return the result as JSON with keys "answer" and "citations".`++Based solely on the conversation history and the relevant context above, provide a clear and professional answer to the user's query. In your answer, include exact quotes from the provided context that support your answer.++Return your response as a JSON object with the following structure:+{{+ "answer": <your answer as a string>,+ "citations": [+ {{ "sourceId": <source id>, "quote": <exact quote from the context> }},+ ...+ ]+}}+`);const intentRecognitionPrompt = PromptTemplate.fromTemplate(@@ -221,33 +227,33 @@ For each filter, use the exact field IDs from the database schema.Use appropriate operators based on field types.`);-export async function rewriteQuery(query: string): Promise<string> {+export const rewriteQuery = async (query: string): Promise<string> => {const task = 'queryRewrite';const model = getChatModel(task);return queryRewritePrompt.pipe(model).pipe(new StringOutputParser()).invoke({ query });-}+};-export async function summarizeDocument(+export const summarizeDocument = async (document: Document,query: string-): Promise<string> {+): Promise<string> => {const task = 'summarization';const model = getChatModel(task);return summarizationPrompt.pipe(model).pipe(new StringOutputParser()).invoke({ text: document.pageContent, query });-}+};-export async function rerankDocuments(+export const rerankDocuments = async (documents: { content: string; type: string; sourceId: string }[],query: string): Promise<Array<{ index: number; score: number; type: string; sourceId: string }>-> {+> => {const task = 'rerank';const model = getChatModel(task).withStructuredOutput(rerankedDocumentsSchema@@ -262,9 +268,9 @@ export async function rerankDocuments(.pipe(model).invoke({ query, context: formattedContext })) as RerankedDocuments;return result.rankings;-}+};-export async function generateFinalAnswer(promptArgs: {+export const generateFinalAnswer = async (promptArgs: {currentTimestamp: string;workspaceName: string;userName: string;@@ -276,28 +282,28 @@ export async function generateFinalAnswer(promptArgs: {}): Promise<{answer: string;citations: Array<{ sourceId: string; quote: string }>;-}> {+}> => {const task = 'response';const model = getChatModel(task).withStructuredOutput(citedAnswerSchema);return (await answerPrompt.pipe(model).invoke(promptArgs)) as CitedAnswer;-}+};-export async function generateNoContextAnswer(+export const generateNoContextAnswer = async (query: string,chatHistory: string = ''-): Promise<string> {+): Promise<string> => {const task = 'noContext';const model = getChatModel(task);return noContextPrompt.pipe(model).pipe(new StringOutputParser()).invoke({ question: query, formattedChatHistory: chatHistory });-}+};-export async function assessUserIntent(+export const assessUserIntent = async (query: string,chatHistory: string-): Promise<'retrieve' | 'no_context'> {+): Promise<'retrieve' | 'no_context'> => {const task = 'intentRecognition';const model = getChatModel(task);const result = await intentRecognitionPrompt@@ -307,75 +313,78 @@ export async function assessUserIntent(return result.trim().toLowerCase() === 'no_context'? 'no_context': 'retrieve';-}+};const getNodeContextPrompt = (metadata: NodeMetadata): string => {- const basePrompt = `Given the following context about a {nodeType}:-Name: {name}-Created by: {authorName} on {createdAt}-Last updated: {updatedAt} by {lastAuthorName}-Path: {path}-Workspace: {workspaceName}-{additionalContext}--Full content:-{fullText}--Current chunk:-{chunk}--Generate a brief (50-100 tokens) contextual prefix that:-1. Explains what this chunk is part of-2. Provides relevant context from the metadata-3. Makes the chunk more understandable in isolation-Do not repeat the chunk content. Return only the contextual prefix.`;-- const collaborators =- metadata.collaborators?.map((c) => `${c.name}`).join(', ') ?? '';+ const metadataLines = [+ metadata.name && `Name: ${metadata.name}`,+ metadata.author?.name &&+ `Created by: ${metadata.author.name} on ${metadata.createdAt}`,+ metadata.lastAuthor?.name &&+ metadata.updatedAt &&+ `Last updated: ${metadata.updatedAt} by ${metadata.lastAuthor.name}`,+ metadata.parentContext?.path && `Path: ${metadata.parentContext.path}`,+ metadata.workspace?.name && `Workspace: ${metadata.workspace.name}`,+ ].filter(Boolean);++ let additionalLines: string[] = [];+ const collaborators = metadata.collaborators+ ?.filter((c) => c.name)+ .map((c) => c.name)+ .join(', ');switch (metadata.nodeType) {+ case 'page':+ if (collaborators) {+ additionalLines.push(`Collaborators: ${collaborators}`);+ }+ break;+case 'message':- return basePrompt.replace(- '{additionalContext}',- `In: ${metadata.parentContext?.type ?? 'unknown'} "${metadata.parentContext?.name ?? 'unknown'}"-Path: ${metadata.parentContext?.path ?? 'unknown'}-Participants: ${collaborators}`- );+ if (metadata.parentContext?.type || metadata.parentContext?.name) {+ additionalLines.push(+ `In: ${metadata.parentContext.type ?? 'channel'} "${metadata.parentContext.name ?? 'Untitled'}"`+ );+ }+ if (collaborators) {+ additionalLines.push(`Participants: ${collaborators}`);+ }+ break;case 'record':- return basePrompt.replace(- '{additionalContext}',- `Database: ${metadata.parentContext?.name ?? 'unknown'}-Path: ${metadata.parentContext?.path ?? 'unknown'}-Fields: ${Object.keys(metadata.fields ?? {}).join(', ')}`- );+ if (metadata.parentContext?.name) {+ additionalLines.push(`Database: ${metadata.parentContext.name}`);+ }+ if (metadata.fieldInfo) {+ const fieldDescriptions = Object.entries(metadata.fieldInfo)+ .map(([fieldId, info]) => `${info.name} (${info.type})`)+ .join(', ');+ additionalLines.push(`Fields: ${fieldDescriptions}`);+ }+ break;+ }- case 'page':- return basePrompt.replace(- '{additionalContext}',- `Location: ${metadata.parentContext?.path ?? 'root level'}-Collaborators: ${collaborators}`- );-- case 'database':- return basePrompt.replace(- '{additionalContext}',- `Path: ${metadata.parentContext?.path ?? 'root level'}-Fields: ${Object.keys(metadata.fields ?? {}).join(', ')}-Collaborators: ${collaborators}`- );-- case 'channel':- return basePrompt.replace(- '{additionalContext}',- `Type: Channel-Path: ${metadata.parentContext?.path ?? 'root level'}-Members: ${collaborators}`- );+ const contextLines = [...metadataLines, ...additionalLines].filter(Boolean);+ const contextSection = `Given the following context about a ${metadata.nodeType === 'record' ? 'database record' : metadata.nodeType}:+${contextLines.join('\n')}`;++ const contentSection =+ metadata.nodeType === 'record'+ ? `\n\nContent:\n{chunk}`+ : `\n\nFull content:\n{fullText}\n\nCurrent chunk:\n{chunk}`;++ const instructionsSection =+ metadata.nodeType === 'record'+ ? `\n\nGenerate a brief (50-100 tokens) contextual prefix that:+1. Explains what this record represents+2. Highlights key information from its fields+3. Provides relevant context about which database it belongs to`+ : `\n\nGenerate a brief (50-100 tokens) contextual prefix that:+1. Explains what this chunk is part of+2. Provides relevant context from the metadata+3. Makes the chunk more understandable in isolation`;- default:- return basePrompt.replace('{additionalContext}', '');- }+ return `${contextSection}${contentSection}${instructionsSection}\nDo not repeat the content. Return only the contextual prefix.`;};interface PromptVariables {@@ -414,11 +423,11 @@ Generate a brief (50-100 tokens) contextual prefix that:Do not repeat the chunk content. Return only the contextual prefix.`);-export async function addContextToChunk(+export const addContextToChunk = async (chunk: string,fullText: string,metadata?: ChunkingMetadata-): Promise<string> {+): Promise<string> => {try {if (!chunk || chunk.trim() === '') {return chunk;@@ -484,9 +493,9 @@ export async function addContextToChunk(console.error('Error in addContextToChunk:', err);return chunk;}-}+};-export async function generateDatabaseFilters(args: {+export const generateDatabaseFilters = async (args: {query: string;databases: Array<{id: string;@@ -494,7 +503,7 @@ export async function generateDatabaseFilters(args: {fields: Record<string, { type: string; name: string }>;sampleRecords: any[];}>;-}): Promise<DatabaseFilterResult> {+}): Promise<DatabaseFilterResult> => {const task = 'databaseFilter';const model = getChatModel(task).withStructuredOutput(databaseFilterSchema);@@ -524,4 +533,4 @@ ${db.sampleRecordsreturn databaseFilterPrompt.pipe(model).invoke({ query: args.query, databasesInfo });-}+};