Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Enhance document metadata with database context and improve enrichment prompts

Ylber Gashi415922d73f3c
diff --git a/apps/server/src/lib/chunking.ts b/apps/server/src/lib/chunking.ts
index 67ebed0e..55e52cd9 100644
--- a/apps/server/src/lib/chunking.ts
+++ b/apps/server/src/lib/chunking.ts
@@ -3,7 +3,7 @@ import { configuration } from '@/lib/configuration';
import { database } from '@/data/database';
import { getNodeModel, FieldAttributes } from '@colanode/core';
import type { SelectNode, SelectDocument } from '@/data/schema';
-import { getNodeContextPrompt, documentContextPrompt } from '@/lib/llm-prompts';
+import { prepareEnrichmentPrompt } from '@/lib/llm-prompts';
import { BaseMetadata, NodeMetadata, DocumentMetadata } from '@/types/chunking';
async function buildBaseMetadata(
@@ -158,6 +158,31 @@ async function buildDocumentMetadata(
const parentContext = await buildParentContext(node);
if (parentContext) baseMetadata.parentContext = parentContext;
}
+
+ // Add database info for record nodes
+ if (node.attributes.type === 'record') {
+ const databaseNode = await database
+ .selectFrom('nodes')
+ .selectAll()
+ .where('id', '=', node.attributes.databaseId)
+ .executeTakeFirst();
+ if (databaseNode?.attributes.type === 'database') {
+ baseMetadata.databaseInfo = {
+ id: databaseNode.id,
+ name: databaseNode.attributes.name || 'Untitled Database',
+ fields: Object.entries(databaseNode.attributes.fields).reduce(
+ (acc, [fieldId, field]) => ({
+ ...acc,
+ [fieldId]: {
+ type: (field as FieldAttributes).type,
+ name: (field as FieldAttributes).name,
+ },
+ }),
+ {} as Record<string, { type: string; name: string }>
+ ),
+ };
+ }
+ }
}
return { type: 'document', ...baseMetadata };
}
@@ -180,43 +205,6 @@ async function fetchMetadata(metadata?: {
}
}
-export async function prepareEnrichmentPrompt(
- chunk: string,
- fullText: string,
- metadata: NodeMetadata | DocumentMetadata
-): Promise<{ prompt: string; baseVars: Record<string, string> }> {
- const formatDate = (date?: Date) =>
- date ? new Date(date).toUTCString() : 'unknown';
-
- const baseVars: Record<string, string> = {
- nodeType:
- metadata.type === 'node'
- ? (metadata as NodeMetadata).nodeType
- : metadata.type,
- name: metadata.name ?? 'Untitled',
- createdAt: formatDate(metadata.createdAt),
- updatedAt: metadata.updatedAt ? formatDate(metadata.updatedAt) : '',
- authorName: metadata.author?.name ?? 'Unknown',
- lastAuthorName: metadata.lastAuthor?.name ?? '',
- path: metadata.parentContext?.path ?? '',
- workspaceName: metadata.workspace?.name ?? 'Unknown Workspace',
- fullText,
- chunk,
- };
-
- let prompt: string;
- if (metadata.type === 'node') {
- prompt = getNodeContextPrompt(metadata as NodeMetadata);
- } else {
- prompt = documentContextPrompt.template.toString();
- }
- Object.entries(baseVars).forEach(([key, value]) => {
- prompt = prompt.replace(new RegExp(`{${key}}`, 'g'), value);
- });
-
- return { prompt, baseVars };
-}
-
export async function chunkText(
text: string,
metadata?: { type: 'node' | 'document'; node: SelectNode },
@@ -243,13 +231,13 @@ export async function chunkText(
return chunks;
}
for (const chunk of chunks) {
- const { prompt } = await prepareEnrichmentPrompt(
+ const { prompt, baseVars } = prepareEnrichmentPrompt(
chunk,
text,
enrichedMetadata
);
- const enrichment = await enrichFn(prompt, {});
+ const enrichment = await enrichFn(prompt, baseVars);
enriched.push(enrichment + '\n\n' + chunk);
}
return enriched;
diff --git a/apps/server/src/lib/llm-prompts.ts b/apps/server/src/lib/llm-prompts.ts
index 8bc49e28..b1d15cc6 100644
--- a/apps/server/src/lib/llm-prompts.ts
+++ b/apps/server/src/lib/llm-prompts.ts
@@ -1,4 +1,5 @@
import { PromptTemplate, ChatPromptTemplate } from '@langchain/core/prompts';
+import { NodeMetadata, DocumentMetadata } from '@/types/chunking';
export const queryRewritePrompt = PromptTemplate.fromTemplate(
`You are an expert at rewriting queries for information retrieval within Colanode.
@@ -177,6 +178,7 @@ Location: {path}
Created by: {authorName} on {createdAt}
Last updated: {updatedAt} by {lastAuthorName}
Workspace: {workspaceName}
+{databaseContext}
Full content:
{fullText}
@@ -185,8 +187,59 @@ Current chunk:
{chunk}
Generate a brief (50-100 tokens) contextual prefix that:
-1. Explains what this document is.
+1. Explains what this document is{recordInstructions}
2. Provides relevant context about its location and purpose.
3. Makes the chunk more understandable in isolation.
Do not repeat the chunk content. Return only the contextual prefix.`
);
+
+export function prepareEnrichmentPrompt(
+ chunk: string,
+ fullText: string,
+ metadata: NodeMetadata | DocumentMetadata
+): { prompt: string; baseVars: Record<string, string> } {
+ const formatDate = (date?: Date) =>
+ date ? new Date(date).toUTCString() : 'unknown';
+
+ let databaseContext = '';
+ let recordInstructions = '';
+ if (metadata.type === 'document' && metadata.databaseInfo) {
+ databaseContext = `Database: ${metadata.databaseInfo.name}
+Fields:
+${Object.entries(metadata.databaseInfo.fields)
+ .map(([_, field]) => `- ${field.name} (${field.type})`)
+ .join('\n')}`;
+ recordInstructions =
+ ', including how it relates to the database record and its fields';
+ }
+
+ const baseVars: Record<string, string> = {
+ nodeType:
+ metadata.type === 'node'
+ ? (metadata as NodeMetadata).nodeType
+ : metadata.type,
+ name: metadata.name ?? 'Untitled',
+ createdAt: metadata.createdAt ? formatDate(metadata.createdAt) : '',
+ updatedAt: metadata.updatedAt ? formatDate(metadata.updatedAt) : '',
+ authorName: metadata.author?.name ?? 'Unknown',
+ lastAuthorName: metadata.lastAuthor?.name ?? '',
+ path: metadata.parentContext?.path ?? '',
+ workspaceName: metadata.workspace?.name ?? 'Unknown Workspace',
+ fullText,
+ chunk,
+ databaseContext,
+ recordInstructions,
+ };
+
+ let prompt: string;
+ if (metadata.type === 'node') {
+ prompt = getNodeContextPrompt(metadata as NodeMetadata);
+ } else {
+ prompt = documentContextPrompt.template.toString();
+ }
+ Object.entries(baseVars).forEach(([key, value]) => {
+ prompt = prompt.replace(new RegExp(`{${key}}`, 'g'), value);
+ });
+
+ return { prompt, baseVars };
+}
diff --git a/apps/server/src/types/chunking.ts b/apps/server/src/types/chunking.ts
index 8dbfd2a6..a5ca9ef0 100644
--- a/apps/server/src/types/chunking.ts
+++ b/apps/server/src/types/chunking.ts
@@ -1,21 +1,26 @@
-export type BaseMetadata = {
+export interface BaseMetadata {
id: string;
name?: string | null;
- createdAt: Date;
+ createdAt: Date | null;
createdBy: string;
updatedAt?: Date | null;
updatedBy?: string | null;
author?: { id: string; name: string };
lastAuthor?: { id: string; name: string };
+ workspace?: { id: string; name: string };
parentContext?: {
id: string;
type: string;
- name?: string;
- path?: string;
+ name?: string | null;
+ path: string;
};
collaborators?: Array<{ id: string; name: string }>;
- workspace?: { id: string; name: string };
-};
+ databaseInfo?: {
+ id: string;
+ name: string;
+ fields: Record<string, { type: string; name: string }>;
+ };
+}
export type NodeMetadata = BaseMetadata & {
type: 'node';