Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Enhance document metadata with database context and improve enrichment prompts
Ylber Gashi415922d73f3c
diff --git a/apps/server/src/lib/chunking.ts b/apps/server/src/lib/chunking.tsindex 67ebed0e..55e52cd9 100644--- a/apps/server/src/lib/chunking.ts+++ b/apps/server/src/lib/chunking.ts@@ -3,7 +3,7 @@ import { configuration } from '@/lib/configuration';import { database } from '@/data/database';import { getNodeModel, FieldAttributes } from '@colanode/core';import type { SelectNode, SelectDocument } from '@/data/schema';-import { getNodeContextPrompt, documentContextPrompt } from '@/lib/llm-prompts';+import { prepareEnrichmentPrompt } from '@/lib/llm-prompts';import { BaseMetadata, NodeMetadata, DocumentMetadata } from '@/types/chunking';async function buildBaseMetadata(@@ -158,6 +158,31 @@ async function buildDocumentMetadata(const parentContext = await buildParentContext(node);if (parentContext) baseMetadata.parentContext = parentContext;}++ // Add database info for record nodes+ if (node.attributes.type === 'record') {+ const databaseNode = await database+ .selectFrom('nodes')+ .selectAll()+ .where('id', '=', node.attributes.databaseId)+ .executeTakeFirst();+ if (databaseNode?.attributes.type === 'database') {+ baseMetadata.databaseInfo = {+ id: databaseNode.id,+ name: databaseNode.attributes.name || 'Untitled Database',+ fields: Object.entries(databaseNode.attributes.fields).reduce(+ (acc, [fieldId, field]) => ({+ ...acc,+ [fieldId]: {+ type: (field as FieldAttributes).type,+ name: (field as FieldAttributes).name,+ },+ }),+ {} as Record<string, { type: string; name: string }>+ ),+ };+ }+ }}return { type: 'document', ...baseMetadata };}@@ -180,43 +205,6 @@ async function fetchMetadata(metadata?: {}}-export async function prepareEnrichmentPrompt(- chunk: string,- fullText: string,- metadata: NodeMetadata | DocumentMetadata-): Promise<{ prompt: string; baseVars: Record<string, string> }> {- const formatDate = (date?: Date) =>- date ? new Date(date).toUTCString() : 'unknown';-- const baseVars: Record<string, string> = {- nodeType:- metadata.type === 'node'- ? (metadata as NodeMetadata).nodeType- : metadata.type,- name: metadata.name ?? 'Untitled',- createdAt: formatDate(metadata.createdAt),- updatedAt: metadata.updatedAt ? formatDate(metadata.updatedAt) : '',- authorName: metadata.author?.name ?? 'Unknown',- lastAuthorName: metadata.lastAuthor?.name ?? '',- path: metadata.parentContext?.path ?? '',- workspaceName: metadata.workspace?.name ?? 'Unknown Workspace',- fullText,- chunk,- };-- let prompt: string;- if (metadata.type === 'node') {- prompt = getNodeContextPrompt(metadata as NodeMetadata);- } else {- prompt = documentContextPrompt.template.toString();- }- Object.entries(baseVars).forEach(([key, value]) => {- prompt = prompt.replace(new RegExp(`{${key}}`, 'g'), value);- });-- return { prompt, baseVars };-}-export async function chunkText(text: string,metadata?: { type: 'node' | 'document'; node: SelectNode },@@ -243,13 +231,13 @@ export async function chunkText(return chunks;}for (const chunk of chunks) {- const { prompt } = await prepareEnrichmentPrompt(+ const { prompt, baseVars } = prepareEnrichmentPrompt(chunk,text,enrichedMetadata);- const enrichment = await enrichFn(prompt, {});+ const enrichment = await enrichFn(prompt, baseVars);enriched.push(enrichment + '\n\n' + chunk);}return enriched;diff --git a/apps/server/src/lib/llm-prompts.ts b/apps/server/src/lib/llm-prompts.tsindex 8bc49e28..b1d15cc6 100644--- a/apps/server/src/lib/llm-prompts.ts+++ b/apps/server/src/lib/llm-prompts.ts@@ -1,4 +1,5 @@import { PromptTemplate, ChatPromptTemplate } from '@langchain/core/prompts';+import { NodeMetadata, DocumentMetadata } from '@/types/chunking';export const queryRewritePrompt = PromptTemplate.fromTemplate(`You are an expert at rewriting queries for information retrieval within Colanode.@@ -177,6 +178,7 @@ Location: {path}Created by: {authorName} on {createdAt}Last updated: {updatedAt} by {lastAuthorName}Workspace: {workspaceName}+{databaseContext}Full content:{fullText}@@ -185,8 +187,59 @@ Current chunk:{chunk}Generate a brief (50-100 tokens) contextual prefix that:-1. Explains what this document is.+1. Explains what this document is{recordInstructions}2. Provides relevant context about its location and purpose.3. Makes the chunk more understandable in isolation.Do not repeat the chunk content. Return only the contextual prefix.`);++export function prepareEnrichmentPrompt(+ chunk: string,+ fullText: string,+ metadata: NodeMetadata | DocumentMetadata+): { prompt: string; baseVars: Record<string, string> } {+ const formatDate = (date?: Date) =>+ date ? new Date(date).toUTCString() : 'unknown';++ let databaseContext = '';+ let recordInstructions = '';+ if (metadata.type === 'document' && metadata.databaseInfo) {+ databaseContext = `Database: ${metadata.databaseInfo.name}+Fields:+${Object.entries(metadata.databaseInfo.fields)+ .map(([_, field]) => `- ${field.name} (${field.type})`)+ .join('\n')}`;+ recordInstructions =+ ', including how it relates to the database record and its fields';+ }++ const baseVars: Record<string, string> = {+ nodeType:+ metadata.type === 'node'+ ? (metadata as NodeMetadata).nodeType+ : metadata.type,+ name: metadata.name ?? 'Untitled',+ createdAt: metadata.createdAt ? formatDate(metadata.createdAt) : '',+ updatedAt: metadata.updatedAt ? formatDate(metadata.updatedAt) : '',+ authorName: metadata.author?.name ?? 'Unknown',+ lastAuthorName: metadata.lastAuthor?.name ?? '',+ path: metadata.parentContext?.path ?? '',+ workspaceName: metadata.workspace?.name ?? 'Unknown Workspace',+ fullText,+ chunk,+ databaseContext,+ recordInstructions,+ };++ let prompt: string;+ if (metadata.type === 'node') {+ prompt = getNodeContextPrompt(metadata as NodeMetadata);+ } else {+ prompt = documentContextPrompt.template.toString();+ }+ Object.entries(baseVars).forEach(([key, value]) => {+ prompt = prompt.replace(new RegExp(`{${key}}`, 'g'), value);+ });++ return { prompt, baseVars };+}diff --git a/apps/server/src/types/chunking.ts b/apps/server/src/types/chunking.tsindex 8dbfd2a6..a5ca9ef0 100644--- a/apps/server/src/types/chunking.ts+++ b/apps/server/src/types/chunking.ts@@ -1,21 +1,26 @@-export type BaseMetadata = {+export interface BaseMetadata {id: string;name?: string | null;- createdAt: Date;+ createdAt: Date | null;createdBy: string;updatedAt?: Date | null;updatedBy?: string | null;author?: { id: string; name: string };lastAuthor?: { id: string; name: string };+ workspace?: { id: string; name: string };parentContext?: {id: string;type: string;- name?: string;- path?: string;+ name?: string | null;+ path: string;};collaborators?: Array<{ id: string; name: string }>;- workspace?: { id: string; name: string };-};+ databaseInfo?: {+ id: string;+ name: string;+ fields: Record<string, { type: string; name: string }>;+ };+}export type NodeMetadata = BaseMetadata & {type: 'node';