Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Refactor document embedding job with improved text extraction and error handling
Ylber Gashi52c5d0dfe3c0
diff --git a/apps/server/src/jobs/embed-document.ts b/apps/server/src/jobs/embed-document.tsindex 243f4da6..73969007 100644--- a/apps/server/src/jobs/embed-document.ts+++ b/apps/server/src/jobs/embed-document.ts@@ -1,52 +1,101 @@-// Updated embed-document.tsimport { OpenAIEmbeddings } from '@langchain/openai';import { ChunkingService } from '@/services/chunking-service';import { database } from '@/data/database';import { configuration } from '@/lib/configuration';import { CreateDocumentEmbedding } from '@/data/schema';import { sql } from 'kysely';-import { fetchNodeWithContext } from '@/services/node-retrieval-service';+import { fetchNode } from '@/lib/nodes';import { DocumentContent, extractBlockTexts } from '@colanode/core';-import { JobHandler } from '@/types/jobs';export type EmbedDocumentInput = {type: 'embed_document';documentId: string;};-export const embedDocumentHandler: JobHandler<EmbedDocumentInput> = async (- input-) => {- if (!configuration.ai.enabled) return;+declare module '@/types/jobs' {+ interface JobMap {+ embed_document: {+ input: EmbedDocumentInput;+ };+ }+}++const getNodeDisplayName = (+ node: Awaited<ReturnType<typeof fetchNode>>+): string => {+ if (!node) {+ return '';+ }++ switch (node.attributes.type) {+ case 'page':+ case 'database':+ case 'folder':+ case 'database_view':+ return node.attributes.name;+ case 'record':+ return node.attributes.name;+ default:+ return '';+ }+};++const extractDocumentText = async (+ documentId: string,+ content: DocumentContent+): Promise<string> => {+ const sections: string[] = [];++ if (documentId) {+ const node = await fetchNode(documentId);+ if (!node) return '';++ const nodeName = getNodeDisplayName(node);++ if (node.attributes.type === 'record') {+ const databaseNode = await fetchNode(node.attributes.databaseId);+ const databaseName =+ databaseNode?.attributes.type === 'database'+ ? databaseNode.attributes.name+ : node.attributes.databaseId;++ sections.push(`Record "${nodeName}" in database "${databaseName}"`);+ } else if (nodeName) {+ sections.push(`${node.attributes.type} "${nodeName}"`);+ }+ }++ const blocksText = extractBlockTexts(documentId, content.blocks);+ if (blocksText) {+ sections.push('Content:');+ sections.push(blocksText);+ }++ return sections.join('\n\n');+};++export const embedDocumentHandler = async (input: {+ type: 'embed_document';+ documentId: string;+}) => {+ if (!configuration.ai.enabled) {+ return;+ }+const { documentId } = input;- // Retrieve document along with its associated node in one queryconst document = await database.selectFrom('documents').select(['id', 'content', 'workspace_id', 'created_at']).where('id', '=', documentId).executeTakeFirst();- if (!document) return;-- // Fetch associated node for context (page node, etc.)- const nodeContext = await fetchNodeWithContext(documentId);- // If available, include node type and parent (space) name in the context.- let header = '';- if (nodeContext) {- const { node, parent, root } = nodeContext;- header = `${node.attributes.type} "${(node.attributes as any).name || ''}"`;- if (parent && parent.attributes.name) {- header += ` in "${parent.attributes.name}"`;- }- if (root && root.attributes.name) {- header += ` [${root.attributes.name}]`;- }++ if (!document) {+ return;}- // Extract text using document content. (For pages and rich–text, use extractBlockTexts.)- const docText = extractBlockTexts(documentId, document.content.blocks) || '';- const fullText = header ? `${header}\n\nContent:\n${docText}` : docText;- if (!fullText.trim()) {+ const text = await extractDocumentText(documentId, document.content);+ if (!text || text.trim() === '') {await database.deleteFrom('document_embeddings').where('document_id', '=', documentId)@@ -55,7 +104,7 @@ export const embedDocumentHandler: JobHandler<EmbedDocumentInput> = async (}const chunkingService = new ChunkingService();- const chunks = await chunkingService.chunkText(fullText, {+ const chunks = await chunkingService.chunkText(text, {type: 'document',id: documentId,});@@ -71,12 +120,13 @@ export const embedDocumentHandler: JobHandler<EmbedDocumentInput> = async (.where('document_id', '=', documentId).execute();- const embeddingsToUpsert: CreateDocumentEmbedding[] = [];+ const embeddingsToCreateOrUpdate: CreateDocumentEmbedding[] = [];for (let i = 0; i < chunks.length; i++) {const chunk = chunks[i];+ if (!chunk) continue;const existing = existingEmbeddings.find((e) => e.chunk === i);if (existing && existing.text === chunk) continue;- embeddingsToUpsert.push({+ embeddingsToCreateOrUpdate.push({document_id: documentId,chunk: i,workspace_id: document.workspace_id,@@ -87,16 +137,24 @@ export const embedDocumentHandler: JobHandler<EmbedDocumentInput> = async (}const batchSize = configuration.ai.embedding.batchSize;- for (let i = 0; i < embeddingsToUpsert.length; i += batchSize) {- const batch = embeddingsToUpsert.slice(i, i + batchSize);+ for (let i = 0; i < embeddingsToCreateOrUpdate.length; i += batchSize) {+ const batch = embeddingsToCreateOrUpdate.slice(i, i + batchSize);const textsToEmbed = batch.map((item) => item.text);const embeddingVectors = await embeddings.embedDocuments(textsToEmbed);for (let j = 0; j < batch.length; j++) {- batch[j].embedding_vector = embeddingVectors[j] ?? [];+ const vector = embeddingVectors[j];+ const batchItem = batch[j];+ if (batchItem) {+ batchItem.embedding_vector = vector ?? [];+ }}}- for (const embedding of embeddingsToUpsert) {+ if (embeddingsToCreateOrUpdate.length === 0) {+ return;+ }++ for (const embedding of embeddingsToCreateOrUpdate) {await database.insertInto('document_embeddings').values({