Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Refactor document embedding job with improved text extraction and error handling

Ylber Gashi52c5d0dfe3c0
diff --git a/apps/server/src/jobs/embed-document.ts b/apps/server/src/jobs/embed-document.ts
index 243f4da6..73969007 100644
--- a/apps/server/src/jobs/embed-document.ts
+++ b/apps/server/src/jobs/embed-document.ts
@@ -1,52 +1,101 @@
-// Updated embed-document.ts
import { OpenAIEmbeddings } from '@langchain/openai';
import { ChunkingService } from '@/services/chunking-service';
import { database } from '@/data/database';
import { configuration } from '@/lib/configuration';
import { CreateDocumentEmbedding } from '@/data/schema';
import { sql } from 'kysely';
-import { fetchNodeWithContext } from '@/services/node-retrieval-service';
+import { fetchNode } from '@/lib/nodes';
import { DocumentContent, extractBlockTexts } from '@colanode/core';
-import { JobHandler } from '@/types/jobs';
export type EmbedDocumentInput = {
type: 'embed_document';
documentId: string;
};
-export const embedDocumentHandler: JobHandler<EmbedDocumentInput> = async (
- input
-) => {
- if (!configuration.ai.enabled) return;
+declare module '@/types/jobs' {
+ interface JobMap {
+ embed_document: {
+ input: EmbedDocumentInput;
+ };
+ }
+}
+
+const getNodeDisplayName = (
+ node: Awaited<ReturnType<typeof fetchNode>>
+): string => {
+ if (!node) {
+ return '';
+ }
+
+ switch (node.attributes.type) {
+ case 'page':
+ case 'database':
+ case 'folder':
+ case 'database_view':
+ return node.attributes.name;
+ case 'record':
+ return node.attributes.name;
+ default:
+ return '';
+ }
+};
+
+const extractDocumentText = async (
+ documentId: string,
+ content: DocumentContent
+): Promise<string> => {
+ const sections: string[] = [];
+
+ if (documentId) {
+ const node = await fetchNode(documentId);
+ if (!node) return '';
+
+ const nodeName = getNodeDisplayName(node);
+
+ if (node.attributes.type === 'record') {
+ const databaseNode = await fetchNode(node.attributes.databaseId);
+ const databaseName =
+ databaseNode?.attributes.type === 'database'
+ ? databaseNode.attributes.name
+ : node.attributes.databaseId;
+
+ sections.push(`Record "${nodeName}" in database "${databaseName}"`);
+ } else if (nodeName) {
+ sections.push(`${node.attributes.type} "${nodeName}"`);
+ }
+ }
+
+ const blocksText = extractBlockTexts(documentId, content.blocks);
+ if (blocksText) {
+ sections.push('Content:');
+ sections.push(blocksText);
+ }
+
+ return sections.join('\n\n');
+};
+
+export const embedDocumentHandler = async (input: {
+ type: 'embed_document';
+ documentId: string;
+}) => {
+ if (!configuration.ai.enabled) {
+ return;
+ }
+
const { documentId } = input;
- // Retrieve document along with its associated node in one query
const document = await database
.selectFrom('documents')
.select(['id', 'content', 'workspace_id', 'created_at'])
.where('id', '=', documentId)
.executeTakeFirst();
- if (!document) return;
-
- // Fetch associated node for context (page node, etc.)
- const nodeContext = await fetchNodeWithContext(documentId);
- // If available, include node type and parent (space) name in the context.
- let header = '';
- if (nodeContext) {
- const { node, parent, root } = nodeContext;
- header = `${node.attributes.type} "${(node.attributes as any).name || ''}"`;
- if (parent && parent.attributes.name) {
- header += ` in "${parent.attributes.name}"`;
- }
- if (root && root.attributes.name) {
- header += ` [${root.attributes.name}]`;
- }
+
+ if (!document) {
+ return;
}
- // Extract text using document content. (For pages and rich–text, use extractBlockTexts.)
- const docText = extractBlockTexts(documentId, document.content.blocks) || '';
- const fullText = header ? `${header}\n\nContent:\n${docText}` : docText;
- if (!fullText.trim()) {
+ const text = await extractDocumentText(documentId, document.content);
+ if (!text || text.trim() === '') {
await database
.deleteFrom('document_embeddings')
.where('document_id', '=', documentId)
@@ -55,7 +104,7 @@ export const embedDocumentHandler: JobHandler<EmbedDocumentInput> = async (
}
const chunkingService = new ChunkingService();
- const chunks = await chunkingService.chunkText(fullText, {
+ const chunks = await chunkingService.chunkText(text, {
type: 'document',
id: documentId,
});
@@ -71,12 +120,13 @@ export const embedDocumentHandler: JobHandler<EmbedDocumentInput> = async (
.where('document_id', '=', documentId)
.execute();
- const embeddingsToUpsert: CreateDocumentEmbedding[] = [];
+ const embeddingsToCreateOrUpdate: CreateDocumentEmbedding[] = [];
for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i];
+ if (!chunk) continue;
const existing = existingEmbeddings.find((e) => e.chunk === i);
if (existing && existing.text === chunk) continue;
- embeddingsToUpsert.push({
+ embeddingsToCreateOrUpdate.push({
document_id: documentId,
chunk: i,
workspace_id: document.workspace_id,
@@ -87,16 +137,24 @@ export const embedDocumentHandler: JobHandler<EmbedDocumentInput> = async (
}
const batchSize = configuration.ai.embedding.batchSize;
- for (let i = 0; i < embeddingsToUpsert.length; i += batchSize) {
- const batch = embeddingsToUpsert.slice(i, i + batchSize);
+ for (let i = 0; i < embeddingsToCreateOrUpdate.length; i += batchSize) {
+ const batch = embeddingsToCreateOrUpdate.slice(i, i + batchSize);
const textsToEmbed = batch.map((item) => item.text);
const embeddingVectors = await embeddings.embedDocuments(textsToEmbed);
for (let j = 0; j < batch.length; j++) {
- batch[j].embedding_vector = embeddingVectors[j] ?? [];
+ const vector = embeddingVectors[j];
+ const batchItem = batch[j];
+ if (batchItem) {
+ batchItem.embedding_vector = vector ?? [];
+ }
}
}
- for (const embedding of embeddingsToUpsert) {
+ if (embeddingsToCreateOrUpdate.length === 0) {
+ return;
+ }
+
+ for (const embedding of embeddingsToCreateOrUpdate) {
await database
.insertInto('document_embeddings')
.values({