Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Refactor embedEntryHandler to use extractEntryText and improve error handling for missing entries
Ylber Gashie7d20f47f027
diff --git a/apps/server/src/jobs/embed-entry.ts b/apps/server/src/jobs/embed-entry.tsindex 3d054cce..b2d87f6e 100644--- a/apps/server/src/jobs/embed-entry.ts+++ b/apps/server/src/jobs/embed-entry.ts@@ -1,8 +1,9 @@import { JobHandler } from '@/types/jobs';import { ChunkingService } from '@/services/chunking-service';-import { Document } from 'langchain/document';import { database } from '@/data/database';-import { generateId, IdType } from '@colanode/core';+import { OpenAIEmbeddings } from '@langchain/openai';+import { aiSettings } from '@/lib/ai-settings';+import { generateId, IdType, extractEntryText } from '@colanode/core';export type EmbedEntryInput = {type: 'embed_entry';@@ -22,46 +23,60 @@ export const embedEntryHandler: JobHandler<EmbedEntryInput> = async () => {const { entryId } = input;- //TODO: use extractEntryText from packages/core/src/lib/texts.ts+ const entry = await database+ .selectFrom('entries')+ .select(['id', 'attributes'])+ .where('id', '=', entryId)+ .executeTakeFirst();++ if (!entry) {+ throw new Error(`Entry ${entryId} not found`);+ }++ const textResult = extractEntryText(entryId, entry.attributes);+ if (!textResult?.text) {+ return;+ }+const chunkingService = new ChunkingService();- const chunks = await chunkingService.chunkText(entryText);-- const embeddings = new OpenAIEmbeddings({- apiKey: aiSettings.openai.apiKey,- modelName: aiSettings.openai.embeddingModel,- dimensions: aiSettings.openai.embeddingDimensions,- });-- const embeddingVectors = await embeddings.embedDocuments(chunks)-- for (let i = 0; i < chunks.length; i++) {- const chunk = chunks[i];- if (!chunk) {- continue;- }-- const embedding = embeddingVectors[i]- if (!embedding) {- continue;- }-- const id = generateId(IdType.AiEmbedding);- const metadata = JSON.stringify({- chunk_index: i,- });-- await database- .insertInto('ai_embeddings')- .values(- {- id,- entity_id: entryId,- entity_type: 'message',- embedding: embedding,- content: chunk,- metadata,- created_at: new Date(),- })- .execute();+ const chunks = await chunkingService.chunkText(textResult.text);++ const embeddings = new OpenAIEmbeddings({+ apiKey: aiSettings.openai.apiKey,+ modelName: aiSettings.openai.embeddingModel,+ dimensions: aiSettings.openai.embeddingDimensions,+ });++ const embeddingVectors = await embeddings.embedDocuments(chunks)++ for (let i = 0; i < chunks.length; i++) {+ const chunk = chunks[i];+ if (!chunk) {+ continue;+ }++ const embedding = embeddingVectors[i]+ if (!embedding) {+ continue;}++ const id = generateId(IdType.AiEmbedding);+ const metadata = JSON.stringify({+ chunk_index: i,+ });++ await database+ .insertInto('ai_embeddings')+ .values(+ {+ id,+ entity_id: entryId,+ entity_type: 'entry',+ embedding: embedding,+ content: chunk,+ metadata,+ created_at: new Date(),+ })+ .execute();+ }};