Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Refactor embedEntryHandler to use extractEntryText and improve error handling for missing entries

Ylber Gashie7d20f47f027
diff --git a/apps/server/src/jobs/embed-entry.ts b/apps/server/src/jobs/embed-entry.ts
index 3d054cce..b2d87f6e 100644
--- a/apps/server/src/jobs/embed-entry.ts
+++ b/apps/server/src/jobs/embed-entry.ts
@@ -1,8 +1,9 @@
import { JobHandler } from '@/types/jobs';
import { ChunkingService } from '@/services/chunking-service';
-import { Document } from 'langchain/document';
import { database } from '@/data/database';
-import { generateId, IdType } from '@colanode/core';
+import { OpenAIEmbeddings } from '@langchain/openai';
+import { aiSettings } from '@/lib/ai-settings';
+import { generateId, IdType, extractEntryText } from '@colanode/core';
export type EmbedEntryInput = {
type: 'embed_entry';
@@ -22,46 +23,60 @@ export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (
) => {
const { entryId } = input;
- //TODO: use extractEntryText from packages/core/src/lib/texts.ts
+ const entry = await database
+ .selectFrom('entries')
+ .select(['id', 'attributes'])
+ .where('id', '=', entryId)
+ .executeTakeFirst();
+
+ if (!entry) {
+ throw new Error(`Entry ${entryId} not found`);
+ }
+
+ const textResult = extractEntryText(entryId, entry.attributes);
+ if (!textResult?.text) {
+ return;
+ }
+
const chunkingService = new ChunkingService();
- const chunks = await chunkingService.chunkText(entryText);
-
- const embeddings = new OpenAIEmbeddings({
- apiKey: aiSettings.openai.apiKey,
- modelName: aiSettings.openai.embeddingModel,
- dimensions: aiSettings.openai.embeddingDimensions,
- });
-
- const embeddingVectors = await embeddings.embedDocuments(chunks)
-
- for (let i = 0; i < chunks.length; i++) {
- const chunk = chunks[i];
- if (!chunk) {
- continue;
- }
-
- const embedding = embeddingVectors[i]
- if (!embedding) {
- continue;
- }
-
- const id = generateId(IdType.AiEmbedding);
- const metadata = JSON.stringify({
- chunk_index: i,
- });
-
- await database
- .insertInto('ai_embeddings')
- .values(
- {
- id,
- entity_id: entryId,
- entity_type: 'message',
- embedding: embedding,
- content: chunk,
- metadata,
- created_at: new Date(),
- })
- .execute();
+ const chunks = await chunkingService.chunkText(textResult.text);
+
+ const embeddings = new OpenAIEmbeddings({
+ apiKey: aiSettings.openai.apiKey,
+ modelName: aiSettings.openai.embeddingModel,
+ dimensions: aiSettings.openai.embeddingDimensions,
+ });
+
+ const embeddingVectors = await embeddings.embedDocuments(chunks)
+
+ for (let i = 0; i < chunks.length; i++) {
+ const chunk = chunks[i];
+ if (!chunk) {
+ continue;
+ }
+
+ const embedding = embeddingVectors[i]
+ if (!embedding) {
+ continue;
}
+
+ const id = generateId(IdType.AiEmbedding);
+ const metadata = JSON.stringify({
+ chunk_index: i,
+ });
+
+ await database
+ .insertInto('ai_embeddings')
+ .values(
+ {
+ id,
+ entity_id: entryId,
+ entity_type: 'entry',
+ embedding: embedding,
+ content: chunk,
+ metadata,
+ created_at: new Date(),
+ })
+ .execute();
+ }
};