Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Refactor AI embedding handling: update schema, add entry_id to message embeddings, and implement embedding scheduling with configurable delay
Ylber Gashi2d81a0a11315
diff --git a/apps/server/src/data/migrations.ts b/apps/server/src/data/migrations.tsindex 4063d047..31c19cd1 100644--- a/apps/server/src/data/migrations.ts+++ b/apps/server/src/data/migrations.ts@@ -854,20 +854,20 @@ const createMessageEmbeddingsTable: Migration = {.addColumn('message_id', 'varchar(30)', (col) => col.notNull()).addColumn('chunk', 'integer', (col) => col.notNull()).addColumn('parent_id', 'varchar(30)', (col) => col.notNull())+ .addColumn('entry_id', 'varchar(30)', (col) => col.notNull()).addColumn('root_id', 'varchar(30)', (col) => col.notNull()).addColumn('workspace_id', 'varchar(30)', (col) => col.notNull())- .addColumn('content', 'text', (col) => col.notNull())- .addColumn('embedding', sql`vector(2000)`, (col) => col.notNull())- .addColumn('fts', sql`tsvector GENERATED ALWAYS AS (to_tsvector('english', content)) STORED`)- .addColumn('metadata', 'jsonb')+ .addColumn('text', 'text', (col) => col.notNull())+ .addColumn('embedding_vector', sql`vector(2000)`, (col) => col.notNull())+ .addColumn('search_vector', sql`tsvector GENERATED ALWAYS AS (to_tsvector('english', text)) STORED`).addColumn('created_at', 'timestamptz', (col) => col.notNull()).addColumn('updated_at', 'timestamptz').execute();await sql`- CREATE INDEX message_embeddings_embedding_idx+ CREATE INDEX message_embeddings_embedding_vector_idxON message_embeddings- USING hnsw(embedding vector_cosine_ops)+ USING hnsw(embedding_vector vector_cosine_ops)WITH (m = 16,ef_construction = 64@@ -875,9 +875,9 @@ const createMessageEmbeddingsTable: Migration = {`.execute(db);await sql`- CREATE INDEX message_embeddings_fts_idx+ CREATE INDEX message_embeddings_search_vector_idxON message_embeddings- USING GIN (fts);+ USING GIN (search_vector);`.execute(db);},down: async (db) => {@@ -891,21 +891,20 @@ const createEntryEmbeddingsTable: Migration = {.createTable('entry_embeddings').addColumn('entry_id', 'varchar(30)', (col) => col.notNull()).addColumn('chunk', 'integer', (col) => col.notNull())- .addColumn('parent_id', 'varchar(30)', (col) => col.notNull())+ .addColumn('parent_id', 'varchar(30)').addColumn('root_id', 'varchar(30)', (col) => col.notNull()).addColumn('workspace_id', 'varchar(30)', (col) => col.notNull())- .addColumn('content', 'text', (col) => col.notNull())- .addColumn('embedding', sql`vector(2000)`, (col) => col.notNull())- .addColumn('fts', sql`tsvector GENERATED ALWAYS AS (to_tsvector('english', content)) STORED`)- .addColumn('metadata', 'jsonb')+ .addColumn('text', 'text', (col) => col.notNull())+ .addColumn('embedding_vector', sql`vector(2000)`, (col) => col.notNull())+ .addColumn('search_vector', sql`tsvector GENERATED ALWAYS AS (to_tsvector('english', text)) STORED`).addColumn('created_at', 'timestamptz', (col) => col.notNull()).addColumn('updated_at', 'timestamptz').execute();await sql`- CREATE INDEX entry_embeddings_embedding_idx+ CREATE INDEX entry_embeddings_embedding_vector_idxON entry_embeddings- USING hnsw(embedding vector_cosine_ops)+ USING hnsw(embedding_vector vector_cosine_ops)WITH (m = 16,ef_construction = 64@@ -913,9 +912,9 @@ const createEntryEmbeddingsTable: Migration = {`.execute(db);await sql`- CREATE INDEX entry_embeddings_fts_idx+ CREATE INDEX entry_embeddings_search_vector_idxON entry_embeddings- USING GIN (fts);+ USING GIN (search_vector);`.execute(db);},down: async (db) => {diff --git a/apps/server/src/data/schema.ts b/apps/server/src/data/schema.tsindex a5133445..9d657c71 100644--- a/apps/server/src/data/schema.ts+++ b/apps/server/src/data/schema.ts@@ -290,13 +290,12 @@ export type UpdateFileTombstone = Updateable<FileTombstoneTable>;interface EntryEmbeddingTable {entry_id: ColumnType<string, string, never>;chunk: ColumnType<number, number, number>;- parent_id: ColumnType<string, string, never>;+ parent_id: ColumnType<string | null, string | null, never>;root_id: ColumnType<string, string, never>;workspace_id: ColumnType<string, string, never>;- content: ColumnType<string, string, string>;- embedding: ColumnType<number[], number[], number[]>;- fts: ColumnType<never, never, never>;- metadata: ColumnType<string | null, string | null, string | null>;+ text: ColumnType<string, string, string>;+ embedding_vector: ColumnType<number[], number[], number[]>;+ search_vector: ColumnType<never, never, never>;created_at: ColumnType<Date, Date, never>;updated_at: ColumnType<Date | null, Date | null, Date | null>;}@@ -309,12 +308,12 @@ interface MessageEmbeddingTable {message_id: ColumnType<string, string, never>;chunk: ColumnType<number, number, number>;parent_id: ColumnType<string, string, never>;+ entry_id: ColumnType<string, string, never>;root_id: ColumnType<string, string, never>;workspace_id: ColumnType<string, string, never>;- content: ColumnType<string, string, string>;- embedding: ColumnType<number[], number[], number[]>;- fts: ColumnType<never, never, never>;- metadata: ColumnType<string | null, string | null, string | null>;+ text: ColumnType<string, string, string>;+ embedding_vector: ColumnType<number[], number[], number[]>;+ search_vector: ColumnType<never, never, never>;created_at: ColumnType<Date, Date, never>;updated_at: ColumnType<Date | null, Date | null, Date | null>;}diff --git a/apps/server/src/jobs/clean-entry-data.ts b/apps/server/src/jobs/clean-entry-data.tsindex deb9f318..3887a948 100644--- a/apps/server/src/jobs/clean-entry-data.ts+++ b/apps/server/src/jobs/clean-entry-data.ts@@ -119,6 +119,17 @@ const deleteChildren = async (parentIds: string[], userId: string) => {}await trx.deleteFrom('entries').where('id', 'in', entryIds).execute();++ await trx+ .deleteFrom('entry_interactions')+ .where('entry_id', 'in', entryIds)+ .execute();++ await trx+ .deleteFrom('entry_embeddings')+ .where('entry_id', 'in', entryIds)+ .execute();+await trx.updateTable('collaborations').set({@@ -195,6 +206,11 @@ const deleteMessages = async (entryIds: string[], userId: string) => {.where('message_id', 'in', messageIds).execute();+ await trx+ .deleteFrom('message_embeddings')+ .where('message_id', 'in', messageIds)+ .execute();+await trx.insertInto('message_tombstones').values(messageTombstonesToCreate)diff --git a/apps/server/src/jobs/embed-entry.ts b/apps/server/src/jobs/embed-entry.tsindex 15d12651..50adcfc1 100644--- a/apps/server/src/jobs/embed-entry.ts+++ b/apps/server/src/jobs/embed-entry.ts@@ -4,6 +4,7 @@ import { database } from '@/data/database';import { OpenAIEmbeddings } from '@langchain/openai';import { aiSettings } from '@/lib/ai-settings';import { extractEntryText } from '@colanode/core';+import { CreateEntryEmbedding } from '@/data/schema';export type EmbedEntryInput = {type: 'embed_entry';@@ -18,9 +19,11 @@ declare module '@/types/jobs' {}}-export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (- input-) => {+export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (input) => {+ if (!aiSettings.enabled) {+ return;+ }+const { entryId } = input;const entry = await database@@ -30,11 +33,20 @@ export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (.executeTakeFirst();if (!entry) {- throw new Error(`Entry ${entryId} not found`);+ return;}const textResult = extractEntryText(entryId, entry.attributes);- if (!textResult?.text) {+ if (textResult === undefined) {+ return;+ }++ if (textResult.text === null || textResult.text === '') {+ await database+ .deleteFrom('entry_embeddings')+ .where('entry_id', '=', entryId)+ .execute();+return;}@@ -49,20 +61,11 @@ export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (const existingEmbeddings = await database.selectFrom('entry_embeddings')- .select(['chunk', 'content'])+ .select(['chunk', 'text']).where('entry_id', '=', entryId).execute();- const embeddingsToCreateOrUpdate: {- entry_id: string;- chunk: number;- parent_id: string;- root_id: string;- workspace_id: string;- content: string;- embedding: number[];- metadata: string | null;- }[] = [];+ const embeddingsToCreateOrUpdate: CreateEntryEmbedding[] = [];for (let i = 0; i < chunks.length; i++) {const chunk = chunks[i];@@ -70,11 +73,9 @@ export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (continue;}- const existingEmbedding = existingEmbeddings.find(- (e) => e.chunk === i- );+ const existingEmbedding = existingEmbeddings.find((e) => e.chunk === i);- if (existingEmbedding && existingEmbedding.content === chunk) {+ if (existingEmbedding && existingEmbedding.text === chunk) {continue;}@@ -84,52 +85,40 @@ export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (parent_id: entry.parent_id,root_id: entry.root_id,workspace_id: entry.workspace_id,- content: chunk,- embedding: [],- metadata: null,+ text: chunk,+ embedding_vector: [],+ created_at: new Date(),});}const batchSize = aiSettings.openai.embeddingBatchSize;for (let i = 0; i < embeddingsToCreateOrUpdate.length; i += batchSize) {const batch = embeddingsToCreateOrUpdate.slice(i, i + batchSize);- const textsToEmbed = batch.map((item) => item.content);+ const textsToEmbed = batch.map((item) => item.text);const embeddingVectors = await embeddings.embedDocuments(textsToEmbed);for (let j = 0; j < batch.length; j++) {const vector = embeddingVectors[j];const batchItem = batch[j];if (vector && batchItem) {- batchItem.embedding = vector;+ batchItem.embedding_vector = vector;}}}- for (const item of embeddingsToCreateOrUpdate) {- if (item.embedding.length === 0) continue;+ if (embeddingsToCreateOrUpdate.length == 0) {+ return;+ }- await database- .insertInto('entry_embeddings')- .values({- entry_id: item.entry_id,- chunk: item.chunk,- parent_id: item.parent_id,- root_id: item.root_id,- workspace_id: item.workspace_id,- content: item.content,- embedding: item.embedding,- metadata: item.metadata,- created_at: new Date(),+ await database+ .insertInto('entry_embeddings')+ .values(embeddingsToCreateOrUpdate)+ .onConflict((oc) =>+ oc.columns(['entry_id', 'chunk']).doUpdateSet((eb) => ({+ text: eb.ref('excluded.text'),+ embedding_vector: eb.ref('excluded.embedding_vector'),updated_at: new Date(),- })- .onConflict((oc) =>- oc.columns(['entry_id', 'chunk']).doUpdateSet({- content: item.content,- embedding: item.embedding,- metadata: item.metadata,- updated_at: new Date(),- })- )- .execute();- }-};\ No newline at end of file+ }))+ )+ .execute();+};diff --git a/apps/server/src/jobs/embed-message.ts b/apps/server/src/jobs/embed-message.tsindex 4be77598..9264cadf 100644--- a/apps/server/src/jobs/embed-message.ts+++ b/apps/server/src/jobs/embed-message.ts@@ -3,6 +3,7 @@ import { ChunkingService } from '@/services/chunking-service';import { database } from '@/data/database';import { OpenAIEmbeddings } from '@langchain/openai';import { aiSettings } from '@/lib/ai-settings';+import { CreateMessageEmbedding } from '@/data/schema';export type EmbedMessageInput = {type: 'embed_message';@@ -20,6 +21,10 @@ declare module '@/types/jobs' {export const embedMessageHandler: JobHandler<EmbedMessageInput> = async (input) => {+ if (!aiSettings.enabled) {+ return;+ }+const { messageId } = input;const message = await database@@ -43,20 +48,11 @@ export const embedMessageHandler: JobHandler<EmbedMessageInput> = async (const existingEmbeddings = await database.selectFrom('message_embeddings')- .select(['chunk', 'content'])+ .select(['chunk', 'text']).where('message_id', '=', messageId).execute();- const embeddingsToCreateOrUpdate: {- message_id: string;- chunk: number;- parent_id: string;- root_id: string;- workspace_id: string;- content: string;- embedding: number[];- metadata: string | null;- }[] = [];+ const embeddingsToCreateOrUpdate: CreateMessageEmbedding[] = [];for (let i = 0; i < chunks.length; i++) {const chunk = chunks[i];@@ -68,7 +64,7 @@ export const embedMessageHandler: JobHandler<EmbedMessageInput> = async ((e) => e.chunk === i);- if (existingEmbedding && existingEmbedding.content === chunk) {+ if (existingEmbedding && existingEmbedding.text === chunk) {continue;}@@ -76,54 +72,43 @@ export const embedMessageHandler: JobHandler<EmbedMessageInput> = async (message_id: messageId,chunk: i,parent_id: message.parent_id,+ entry_id: message.entry_id,root_id: message.root_id,workspace_id: message.workspace_id,- content: chunk,- embedding: [],- metadata: null,+ text: chunk,+ embedding_vector: [],+ created_at: new Date(),});}const batchSize = aiSettings.openai.embeddingBatchSize;for (let i = 0; i < embeddingsToCreateOrUpdate.length; i += batchSize) {const batch = embeddingsToCreateOrUpdate.slice(i, i + batchSize);- const textsToEmbed = batch.map((item) => item.content);+ const textsToEmbed = batch.map((item) => item.text);const embeddingVectors = await embeddings.embedDocuments(textsToEmbed);for (let j = 0; j < batch.length; j++) {const vector = embeddingVectors[j];const batchItem = batch[j];if (vector && batchItem) {- batchItem.embedding = vector;+ batchItem.embedding_vector = vector;}}}- for (const item of embeddingsToCreateOrUpdate) {- if (item.embedding.length === 0) continue;-- await database- .insertInto('message_embeddings')- .values({- message_id: item.message_id,- chunk: item.chunk,- parent_id: item.parent_id,- root_id: item.root_id,- workspace_id: item.workspace_id,- content: item.content,- embedding: item.embedding,- metadata: item.metadata,- created_at: new Date(),- updated_at: new Date(),- })- .onConflict((oc) =>- oc.columns(['message_id', 'chunk']).doUpdateSet({- content: item.content,- embedding: item.embedding,- metadata: item.metadata,- updated_at: new Date(),- })- )- .execute();+ if (embeddingsToCreateOrUpdate.length == 0) {+ return;}++ await database+ .insertInto('message_embeddings')+ .values(embeddingsToCreateOrUpdate)+ .onConflict((oc) =>+ oc.columns(['message_id', 'chunk']).doUpdateSet((eb) => ({+ text: eb.ref('excluded.text'),+ embedding_vector: eb.ref('excluded.embedding_vector'),+ updated_at: new Date(),+ }))+ )+ .execute();};\ No newline at end of filediff --git a/apps/server/src/lib/ai-settings.ts b/apps/server/src/lib/ai-settings.tsindex 47b4d5bc..c401c30b 100644--- a/apps/server/src/lib/ai-settings.ts+++ b/apps/server/src/lib/ai-settings.ts@@ -1,4 +1,7 @@export const aiSettings = {+ enabled: false,+ embedDelay: 60000,+openai: {apiKey: process.env.OPENAI_API_KEY || '',embeddingModel: 'text-embedding-3-large',@@ -13,4 +16,4 @@ export const aiSettings = {contextEnhancerModel: 'gpt-4o-mini',contextEnhancerTemperature: 0.3,},-};\ No newline at end of file+};diff --git a/apps/server/src/services/entry-service.ts b/apps/server/src/services/entry-service.tsindex 924961a0..63af12b8 100644--- a/apps/server/src/services/entry-service.ts+++ b/apps/server/src/services/entry-service.ts@@ -42,6 +42,7 @@ import {UpdateEntryInput,UpdateEntryOutput,} from '@/types/entries';+import { aiSettings } from '@/lib/ai-settings';const UPDATE_RETRIES_LIMIT = 10;@@ -120,10 +121,7 @@ class EntryService {});}- await jobService.addJob({- type: 'embed_entry',- entryId: input.entryId,- });+ await this.scheduleEntryEmbedding(input.entryId);return {entry: createdEntry,@@ -287,10 +285,7 @@ class EntryService {});}- await jobService.addJob({- type: 'embed_entry',- entryId: input.entryId,- });+ await this.scheduleEntryEmbedding(input.entryId);return {type: 'success',@@ -376,6 +371,8 @@ class EntryService {});}+ await this.scheduleEntryEmbedding(input.entryId);+return {entry: createdEntry,transaction: createdTransaction,@@ -551,6 +548,8 @@ class EntryService {});}+ await this.scheduleEntryEmbedding(input.entryId);+return {type: 'success',output: {@@ -1014,6 +1013,19 @@ class EntryService {removedCollaborators,};}++ private async scheduleEntryEmbedding(entryId: string) {+ await jobService.addJob(+ {+ type: 'embed_entry',+ entryId,+ },+ {+ jobId: `embed_entry:${entryId}`,+ delay: aiSettings.embedDelay,+ }+ );+ }}export const entryService = new EntryService();diff --git a/apps/server/src/services/message-service.ts b/apps/server/src/services/message-service.tsindex 8b85199b..170dac0f 100644--- a/apps/server/src/services/message-service.ts+++ b/apps/server/src/services/message-service.ts@@ -71,12 +71,10 @@ class MessageService {return false;}- await jobService.addJob(- {- type: 'embed_message',- messageId: createdMessage.id,- }- );+ await jobService.addJob({+ type: 'embed_message',+ messageId: createdMessage.id,+ });eventBus.publish({type: 'message_created',@@ -142,6 +140,16 @@ class MessageService {.where('message_id', '=', deletedMessage.id).execute();+ await tx+ .deleteFrom('message_reactions')+ .where('message_id', '=', deletedMessage.id)+ .execute();++ await tx+ .deleteFrom('message_embeddings')+ .where('message_id', '=', deletedMessage.id)+ .execute();+await tx.insertInto('message_tombstones').values({diff --git a/packages/core/src/lib/id.ts b/packages/core/src/lib/id.tsindex b26630f9..63475012 100644--- a/packages/core/src/lib/id.ts+++ b/packages/core/src/lib/id.ts@@ -36,7 +36,6 @@ export enum IdType {Event = 'ev',Host = 'ht',Block = 'bl',- AiEmbedding = 'ae',}export const generateId = (type: IdType): string => {