Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Refactor AI embedding handling: update schema, add entry_id to message embeddings, and implement embedding scheduling with configurable delay

Ylber Gashi2d81a0a11315
diff --git a/apps/server/src/data/migrations.ts b/apps/server/src/data/migrations.ts
index 4063d047..31c19cd1 100644
--- a/apps/server/src/data/migrations.ts
+++ b/apps/server/src/data/migrations.ts
@@ -854,20 +854,20 @@ const createMessageEmbeddingsTable: Migration = {
.addColumn('message_id', 'varchar(30)', (col) => col.notNull())
.addColumn('chunk', 'integer', (col) => col.notNull())
.addColumn('parent_id', 'varchar(30)', (col) => col.notNull())
+ .addColumn('entry_id', 'varchar(30)', (col) => col.notNull())
.addColumn('root_id', 'varchar(30)', (col) => col.notNull())
.addColumn('workspace_id', 'varchar(30)', (col) => col.notNull())
- .addColumn('content', 'text', (col) => col.notNull())
- .addColumn('embedding', sql`vector(2000)`, (col) => col.notNull())
- .addColumn('fts', sql`tsvector GENERATED ALWAYS AS (to_tsvector('english', content)) STORED`)
- .addColumn('metadata', 'jsonb')
+ .addColumn('text', 'text', (col) => col.notNull())
+ .addColumn('embedding_vector', sql`vector(2000)`, (col) => col.notNull())
+ .addColumn('search_vector', sql`tsvector GENERATED ALWAYS AS (to_tsvector('english', text)) STORED`)
.addColumn('created_at', 'timestamptz', (col) => col.notNull())
.addColumn('updated_at', 'timestamptz')
.execute();
await sql`
- CREATE INDEX message_embeddings_embedding_idx
+ CREATE INDEX message_embeddings_embedding_vector_idx
ON message_embeddings
- USING hnsw(embedding vector_cosine_ops)
+ USING hnsw(embedding_vector vector_cosine_ops)
WITH (
m = 16,
ef_construction = 64
@@ -875,9 +875,9 @@ const createMessageEmbeddingsTable: Migration = {
`.execute(db);
await sql`
- CREATE INDEX message_embeddings_fts_idx
+ CREATE INDEX message_embeddings_search_vector_idx
ON message_embeddings
- USING GIN (fts);
+ USING GIN (search_vector);
`.execute(db);
},
down: async (db) => {
@@ -891,21 +891,20 @@ const createEntryEmbeddingsTable: Migration = {
.createTable('entry_embeddings')
.addColumn('entry_id', 'varchar(30)', (col) => col.notNull())
.addColumn('chunk', 'integer', (col) => col.notNull())
- .addColumn('parent_id', 'varchar(30)', (col) => col.notNull())
+ .addColumn('parent_id', 'varchar(30)')
.addColumn('root_id', 'varchar(30)', (col) => col.notNull())
.addColumn('workspace_id', 'varchar(30)', (col) => col.notNull())
- .addColumn('content', 'text', (col) => col.notNull())
- .addColumn('embedding', sql`vector(2000)`, (col) => col.notNull())
- .addColumn('fts', sql`tsvector GENERATED ALWAYS AS (to_tsvector('english', content)) STORED`)
- .addColumn('metadata', 'jsonb')
+ .addColumn('text', 'text', (col) => col.notNull())
+ .addColumn('embedding_vector', sql`vector(2000)`, (col) => col.notNull())
+ .addColumn('search_vector', sql`tsvector GENERATED ALWAYS AS (to_tsvector('english', text)) STORED`)
.addColumn('created_at', 'timestamptz', (col) => col.notNull())
.addColumn('updated_at', 'timestamptz')
.execute();
await sql`
- CREATE INDEX entry_embeddings_embedding_idx
+ CREATE INDEX entry_embeddings_embedding_vector_idx
ON entry_embeddings
- USING hnsw(embedding vector_cosine_ops)
+ USING hnsw(embedding_vector vector_cosine_ops)
WITH (
m = 16,
ef_construction = 64
@@ -913,9 +912,9 @@ const createEntryEmbeddingsTable: Migration = {
`.execute(db);
await sql`
- CREATE INDEX entry_embeddings_fts_idx
+ CREATE INDEX entry_embeddings_search_vector_idx
ON entry_embeddings
- USING GIN (fts);
+ USING GIN (search_vector);
`.execute(db);
},
down: async (db) => {
diff --git a/apps/server/src/data/schema.ts b/apps/server/src/data/schema.ts
index a5133445..9d657c71 100644
--- a/apps/server/src/data/schema.ts
+++ b/apps/server/src/data/schema.ts
@@ -290,13 +290,12 @@ export type UpdateFileTombstone = Updateable<FileTombstoneTable>;
interface EntryEmbeddingTable {
entry_id: ColumnType<string, string, never>;
chunk: ColumnType<number, number, number>;
- parent_id: ColumnType<string, string, never>;
+ parent_id: ColumnType<string | null, string | null, never>;
root_id: ColumnType<string, string, never>;
workspace_id: ColumnType<string, string, never>;
- content: ColumnType<string, string, string>;
- embedding: ColumnType<number[], number[], number[]>;
- fts: ColumnType<never, never, never>;
- metadata: ColumnType<string | null, string | null, string | null>;
+ text: ColumnType<string, string, string>;
+ embedding_vector: ColumnType<number[], number[], number[]>;
+ search_vector: ColumnType<never, never, never>;
created_at: ColumnType<Date, Date, never>;
updated_at: ColumnType<Date | null, Date | null, Date | null>;
}
@@ -309,12 +308,12 @@ interface MessageEmbeddingTable {
message_id: ColumnType<string, string, never>;
chunk: ColumnType<number, number, number>;
parent_id: ColumnType<string, string, never>;
+ entry_id: ColumnType<string, string, never>;
root_id: ColumnType<string, string, never>;
workspace_id: ColumnType<string, string, never>;
- content: ColumnType<string, string, string>;
- embedding: ColumnType<number[], number[], number[]>;
- fts: ColumnType<never, never, never>;
- metadata: ColumnType<string | null, string | null, string | null>;
+ text: ColumnType<string, string, string>;
+ embedding_vector: ColumnType<number[], number[], number[]>;
+ search_vector: ColumnType<never, never, never>;
created_at: ColumnType<Date, Date, never>;
updated_at: ColumnType<Date | null, Date | null, Date | null>;
}
diff --git a/apps/server/src/jobs/clean-entry-data.ts b/apps/server/src/jobs/clean-entry-data.ts
index deb9f318..3887a948 100644
--- a/apps/server/src/jobs/clean-entry-data.ts
+++ b/apps/server/src/jobs/clean-entry-data.ts
@@ -119,6 +119,17 @@ const deleteChildren = async (parentIds: string[], userId: string) => {
}
await trx.deleteFrom('entries').where('id', 'in', entryIds).execute();
+
+ await trx
+ .deleteFrom('entry_interactions')
+ .where('entry_id', 'in', entryIds)
+ .execute();
+
+ await trx
+ .deleteFrom('entry_embeddings')
+ .where('entry_id', 'in', entryIds)
+ .execute();
+
await trx
.updateTable('collaborations')
.set({
@@ -195,6 +206,11 @@ const deleteMessages = async (entryIds: string[], userId: string) => {
.where('message_id', 'in', messageIds)
.execute();
+ await trx
+ .deleteFrom('message_embeddings')
+ .where('message_id', 'in', messageIds)
+ .execute();
+
await trx
.insertInto('message_tombstones')
.values(messageTombstonesToCreate)
diff --git a/apps/server/src/jobs/embed-entry.ts b/apps/server/src/jobs/embed-entry.ts
index 15d12651..50adcfc1 100644
--- a/apps/server/src/jobs/embed-entry.ts
+++ b/apps/server/src/jobs/embed-entry.ts
@@ -4,6 +4,7 @@ import { database } from '@/data/database';
import { OpenAIEmbeddings } from '@langchain/openai';
import { aiSettings } from '@/lib/ai-settings';
import { extractEntryText } from '@colanode/core';
+import { CreateEntryEmbedding } from '@/data/schema';
export type EmbedEntryInput = {
type: 'embed_entry';
@@ -18,9 +19,11 @@ declare module '@/types/jobs' {
}
}
-export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (
- input
-) => {
+export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (input) => {
+ if (!aiSettings.enabled) {
+ return;
+ }
+
const { entryId } = input;
const entry = await database
@@ -30,11 +33,20 @@ export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (
.executeTakeFirst();
if (!entry) {
- throw new Error(`Entry ${entryId} not found`);
+ return;
}
const textResult = extractEntryText(entryId, entry.attributes);
- if (!textResult?.text) {
+ if (textResult === undefined) {
+ return;
+ }
+
+ if (textResult.text === null || textResult.text === '') {
+ await database
+ .deleteFrom('entry_embeddings')
+ .where('entry_id', '=', entryId)
+ .execute();
+
return;
}
@@ -49,20 +61,11 @@ export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (
const existingEmbeddings = await database
.selectFrom('entry_embeddings')
- .select(['chunk', 'content'])
+ .select(['chunk', 'text'])
.where('entry_id', '=', entryId)
.execute();
- const embeddingsToCreateOrUpdate: {
- entry_id: string;
- chunk: number;
- parent_id: string;
- root_id: string;
- workspace_id: string;
- content: string;
- embedding: number[];
- metadata: string | null;
- }[] = [];
+ const embeddingsToCreateOrUpdate: CreateEntryEmbedding[] = [];
for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i];
@@ -70,11 +73,9 @@ export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (
continue;
}
- const existingEmbedding = existingEmbeddings.find(
- (e) => e.chunk === i
- );
+ const existingEmbedding = existingEmbeddings.find((e) => e.chunk === i);
- if (existingEmbedding && existingEmbedding.content === chunk) {
+ if (existingEmbedding && existingEmbedding.text === chunk) {
continue;
}
@@ -84,52 +85,40 @@ export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (
parent_id: entry.parent_id,
root_id: entry.root_id,
workspace_id: entry.workspace_id,
- content: chunk,
- embedding: [],
- metadata: null,
+ text: chunk,
+ embedding_vector: [],
+ created_at: new Date(),
});
}
const batchSize = aiSettings.openai.embeddingBatchSize;
for (let i = 0; i < embeddingsToCreateOrUpdate.length; i += batchSize) {
const batch = embeddingsToCreateOrUpdate.slice(i, i + batchSize);
- const textsToEmbed = batch.map((item) => item.content);
+ const textsToEmbed = batch.map((item) => item.text);
const embeddingVectors = await embeddings.embedDocuments(textsToEmbed);
for (let j = 0; j < batch.length; j++) {
const vector = embeddingVectors[j];
const batchItem = batch[j];
if (vector && batchItem) {
- batchItem.embedding = vector;
+ batchItem.embedding_vector = vector;
}
}
}
- for (const item of embeddingsToCreateOrUpdate) {
- if (item.embedding.length === 0) continue;
+ if (embeddingsToCreateOrUpdate.length == 0) {
+ return;
+ }
- await database
- .insertInto('entry_embeddings')
- .values({
- entry_id: item.entry_id,
- chunk: item.chunk,
- parent_id: item.parent_id,
- root_id: item.root_id,
- workspace_id: item.workspace_id,
- content: item.content,
- embedding: item.embedding,
- metadata: item.metadata,
- created_at: new Date(),
+ await database
+ .insertInto('entry_embeddings')
+ .values(embeddingsToCreateOrUpdate)
+ .onConflict((oc) =>
+ oc.columns(['entry_id', 'chunk']).doUpdateSet((eb) => ({
+ text: eb.ref('excluded.text'),
+ embedding_vector: eb.ref('excluded.embedding_vector'),
updated_at: new Date(),
- })
- .onConflict((oc) =>
- oc.columns(['entry_id', 'chunk']).doUpdateSet({
- content: item.content,
- embedding: item.embedding,
- metadata: item.metadata,
- updated_at: new Date(),
- })
- )
- .execute();
- }
-};
\ No newline at end of file
+ }))
+ )
+ .execute();
+};
diff --git a/apps/server/src/jobs/embed-message.ts b/apps/server/src/jobs/embed-message.ts
index 4be77598..9264cadf 100644
--- a/apps/server/src/jobs/embed-message.ts
+++ b/apps/server/src/jobs/embed-message.ts
@@ -3,6 +3,7 @@ import { ChunkingService } from '@/services/chunking-service';
import { database } from '@/data/database';
import { OpenAIEmbeddings } from '@langchain/openai';
import { aiSettings } from '@/lib/ai-settings';
+import { CreateMessageEmbedding } from '@/data/schema';
export type EmbedMessageInput = {
type: 'embed_message';
@@ -20,6 +21,10 @@ declare module '@/types/jobs' {
export const embedMessageHandler: JobHandler<EmbedMessageInput> = async (
input
) => {
+ if (!aiSettings.enabled) {
+ return;
+ }
+
const { messageId } = input;
const message = await database
@@ -43,20 +48,11 @@ export const embedMessageHandler: JobHandler<EmbedMessageInput> = async (
const existingEmbeddings = await database
.selectFrom('message_embeddings')
- .select(['chunk', 'content'])
+ .select(['chunk', 'text'])
.where('message_id', '=', messageId)
.execute();
- const embeddingsToCreateOrUpdate: {
- message_id: string;
- chunk: number;
- parent_id: string;
- root_id: string;
- workspace_id: string;
- content: string;
- embedding: number[];
- metadata: string | null;
- }[] = [];
+ const embeddingsToCreateOrUpdate: CreateMessageEmbedding[] = [];
for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i];
@@ -68,7 +64,7 @@ export const embedMessageHandler: JobHandler<EmbedMessageInput> = async (
(e) => e.chunk === i
);
- if (existingEmbedding && existingEmbedding.content === chunk) {
+ if (existingEmbedding && existingEmbedding.text === chunk) {
continue;
}
@@ -76,54 +72,43 @@ export const embedMessageHandler: JobHandler<EmbedMessageInput> = async (
message_id: messageId,
chunk: i,
parent_id: message.parent_id,
+ entry_id: message.entry_id,
root_id: message.root_id,
workspace_id: message.workspace_id,
- content: chunk,
- embedding: [],
- metadata: null,
+ text: chunk,
+ embedding_vector: [],
+ created_at: new Date(),
});
}
const batchSize = aiSettings.openai.embeddingBatchSize;
for (let i = 0; i < embeddingsToCreateOrUpdate.length; i += batchSize) {
const batch = embeddingsToCreateOrUpdate.slice(i, i + batchSize);
- const textsToEmbed = batch.map((item) => item.content);
+ const textsToEmbed = batch.map((item) => item.text);
const embeddingVectors = await embeddings.embedDocuments(textsToEmbed);
for (let j = 0; j < batch.length; j++) {
const vector = embeddingVectors[j];
const batchItem = batch[j];
if (vector && batchItem) {
- batchItem.embedding = vector;
+ batchItem.embedding_vector = vector;
}
}
}
- for (const item of embeddingsToCreateOrUpdate) {
- if (item.embedding.length === 0) continue;
-
- await database
- .insertInto('message_embeddings')
- .values({
- message_id: item.message_id,
- chunk: item.chunk,
- parent_id: item.parent_id,
- root_id: item.root_id,
- workspace_id: item.workspace_id,
- content: item.content,
- embedding: item.embedding,
- metadata: item.metadata,
- created_at: new Date(),
- updated_at: new Date(),
- })
- .onConflict((oc) =>
- oc.columns(['message_id', 'chunk']).doUpdateSet({
- content: item.content,
- embedding: item.embedding,
- metadata: item.metadata,
- updated_at: new Date(),
- })
- )
- .execute();
+ if (embeddingsToCreateOrUpdate.length == 0) {
+ return;
}
+
+ await database
+ .insertInto('message_embeddings')
+ .values(embeddingsToCreateOrUpdate)
+ .onConflict((oc) =>
+ oc.columns(['message_id', 'chunk']).doUpdateSet((eb) => ({
+ text: eb.ref('excluded.text'),
+ embedding_vector: eb.ref('excluded.embedding_vector'),
+ updated_at: new Date(),
+ }))
+ )
+ .execute();
};
\ No newline at end of file
diff --git a/apps/server/src/lib/ai-settings.ts b/apps/server/src/lib/ai-settings.ts
index 47b4d5bc..c401c30b 100644
--- a/apps/server/src/lib/ai-settings.ts
+++ b/apps/server/src/lib/ai-settings.ts
@@ -1,4 +1,7 @@
export const aiSettings = {
+ enabled: false,
+ embedDelay: 60000,
+
openai: {
apiKey: process.env.OPENAI_API_KEY || '',
embeddingModel: 'text-embedding-3-large',
@@ -13,4 +16,4 @@ export const aiSettings = {
contextEnhancerModel: 'gpt-4o-mini',
contextEnhancerTemperature: 0.3,
},
-};
\ No newline at end of file
+};
diff --git a/apps/server/src/services/entry-service.ts b/apps/server/src/services/entry-service.ts
index 924961a0..63af12b8 100644
--- a/apps/server/src/services/entry-service.ts
+++ b/apps/server/src/services/entry-service.ts
@@ -42,6 +42,7 @@ import {
UpdateEntryInput,
UpdateEntryOutput,
} from '@/types/entries';
+import { aiSettings } from '@/lib/ai-settings';
const UPDATE_RETRIES_LIMIT = 10;
@@ -120,10 +121,7 @@ class EntryService {
});
}
- await jobService.addJob({
- type: 'embed_entry',
- entryId: input.entryId,
- });
+ await this.scheduleEntryEmbedding(input.entryId);
return {
entry: createdEntry,
@@ -287,10 +285,7 @@ class EntryService {
});
}
- await jobService.addJob({
- type: 'embed_entry',
- entryId: input.entryId,
- });
+ await this.scheduleEntryEmbedding(input.entryId);
return {
type: 'success',
@@ -376,6 +371,8 @@ class EntryService {
});
}
+ await this.scheduleEntryEmbedding(input.entryId);
+
return {
entry: createdEntry,
transaction: createdTransaction,
@@ -551,6 +548,8 @@ class EntryService {
});
}
+ await this.scheduleEntryEmbedding(input.entryId);
+
return {
type: 'success',
output: {
@@ -1014,6 +1013,19 @@ class EntryService {
removedCollaborators,
};
}
+
+ private async scheduleEntryEmbedding(entryId: string) {
+ await jobService.addJob(
+ {
+ type: 'embed_entry',
+ entryId,
+ },
+ {
+ jobId: `embed_entry:${entryId}`,
+ delay: aiSettings.embedDelay,
+ }
+ );
+ }
}
export const entryService = new EntryService();
diff --git a/apps/server/src/services/message-service.ts b/apps/server/src/services/message-service.ts
index 8b85199b..170dac0f 100644
--- a/apps/server/src/services/message-service.ts
+++ b/apps/server/src/services/message-service.ts
@@ -71,12 +71,10 @@ class MessageService {
return false;
}
- await jobService.addJob(
- {
- type: 'embed_message',
- messageId: createdMessage.id,
- }
- );
+ await jobService.addJob({
+ type: 'embed_message',
+ messageId: createdMessage.id,
+ });
eventBus.publish({
type: 'message_created',
@@ -142,6 +140,16 @@ class MessageService {
.where('message_id', '=', deletedMessage.id)
.execute();
+ await tx
+ .deleteFrom('message_reactions')
+ .where('message_id', '=', deletedMessage.id)
+ .execute();
+
+ await tx
+ .deleteFrom('message_embeddings')
+ .where('message_id', '=', deletedMessage.id)
+ .execute();
+
await tx
.insertInto('message_tombstones')
.values({
diff --git a/packages/core/src/lib/id.ts b/packages/core/src/lib/id.ts
index b26630f9..63475012 100644
--- a/packages/core/src/lib/id.ts
+++ b/packages/core/src/lib/id.ts
@@ -36,7 +36,6 @@ export enum IdType {
Event = 'ev',
Host = 'ht',
Block = 'bl',
- AiEmbedding = 'ae',
}
export const generateId = (type: IdType): string => {