Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Add AI embedding functionality for entries and messages

Ylber Gashi6ac99f17fb29
diff --git a/apps/server/package.json b/apps/server/package.json
index b0d1d26a..05d3f046 100644
--- a/apps/server/package.json
+++ b/apps/server/package.json
@@ -36,6 +36,8 @@
"@aws-sdk/s3-request-presigner": "^3.701.0",
"@colanode/core": "*",
"@colanode/crdt": "*",
+ "@langchain/core": "^0.3.26",
+ "@langchain/openai": "^0.3.16",
"axios": "^1.7.8",
"bcrypt": "^5.1.1",
"bullmq": "^5.30.1",
@@ -46,6 +48,7 @@
"js-sha256": "^0.11.0",
"kafkajs": "^2.2.4",
"kysely": "^0.27.4",
+ "langchain": "^0.3.8",
"multer": "^1.4.5-lts.1",
"nodemailer": "^6.9.16",
"pg": "^8.13.0",
diff --git a/apps/server/src/data/migrations.ts b/apps/server/src/data/migrations.ts
index 6165405c..974a5831 100644
--- a/apps/server/src/data/migrations.ts
+++ b/apps/server/src/data/migrations.ts
@@ -673,6 +673,42 @@ const createEntryPathsTable: Migration = {
},
};
+const createAIEmbeddingsTable: Migration = {
+ up: async (db) => {
+ await db.schema
+ .createTable('ai_embeddings')
+ .addColumn('id', 'varchar(30)', (col) => col.notNull().primaryKey())
+ .addColumn('entity_id', 'varchar(30)', (col) => col.notNull())
+ .addColumn('entity_type', 'varchar(30)', (col) => col.notNull())
+ .addColumn('content', 'text', (col) => col.notNull())
+ .addColumn('embedding', sql`vector(2000)`, (col) => col.notNull())
+ .addColumn('fts', sql`tsvector GENERATED ALWAYS AS (to_tsvector('english', content)) STORED`)
+ .addColumn('metadata', 'jsonb')
+ .addColumn('created_at', 'timestamptz', (col) => col.notNull())
+ .addColumn('updated_at', 'timestamptz')
+ .execute();
+
+ await sql`
+ CREATE INDEX ai_embeddings_embedding_idx
+ ON ai_embeddings
+ USING hnsw(embedding vector_cosine_ops)
+ WITH (
+ m = 16,
+ ef_construction = 64
+ );
+ `.execute(db);
+
+ await sql`
+ CREATE INDEX ai_embeddings_fts_idx
+ ON ai_embeddings
+ USING GIN (fts);
+ `.execute(db);
+ },
+ down: async (db) => {
+ await db.schema.dropTable('ai_embeddings').execute();
+ },
+};
+
export const databaseMigrations: Record<string, Migration> = {
'00001_create_accounts_table': createAccountsTable,
'00002_create_devices_table': createDevicesTable,
@@ -688,4 +724,5 @@ export const databaseMigrations: Record<string, Migration> = {
'00012_create_file_interactions_table': createFileInteractionsTable,
'00013_create_collaborations_table': createCollaborationsTable,
'00014_create_entry_paths_table': createEntryPathsTable,
+ '00015_create_ai_embeddings_table': createAIEmbeddingsTable,
};
diff --git a/apps/server/src/data/schema.ts b/apps/server/src/data/schema.ts
index 48ac0857..791ebb5b 100644
--- a/apps/server/src/data/schema.ts
+++ b/apps/server/src/data/schema.ts
@@ -266,6 +266,24 @@ export type SelectEntryPath = Selectable<EntryPathTable>;
export type CreateEntryPath = Insertable<EntryPathTable>;
export type UpdateEntryPath = Updateable<EntryPathTable>;
+interface AIEmbeddingsTable {
+ id: ColumnType<string, string, never>;
+ entity_id: ColumnType<string, string, never>;
+ entity_type: ColumnType<string, string, never>;
+ content: ColumnType<string, string, string>;
+ embedding: ColumnType<number[], number[], number[]>;
+ fts: ColumnType<never, never, never>;
+ metadata: ColumnType<string | null, string | null, string | null>;
+ created_at: ColumnType<Date, Date, never>;
+ updated_at: ColumnType<Date | null, Date | null, Date | null>;
+}
+
+
+export type SelectAIEmbedding = Selectable<AIEmbeddingsTable>;
+export type CreateAIEmbedding = Insertable<AIEmbeddingsTable>;
+export type UpdateAIEmbedding = Updateable<AIEmbeddingsTable>;
+
+
export interface DatabaseSchema {
accounts: AccountTable;
devices: DeviceTable;
@@ -281,4 +299,5 @@ export interface DatabaseSchema {
files: FileTable;
file_interactions: FileInteractionTable;
entry_paths: EntryPathTable;
+ ai_embeddings: AIEmbeddingsTable
}
diff --git a/apps/server/src/jobs/embed-entry.ts b/apps/server/src/jobs/embed-entry.ts
new file mode 100644
index 00000000..3d054cce
--- /dev/null
+++ b/apps/server/src/jobs/embed-entry.ts
@@ -0,0 +1,67 @@
+import { JobHandler } from '@/types/jobs';
+import { ChunkingService } from '@/services/chunking-service';
+import { Document } from 'langchain/document';
+import { database } from '@/data/database';
+import { generateId, IdType } from '@colanode/core';
+
+export type EmbedEntryInput = {
+ type: 'embed_entry';
+ entryId: string;
+};
+
+declare module '@/types/jobs' {
+ interface JobMap {
+ embed_entry: {
+ input: EmbedEntryInput;
+ };
+ }
+}
+
+export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (
+ input
+) => {
+ const { entryId } = input;
+
+ //TODO: use extractEntryText from packages/core/src/lib/texts.ts
+ const chunkingService = new ChunkingService();
+ const chunks = await chunkingService.chunkText(entryText);
+
+ const embeddings = new OpenAIEmbeddings({
+ apiKey: aiSettings.openai.apiKey,
+ modelName: aiSettings.openai.embeddingModel,
+ dimensions: aiSettings.openai.embeddingDimensions,
+ });
+
+ const embeddingVectors = await embeddings.embedDocuments(chunks)
+
+ for (let i = 0; i < chunks.length; i++) {
+ const chunk = chunks[i];
+ if (!chunk) {
+ continue;
+ }
+
+ const embedding = embeddingVectors[i]
+ if (!embedding) {
+ continue;
+ }
+
+ const id = generateId(IdType.AiEmbedding);
+ const metadata = JSON.stringify({
+ chunk_index: i,
+ });
+
+ await database
+ .insertInto('ai_embeddings')
+ .values(
+ {
+ id,
+ entity_id: entryId,
+ entity_type: 'message',
+ embedding: embedding,
+ content: chunk,
+ metadata,
+ created_at: new Date(),
+ })
+ .execute();
+ }
+};
diff --git a/apps/server/src/jobs/embed-message.ts b/apps/server/src/jobs/embed-message.ts
new file mode 100644
index 00000000..86157f84
--- /dev/null
+++ b/apps/server/src/jobs/embed-message.ts
@@ -0,0 +1,80 @@
+import { JobHandler } from '@/types/jobs';
+import { ChunkingService } from '@/services/chunking-service';
+import { database } from '@/data/database';
+import { OpenAIEmbeddings } from '@langchain/openai';
+import { aiSettings } from '@/lib/ai-settings';
+import { generateId, IdType } from '@colanode/core';
+
+export type EmbedMessageInput = {
+ type: 'embed_message';
+ messageId: string;
+};
+
+declare module '@/types/jobs' {
+ interface JobMap {
+ embed_message: {
+ input: EmbedMessageInput;
+ };
+ }
+}
+
+export const embedMessageHandler: JobHandler<EmbedMessageInput> = async (
+ input
+) => {
+ const { messageId } = input;
+
+ const message = await database
+ .selectFrom('messages')
+ .selectAll()
+ .where('id', '=', messageId)
+ .executeTakeFirst();
+
+ if (!message) {
+ return;
+ }
+ if (!message.content) {
+ return;
+ }
+
+ const chunkingService = new ChunkingService();
+ const chunks = await chunkingService.chunkText(message.content);
+
+ const embeddings = new OpenAIEmbeddings({
+ apiKey: aiSettings.openai.apiKey,
+ modelName: aiSettings.openai.embeddingModel,
+ dimensions: aiSettings.openai.embeddingDimensions,
+ });
+
+ const embeddingVectors = await embeddings.embedDocuments(chunks)
+
+ for (let i = 0; i < chunks.length; i++) {
+ const chunk = chunks[i];
+ if (!chunk) {
+ continue;
+ }
+
+ const embedding = embeddingVectors[i]
+ if (!embedding) {
+ continue;
+ }
+
+ const id = generateId(IdType.AiEmbedding);
+ const metadata = JSON.stringify({
+ chunk_index: i,
+ });
+
+ await database
+ .insertInto('ai_embeddings')
+ .values(
+ {
+ id,
+ entity_id: messageId,
+ entity_type: 'message',
+ embedding: embedding,
+ content: chunk,
+ metadata,
+ created_at: new Date(),
+ })
+ .execute();
+ }
+};
\ No newline at end of file
diff --git a/apps/server/src/jobs/index.ts b/apps/server/src/jobs/index.ts
index a89f604b..d8da986a 100644
--- a/apps/server/src/jobs/index.ts
+++ b/apps/server/src/jobs/index.ts
@@ -2,6 +2,8 @@ import { cleanEntryDataHandler } from '@/jobs/clean-entry-data';
import { cleanWorkspaceDataHandler } from '@/jobs/clean-workspace-data';
import { sendEmailHandler } from '@/jobs/send-email';
import { JobHandler, JobMap } from '@/types/jobs';
+import { embedMessageHandler } from '@/jobs/embed-message';
+import { embedEntryHandler } from '@/jobs/embed-entry';
type JobHandlerMap = {
[K in keyof JobMap]: JobHandler<JobMap[K]['input']>;
@@ -11,4 +13,6 @@ export const jobHandlerMap: JobHandlerMap = {
send_email: sendEmailHandler,
clean_workspace_data: cleanWorkspaceDataHandler,
clean_entry_data: cleanEntryDataHandler,
+ embed_message: embedMessageHandler,
+ embed_entry: embedEntryHandler,
};
diff --git a/apps/server/src/lib/ai-settings.ts b/apps/server/src/lib/ai-settings.ts
new file mode 100644
index 00000000..40cb4fd5
--- /dev/null
+++ b/apps/server/src/lib/ai-settings.ts
@@ -0,0 +1,15 @@
+export const aiSettings = {
+ openai: {
+ apiKey: process.env.OPENAI_API_KEY || '',
+ embeddingModel: 'text-embedding-3-large',
+ embeddingDimensions: 2000,
+ },
+
+ chunking: {
+ defaultChunkSize: 1000,
+ defaultOverlap: 200,
+ enhanceWithContext: true,
+ contextEnhancerModel: 'gpt-4o-mini',
+ contextEnhancerTemperature: 0.3,
+ },
+};
\ No newline at end of file
diff --git a/apps/server/src/services/chunking-service.ts b/apps/server/src/services/chunking-service.ts
new file mode 100644
index 00000000..3efba6f7
--- /dev/null
+++ b/apps/server/src/services/chunking-service.ts
@@ -0,0 +1,70 @@
+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
+import { ChatOpenAI } from '@langchain/openai';
+import { SystemMessage } from '@langchain/core/messages';
+import { aiSettings } from '@/lib/ai-settings';
+
+export class ChunkingService {
+ public async chunkText(
+ text: string
+ ): Promise<string[]> {
+
+ const chunkSize = aiSettings.chunking.defaultChunkSize;
+ const chunkOverlap = aiSettings.chunking.defaultOverlap;
+
+ const splitter = new RecursiveCharacterTextSplitter({
+ chunkSize,
+ chunkOverlap,
+ });
+
+ const docs = await splitter.createDocuments([text]);
+ let chunks = docs.map((doc) => doc.pageContent);
+
+ chunks = chunks.filter((c) => c.trim().length > 10);
+
+ if (aiSettings.chunking.enhanceWithContext) {
+ const enriched: string[] = [];
+ for (const chunk of chunks) {
+ const c = await this.addContextToChunk(chunk, text);
+ enriched.push(c);
+ }
+ return enriched;
+ }
+
+ return chunks;
+ }
+
+ private async addContextToChunk(
+ chunk: string,
+ fullText: string
+ ): Promise<string> {
+ try {
+ const chat = new ChatOpenAI({
+ openAIApiKey: aiSettings.openai.apiKey,
+ modelName: aiSettings.chunking.contextEnhancerModel,
+ temperature: aiSettings.chunking.contextEnhancerTemperature,
+ maxTokens: 200,
+ });
+
+ const prompt = `
+<document>
+${fullText}
+</document>
+
+Here is the chunk we want to situate in context:
+<chunk>
+${chunk}
+</chunk>
+
+Generate a short (50-100 tokens) contextual prefix that seamlessly provides background or location info for the chunk. Then include the original chunk text below it without any extra separator.
+ `;
+
+ const response = await chat.invoke([new SystemMessage(prompt)]);
+ const context = (response.content.toString() ?? '').trim();
+
+ return `${context} ${chunk}`;
+ } catch (err) {
+ console.error('Error adding context to chunk:', err);
+ return chunk;
+ }
+ }
+}
\ No newline at end of file
diff --git a/apps/server/src/services/entry-service.ts b/apps/server/src/services/entry-service.ts
index d7e4e73f..91aa3c70 100644
--- a/apps/server/src/services/entry-service.ts
+++ b/apps/server/src/services/entry-service.ts
@@ -116,6 +116,11 @@ class EntryService {
});
}
+ await jobService.addJob({
+ type: 'embed_entry',
+ entryId: input.entryId,
+ });
+
return {
entry: createdEntry,
transaction: createdTransaction,
@@ -279,6 +284,11 @@ class EntryService {
});
}
+ await jobService.addJob({
+ type: 'embed_entry',
+ entryId: input.entryId,
+ });
+
return {
type: 'success',
output: {
diff --git a/apps/server/src/services/message-service.ts b/apps/server/src/services/message-service.ts
index a357783f..a21de5bf 100644
--- a/apps/server/src/services/message-service.ts
+++ b/apps/server/src/services/message-service.ts
@@ -13,6 +13,7 @@ import { database } from '@/data/database';
import { SelectUser } from '@/data/schema';
import { mapEntry } from '@/lib/entries';
import { eventBus } from '@/lib/event-bus';
+import { jobService } from '@/services/job-service';
class MessageService {
public async createMessage(
@@ -64,6 +65,13 @@ class MessageService {
return false;
}
+ await jobService.addJob(
+ {
+ type: 'embed_message',
+ messageId: createdMessage.id,
+ }
+ );
+
eventBus.publish({
type: 'message_created',
messageId: createdMessage.id,
diff --git a/package-lock.json b/package-lock.json
index d3ef2592..d2f8597a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -214,6 +214,8 @@
"@aws-sdk/s3-request-presigner": "^3.701.0",
"@colanode/core": "*",
"@colanode/crdt": "*",
+ "@langchain/core": "^0.3.26",
+ "@langchain/openai": "^0.3.16",
"axios": "^1.7.8",
"bcrypt": "^5.1.1",
"bullmq": "^5.30.1",
@@ -224,6 +226,7 @@
"js-sha256": "^0.11.0",
"kafkajs": "^2.2.4",
"kysely": "^0.27.4",
+ "langchain": "^0.3.8",
"multer": "^1.4.5-lts.1",
"nodemailer": "^6.9.16",
"pg": "^8.13.0",
@@ -440,6 +443,86 @@
"node": ">=0.10.0"
}
},
+ "apps/server/node_modules/langchain": {
+ "version": "0.3.8",
+ "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.3.8.tgz",
+ "integrity": "sha512-EiAHFgBdThuXFmIx9j81wjdPItpRsw0Ck4r5dyhB74gyhehRGna/UK2CTqeKVnIUM/f4g4JbxUgAU4voXljDMw==",
+ "license": "MIT",
+ "dependencies": {
+ "@langchain/openai": ">=0.1.0 <0.4.0",
+ "@langchain/textsplitters": ">=0.0.0 <0.2.0",
+ "js-tiktoken": "^1.0.12",
+ "js-yaml": "^4.1.0",
+ "jsonpointer": "^5.0.1",
+ "langsmith": "^0.2.8",
+ "openapi-types": "^12.1.3",
+ "p-retry": "4",
+ "uuid": "^10.0.0",
+ "yaml": "^2.2.1",
+ "zod": "^3.22.4",
+ "zod-to-json-schema": "^3.22.3"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "peerDependencies": {
+ "@langchain/anthropic": "*",
+ "@langchain/aws": "*",
+ "@langchain/cohere": "*",
+ "@langchain/core": ">=0.2.21 <0.4.0",
+ "@langchain/google-genai": "*",
+ "@langchain/google-vertexai": "*",
+ "@langchain/groq": "*",
+ "@langchain/mistralai": "*",
+ "@langchain/ollama": "*",
+ "axios": "*",
+ "cheerio": "*",
+ "handlebars": "^4.7.8",
+ "peggy": "^3.0.2",
+ "typeorm": "*"
+ },
+ "peerDependenciesMeta": {
+ "@langchain/anthropic": {
+ "optional": true
+ },
+ "@langchain/aws": {
+ "optional": true
+ },
+ "@langchain/cohere": {
+ "optional": true
+ },
+ "@langchain/google-genai": {
+ "optional": true
+ },
+ "@langchain/google-vertexai": {
+ "optional": true
+ },
+ "@langchain/groq": {
+ "optional": true
+ },
+ "@langchain/mistralai": {
+ "optional": true
+ },
+ "@langchain/ollama": {
+ "optional": true
+ },
+ "axios": {
+ "optional": true
+ },
+ "cheerio": {
+ "optional": true
+ },
+ "handlebars": {
+ "optional": true
+ },
+ "peggy": {
+ "optional": true
+ },
+ "typeorm": {
+ "optional": true
+ }
+ }
+ },
"apps/server/node_modules/media-typer": {
"version": "1.1.0",
"license": "MIT",
@@ -562,6 +645,19 @@
"node": ">= 18"
}
},
+ "apps/server/node_modules/uuid": {
+ "version": "10.0.0",
+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
+ "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
+ "funding": [
+ "https://github.com/sponsors/broofa",
+ "https://github.com/sponsors/ctavan"
+ ],
+ "license": "MIT",
+ "bin": {
+ "uuid": "dist/bin/uuid"
+ }
+ },
"node_modules/@alloc/quick-lru": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz",
@@ -1533,6 +1629,12 @@
"node": ">=6.9.0"
}
},
+ "node_modules/@cfworker/json-schema": {
+ "version": "4.0.3",
+ "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.0.3.tgz",
+ "integrity": "sha512-ZykIcDTVv5UNmKWSTLAs3VukO6NDJkkSKxrgUTDPBkAlORVT3H9n5DbRjRl8xIotklscHdbLIa0b9+y3mQq73g==",
+ "license": "MIT"
+ },
"node_modules/@colanode/core": {
"resolved": "packages/core",
"link": true
@@ -3527,6 +3629,87 @@
"@jridgewell/sourcemap-codec": "^1.4.10"
}
},
+ "node_modules/@langchain/core": {
+ "version": "0.3.26",
+ "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.26.tgz",
+ "integrity": "sha512-6RUQHEp8wv+JwtYIIEBYBzbLlcAQZFc7EDOgAM0ukExjh9HiXoJzoWpgMRRCrr/koIbtwXPJUqBprZK1I1CXHQ==",
+ "license": "MIT",
+ "dependencies": {
+ "@cfworker/json-schema": "^4.0.2",
+ "ansi-styles": "^5.0.0",
+ "camelcase": "6",
+ "decamelize": "1.2.0",
+ "js-tiktoken": "^1.0.12",
+ "langsmith": "^0.2.8",
+ "mustache": "^4.2.0",
+ "p-queue": "^6.6.2",
+ "p-retry": "4",
+ "uuid": "^10.0.0",
+ "zod": "^3.22.4",
+ "zod-to-json-schema": "^3.22.3"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/@langchain/core/node_modules/ansi-styles": {
+ "version": "5.2.0",
+ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",
+ "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+ }
+ },
+ "node_modules/@langchain/core/node_modules/uuid": {
+ "version": "10.0.0",
+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
+ "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
+ "funding": [
+ "https://github.com/sponsors/broofa",
+ "https://github.com/sponsors/ctavan"
+ ],
+ "license": "MIT",
+ "bin": {
+ "uuid": "dist/bin/uuid"
+ }
+ },
+ "node_modules/@langchain/openai": {
+ "version": "0.3.16",
+ "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.3.16.tgz",
+ "integrity": "sha512-Om9HRlTeI0Ou6D4pfxbWHop4WGfkCdV/7v1W/+Jr7NSf0BNoA9jk5GqGms8ZtOYSGgPvizDu3i0TrM3B4cN4NA==",
+ "license": "MIT",
+ "dependencies": {
+ "js-tiktoken": "^1.0.12",
+ "openai": "^4.77.0",
+ "zod": "^3.22.4",
+ "zod-to-json-schema": "^3.22.3"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "peerDependencies": {
+ "@langchain/core": ">=0.2.26 <0.4.0"
+ }
+ },
+ "node_modules/@langchain/textsplitters": {
+ "version": "0.1.0",
+ "resolved": "https://registry.npmjs.org/@langchain/textsplitters/-/textsplitters-0.1.0.tgz",
+ "integrity": "sha512-djI4uw9rlkAb5iMhtLED+xJebDdAG935AdP4eRTB02R7OB/act55Bj9wsskhZsvuyQRpO4O1wQOp85s6T6GWmw==",
+ "license": "MIT",
+ "dependencies": {
+ "js-tiktoken": "^1.0.12"
+ },
+ "engines": {
+ "node": ">=18"
+ },
+ "peerDependencies": {
+ "@langchain/core": ">=0.2.21 <0.4.0"
+ }
+ },
"node_modules/@malept/cross-spawn-promise": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@malept/cross-spawn-promise/-/cross-spawn-promise-2.0.0.tgz",
@@ -6706,12 +6889,21 @@
"version": "22.10.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz",
"integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==",
- "devOptional": true,
"license": "MIT",
"dependencies": {
"undici-types": "~6.20.0"
}
},
+ "node_modules/@types/node-fetch": {
+ "version": "2.6.12",
+ "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz",
+ "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==",
+ "license": "MIT",
+ "dependencies": {
+ "@types/node": "*",
+ "form-data": "^4.0.0"
+ }
+ },
"node_modules/@types/nodemailer": {
"version": "6.4.17",
"resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.17.tgz",
@@ -6796,6 +6988,12 @@
"@types/node": "*"
}
},
+ "node_modules/@types/retry": {
+ "version": "0.12.0",
+ "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",
+ "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==",
+ "license": "MIT"
+ },
"node_modules/@types/send": {
"version": "0.17.4",
"resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz",
@@ -6841,6 +7039,12 @@
"integrity": "sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg==",
"license": "MIT"
},
+ "node_modules/@types/uuid": {
+ "version": "10.0.0",
+ "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz",
+ "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==",
+ "license": "MIT"
+ },
"node_modules/@types/ws": {
"version": "8.5.13",
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.13.tgz",
@@ -7192,6 +7396,18 @@
"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
"license": "ISC"
},
+ "node_modules/abort-controller": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
+ "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
+ "license": "MIT",
+ "dependencies": {
+ "event-target-shim": "^5.0.0"
+ },
+ "engines": {
+ "node": ">=6.5"
+ }
+ },
"node_modules/accepts": {
"version": "1.3.8",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@@ -7267,7 +7483,6 @@
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
"integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
- "dev": true,
"license": "MIT",
"dependencies": {
"humanize-ms": "^1.2.1"
@@ -8321,6 +8536,18 @@
"node": ">=6"
}
},
+ "node_modules/camelcase": {
+ "version": "6.3.0",
+ "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",
+ "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/camelcase-css": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz",
@@ -9046,6 +9273,15 @@
}
}
},
+ "node_modules/decamelize": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
+ "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/decompress-response": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
@@ -10790,6 +11026,15 @@
"node": ">= 0.6"
}
},
+ "node_modules/event-target-shim": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
+ "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=6"
+ }
+ },
"node_modules/eventemitter3": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",
@@ -11408,6 +11653,25 @@
"node": ">= 6"
}
},
+ "node_modules/form-data-encoder": {
+ "version": "1.7.2",
+ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
+ "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",
+ "license": "MIT"
+ },
+ "node_modules/formdata-node": {
+ "version": "4.4.1",
+ "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
+ "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
+ "license": "MIT",
+ "dependencies": {
+ "node-domexception": "1.0.0",
+ "web-streams-polyfill": "4.0.0-beta.3"
+ },
+ "engines": {
+ "node": ">= 12.20"
+ }
+ },
"node_modules/forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -12237,7 +12501,6 @@
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
"integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
- "dev": true,
"license": "MIT",
"dependencies": {
"ms": "^2.0.0"
@@ -13068,6 +13331,15 @@
"integrity": "sha512-6xNlKayMZvds9h1Y1VWc0fQHQ82BxTXizWPEtEeGvmOUYpBRy4gbWroHLpzowe6xiQhHpelCQiE7HEdznyBL9Q==",
"license": "MIT"
},
+ "node_modules/js-tiktoken": {
+ "version": "1.0.16",
+ "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.16.tgz",
+ "integrity": "sha512-nUVdO5k/M9llWpiaZlBBDdtmr6qWXwSD6fgaDu2zM8UP+OXxx9V37lFkI6w0/1IuaDx7WffZ37oYd9KvcWKElg==",
+ "license": "MIT",
+ "dependencies": {
+ "base64-js": "^1.5.1"
+ }
+ },
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -13078,7 +13350,6 @@
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
"integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
- "dev": true,
"license": "MIT",
"dependencies": {
"argparse": "^2.0.1"
@@ -13153,9 +13424,7 @@
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz",
"integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==",
- "dev": true,
"license": "MIT",
- "optional": true,
"engines": {
"node": ">=0.10.0"
}
@@ -13214,6 +13483,50 @@
"node": ">=14.0.0"
}
},
+ "node_modules/langsmith": {
+ "version": "0.2.14",
+ "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.2.14.tgz",
+ "integrity": "sha512-ClAuAgSf3m9miMYotLEaZKQyKdaWlfjhebCuYco8bc6g72dU2VwTg31Bv4YINBq7EH2i1cMwbOiJxbOXPqjGig==",
+ "license": "MIT",
+ "dependencies": {
+ "@types/uuid": "^10.0.0",
+ "commander": "^10.0.1",
+ "p-queue": "^6.6.2",
+ "p-retry": "4",
+ "semver": "^7.6.3",
+ "uuid": "^10.0.0"
+ },
+ "peerDependencies": {
+ "openai": "*"
+ },
+ "peerDependenciesMeta": {
+ "openai": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/langsmith/node_modules/commander": {
+ "version": "10.0.1",
+ "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz",
+ "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=14"
+ }
+ },
+ "node_modules/langsmith/node_modules/uuid": {
+ "version": "10.0.0",
+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
+ "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
+ "funding": [
+ "https://github.com/sponsors/broofa",
+ "https://github.com/sponsors/ctavan"
+ ],
+ "license": "MIT",
+ "bin": {
+ "uuid": "dist/bin/uuid"
+ }
+ },
"node_modules/levn": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
@@ -14014,6 +14327,15 @@
"imul": "^1.0.0"
}
},
+ "node_modules/mustache": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz",
+ "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==",
+ "license": "MIT",
+ "bin": {
+ "mustache": "bin/mustache"
+ }
+ },
"node_modules/mz": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -14115,6 +14437,25 @@
"semver": "^7.3.5"
}
},
+ "node_modules/node-domexception": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
+ "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/jimmywarting"
+ },
+ {
+ "type": "github",
+ "url": "https://paypal.me/jimmywarting"
+ }
+ ],
+ "license": "MIT",
+ "engines": {
+ "node": ">=10.5.0"
+ }
+ },
"node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
@@ -14542,6 +14883,53 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/openai": {
+ "version": "4.77.0",
+ "resolved": "https://registry.npmjs.org/openai/-/openai-4.77.0.tgz",
+ "integrity": "sha512-WWacavtns/7pCUkOWvQIjyOfcdr9X+9n9Vvb0zFeKVDAqwCMDHB+iSr24SVaBAhplvSG6JrRXFpcNM9gWhOGIw==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "@types/node": "^18.11.18",
+ "@types/node-fetch": "^2.6.4",
+ "abort-controller": "^3.0.0",
+ "agentkeepalive": "^4.2.1",
+ "form-data-encoder": "1.7.2",
+ "formdata-node": "^4.3.2",
+ "node-fetch": "^2.6.7"
+ },
+ "bin": {
+ "openai": "bin/cli"
+ },
+ "peerDependencies": {
+ "zod": "^3.23.8"
+ },
+ "peerDependenciesMeta": {
+ "zod": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/openai/node_modules/@types/node": {
+ "version": "18.19.68",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.68.tgz",
+ "integrity": "sha512-QGtpFH1vB99ZmTa63K4/FU8twThj4fuVSBkGddTp7uIL/cuoLWIUSL2RcOaigBhfR+hg5pgGkBnkoOxrTVBMKw==",
+ "license": "MIT",
+ "dependencies": {
+ "undici-types": "~5.26.4"
+ }
+ },
+ "node_modules/openai/node_modules/undici-types": {
+ "version": "5.26.5",
+ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
+ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
+ "license": "MIT"
+ },
+ "node_modules/openapi-types": {
+ "version": "12.1.3",
+ "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz",
+ "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==",
+ "license": "MIT"
+ },
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@@ -14642,7 +15030,6 @@
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
"integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==",
- "dev": true,
"license": "MIT",
"engines": {
"node": ">=4"
@@ -14706,6 +15093,62 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/p-queue": {
+ "version": "6.6.2",
+ "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",
+ "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==",
+ "license": "MIT",
+ "dependencies": {
+ "eventemitter3": "^4.0.4",
+ "p-timeout": "^3.2.0"
+ },
+ "engines": {
+ "node": ">=8"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/p-queue/node_modules/eventemitter3": {
+ "version": "4.0.7",
+ "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
+ "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",
+ "license": "MIT"
+ },
+ "node_modules/p-retry": {
+ "version": "4.6.2",
+ "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz",
+ "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==",
+ "license": "MIT",
+ "dependencies": {
+ "@types/retry": "0.12.0",
+ "retry": "^0.13.1"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/p-retry/node_modules/retry": {
+ "version": "0.13.1",
+ "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",
+ "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",
+ "license": "MIT",
+ "engines": {
+ "node": ">= 4"
+ }
+ },
+ "node_modules/p-timeout": {
+ "version": "3.2.0",
+ "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",
+ "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",
+ "license": "MIT",
+ "dependencies": {
+ "p-finally": "^1.0.0"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
"node_modules/p-try": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
@@ -19368,7 +19811,6 @@
"version": "6.20.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==",
- "devOptional": true,
"license": "MIT"
},
"node_modules/unique-filename": {
@@ -20205,6 +20647,15 @@
"defaults": "^1.0.3"
}
},
+ "node_modules/web-streams-polyfill": {
+ "version": "4.0.0-beta.3",
+ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
+ "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
+ "license": "MIT",
+ "engines": {
+ "node": ">= 14"
+ }
+ },
"node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
@@ -20808,6 +21259,15 @@
"url": "https://github.com/sponsors/colinhacks"
}
},
+ "node_modules/zod-to-json-schema": {
+ "version": "3.24.1",
+ "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.1.tgz",
+ "integrity": "sha512-3h08nf3Vw3Wl3PK+q3ow/lIil81IT2Oa7YpQyUUDsEWbXveMesdfK1xBd2RhCkynwZndAxixji/7SYJJowr62w==",
+ "license": "ISC",
+ "peerDependencies": {
+ "zod": "^3.24.1"
+ }
+ },
"packages/core": {
"name": "@colanode/core",
"version": "1.0.0",
diff --git a/packages/core/src/lib/id.ts b/packages/core/src/lib/id.ts
index 63475012..b26630f9 100644
--- a/packages/core/src/lib/id.ts
+++ b/packages/core/src/lib/id.ts
@@ -36,6 +36,7 @@ export enum IdType {
Event = 'ev',
Host = 'ht',
Block = 'bl',
+ AiEmbedding = 'ae',
}
export const generateId = (type: IdType): string => {