Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Add AI embedding functionality for entries and messages
Ylber Gashi6ac99f17fb29
diff --git a/apps/server/package.json b/apps/server/package.jsonindex b0d1d26a..05d3f046 100644--- a/apps/server/package.json+++ b/apps/server/package.json@@ -36,6 +36,8 @@"@aws-sdk/s3-request-presigner": "^3.701.0","@colanode/core": "*","@colanode/crdt": "*",+ "@langchain/core": "^0.3.26",+ "@langchain/openai": "^0.3.16","axios": "^1.7.8","bcrypt": "^5.1.1","bullmq": "^5.30.1",@@ -46,6 +48,7 @@"js-sha256": "^0.11.0","kafkajs": "^2.2.4","kysely": "^0.27.4",+ "langchain": "^0.3.8","multer": "^1.4.5-lts.1","nodemailer": "^6.9.16","pg": "^8.13.0",diff --git a/apps/server/src/data/migrations.ts b/apps/server/src/data/migrations.tsindex 6165405c..974a5831 100644--- a/apps/server/src/data/migrations.ts+++ b/apps/server/src/data/migrations.ts@@ -673,6 +673,42 @@ const createEntryPathsTable: Migration = {},};+const createAIEmbeddingsTable: Migration = {+ up: async (db) => {+ await db.schema+ .createTable('ai_embeddings')+ .addColumn('id', 'varchar(30)', (col) => col.notNull().primaryKey())+ .addColumn('entity_id', 'varchar(30)', (col) => col.notNull())+ .addColumn('entity_type', 'varchar(30)', (col) => col.notNull())+ .addColumn('content', 'text', (col) => col.notNull())+ .addColumn('embedding', sql`vector(2000)`, (col) => col.notNull())+ .addColumn('fts', sql`tsvector GENERATED ALWAYS AS (to_tsvector('english', content)) STORED`)+ .addColumn('metadata', 'jsonb')+ .addColumn('created_at', 'timestamptz', (col) => col.notNull())+ .addColumn('updated_at', 'timestamptz')+ .execute();++ await sql`+ CREATE INDEX ai_embeddings_embedding_idx+ ON ai_embeddings+ USING hnsw(embedding vector_cosine_ops)+ WITH (+ m = 16,+ ef_construction = 64+ );+ `.execute(db);++ await sql`+ CREATE INDEX ai_embeddings_fts_idx+ ON ai_embeddings+ USING GIN (fts);+ `.execute(db);+ },+ down: async (db) => {+ await db.schema.dropTable('ai_embeddings').execute();+ },+};+export const databaseMigrations: Record<string, Migration> = {'00001_create_accounts_table': createAccountsTable,'00002_create_devices_table': createDevicesTable,@@ -688,4 +724,5 @@ export const databaseMigrations: Record<string, Migration> = {'00012_create_file_interactions_table': createFileInteractionsTable,'00013_create_collaborations_table': createCollaborationsTable,'00014_create_entry_paths_table': createEntryPathsTable,+ '00015_create_ai_embeddings_table': createAIEmbeddingsTable,};diff --git a/apps/server/src/data/schema.ts b/apps/server/src/data/schema.tsindex 48ac0857..791ebb5b 100644--- a/apps/server/src/data/schema.ts+++ b/apps/server/src/data/schema.ts@@ -266,6 +266,24 @@ export type SelectEntryPath = Selectable<EntryPathTable>;export type CreateEntryPath = Insertable<EntryPathTable>;export type UpdateEntryPath = Updateable<EntryPathTable>;+interface AIEmbeddingsTable {+ id: ColumnType<string, string, never>;+ entity_id: ColumnType<string, string, never>;+ entity_type: ColumnType<string, string, never>;+ content: ColumnType<string, string, string>;+ embedding: ColumnType<number[], number[], number[]>;+ fts: ColumnType<never, never, never>;+ metadata: ColumnType<string | null, string | null, string | null>;+ created_at: ColumnType<Date, Date, never>;+ updated_at: ColumnType<Date | null, Date | null, Date | null>;+}+++export type SelectAIEmbedding = Selectable<AIEmbeddingsTable>;+export type CreateAIEmbedding = Insertable<AIEmbeddingsTable>;+export type UpdateAIEmbedding = Updateable<AIEmbeddingsTable>;++export interface DatabaseSchema {accounts: AccountTable;devices: DeviceTable;@@ -281,4 +299,5 @@ export interface DatabaseSchema {files: FileTable;file_interactions: FileInteractionTable;entry_paths: EntryPathTable;+ ai_embeddings: AIEmbeddingsTable}diff --git a/apps/server/src/jobs/embed-entry.ts b/apps/server/src/jobs/embed-entry.tsnew file mode 100644index 00000000..3d054cce--- /dev/null+++ b/apps/server/src/jobs/embed-entry.ts@@ -0,0 +1,67 @@+import { JobHandler } from '@/types/jobs';+import { ChunkingService } from '@/services/chunking-service';+import { Document } from 'langchain/document';+import { database } from '@/data/database';+import { generateId, IdType } from '@colanode/core';++export type EmbedEntryInput = {+ type: 'embed_entry';+ entryId: string;+};++declare module '@/types/jobs' {+ interface JobMap {+ embed_entry: {+ input: EmbedEntryInput;+ };+ }+}++export const embedEntryHandler: JobHandler<EmbedEntryInput> = async (+ input+) => {+ const { entryId } = input;++ //TODO: use extractEntryText from packages/core/src/lib/texts.ts+ const chunkingService = new ChunkingService();+ const chunks = await chunkingService.chunkText(entryText);++ const embeddings = new OpenAIEmbeddings({+ apiKey: aiSettings.openai.apiKey,+ modelName: aiSettings.openai.embeddingModel,+ dimensions: aiSettings.openai.embeddingDimensions,+ });++ const embeddingVectors = await embeddings.embedDocuments(chunks)++ for (let i = 0; i < chunks.length; i++) {+ const chunk = chunks[i];+ if (!chunk) {+ continue;+ }++ const embedding = embeddingVectors[i]+ if (!embedding) {+ continue;+ }++ const id = generateId(IdType.AiEmbedding);+ const metadata = JSON.stringify({+ chunk_index: i,+ });++ await database+ .insertInto('ai_embeddings')+ .values(+ {+ id,+ entity_id: entryId,+ entity_type: 'message',+ embedding: embedding,+ content: chunk,+ metadata,+ created_at: new Date(),+ })+ .execute();+ }+};diff --git a/apps/server/src/jobs/embed-message.ts b/apps/server/src/jobs/embed-message.tsnew file mode 100644index 00000000..86157f84--- /dev/null+++ b/apps/server/src/jobs/embed-message.ts@@ -0,0 +1,80 @@+import { JobHandler } from '@/types/jobs';+import { ChunkingService } from '@/services/chunking-service';+import { database } from '@/data/database';+import { OpenAIEmbeddings } from '@langchain/openai';+import { aiSettings } from '@/lib/ai-settings';+import { generateId, IdType } from '@colanode/core';++export type EmbedMessageInput = {+ type: 'embed_message';+ messageId: string;+};++declare module '@/types/jobs' {+ interface JobMap {+ embed_message: {+ input: EmbedMessageInput;+ };+ }+}++export const embedMessageHandler: JobHandler<EmbedMessageInput> = async (+ input+) => {+ const { messageId } = input;++ const message = await database+ .selectFrom('messages')+ .selectAll()+ .where('id', '=', messageId)+ .executeTakeFirst();++ if (!message) {+ return;+ }+ if (!message.content) {+ return;+ }++ const chunkingService = new ChunkingService();+ const chunks = await chunkingService.chunkText(message.content);++ const embeddings = new OpenAIEmbeddings({+ apiKey: aiSettings.openai.apiKey,+ modelName: aiSettings.openai.embeddingModel,+ dimensions: aiSettings.openai.embeddingDimensions,+ });++ const embeddingVectors = await embeddings.embedDocuments(chunks)++ for (let i = 0; i < chunks.length; i++) {+ const chunk = chunks[i];+ if (!chunk) {+ continue;+ }++ const embedding = embeddingVectors[i]+ if (!embedding) {+ continue;+ }++ const id = generateId(IdType.AiEmbedding);+ const metadata = JSON.stringify({+ chunk_index: i,+ });++ await database+ .insertInto('ai_embeddings')+ .values(+ {+ id,+ entity_id: messageId,+ entity_type: 'message',+ embedding: embedding,+ content: chunk,+ metadata,+ created_at: new Date(),+ })+ .execute();+ }+};\ No newline at end of filediff --git a/apps/server/src/jobs/index.ts b/apps/server/src/jobs/index.tsindex a89f604b..d8da986a 100644--- a/apps/server/src/jobs/index.ts+++ b/apps/server/src/jobs/index.ts@@ -2,6 +2,8 @@ import { cleanEntryDataHandler } from '@/jobs/clean-entry-data';import { cleanWorkspaceDataHandler } from '@/jobs/clean-workspace-data';import { sendEmailHandler } from '@/jobs/send-email';import { JobHandler, JobMap } from '@/types/jobs';+import { embedMessageHandler } from '@/jobs/embed-message';+import { embedEntryHandler } from '@/jobs/embed-entry';type JobHandlerMap = {[K in keyof JobMap]: JobHandler<JobMap[K]['input']>;@@ -11,4 +13,6 @@ export const jobHandlerMap: JobHandlerMap = {send_email: sendEmailHandler,clean_workspace_data: cleanWorkspaceDataHandler,clean_entry_data: cleanEntryDataHandler,+ embed_message: embedMessageHandler,+ embed_entry: embedEntryHandler,};diff --git a/apps/server/src/lib/ai-settings.ts b/apps/server/src/lib/ai-settings.tsnew file mode 100644index 00000000..40cb4fd5--- /dev/null+++ b/apps/server/src/lib/ai-settings.ts@@ -0,0 +1,15 @@+export const aiSettings = {+ openai: {+ apiKey: process.env.OPENAI_API_KEY || '',+ embeddingModel: 'text-embedding-3-large',+ embeddingDimensions: 2000,+ },++ chunking: {+ defaultChunkSize: 1000,+ defaultOverlap: 200,+ enhanceWithContext: true,+ contextEnhancerModel: 'gpt-4o-mini',+ contextEnhancerTemperature: 0.3,+ },+};\ No newline at end of filediff --git a/apps/server/src/services/chunking-service.ts b/apps/server/src/services/chunking-service.tsnew file mode 100644index 00000000..3efba6f7--- /dev/null+++ b/apps/server/src/services/chunking-service.ts@@ -0,0 +1,70 @@+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';+import { ChatOpenAI } from '@langchain/openai';+import { SystemMessage } from '@langchain/core/messages';+import { aiSettings } from '@/lib/ai-settings';++export class ChunkingService {+ public async chunkText(+ text: string+ ): Promise<string[]> {++ const chunkSize = aiSettings.chunking.defaultChunkSize;+ const chunkOverlap = aiSettings.chunking.defaultOverlap;++ const splitter = new RecursiveCharacterTextSplitter({+ chunkSize,+ chunkOverlap,+ });++ const docs = await splitter.createDocuments([text]);+ let chunks = docs.map((doc) => doc.pageContent);++ chunks = chunks.filter((c) => c.trim().length > 10);++ if (aiSettings.chunking.enhanceWithContext) {+ const enriched: string[] = [];+ for (const chunk of chunks) {+ const c = await this.addContextToChunk(chunk, text);+ enriched.push(c);+ }+ return enriched;+ }++ return chunks;+ }++ private async addContextToChunk(+ chunk: string,+ fullText: string+ ): Promise<string> {+ try {+ const chat = new ChatOpenAI({+ openAIApiKey: aiSettings.openai.apiKey,+ modelName: aiSettings.chunking.contextEnhancerModel,+ temperature: aiSettings.chunking.contextEnhancerTemperature,+ maxTokens: 200,+ });++ const prompt = `+<document>+${fullText}+</document>++Here is the chunk we want to situate in context:+<chunk>+${chunk}+</chunk>++Generate a short (50-100 tokens) contextual prefix that seamlessly provides background or location info for the chunk. Then include the original chunk text below it without any extra separator.+ `;++ const response = await chat.invoke([new SystemMessage(prompt)]);+ const context = (response.content.toString() ?? '').trim();++ return `${context} ${chunk}`;+ } catch (err) {+ console.error('Error adding context to chunk:', err);+ return chunk;+ }+ }+}\ No newline at end of filediff --git a/apps/server/src/services/entry-service.ts b/apps/server/src/services/entry-service.tsindex d7e4e73f..91aa3c70 100644--- a/apps/server/src/services/entry-service.ts+++ b/apps/server/src/services/entry-service.ts@@ -116,6 +116,11 @@ class EntryService {});}+ await jobService.addJob({+ type: 'embed_entry',+ entryId: input.entryId,+ });+return {entry: createdEntry,transaction: createdTransaction,@@ -279,6 +284,11 @@ class EntryService {});}+ await jobService.addJob({+ type: 'embed_entry',+ entryId: input.entryId,+ });+return {type: 'success',output: {diff --git a/apps/server/src/services/message-service.ts b/apps/server/src/services/message-service.tsindex a357783f..a21de5bf 100644--- a/apps/server/src/services/message-service.ts+++ b/apps/server/src/services/message-service.ts@@ -13,6 +13,7 @@ import { database } from '@/data/database';import { SelectUser } from '@/data/schema';import { mapEntry } from '@/lib/entries';import { eventBus } from '@/lib/event-bus';+import { jobService } from '@/services/job-service';class MessageService {public async createMessage(@@ -64,6 +65,13 @@ class MessageService {return false;}+ await jobService.addJob(+ {+ type: 'embed_message',+ messageId: createdMessage.id,+ }+ );+eventBus.publish({type: 'message_created',messageId: createdMessage.id,diff --git a/package-lock.json b/package-lock.jsonindex d3ef2592..d2f8597a 100644--- a/package-lock.json+++ b/package-lock.json@@ -214,6 +214,8 @@"@aws-sdk/s3-request-presigner": "^3.701.0","@colanode/core": "*","@colanode/crdt": "*",+ "@langchain/core": "^0.3.26",+ "@langchain/openai": "^0.3.16","axios": "^1.7.8","bcrypt": "^5.1.1","bullmq": "^5.30.1",@@ -224,6 +226,7 @@"js-sha256": "^0.11.0","kafkajs": "^2.2.4","kysely": "^0.27.4",+ "langchain": "^0.3.8","multer": "^1.4.5-lts.1","nodemailer": "^6.9.16","pg": "^8.13.0",@@ -440,6 +443,86 @@"node": ">=0.10.0"}},+ "apps/server/node_modules/langchain": {+ "version": "0.3.8",+ "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.3.8.tgz",+ "integrity": "sha512-EiAHFgBdThuXFmIx9j81wjdPItpRsw0Ck4r5dyhB74gyhehRGna/UK2CTqeKVnIUM/f4g4JbxUgAU4voXljDMw==",+ "license": "MIT",+ "dependencies": {+ "@langchain/openai": ">=0.1.0 <0.4.0",+ "@langchain/textsplitters": ">=0.0.0 <0.2.0",+ "js-tiktoken": "^1.0.12",+ "js-yaml": "^4.1.0",+ "jsonpointer": "^5.0.1",+ "langsmith": "^0.2.8",+ "openapi-types": "^12.1.3",+ "p-retry": "4",+ "uuid": "^10.0.0",+ "yaml": "^2.2.1",+ "zod": "^3.22.4",+ "zod-to-json-schema": "^3.22.3"+ },+ "engines": {+ "node": ">=18"+ },+ "peerDependencies": {+ "@langchain/anthropic": "*",+ "@langchain/aws": "*",+ "@langchain/cohere": "*",+ "@langchain/core": ">=0.2.21 <0.4.0",+ "@langchain/google-genai": "*",+ "@langchain/google-vertexai": "*",+ "@langchain/groq": "*",+ "@langchain/mistralai": "*",+ "@langchain/ollama": "*",+ "axios": "*",+ "cheerio": "*",+ "handlebars": "^4.7.8",+ "peggy": "^3.0.2",+ "typeorm": "*"+ },+ "peerDependenciesMeta": {+ "@langchain/anthropic": {+ "optional": true+ },+ "@langchain/aws": {+ "optional": true+ },+ "@langchain/cohere": {+ "optional": true+ },+ "@langchain/google-genai": {+ "optional": true+ },+ "@langchain/google-vertexai": {+ "optional": true+ },+ "@langchain/groq": {+ "optional": true+ },+ "@langchain/mistralai": {+ "optional": true+ },+ "@langchain/ollama": {+ "optional": true+ },+ "axios": {+ "optional": true+ },+ "cheerio": {+ "optional": true+ },+ "handlebars": {+ "optional": true+ },+ "peggy": {+ "optional": true+ },+ "typeorm": {+ "optional": true+ }+ }+ },"apps/server/node_modules/media-typer": {"version": "1.1.0","license": "MIT",@@ -562,6 +645,19 @@"node": ">= 18"}},+ "apps/server/node_modules/uuid": {+ "version": "10.0.0",+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",+ "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",+ "funding": [+ "https://github.com/sponsors/broofa",+ "https://github.com/sponsors/ctavan"+ ],+ "license": "MIT",+ "bin": {+ "uuid": "dist/bin/uuid"+ }+ },"node_modules/@alloc/quick-lru": {"version": "5.2.0","resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz",@@ -1533,6 +1629,12 @@"node": ">=6.9.0"}},+ "node_modules/@cfworker/json-schema": {+ "version": "4.0.3",+ "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.0.3.tgz",+ "integrity": "sha512-ZykIcDTVv5UNmKWSTLAs3VukO6NDJkkSKxrgUTDPBkAlORVT3H9n5DbRjRl8xIotklscHdbLIa0b9+y3mQq73g==",+ "license": "MIT"+ },"node_modules/@colanode/core": {"resolved": "packages/core","link": true@@ -3527,6 +3629,87 @@"@jridgewell/sourcemap-codec": "^1.4.10"}},+ "node_modules/@langchain/core": {+ "version": "0.3.26",+ "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.26.tgz",+ "integrity": "sha512-6RUQHEp8wv+JwtYIIEBYBzbLlcAQZFc7EDOgAM0ukExjh9HiXoJzoWpgMRRCrr/koIbtwXPJUqBprZK1I1CXHQ==",+ "license": "MIT",+ "dependencies": {+ "@cfworker/json-schema": "^4.0.2",+ "ansi-styles": "^5.0.0",+ "camelcase": "6",+ "decamelize": "1.2.0",+ "js-tiktoken": "^1.0.12",+ "langsmith": "^0.2.8",+ "mustache": "^4.2.0",+ "p-queue": "^6.6.2",+ "p-retry": "4",+ "uuid": "^10.0.0",+ "zod": "^3.22.4",+ "zod-to-json-schema": "^3.22.3"+ },+ "engines": {+ "node": ">=18"+ }+ },+ "node_modules/@langchain/core/node_modules/ansi-styles": {+ "version": "5.2.0",+ "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz",+ "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==",+ "license": "MIT",+ "engines": {+ "node": ">=10"+ },+ "funding": {+ "url": "https://github.com/chalk/ansi-styles?sponsor=1"+ }+ },+ "node_modules/@langchain/core/node_modules/uuid": {+ "version": "10.0.0",+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",+ "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",+ "funding": [+ "https://github.com/sponsors/broofa",+ "https://github.com/sponsors/ctavan"+ ],+ "license": "MIT",+ "bin": {+ "uuid": "dist/bin/uuid"+ }+ },+ "node_modules/@langchain/openai": {+ "version": "0.3.16",+ "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.3.16.tgz",+ "integrity": "sha512-Om9HRlTeI0Ou6D4pfxbWHop4WGfkCdV/7v1W/+Jr7NSf0BNoA9jk5GqGms8ZtOYSGgPvizDu3i0TrM3B4cN4NA==",+ "license": "MIT",+ "dependencies": {+ "js-tiktoken": "^1.0.12",+ "openai": "^4.77.0",+ "zod": "^3.22.4",+ "zod-to-json-schema": "^3.22.3"+ },+ "engines": {+ "node": ">=18"+ },+ "peerDependencies": {+ "@langchain/core": ">=0.2.26 <0.4.0"+ }+ },+ "node_modules/@langchain/textsplitters": {+ "version": "0.1.0",+ "resolved": "https://registry.npmjs.org/@langchain/textsplitters/-/textsplitters-0.1.0.tgz",+ "integrity": "sha512-djI4uw9rlkAb5iMhtLED+xJebDdAG935AdP4eRTB02R7OB/act55Bj9wsskhZsvuyQRpO4O1wQOp85s6T6GWmw==",+ "license": "MIT",+ "dependencies": {+ "js-tiktoken": "^1.0.12"+ },+ "engines": {+ "node": ">=18"+ },+ "peerDependencies": {+ "@langchain/core": ">=0.2.21 <0.4.0"+ }+ },"node_modules/@malept/cross-spawn-promise": {"version": "2.0.0","resolved": "https://registry.npmjs.org/@malept/cross-spawn-promise/-/cross-spawn-promise-2.0.0.tgz",@@ -6706,12 +6889,21 @@"version": "22.10.2","resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.2.tgz","integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==",- "devOptional": true,"license": "MIT","dependencies": {"undici-types": "~6.20.0"}},+ "node_modules/@types/node-fetch": {+ "version": "2.6.12",+ "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz",+ "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==",+ "license": "MIT",+ "dependencies": {+ "@types/node": "*",+ "form-data": "^4.0.0"+ }+ },"node_modules/@types/nodemailer": {"version": "6.4.17","resolved": "https://registry.npmjs.org/@types/nodemailer/-/nodemailer-6.4.17.tgz",@@ -6796,6 +6988,12 @@"@types/node": "*"}},+ "node_modules/@types/retry": {+ "version": "0.12.0",+ "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",+ "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==",+ "license": "MIT"+ },"node_modules/@types/send": {"version": "0.17.4","resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz",@@ -6841,6 +7039,12 @@"integrity": "sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg==","license": "MIT"},+ "node_modules/@types/uuid": {+ "version": "10.0.0",+ "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz",+ "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==",+ "license": "MIT"+ },"node_modules/@types/ws": {"version": "8.5.13","resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.13.tgz",@@ -7192,6 +7396,18 @@"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==","license": "ISC"},+ "node_modules/abort-controller": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",+ "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",+ "license": "MIT",+ "dependencies": {+ "event-target-shim": "^5.0.0"+ },+ "engines": {+ "node": ">=6.5"+ }+ },"node_modules/accepts": {"version": "1.3.8","resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",@@ -7267,7 +7483,6 @@"version": "4.5.0","resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz","integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",- "dev": true,"license": "MIT","dependencies": {"humanize-ms": "^1.2.1"@@ -8321,6 +8536,18 @@"node": ">=6"}},+ "node_modules/camelcase": {+ "version": "6.3.0",+ "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz",+ "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==",+ "license": "MIT",+ "engines": {+ "node": ">=10"+ },+ "funding": {+ "url": "https://github.com/sponsors/sindresorhus"+ }+ },"node_modules/camelcase-css": {"version": "2.0.1","resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz",@@ -9046,6 +9273,15 @@}}},+ "node_modules/decamelize": {+ "version": "1.2.0",+ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",+ "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==",+ "license": "MIT",+ "engines": {+ "node": ">=0.10.0"+ }+ },"node_modules/decompress-response": {"version": "6.0.0","resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",@@ -10790,6 +11026,15 @@"node": ">= 0.6"}},+ "node_modules/event-target-shim": {+ "version": "5.0.1",+ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",+ "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",+ "license": "MIT",+ "engines": {+ "node": ">=6"+ }+ },"node_modules/eventemitter3": {"version": "5.0.1","resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",@@ -11408,6 +11653,25 @@"node": ">= 6"}},+ "node_modules/form-data-encoder": {+ "version": "1.7.2",+ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",+ "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",+ "license": "MIT"+ },+ "node_modules/formdata-node": {+ "version": "4.4.1",+ "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",+ "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",+ "license": "MIT",+ "dependencies": {+ "node-domexception": "1.0.0",+ "web-streams-polyfill": "4.0.0-beta.3"+ },+ "engines": {+ "node": ">= 12.20"+ }+ },"node_modules/forwarded": {"version": "0.2.0","resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",@@ -12237,7 +12501,6 @@"version": "1.2.1","resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz","integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",- "dev": true,"license": "MIT","dependencies": {"ms": "^2.0.0"@@ -13068,6 +13331,15 @@"integrity": "sha512-6xNlKayMZvds9h1Y1VWc0fQHQ82BxTXizWPEtEeGvmOUYpBRy4gbWroHLpzowe6xiQhHpelCQiE7HEdznyBL9Q==","license": "MIT"},+ "node_modules/js-tiktoken": {+ "version": "1.0.16",+ "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.16.tgz",+ "integrity": "sha512-nUVdO5k/M9llWpiaZlBBDdtmr6qWXwSD6fgaDu2zM8UP+OXxx9V37lFkI6w0/1IuaDx7WffZ37oYd9KvcWKElg==",+ "license": "MIT",+ "dependencies": {+ "base64-js": "^1.5.1"+ }+ },"node_modules/js-tokens": {"version": "4.0.0","resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",@@ -13078,7 +13350,6 @@"version": "4.1.0","resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz","integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",- "dev": true,"license": "MIT","dependencies": {"argparse": "^2.0.1"@@ -13153,9 +13424,7 @@"version": "5.0.1","resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz","integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==",- "dev": true,"license": "MIT",- "optional": true,"engines": {"node": ">=0.10.0"}@@ -13214,6 +13483,50 @@"node": ">=14.0.0"}},+ "node_modules/langsmith": {+ "version": "0.2.14",+ "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.2.14.tgz",+ "integrity": "sha512-ClAuAgSf3m9miMYotLEaZKQyKdaWlfjhebCuYco8bc6g72dU2VwTg31Bv4YINBq7EH2i1cMwbOiJxbOXPqjGig==",+ "license": "MIT",+ "dependencies": {+ "@types/uuid": "^10.0.0",+ "commander": "^10.0.1",+ "p-queue": "^6.6.2",+ "p-retry": "4",+ "semver": "^7.6.3",+ "uuid": "^10.0.0"+ },+ "peerDependencies": {+ "openai": "*"+ },+ "peerDependenciesMeta": {+ "openai": {+ "optional": true+ }+ }+ },+ "node_modules/langsmith/node_modules/commander": {+ "version": "10.0.1",+ "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz",+ "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==",+ "license": "MIT",+ "engines": {+ "node": ">=14"+ }+ },+ "node_modules/langsmith/node_modules/uuid": {+ "version": "10.0.0",+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",+ "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",+ "funding": [+ "https://github.com/sponsors/broofa",+ "https://github.com/sponsors/ctavan"+ ],+ "license": "MIT",+ "bin": {+ "uuid": "dist/bin/uuid"+ }+ },"node_modules/levn": {"version": "0.4.1","resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",@@ -14014,6 +14327,15 @@"imul": "^1.0.0"}},+ "node_modules/mustache": {+ "version": "4.2.0",+ "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz",+ "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==",+ "license": "MIT",+ "bin": {+ "mustache": "bin/mustache"+ }+ },"node_modules/mz": {"version": "2.7.0","resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",@@ -14115,6 +14437,25 @@"semver": "^7.3.5"}},+ "node_modules/node-domexception": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",+ "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",+ "funding": [+ {+ "type": "github",+ "url": "https://github.com/sponsors/jimmywarting"+ },+ {+ "type": "github",+ "url": "https://paypal.me/jimmywarting"+ }+ ],+ "license": "MIT",+ "engines": {+ "node": ">=10.5.0"+ }+ },"node_modules/node-fetch": {"version": "2.7.0","resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",@@ -14542,6 +14883,53 @@"url": "https://github.com/sponsors/sindresorhus"}},+ "node_modules/openai": {+ "version": "4.77.0",+ "resolved": "https://registry.npmjs.org/openai/-/openai-4.77.0.tgz",+ "integrity": "sha512-WWacavtns/7pCUkOWvQIjyOfcdr9X+9n9Vvb0zFeKVDAqwCMDHB+iSr24SVaBAhplvSG6JrRXFpcNM9gWhOGIw==",+ "license": "Apache-2.0",+ "dependencies": {+ "@types/node": "^18.11.18",+ "@types/node-fetch": "^2.6.4",+ "abort-controller": "^3.0.0",+ "agentkeepalive": "^4.2.1",+ "form-data-encoder": "1.7.2",+ "formdata-node": "^4.3.2",+ "node-fetch": "^2.6.7"+ },+ "bin": {+ "openai": "bin/cli"+ },+ "peerDependencies": {+ "zod": "^3.23.8"+ },+ "peerDependenciesMeta": {+ "zod": {+ "optional": true+ }+ }+ },+ "node_modules/openai/node_modules/@types/node": {+ "version": "18.19.68",+ "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.68.tgz",+ "integrity": "sha512-QGtpFH1vB99ZmTa63K4/FU8twThj4fuVSBkGddTp7uIL/cuoLWIUSL2RcOaigBhfR+hg5pgGkBnkoOxrTVBMKw==",+ "license": "MIT",+ "dependencies": {+ "undici-types": "~5.26.4"+ }+ },+ "node_modules/openai/node_modules/undici-types": {+ "version": "5.26.5",+ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",+ "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",+ "license": "MIT"+ },+ "node_modules/openapi-types": {+ "version": "12.1.3",+ "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz",+ "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==",+ "license": "MIT"+ },"node_modules/optionator": {"version": "0.9.4","resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",@@ -14642,7 +15030,6 @@"version": "1.0.0","resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz","integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==",- "dev": true,"license": "MIT","engines": {"node": ">=4"@@ -14706,6 +15093,62 @@"url": "https://github.com/sponsors/sindresorhus"}},+ "node_modules/p-queue": {+ "version": "6.6.2",+ "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz",+ "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==",+ "license": "MIT",+ "dependencies": {+ "eventemitter3": "^4.0.4",+ "p-timeout": "^3.2.0"+ },+ "engines": {+ "node": ">=8"+ },+ "funding": {+ "url": "https://github.com/sponsors/sindresorhus"+ }+ },+ "node_modules/p-queue/node_modules/eventemitter3": {+ "version": "4.0.7",+ "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",+ "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==",+ "license": "MIT"+ },+ "node_modules/p-retry": {+ "version": "4.6.2",+ "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz",+ "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==",+ "license": "MIT",+ "dependencies": {+ "@types/retry": "0.12.0",+ "retry": "^0.13.1"+ },+ "engines": {+ "node": ">=8"+ }+ },+ "node_modules/p-retry/node_modules/retry": {+ "version": "0.13.1",+ "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz",+ "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==",+ "license": "MIT",+ "engines": {+ "node": ">= 4"+ }+ },+ "node_modules/p-timeout": {+ "version": "3.2.0",+ "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz",+ "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==",+ "license": "MIT",+ "dependencies": {+ "p-finally": "^1.0.0"+ },+ "engines": {+ "node": ">=8"+ }+ },"node_modules/p-try": {"version": "2.2.0","resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",@@ -19368,7 +19811,6 @@"version": "6.20.0","resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz","integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==",- "devOptional": true,"license": "MIT"},"node_modules/unique-filename": {@@ -20205,6 +20647,15 @@"defaults": "^1.0.3"}},+ "node_modules/web-streams-polyfill": {+ "version": "4.0.0-beta.3",+ "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",+ "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",+ "license": "MIT",+ "engines": {+ "node": ">= 14"+ }+ },"node_modules/webidl-conversions": {"version": "3.0.1","resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",@@ -20808,6 +21259,15 @@"url": "https://github.com/sponsors/colinhacks"}},+ "node_modules/zod-to-json-schema": {+ "version": "3.24.1",+ "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.1.tgz",+ "integrity": "sha512-3h08nf3Vw3Wl3PK+q3ow/lIil81IT2Oa7YpQyUUDsEWbXveMesdfK1xBd2RhCkynwZndAxixji/7SYJJowr62w==",+ "license": "ISC",+ "peerDependencies": {+ "zod": "^3.24.1"+ }+ },"packages/core": {"name": "@colanode/core","version": "1.0.0",diff --git a/packages/core/src/lib/id.ts b/packages/core/src/lib/id.tsindex 63475012..b26630f9 100644--- a/packages/core/src/lib/id.ts+++ b/packages/core/src/lib/id.ts@@ -36,6 +36,7 @@ export enum IdType {Event = 'ev',Host = 'ht',Block = 'bl',+ AiEmbedding = 'ae',}export const generateId = (type: IdType): string => {