Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Add recurring jobs to check node and document embeddings

Hakan Shehud095dbbd16b2
diff --git a/apps/server/src/data/migrations/00018-create-node-embeddings-table.ts b/apps/server/src/data/migrations/00018-create-node-embeddings-table.ts
index e0bd82e1..ff137263 100644
--- a/apps/server/src/data/migrations/00018-create-node-embeddings-table.ts
+++ b/apps/server/src/data/migrations/00018-create-node-embeddings-table.ts
@@ -6,6 +6,7 @@ export const createNodeEmbeddingsTable: Migration = {
.createTable('node_embeddings')
.addColumn('node_id', 'varchar(30)', (col) => col.notNull())
.addColumn('chunk', 'integer', (col) => col.notNull())
+ .addColumn('revision', 'bigint', (col) => col.notNull())
.addColumn('workspace_id', 'varchar(30)', (col) => col.notNull())
.addColumn('text', 'text', (col) => col.notNull())
.addColumn('summary', 'text')
diff --git a/apps/server/src/data/migrations/00019-create-document-embeddings-table.ts b/apps/server/src/data/migrations/00019-create-document-embeddings-table.ts
index 771751db..a5921200 100644
--- a/apps/server/src/data/migrations/00019-create-document-embeddings-table.ts
+++ b/apps/server/src/data/migrations/00019-create-document-embeddings-table.ts
@@ -6,6 +6,7 @@ export const createDocumentEmbeddingsTable: Migration = {
.createTable('document_embeddings')
.addColumn('document_id', 'varchar(30)', (col) => col.notNull())
.addColumn('chunk', 'integer', (col) => col.notNull())
+ .addColumn('revision', 'bigint', (col) => col.notNull())
.addColumn('workspace_id', 'varchar(30)', (col) => col.notNull())
.addColumn('text', 'text', (col) => col.notNull())
.addColumn('summary', 'text')
diff --git a/apps/server/src/data/schema.ts b/apps/server/src/data/schema.ts
index c74f2723..f86735e7 100644
--- a/apps/server/src/data/schema.ts
+++ b/apps/server/src/data/schema.ts
@@ -261,6 +261,7 @@ export type UpdateUpload = Updateable<UploadTable>;
interface NodeEmbeddingTable {
node_id: ColumnType<string, string, never>;
chunk: ColumnType<number, number, number>;
+ revision: ColumnType<bigint, bigint, bigint>;
workspace_id: ColumnType<string, string, never>;
text: ColumnType<string, string, string>;
summary: ColumnType<string | null, string | null, string | null>;
@@ -277,6 +278,7 @@ export type UpdateNodeEmbedding = Updateable<NodeEmbeddingTable>;
interface DocumentEmbeddingTable {
document_id: ColumnType<string, string, never>;
chunk: ColumnType<number, number, number>;
+ revision: ColumnType<bigint, bigint, bigint>;
workspace_id: ColumnType<string, string, never>;
text: ColumnType<string, string, string>;
summary: ColumnType<string | null, string | null, string | null>;
diff --git a/apps/server/src/jobs/check-document-embeddings.ts b/apps/server/src/jobs/check-document-embeddings.ts
new file mode 100644
index 00000000..9f2fe7fa
--- /dev/null
+++ b/apps/server/src/jobs/check-document-embeddings.ts
@@ -0,0 +1,84 @@
+import { extractDocumentText } from '@colanode/core';
+
+import { database } from '@/data/database';
+import { configuration } from '@/lib/configuration';
+import {
+ fetchEmbeddingCursor,
+ scheduleDocumentEmbedding,
+ updateEmbeddingCursor,
+} from '@/lib/embeddings';
+
+const BATCH_SIZE = 100;
+
+export type CheckDocumentEmbeddingsInput = {
+ type: 'check_document_embeddings';
+};
+
+declare module '@/types/jobs' {
+ interface JobMap {
+ check_document_embeddings: {
+ input: CheckDocumentEmbeddingsInput;
+ };
+ }
+}
+
+export const checkDocumentEmbeddingsHandler = async () => {
+ if (!configuration.ai.enabled) {
+ return;
+ }
+
+ const cursor = await fetchEmbeddingCursor('document_embeddings');
+ let hasMore = true;
+ let lastRevision = cursor;
+
+ while (hasMore) {
+ const documents = await database
+ .selectFrom('documents')
+ .selectAll()
+ .where('revision', '>=', lastRevision)
+ .orderBy('revision', 'asc')
+ .limit(BATCH_SIZE)
+ .execute();
+
+ if (documents.length === 0) {
+ hasMore = false;
+ continue;
+ }
+
+ for (const document of documents) {
+ const text = extractDocumentText(document.id, document.content);
+ if (!text || text.trim() === '') {
+ await database
+ .deleteFrom('document_embeddings')
+ .where('document_id', '=', document.id)
+ .execute();
+
+ return;
+ }
+
+ const firstEmbedding = await database
+ .selectFrom('document_embeddings')
+ .select(['revision'])
+ .where('document_id', '=', document.id)
+ .orderBy('created_at', 'asc')
+ .executeTakeFirst();
+
+ const revision = firstEmbedding?.revision ?? 0n;
+ if (revision >= document.revision) {
+ continue;
+ }
+
+ await scheduleDocumentEmbedding(document.id);
+ }
+
+ if (documents.length > 0) {
+ lastRevision = documents[documents.length - 1]?.revision ?? 0n;
+ }
+
+ if (documents.length < BATCH_SIZE) {
+ hasMore = false;
+ }
+ }
+
+ await updateEmbeddingCursor('document_embeddings', lastRevision);
+};
diff --git a/apps/server/src/jobs/check-node-embeddings.ts b/apps/server/src/jobs/check-node-embeddings.ts
new file mode 100644
index 00000000..b9a84d19
--- /dev/null
+++ b/apps/server/src/jobs/check-node-embeddings.ts
@@ -0,0 +1,97 @@
+import { getNodeModel } from '@colanode/core';
+
+import { database } from '@/data/database';
+import { configuration } from '@/lib/configuration';
+import {
+ fetchEmbeddingCursor,
+ scheduleNodeEmbedding,
+ updateEmbeddingCursor,
+} from '@/lib/embeddings';
+
+const BATCH_SIZE = 100;
+
+export type CheckNodeEmbeddingsInput = {
+ type: 'check_node_embeddings';
+};
+
+declare module '@/types/jobs' {
+ interface JobMap {
+ check_node_embeddings: {
+ input: CheckNodeEmbeddingsInput;
+ };
+ }
+}
+
+export const checkNodeEmbeddingsHandler = async () => {
+ if (!configuration.ai.enabled) {
+ return;
+ }
+
+ const cursor = await fetchEmbeddingCursor('node_embeddings');
+ let hasMore = true;
+ let lastRevision = cursor;
+
+ while (hasMore) {
+ const nodes = await database
+ .selectFrom('nodes')
+ .selectAll()
+ .where('revision', '>=', lastRevision)
+ .orderBy('revision', 'asc')
+ .limit(BATCH_SIZE)
+ .execute();
+
+ if (nodes.length === 0) {
+ hasMore = false;
+ continue;
+ }
+
+ for (const node of nodes) {
+ const nodeModel = getNodeModel(node.attributes.type);
+ if (!nodeModel) {
+ continue;
+ }
+
+ const nodeText = nodeModel.extractNodeText(node.id, node.attributes);
+ if (!nodeText) {
+ continue;
+ }
+
+ if (nodeText === null) {
+ continue;
+ }
+
+ if (!nodeText.attributes || nodeText.attributes.trim() === '') {
+ await database
+ .deleteFrom('node_embeddings')
+ .where('node_id', '=', node.id)
+ .execute();
+
+ continue;
+ }
+
+ const firstEmbedding = await database
+ .selectFrom('node_embeddings')
+ .select(['revision'])
+ .where('node_id', '=', node.id)
+ .orderBy('created_at', 'asc')
+ .executeTakeFirst();
+
+ const revision = firstEmbedding?.revision ?? 0n;
+ if (revision >= node.revision) {
+ continue;
+ }
+
+ await scheduleNodeEmbedding(node);
+ }
+
+ if (nodes.length > 0) {
+ lastRevision = nodes[nodes.length - 1]?.revision ?? 0n;
+ }
+
+ if (nodes.length < BATCH_SIZE) {
+ hasMore = false;
+ }
+ }
+
+ await updateEmbeddingCursor('node_embeddings', lastRevision);
+};
diff --git a/apps/server/src/jobs/embed-document.ts b/apps/server/src/jobs/embed-document.ts
index 14f31f44..9da9477e 100644
--- a/apps/server/src/jobs/embed-document.ts
+++ b/apps/server/src/jobs/embed-document.ts
@@ -32,7 +32,7 @@ export const embedDocumentHandler = async (input: {
const { documentId } = input;
const document = await database
.selectFrom('documents')
- .select(['id', 'content', 'workspace_id', 'created_at'])
+ .select(['id', 'content', 'workspace_id', 'created_at', 'revision'])
.where('id', '=', documentId)
.executeTakeFirst();
@@ -68,10 +68,17 @@ export const embedDocumentHandler = async (input: {
const existingEmbeddings = await database
.selectFrom('document_embeddings')
- .select(['chunk', 'text', 'summary'])
+ .select(['chunk', 'revision', 'text', 'summary'])
.where('document_id', '=', documentId)
.execute();
+ const revision =
+ existingEmbeddings.length > 0 ? existingEmbeddings[0]!.revision : 0n;
+
+ if (revision >= document.revision) {
+ return;
+ }
+
const textChunks = await chunkText(
text,
existingEmbeddings.map((e) => ({
@@ -96,6 +103,7 @@ export const embedDocumentHandler = async (input: {
embeddingsToUpsert.push({
document_id: documentId,
chunk: i,
+ revision: document.revision,
workspace_id: document.workspace_id,
text: chunk.text,
summary: chunk.summary,
@@ -131,6 +139,7 @@ export const embedDocumentHandler = async (input: {
embeddingsToUpsert.map((embedding) => ({
document_id: embedding.document_id,
chunk: embedding.chunk,
+ revision: embedding.revision,
workspace_id: embedding.workspace_id,
text: embedding.text,
summary: embedding.summary,
diff --git a/apps/server/src/jobs/embed-node.ts b/apps/server/src/jobs/embed-node.ts
index 74665185..b5ee0558 100644
--- a/apps/server/src/jobs/embed-node.ts
+++ b/apps/server/src/jobs/embed-node.ts
@@ -67,12 +67,20 @@ export const embedNodeHandler = async (input: {
const existingEmbeddings = await database
.selectFrom('node_embeddings')
- .select(['chunk', 'text', 'summary'])
+ .select(['chunk', 'revision', 'text', 'summary'])
.where('node_id', '=', nodeId)
.execute();
+ const revision =
+ existingEmbeddings.length > 0 ? existingEmbeddings[0]!.revision : 0n;
+
+ if (revision >= node.revision) {
+ return;
+ }
+
const fullText =
`${nodeText.name ?? ''}\n\n${nodeText.attributes ?? ''}`.trim();
+
const textChunks = await chunkText(
fullText,
existingEmbeddings.map((e) => ({
@@ -97,6 +105,7 @@ export const embedNodeHandler = async (input: {
embeddingsToUpsert.push({
node_id: nodeId,
chunk: i,
+ revision: node.revision,
workspace_id: node.workspace_id,
text: chunk.text,
summary: chunk.summary,
@@ -132,6 +141,7 @@ export const embedNodeHandler = async (input: {
embeddingsToUpsert.map((embedding) => ({
node_id: embedding.node_id,
chunk: embedding.chunk,
+ revision: embedding.revision,
workspace_id: embedding.workspace_id,
text: embedding.text,
summary: embedding.summary,
diff --git a/apps/server/src/jobs/index.ts b/apps/server/src/jobs/index.ts
index 24829312..50bc6c03 100644
--- a/apps/server/src/jobs/index.ts
+++ b/apps/server/src/jobs/index.ts
@@ -2,9 +2,12 @@ import { cleanNodeDataHandler } from '@/jobs/clean-node-data';
import { cleanWorkspaceDataHandler } from '@/jobs/clean-workspace-data';
import { JobHandler, JobMap } from '@/types/jobs';
import { sendEmailVerifyEmailHandler } from '@/jobs/send-email-verify-email';
-import { embedNodeHandler } from './embed-node';
-import { embedDocumentHandler } from './embed-document';
-import { assistantResponseHandler } from './assistant-response';
+import { embedNodeHandler } from '@/jobs/embed-node';
+import { embedDocumentHandler } from '@/jobs/embed-document';
+import { assistantResponseHandler } from '@/jobs/assistant-response';
+import { checkNodeEmbeddingsHandler } from '@/jobs/check-node-embeddings';
+import { checkDocumentEmbeddingsHandler } from '@/jobs/check-document-embeddings';
+
type JobHandlerMap = {
[K in keyof JobMap]: JobHandler<JobMap[K]['input']>;
};
@@ -16,4 +19,6 @@ export const jobHandlerMap: JobHandlerMap = {
embed_node: embedNodeHandler,
embed_document: embedDocumentHandler,
assistant_response: assistantResponseHandler,
+ check_node_embeddings: checkNodeEmbeddingsHandler,
+ check_document_embeddings: checkDocumentEmbeddingsHandler,
};
diff --git a/apps/server/src/lib/ai-utils.ts b/apps/server/src/lib/ai-utils.ts
index 3e3e61d5..967b41e2 100644
--- a/apps/server/src/lib/ai-utils.ts
+++ b/apps/server/src/lib/ai-utils.ts
@@ -1,4 +1,5 @@
import { Document } from '@langchain/core/documents';
+
import { SearchResult } from '@/types/retrieval';
import { RerankedContextItem } from '@/types/assistant';
import { NodeMetadata, DocumentMetadata } from '@/types/metadata';
@@ -151,7 +152,7 @@ const processSearchResult = (
) => {
const key = createKey(result);
const recencyBoost = calculateRecencyBoost(result.createdAt);
- let normalizedScore = isKeyword
+ const normalizedScore = isKeyword
? (result.score / maxScore) * weight
: ((maxScore - result.score) / maxScore) * weight;
diff --git a/apps/server/src/lib/assistant.ts b/apps/server/src/lib/assistant.ts
index e715a4e3..fe6d1c97 100644
--- a/apps/server/src/lib/assistant.ts
+++ b/apps/server/src/lib/assistant.ts
@@ -1,8 +1,14 @@
import { StateGraph } from '@langchain/langgraph';
import { Document } from '@langchain/core/documents';
+import { CallbackHandler } from 'langfuse-langchain';
+import {
+ DatabaseAttributes,
+ getNodeModel,
+ RecordAttributes,
+} from '@colanode/core';
+
import { database } from '@/data/database';
import { configuration } from '@/lib/configuration';
-import { CallbackHandler } from 'langfuse-langchain';
import { fetchNode, fetchNodeDescendants } from '@/lib/nodes';
import {
rewriteQuery,
@@ -15,11 +21,6 @@ import {
import { nodeRetrievalService } from '@/services/node-retrieval-service';
import { documentRetrievalService } from '@/services/document-retrieval-service';
import { recordsRetrievalService } from '@/services/records-retrieval-service';
-import {
- DatabaseAttributes,
- getNodeModel,
- RecordAttributes,
-} from '@colanode/core';
import {
AssistantChainState,
ResponseState,
diff --git a/apps/server/src/lib/chunking.ts b/apps/server/src/lib/chunking.ts
index d3442670..04f3e23b 100644
--- a/apps/server/src/lib/chunking.ts
+++ b/apps/server/src/lib/chunking.ts
@@ -1,6 +1,7 @@
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
-import { configuration } from '@/lib/configuration';
import type { NodeType } from '@colanode/core';
+
+import { configuration } from '@/lib/configuration';
import { enrichChunk } from '@/services/llm-service';
import { TextChunk } from '@/types/chunking';
@@ -21,24 +22,25 @@ export const chunkText = async (
.map((doc) => ({ text: doc.pageContent }))
.filter((c) => c.text.trim().length > 5);
- if (configuration.ai.chunking.enhanceWithContext) {
- const enrichedChunks: TextChunk[] = [];
-
- for (const chunk of chunks) {
- const existingChunk = existingChunks.find((ec) => ec.text === chunk.text);
- if (existingChunk?.summary) {
- enrichedChunks.push({
- text: chunk.text,
- summary: existingChunk.summary,
- });
- continue;
- }
-
- const summary = await enrichChunk(chunk.text, text, nodeType);
- enrichedChunks.push({ text: chunk.text, summary });
+ if (!configuration.ai.chunking.enhanceWithContext) {
+ return chunks;
+ }
+
+ const enrichedChunks: TextChunk[] = [];
+ for (const chunk of chunks) {
+ const existingChunk = existingChunks.find((ec) => ec.text === chunk.text);
+ if (existingChunk?.summary) {
+ enrichedChunks.push({
+ text: chunk.text,
+ summary: existingChunk.summary,
+ });
+
+ continue;
}
- return enrichedChunks;
+
+ const summary = await enrichChunk(chunk.text, text, nodeType);
+ enrichedChunks.push({ text: chunk.text, summary });
}
- return chunks;
+ return enrichedChunks;
};
diff --git a/apps/server/src/lib/documents.ts b/apps/server/src/lib/documents.ts
index d3234684..fbeac13c 100644
--- a/apps/server/src/lib/documents.ts
+++ b/apps/server/src/lib/documents.ts
@@ -15,8 +15,7 @@ import { ConcurrentUpdateResult, UpdateDocumentOutput } from '@/types/nodes';
import { eventBus } from '@/lib/event-bus';
import { fetchNode, fetchNodeTree, mapNode } from '@/lib/nodes';
import { CreateDocumentInput, CreateDocumentOutput } from '@/types/documents';
-import { jobService } from '@/services/job-service';
-import { configuration } from '@/lib/configuration';
+import { scheduleDocumentEmbedding } from '@/lib/embeddings';
const debug = createDebugger('server:lib:documents');
@@ -299,16 +298,3 @@ const tryUpdateDocumentFromMutation = async (
return { type: 'retry', output: null };
}
};
-
-const scheduleDocumentEmbedding = async (documentId: string) => {
- await jobService.addJob(
- {
- type: 'embed_document',
- documentId,
- },
- {
- jobId: `embed_document:${documentId}`,
- delay: configuration.ai.documentEmbeddingDelay,
- }
- );
-};
diff --git a/apps/server/src/lib/embeddings.ts b/apps/server/src/lib/embeddings.ts
new file mode 100644
index 00000000..0b950a1f
--- /dev/null
+++ b/apps/server/src/lib/embeddings.ts
@@ -0,0 +1,67 @@
+import { MessageAttributes } from '@colanode/core';
+
+import { redis } from '@/data/redis';
+import { SelectNode } from '@/data/schema';
+import { configuration } from '@/lib/configuration';
+import { jobService } from '@/services/job-service';
+
+export const fetchEmbeddingCursor = async (
+ cursorId: string
+): Promise<bigint> => {
+ const cursorStringValue = await redis.get(`embedding_cursor:${cursorId}`);
+ if (!cursorStringValue) {
+ return 0n;
+ }
+
+ return BigInt(cursorStringValue);
+};
+
+export const updateEmbeddingCursor = async (
+ cursorId: string,
+ value: bigint
+) => {
+ await redis.set(`embedding_cursor:${cursorId}`, value.toString());
+};
+
+export const deleteEmbeddingCursor = async (cursorId: string) => {
+ await redis.del(`embedding_cursor:${cursorId}`);
+};
+
+export const scheduleNodeEmbedding = async (node: SelectNode) => {
+ if (node.type === 'message') {
+ const attributes = node.attributes as MessageAttributes;
+ if (attributes.subtype === 'question' || attributes.subtype === 'answer') {
+ return;
+ }
+ }
+
+ const jobOptions: { jobId: string; delay?: number } = {
+ jobId: `embed_node:${node.id}`,
+ };
+
+ // Only add delay for non-message nodes
+ if (node.type !== 'message') {
+ jobOptions.delay = configuration.ai.nodeEmbeddingDelay;
+ }
+
+ await jobService.addJob(
+ {
+ type: 'embed_node',
+ nodeId: node.id,
+ },
+ jobOptions
+ );
+};
+
+export const scheduleDocumentEmbedding = async (documentId: string) => {
+ await jobService.addJob(
+ {
+ type: 'embed_document',
+ documentId,
+ },
+ {
+ jobId: `embed_document:${documentId}`,
+ delay: configuration.ai.documentEmbeddingDelay,
+ }
+ );
+};
diff --git a/apps/server/src/lib/metadata.ts b/apps/server/src/lib/metadata.ts
index 0a0f6479..ad4eef70 100644
--- a/apps/server/src/lib/metadata.ts
+++ b/apps/server/src/lib/metadata.ts
@@ -2,7 +2,9 @@ import {
getNodeModel,
FieldAttributes,
RecordAttributes,
+ DatabaseAttributes,
} from '@colanode/core';
+
import { database } from '@/data/database';
import {
NodeMetadata,
@@ -14,7 +16,6 @@ import {
DatabaseInfo,
DatabaseFieldInfo,
} from '@/types/metadata';
-import { DatabaseAttributes } from '@colanode/core';
const fetchBaseMetadata = async (
id: string,
diff --git a/apps/server/src/lib/nodes.ts b/apps/server/src/lib/nodes.ts
index 0303f077..a3b5ecbe 100644
--- a/apps/server/src/lib/nodes.ts
+++ b/apps/server/src/lib/nodes.ts
@@ -39,7 +39,7 @@ import {
} from '@/lib/collaborations';
import { jobService } from '@/services/job-service';
import { deleteFile } from '@/lib/files';
-import { configuration } from '@/lib/configuration';
+import { scheduleNodeEmbedding } from '@/lib/embeddings';
const debug = createDebugger('server:lib:nodes');
@@ -803,29 +803,3 @@ export const deleteNode = async (
node: deletedNode,
};
};
-
-const scheduleNodeEmbedding = async (node: SelectNode) => {
- if (node.type === 'message') {
- const attributes = node.attributes as MessageAttributes;
- if (attributes.subtype === 'question' || attributes.subtype === 'answer') {
- return;
- }
- }
-
- const jobOptions: { jobId: string; delay?: number } = {
- jobId: `embed_node:${node.id}`,
- };
-
- // Only add delay for non-message nodes
- if (node.type !== 'message') {
- jobOptions.delay = configuration.ai.nodeEmbeddingDelay;
- }
-
- await jobService.addJob(
- {
- type: 'embed_node',
- nodeId: node.id,
- },
- jobOptions
- );
-};
diff --git a/apps/server/src/services/job-service.ts b/apps/server/src/services/job-service.ts
index 0b5324d9..f52e9787 100644
--- a/apps/server/src/services/job-service.ts
+++ b/apps/server/src/services/job-service.ts
@@ -39,6 +39,34 @@ class JobService {
this.jobQueue.on('error', (error) => {
debug(`Job queue error: ${error}`);
});
+
+ this.jobQueue.upsertJobScheduler(
+ 'check_node_embeddings',
+ { pattern: '0 */30 * * * *' },
+ {
+ name: 'check_node_embeddings',
+ data: { type: 'check_node_embeddings' } as JobInput,
+ opts: {
+ backoff: 3,
+ attempts: 5,
+ removeOnFail: 1000,
+ },
+ }
+ );
+
+ this.jobQueue.upsertJobScheduler(
+ 'check_document_embeddings',
+ { pattern: '0 */30 * * * *' },
+ {
+ name: 'check_document_embeddings',
+ data: { type: 'check_document_embeddings' } as JobInput,
+ opts: {
+ backoff: 3,
+ attempts: 5,
+ removeOnFail: 1000,
+ },
+ }
+ );
}
public async initWorker() {
@@ -67,6 +95,8 @@ class JobService {
const input = job.data as JobInput;
const handler = jobHandlerMap[input.type] as JobHandler<typeof input>;
await handler(input);
+
+ debug(`Job ${job.id} with type ${input.type} completed.`);
};
}
diff --git a/apps/server/src/services/llm-service.ts b/apps/server/src/services/llm-service.ts
index e739db76..c273aa80 100644
--- a/apps/server/src/services/llm-service.ts
+++ b/apps/server/src/services/llm-service.ts
@@ -2,8 +2,7 @@ import { ChatOpenAI } from '@langchain/openai';
import { ChatGoogleGenerativeAI } from '@langchain/google-genai';
import { StringOutputParser } from '@langchain/core/output_parsers';
import { Document } from '@langchain/core/documents';
-import { CallbackHandler } from 'langfuse-langchain';
-import { SystemMessage } from '@langchain/core/messages';
+import { NodeType, RecordNode } from '@colanode/core';
import { configuration } from '@/lib/configuration';
import {
@@ -26,15 +25,6 @@ import {
databaseFilterPrompt,
chunkSummarizationPrompt,
} from '@/lib/llm-prompts';
-import { NodeType } from '@colanode/core';
-
-const langfuseCallback = configuration.ai.langfuse.enabled
- ? new CallbackHandler({
- publicKey: configuration.ai.langfuse.publicKey,
- secretKey: configuration.ai.langfuse.secretKey,
- baseUrl: configuration.ai.langfuse.baseUrl,
- })
- : undefined;
const getChatModel = (
task: keyof typeof configuration.ai.models
@@ -158,7 +148,7 @@ export const generateDatabaseFilters = async (args: {
id: string;
name: string;
fields: Record<string, { type: string; name: string }>;
- sampleRecords: any[];
+ sampleRecords: RecordNode[];
}>;
}): Promise<DatabaseFilterResult> => {
const task = 'databaseFilter';
diff --git a/packages/core/src/registry/nodes/record.ts b/packages/core/src/registry/nodes/record.ts
index 55514c00..3903810c 100644
--- a/packages/core/src/registry/nodes/record.ts
+++ b/packages/core/src/registry/nodes/record.ts
@@ -90,9 +90,18 @@ export const recordModel: NodeModel = {
throw new Error('Invalid node type');
}
+ const texts: string[] = [];
+ for (const field of Object.values(attributes.fields)) {
+ if (field.type === 'text') {
+ texts.push(field.value);
+ } else if (field.type === 'string') {
+ texts.push(field.value);
+ }
+ }
+
return {
name: attributes.name,
- attributes: null,
+ attributes: texts.join('\n'),
};
},
};