Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Refactor node embedding scheduling and clean-up with configuration updates

Ylber Gashi9ea780813f8f
diff --git a/apps/server/src/jobs/clean-node-data.ts b/apps/server/src/jobs/clean-node-data.ts
index 2138bdf6..759d2c29 100644
--- a/apps/server/src/jobs/clean-node-data.ts
+++ b/apps/server/src/jobs/clean-node-data.ts
@@ -79,6 +79,16 @@ const deleteChildren = async (parentIds: string[], userId: string) => {
.where('node_id', 'in', nodeIds)
.execute();
+ await trx
+ .deleteFrom('node_embeddings')
+ .where('node_id', 'in', nodeIds)
+ .execute();
+
+ await trx
+ .deleteFrom('document_embeddings')
+ .where('document_id', 'in', nodeIds)
+ .execute();
+
await trx
.updateTable('collaborations')
.set({
diff --git a/apps/server/src/lib/configuration.ts b/apps/server/src/lib/configuration.ts
index f5b35ae8..a814c9b7 100644
--- a/apps/server/src/lib/configuration.ts
+++ b/apps/server/src/lib/configuration.ts
@@ -84,7 +84,7 @@ export interface AIModelConfiguration {
export interface AiConfiguration {
enabled: boolean;
- entryEmbedDelay: number;
+ nodeEmbedDelay: number;
providers: {
openai: AIProviderConfiguration;
google: AIProviderConfiguration;
@@ -207,9 +207,7 @@ export const configuration: Configuration = {
},
ai: {
enabled: getOptionalEnv('AI_ENABLED') === 'true',
- entryEmbedDelay: parseInt(
- getOptionalEnv('AI_ENTRY_EMBED_DELAY') || '60000'
- ),
+ nodeEmbedDelay: parseInt(getOptionalEnv('AI_NODE_EMBED_DELAY') || '10000'),
providers: {
openai: {
apiKey: getOptionalEnv('OPENAI_API_KEY') || '',
diff --git a/apps/server/src/lib/nodes.ts b/apps/server/src/lib/nodes.ts
index bcae67f5..c3c771c7 100644
--- a/apps/server/src/lib/nodes.ts
+++ b/apps/server/src/lib/nodes.ts
@@ -170,17 +170,7 @@ export const createNode = async (
});
}
- // Schedule node embedding
- await jobService.addJob(
- {
- type: 'embed_node',
- nodeId: input.nodeId,
- },
- {
- jobId: `embed_node:${input.nodeId}`,
- delay: configuration.ai.entryEmbedDelay,
- }
- );
+ await scheduleNodeEmbedding(input.nodeId);
return {
node: createdNode,
@@ -303,17 +293,7 @@ export const tryUpdateNode = async (
});
}
- // Schedule node embedding
- await jobService.addJob(
- {
- type: 'embed_node',
- nodeId: input.nodeId,
- },
- {
- jobId: `embed_node:${input.nodeId}`,
- delay: configuration.ai.entryEmbedDelay,
- }
- );
+ await scheduleNodeEmbedding(input.nodeId);
return {
type: 'success',
@@ -421,17 +401,7 @@ export const createNodeFromMutation = async (
});
}
- // Schedule node embedding
- await jobService.addJob(
- {
- type: 'embed_node',
- nodeId: mutation.id,
- },
- {
- jobId: `embed_node:${mutation.id}`,
- delay: configuration.ai.entryEmbedDelay,
- }
- );
+ await scheduleNodeEmbedding(mutation.id);
return {
node: createdNode,
@@ -564,17 +534,7 @@ const tryUpdateNodeFromMutation = async (
});
}
- // Schedule node embedding
- await jobService.addJob(
- {
- type: 'embed_node',
- nodeId: mutation.id,
- },
- {
- jobId: `embed_node:${mutation.id}`,
- delay: configuration.ai.entryEmbedDelay,
- }
- );
+ await scheduleNodeEmbedding(mutation.id);
return {
type: 'success',
@@ -699,3 +659,16 @@ export const deleteNode = async (
node: deletedNode,
};
};
+
+async function scheduleNodeEmbedding(nodeId: string) {
+ await jobService.addJob(
+ {
+ type: 'embed_node',
+ nodeId,
+ },
+ {
+ jobId: `embed_node:${nodeId}`,
+ delay: configuration.ai.nodeEmbedDelay,
+ }
+ );
+}