Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Implement node data cleanup job
Hakan Shehuad899c1c7c15
diff --git a/apps/desktop/src/main/services/collaboration-service.ts b/apps/desktop/src/main/services/collaboration-service.tsindex 90dfe3f5..454c427f 100644--- a/apps/desktop/src/main/services/collaboration-service.ts+++ b/apps/desktop/src/main/services/collaboration-service.ts@@ -72,6 +72,21 @@ class CollaborationService {.deleteFrom('node_transactions').where('node_id', '=', revocation.nodeId).execute();++ await tx+ .deleteFrom('collaborations')+ .where('node_id', '=', revocation.nodeId)+ .execute();++ await tx+ .deleteFrom('interaction_events')+ .where('node_id', '=', revocation.nodeId)+ .execute();++ await tx+ .deleteFrom('interactions')+ .where('node_id', '=', revocation.nodeId)+ .execute();});}}diff --git a/apps/desktop/src/main/services/node-service.ts b/apps/desktop/src/main/services/node-service.tsindex fc56deb1..63d18257 100644--- a/apps/desktop/src/main/services/node-service.ts+++ b/apps/desktop/src/main/services/node-service.ts@@ -456,6 +456,21 @@ class NodeService {.where('node_id', '=', nodeId).execute();+ await trx+ .deleteFrom('collaborations')+ .where('node_id', '=', nodeId)+ .execute();++ await trx+ .deleteFrom('interaction_events')+ .where('node_id', '=', nodeId)+ .execute();++ await trx+ .deleteFrom('interactions')+ .where('node_id', '=', nodeId)+ .execute();+const createdTransaction = await trx.insertInto('node_transactions').returningAll()@@ -879,6 +894,11 @@ class NodeService {.where('node_id', '=', transaction.nodeId).execute();+ await trx+ .deleteFrom('collaborations')+ .where('node_id', '=', transaction.nodeId)+ .execute();+const nodeRow = await trx.deleteFrom('nodes').returningAll()diff --git a/apps/server/src/data/redis.ts b/apps/server/src/data/redis.tsindex cdcf96dc..0fea673a 100644--- a/apps/server/src/data/redis.ts+++ b/apps/server/src/data/redis.ts@@ -29,7 +29,3 @@ export const initRedis = async () => {console.error('Redis client error:', err);});};--export const CHANNEL_NAMES = {- SYNAPSE: process.env.REDIS_SYNAPSE_CHANNEL_NAME || 'colanode_synapse',-};diff --git a/apps/server/src/data/schema.ts b/apps/server/src/data/schema.tsindex d12c9e8f..f0325502 100644--- a/apps/server/src/data/schema.ts+++ b/apps/server/src/data/schema.ts@@ -174,6 +174,10 @@ interface UploadTable {completed_at: ColumnType<Date, Date, never>;}+export type SelectUpload = Selectable<UploadTable>;+export type CreateUpload = Insertable<UploadTable>;+export type UpdateUpload = Updateable<UploadTable>;+interface InteractionTable {user_id: ColumnType<string, string, never>;node_id: ColumnType<string, string, never>;diff --git a/apps/server/src/jobs/clean-node-data.ts b/apps/server/src/jobs/clean-node-data.tsnew file mode 100644index 00000000..9b35a4ca--- /dev/null+++ b/apps/server/src/jobs/clean-node-data.ts@@ -0,0 +1,186 @@+import { generateId, IdType } from '@colanode/core';+import { DeleteObjectCommand } from '@aws-sdk/client-s3';++import { database } from '@/data/database';+import { CreateNodeTransaction, SelectUpload } from '@/data/schema';+import { JobHandler } from '@/types/jobs';+import { filesStorage, BUCKET_NAMES } from '@/data/storage';+import { eventBus } from '@/lib/event-bus';+import { createLogger } from '@/lib/logger';++const BATCH_SIZE = 100;++const logger = createLogger('clean-node-data');++export type CleanNodeDataInput = {+ type: 'clean_node_data';+ nodeId: string;+ workspaceId: string;+};++declare module '@/types/jobs' {+ interface JobMap {+ clean_node_data: {+ input: CleanNodeDataInput;+ };+ }+}++export const cleanNodeDataHandler: JobHandler<CleanNodeDataInput> = async (+ input+) => {+ logger.trace(`Cleaning node data for ${input.nodeId}`);++ const deleteTransactions = await database+ .selectFrom('node_transactions')+ .selectAll()+ .where('node_id', '=', input.nodeId)+ .execute();++ if (deleteTransactions.length !== 1) {+ logger.error(`Expected 1 delete transaction for ${input.nodeId}`);+ return;+ }++ const deleteTransaction = deleteTransactions[0];+ if (+ !deleteTransaction?.operation ||+ deleteTransaction.operation !== 'delete'+ ) {+ logger.error(`Expected delete transaction for ${input.nodeId}`);+ return;+ }++ const parentIds = [input.nodeId];+ while (parentIds.length > 0) {+ const tempParentIds = parentIds.splice(0, BATCH_SIZE);+ const deletedNodeIds = await deleteChildren(+ tempParentIds,+ input.workspaceId,+ deleteTransaction.created_by+ );++ parentIds.push(...deletedNodeIds);+ }+};++const deleteChildren = async (+ parentIds: string[],+ workspaceId: string,+ userId: string+) => {+ const deletedNodeIds: string[] = [];+ let hasMore = true;+ while (hasMore) {+ try {+ const descendants = await database+ .selectFrom('nodes')+ .selectAll()+ .where('parent_id', 'in', parentIds)+ .orderBy('id', 'asc')+ .limit(BATCH_SIZE)+ .execute();++ if (descendants.length === 0) {+ logger.trace(`No descendants found for ${parentIds}`);+ hasMore = false;+ break;+ }++ const fileIds: string[] = descendants+ .filter((d) => d.type === 'file')+ .map((d) => d.id);++ const uploads: SelectUpload[] =+ fileIds.length > 0+ ? await database+ .selectFrom('uploads')+ .selectAll()+ .where('node_id', 'in', fileIds)+ .execute()+ : [];++ const nodeIds: string[] = descendants.map((d) => d.id);+ const transactionsToCreate: CreateNodeTransaction[] = descendants.map(+ (descendant) => ({+ id: generateId(IdType.Transaction),+ node_id: descendant.id,+ node_type: descendant.type,+ workspace_id: workspaceId,+ operation: 'delete',+ created_at: new Date(),+ created_by: userId,+ server_created_at: new Date(),+ })+ );+ const uploadsToDelete: string[] = uploads.map((u) => u.node_id);++ await database.transaction().execute(async (trx) => {+ await trx+ .deleteFrom('node_transactions')+ .where('node_id', 'in', nodeIds)+ .execute();++ const createdTransactions = await trx+ .insertInto('node_transactions')+ .returningAll()+ .values(transactionsToCreate)+ .execute();++ if (createdTransactions.length !== transactionsToCreate.length) {+ throw new Error('Failed to create transactions');+ }++ if (uploadsToDelete.length > 0) {+ await trx+ .deleteFrom('uploads')+ .where('node_id', 'in', uploadsToDelete)+ .execute();+ }++ await trx.deleteFrom('nodes').where('id', 'in', nodeIds).execute();+ await trx+ .updateTable('collaborations')+ .set({+ roles: '{}',+ updated_at: new Date(),+ })+ .where('node_id', 'in', nodeIds)+ .execute();+ });++ for (const upload of uploads) {+ const command = new DeleteObjectCommand({+ Bucket: BUCKET_NAMES.FILES,+ Key: upload.path,+ });++ logger.trace(+ `Deleting file as a descendant of ${parentIds}: ${upload.path}`+ );++ await filesStorage.send(command);+ }++ for (const node of descendants) {+ logger.trace(`Publishing node deleted event for ${node.id}`);++ eventBus.publish({+ type: 'node_deleted',+ nodeId: node.id,+ nodeType: node.type,+ workspaceId: workspaceId,+ });++ deletedNodeIds.push(node.id);+ }++ hasMore = descendants.length === BATCH_SIZE;+ } catch (error) {+ logger.error(`Error cleaning node data for ${parentIds}: ${error}`);+ hasMore = false;+ }+ }++ return deletedNodeIds;+};diff --git a/apps/server/src/jobs/index.ts b/apps/server/src/jobs/index.tsindex 08b65c7a..6a2b03bf 100644--- a/apps/server/src/jobs/index.ts+++ b/apps/server/src/jobs/index.ts@@ -1,6 +1,7 @@+import { cleanNodeDataHandler } from '@/jobs/clean-node-data';import { cleanWorkspaceDataHandler } from '@/jobs/clean-workspace-data';import { sendEmailHandler } from '@/jobs/send-email';-import { JobHandler , JobMap } from '@/types/jobs';+import { JobHandler, JobMap } from '@/types/jobs';type JobHandlerMap = {[K in keyof JobMap]: JobHandler<JobMap[K]['input']>;@@ -9,4 +10,5 @@ type JobHandlerMap = {export const jobHandlerMap: JobHandlerMap = {send_email: sendEmailHandler,clean_workspace_data: cleanWorkspaceDataHandler,+ clean_node_data: cleanNodeDataHandler,};diff --git a/apps/server/src/services/job-service.ts b/apps/server/src/services/job-service.tsindex 422e23ed..5583a555 100644--- a/apps/server/src/services/job-service.ts+++ b/apps/server/src/services/job-service.ts@@ -1,9 +1,11 @@-import { Job, JobsOptions,Queue, Worker } from 'bullmq';+import { Job, JobsOptions, Queue, Worker } from 'bullmq';import { redisConfig } from '@/data/redis';import { jobHandlerMap } from '@/jobs';import { JobHandler, JobInput } from '@/types/jobs';+const JOBS_QUEUE_NAME = process.env.REDIS_JOBS_QUEUE_NAME ?? 'jobs';+class JobService {private jobQueue: Queue | undefined;private jobWorker: Worker | undefined;@@ -13,7 +15,7 @@ class JobService {return;}- this.jobQueue = new Queue('jobs', {+ this.jobQueue = new Queue(JOBS_QUEUE_NAME, {connection: {host: redisConfig.host,password: redisConfig.password,@@ -31,7 +33,7 @@ class JobService {return;}- this.jobWorker = new Worker('jobs', this.handleJobJob, {+ this.jobWorker = new Worker(JOBS_QUEUE_NAME, this.handleJobJob, {connection: {host: redisConfig.host,password: redisConfig.password,diff --git a/apps/server/src/services/node-service.ts b/apps/server/src/services/node-service.tsindex 6a832bf4..47ef50b8 100644--- a/apps/server/src/services/node-service.ts+++ b/apps/server/src/services/node-service.ts@@ -11,6 +11,7 @@ import { decodeState, YDoc } from '@colanode/crdt';import { sql, Transaction } from 'kysely';import { cloneDeep } from 'lodash-es';+import { jobService } from '@/services/job-service';import { database } from '@/data/database';import {CreateCollaboration,@@ -569,6 +570,15 @@ class NodeService {throw new Error('Failed to create transaction');}+ await trx+ .updateTable('collaborations')+ .set({+ roles: '{}',+ updated_at: new Date(),+ })+ .where('node_id', '=', input.nodeId)+ .execute();+return {deletedNode,createdTransaction,@@ -582,6 +592,12 @@ class NodeService {workspaceId: workspaceUser.workspace_id,});+ await jobService.addJob({+ type: 'clean_node_data',+ nodeId: input.nodeId,+ workspaceId: workspaceUser.workspace_id,+ });+return {node: deletedNode,transaction: createdTransaction,diff --git a/apps/server/src/services/synapse-service.ts b/apps/server/src/services/synapse-service.tsindex 8affcfba..032943d2 100644--- a/apps/server/src/services/synapse-service.ts+++ b/apps/server/src/services/synapse-service.ts@@ -514,8 +514,8 @@ class SynapseService {if (PUBLIC_NODES.includes(event.nodeType)) {usersToSend = userIds;} else {- const collaborations = await database- .selectFrom('collaborations')+ const revocations = await database+ .selectFrom('collaboration_revocations').selectAll().where((eb) =>eb.and([@@ -525,7 +525,7 @@ class SynapseService {).execute();- usersToSend = collaborations.map((c) => c.user_id);+ usersToSend = revocations.map((r) => r.user_id);}if (usersToSend.length === 0) {@@ -541,6 +541,7 @@ class SynapseService {}this.sendPendingTransactions(socketConnection, userId);+ this.sendPendingRevocations(socketConnection, userId);}}}