Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Refactor and improve node cleanups on delete (#24)
Hakan Shehu4998a9d4b2d3
diff --git a/apps/desktop/src/main/lib/utils.ts b/apps/desktop/src/main/lib/utils.tsindex de4ed1de..6ce51dc4 100644--- a/apps/desktop/src/main/lib/utils.ts+++ b/apps/desktop/src/main/lib/utils.ts@@ -176,3 +176,58 @@ export const fetchUserStorageUsed = async (return BigInt(storageUsedRow?.storage_used ?? 0);};++export const deleteNodeRelations = async (+ database:+ | Kysely<WorkspaceDatabaseSchema>+ | Transaction<WorkspaceDatabaseSchema>,+ nodeId: string+) => {+ await database.deleteFrom('node_states').where('id', '=', nodeId).execute();++ await database+ .deleteFrom('node_updates')+ .where('node_id', '=', nodeId)+ .execute();++ await database+ .deleteFrom('node_interactions')+ .where('node_id', '=', nodeId)+ .execute();++ await database+ .deleteFrom('node_reactions')+ .where('node_id', '=', nodeId)+ .execute();++ await database.deleteFrom('node_texts').where('id', '=', nodeId).execute();++ await database+ .deleteFrom('documents')+ .where('id', '=', nodeId)+ .executeTakeFirst();++ await database+ .deleteFrom('document_states')+ .where('id', '=', nodeId)+ .execute();++ await database+ .deleteFrom('document_updates')+ .where('document_id', '=', nodeId)+ .execute();++ await database.deleteFrom('tombstones').where('id', '=', nodeId).execute();++ await database+ .deleteFrom('node_references')+ .where('node_id', '=', nodeId)+ .execute();++ await database+ .deleteFrom('node_counters')+ .where('node_id', '=', nodeId)+ .execute();++ await database.deleteFrom('file_states').where('id', '=', nodeId).execute();+};diff --git a/apps/desktop/src/main/queries/documents/document-get.ts b/apps/desktop/src/main/queries/documents/document-get.tsindex ef805883..f81b42ec 100644--- a/apps/desktop/src/main/queries/documents/document-get.ts+++ b/apps/desktop/src/main/queries/documents/document-get.ts@@ -66,6 +66,19 @@ export class DocumentGetQueryHandler};}+ if (+ event.type === 'node_created' &&+ event.accountId === input.accountId &&+ event.workspaceId === input.workspaceId &&+ event.node.id === input.documentId+ ) {+ const newOutput = await this.handleQuery(input);+ return {+ hasChanges: true,+ result: newOutput,+ };+ }+return {hasChanges: false,};diff --git a/apps/desktop/src/main/queries/documents/document-state-get.ts b/apps/desktop/src/main/queries/documents/document-state-get.tsindex faccfc6c..e9701889 100644--- a/apps/desktop/src/main/queries/documents/document-state-get.ts+++ b/apps/desktop/src/main/queries/documents/document-state-get.ts@@ -66,6 +66,19 @@ export class DocumentStateGetQueryHandler};}+ if (+ event.type === 'node_created' &&+ event.accountId === input.accountId &&+ event.workspaceId === input.workspaceId &&+ event.node.id === input.documentId+ ) {+ const newOutput = await this.handleQuery(input);+ return {+ hasChanges: true,+ result: newOutput,+ };+ }+return {hasChanges: false,};diff --git a/apps/desktop/src/main/queries/documents/document-update-list.ts b/apps/desktop/src/main/queries/documents/document-update-list.tsindex eff05155..fee2ba6b 100644--- a/apps/desktop/src/main/queries/documents/document-update-list.ts+++ b/apps/desktop/src/main/queries/documents/document-update-list.ts@@ -81,6 +81,19 @@ export class DocumentUpdatesListQueryHandler};}+ if (+ event.type === 'node_created' &&+ event.accountId === input.accountId &&+ event.workspaceId === input.workspaceId &&+ event.node.id === input.documentId+ ) {+ const newOutput = await this.handleQuery(input);+ return {+ hasChanges: true,+ result: newOutput,+ };+ }+return {hasChanges: false,};diff --git a/apps/desktop/src/main/queries/files/file-state-get.ts b/apps/desktop/src/main/queries/files/file-state-get.tsindex f51ded07..c64e2de1 100644--- a/apps/desktop/src/main/queries/files/file-state-get.ts+++ b/apps/desktop/src/main/queries/files/file-state-get.ts@@ -56,6 +56,19 @@ export class FileStateGetQueryHandler};}+ if (+ event.type === 'node_created' &&+ event.accountId === input.accountId &&+ event.workspaceId === input.workspaceId &&+ event.node.id === input.id+ ) {+ const newOutput = await this.handleQuery(input);+ return {+ hasChanges: true,+ result: newOutput,+ };+ }+return {hasChanges: false,};diff --git a/apps/desktop/src/main/queries/nodes/node-reaction-list.ts b/apps/desktop/src/main/queries/nodes/node-reaction-list.tsindex 9e68094d..70738eb7 100644--- a/apps/desktop/src/main/queries/nodes/node-reaction-list.ts+++ b/apps/desktop/src/main/queries/nodes/node-reaction-list.ts@@ -58,6 +58,32 @@ export class NodeReactionsListQueryHandler};}+ if (+ event.type === 'node_created' &&+ event.accountId === input.accountId &&+ event.workspaceId === input.workspaceId &&+ event.node.id === input.nodeId+ ) {+ const newResult = await this.handleQuery(input);++ return {+ hasChanges: true,+ result: newResult,+ };+ }++ if (+ event.type === 'node_deleted' &&+ event.accountId === input.accountId &&+ event.workspaceId === input.workspaceId &&+ event.node.id === input.nodeId+ ) {+ return {+ hasChanges: true,+ result: [],+ };+ }+return {hasChanges: false,};diff --git a/apps/desktop/src/main/queries/nodes/node-reactions-aggregate.ts b/apps/desktop/src/main/queries/nodes/node-reactions-aggregate.tsindex 95708e1d..6de36e42 100644--- a/apps/desktop/src/main/queries/nodes/node-reactions-aggregate.ts+++ b/apps/desktop/src/main/queries/nodes/node-reactions-aggregate.ts@@ -66,6 +66,32 @@ export class NodeReactionsAggregateQueryHandler};}+ if (+ event.type === 'node_created' &&+ event.accountId === input.accountId &&+ event.workspaceId === input.workspaceId &&+ event.node.id === input.nodeId+ ) {+ const newResult = await this.handleQuery(input);++ return {+ hasChanges: true,+ result: newResult,+ };+ }++ if (+ event.type === 'node_deleted' &&+ event.accountId === input.accountId &&+ event.workspaceId === input.workspaceId &&+ event.node.id === input.nodeId+ ) {+ return {+ hasChanges: true,+ result: [],+ };+ }+return {hasChanges: false,};diff --git a/apps/desktop/src/main/services/workspaces/file-service.ts b/apps/desktop/src/main/services/workspaces/file-service.tsindex 49594dd4..186534ef 100644--- a/apps/desktop/src/main/services/workspaces/file-service.ts+++ b/apps/desktop/src/main/services/workspaces/file-service.ts@@ -201,7 +201,7 @@ export class FileService {this.triggerUploads();}- public async deleteFile(node: SelectNode): Promise<void> {+ public deleteFile(node: SelectNode): void {const file = mapNode(node);if (file.type !== 'file') {diff --git a/apps/desktop/src/main/services/workspaces/node-service.ts b/apps/desktop/src/main/services/workspaces/node-service.tsindex 33fe42cc..ecde5eeb 100644--- a/apps/desktop/src/main/services/workspaces/node-service.ts+++ b/apps/desktop/src/main/services/workspaces/node-service.ts@@ -15,7 +15,7 @@ import {} from '@colanode/core';import { decodeState, encodeState, YDoc } from '@colanode/crdt';-import { fetchNodeTree } from '@/main/lib/utils';+import { deleteNodeRelations, fetchNodeTree } from '@/main/lib/utils';import { mapNode, mapNodeReference } from '@/main/lib/mappers';import { eventBus } from '@/shared/lib/event-bus';import { WorkspaceService } from '@/main/services/workspaces/workspace-service';@@ -795,7 +795,7 @@ export class NodeService {`Applying server delete transaction ${tombstone.id} for node ${tombstone.id}`);- const { deletedNode } = await this.workspace.database+ const { deletedNode, deletedCollaborations } = await this.workspace.database.transaction().execute(async (trx) => {const deletedNode = await trx@@ -804,52 +804,15 @@ export class NodeService {.where('id', '=', tombstone.id).executeTakeFirst();- await trx- .deleteFrom('node_updates')- .where('node_id', '=', tombstone.id)- .execute();-- await trx- .deleteFrom('node_states')- .where('id', '=', tombstone.id)- .execute();-- await trx- .deleteFrom('node_reactions')- .where('node_id', '=', tombstone.id)- .execute();-- await trx- .deleteFrom('node_interactions')- .where('node_id', '=', tombstone.id)- .execute();-- await trx- .deleteFrom('node_references')+ const deletedCollaborations = await trx+ .deleteFrom('collaborations').where('node_id', '=', tombstone.id)+ .returningAll().execute();- await trx- .deleteFrom('tombstones')- .where('id', '=', tombstone.id)- .execute();-- await trx- .deleteFrom('documents')- .where('id', '=', tombstone.id)- .executeTakeFirst();-- await trx- .deleteFrom('document_updates')- .where('document_id', '=', tombstone.id)- .execute();-- await trx- .deleteFrom('node_texts')- .where('id', '=', tombstone.id)- .execute();+ await deleteNodeRelations(trx, tombstone.id);- return { deletedNode };+ return { deletedNode, deletedCollaborations };});if (!deletedNode) {@@ -858,74 +821,71 @@ export class NodeService {await this.workspace.nodeCounters.checkCountersForDeletedNode(deletedNode);+ if (deletedNode.type === 'file') {+ this.workspace.files.deleteFile(deletedNode);+ }+eventBus.publish({type: 'node_deleted',accountId: this.workspace.accountId,workspaceId: this.workspace.id,node: mapNode(deletedNode),});++ for (const deletedCollaboration of deletedCollaborations) {+ eventBus.publish({+ type: 'collaboration_deleted',+ accountId: this.workspace.accountId,+ workspaceId: this.workspace.id,+ nodeId: deletedCollaboration.node_id,+ });+ }}public async revertNodeCreate(mutation: CreateNodeMutationData) {- const node = await this.workspace.database- .selectFrom('nodes')- .selectAll()- .where('id', '=', mutation.nodeId)- .executeTakeFirst();+ const { deletedNode, deletedCollaborations } = await this.workspace.database+ .transaction()+ .execute(async (tx) => {+ const deletedNode = await tx+ .deleteFrom('nodes')+ .where('id', '=', mutation.nodeId)+ .returningAll()+ .executeTakeFirst();- if (!node) {+ const deletedCollaborations = await tx+ .deleteFrom('collaborations')+ .where('node_id', '=', mutation.nodeId)+ .returningAll()+ .execute();++ await deleteNodeRelations(tx, mutation.nodeId);++ return { deletedNode, deletedCollaborations };+ });++ if (!deletedNode) {return;}- await this.workspace.database.transaction().execute(async (tx) => {- await tx.deleteFrom('nodes').where('id', '=', mutation.nodeId).execute();-- await tx- .deleteFrom('node_updates')- .where('node_id', '=', mutation.nodeId)- .execute();-- await tx- .deleteFrom('node_interactions')- .where('node_id', '=', mutation.nodeId)- .execute();-- await tx- .deleteFrom('node_reactions')- .where('node_id', '=', mutation.nodeId)- .execute();-- await tx- .deleteFrom('node_states')- .where('id', '=', mutation.nodeId)- .execute();-- await tx- .deleteFrom('node_references')- .where('node_id', '=', mutation.nodeId)- .execute();-- await tx- .deleteFrom('documents')- .where('id', '=', mutation.nodeId)- .execute();-- await tx- .deleteFrom('document_updates')- .where('document_id', '=', mutation.nodeId)- .execute();- });-- if (node.type === 'file') {- await this.workspace.files.deleteFile(node);+ if (deletedNode.type === 'file') {+ this.workspace.files.deleteFile(deletedNode);}eventBus.publish({type: 'node_deleted',accountId: this.workspace.accountId,workspaceId: this.workspace.id,- node: mapNode(node),+ node: mapNode(deletedNode),});++ for (const deletedCollaboration of deletedCollaborations) {+ eventBus.publish({+ type: 'collaboration_deleted',+ accountId: this.workspace.accountId,+ workspaceId: this.workspace.id,+ nodeId: deletedCollaboration.node_id,+ });+ }}public async revertNodeUpdate(mutation: UpdateNodeMutationData) {@@ -948,42 +908,7 @@ export class NodeService {.executeTakeFirst();if (!node) {- // Make sure we don't have any data left behind- await this.workspace.database- .deleteFrom('node_updates')- .where('id', '=', mutation.updateId)- .execute();-- await this.workspace.database- .deleteFrom('node_interactions')- .where('node_id', '=', mutation.nodeId)- .execute();-- await this.workspace.database- .deleteFrom('node_reactions')- .where('node_id', '=', mutation.nodeId)- .execute();-- await this.workspace.database- .deleteFrom('node_states')- .where('id', '=', mutation.nodeId)- .execute();-- await this.workspace.database- .deleteFrom('node_references')- .where('node_id', '=', mutation.nodeId)- .execute();-- await this.workspace.database- .deleteFrom('documents')- .where('id', '=', mutation.nodeId)- .execute();-- await this.workspace.database- .deleteFrom('document_updates')- .where('document_id', '=', mutation.nodeId)- .execute();-+ await deleteNodeRelations(this.workspace.database, mutation.nodeId);return true;}diff --git a/apps/server/src/jobs/clean-node-data.ts b/apps/server/src/jobs/clean-node-data.tsindex f5e60511..eba58474 100644--- a/apps/server/src/jobs/clean-node-data.ts+++ b/apps/server/src/jobs/clean-node-data.ts@@ -7,7 +7,6 @@ import { CreateNodeTombstone } from '@/data/schema';import { deleteFile } from '@/lib/files';const BATCH_SIZE = 100;-const debug = createDebugger('server:job:clean-node-data');export type CleanNodeDataInput = {@@ -29,116 +28,142 @@ export const cleanNodeDataHandler: JobHandler<CleanNodeDataInput> = async (input) => {debug(`Cleaning node data for ${input.nodeId}`);- const parentIds = [input.nodeId];- while (parentIds.length > 0) {- const tempParentIds = parentIds.splice(0, BATCH_SIZE);- const deletedNodeIds = await deleteChildren(tempParentIds, input.userId);- parentIds.push(...deletedNodeIds);++ await cleanNodeRelations([input.nodeId]);+ await cleanNodeFiles([input.nodeId]);++ let hasMore = true;+ while (hasMore) {+ const children = await database+ .selectFrom('nodes')+ .select(['id', 'root_id', 'workspace_id'])+ .where('parent_id', '=', input.nodeId)+ .limit(BATCH_SIZE)+ .execute();++ if (children.length === 0) {+ hasMore = false;+ break;+ }++ for (const child of children) {+ await cleanDescendants(child.id, input.userId);+ }}};-const deleteChildren = async (parentIds: string[], userId: string) => {- const deletedNodeIds: string[] = [];+const cleanDescendants = async (nodeId: string, userId: string) => {let hasMore = true;while (hasMore) {- try {- const descendants = await database- .selectFrom('nodes')- .selectAll()- .where('parent_id', 'in', parentIds)- .orderBy('id', 'asc')- .limit(BATCH_SIZE)+ const descendants = await database+ .selectFrom('node_paths')+ .select('descendant_id')+ .where('ancestor_id', '=', nodeId)+ .orderBy('level', 'desc')+ .limit(BATCH_SIZE)+ .execute();++ if (descendants.length === 0) {+ hasMore = false;+ break;+ }++ const nodeIds = descendants.map((d) => d.descendant_id);+ const nodes = await database+ .selectFrom('nodes')+ .select(['id', 'root_id', 'workspace_id'])+ .where('id', 'in', nodeIds)+ .execute();++ const nodeTombstonesToCreate: CreateNodeTombstone[] = nodes.map((node) => ({+ id: node.id,+ root_id: node.root_id,+ workspace_id: node.workspace_id,+ deleted_at: new Date(),+ deleted_by: userId,+ }));++ await cleanNodeRelations(nodeIds);+ await cleanNodeFiles(nodeIds);++ await database.transaction().execute(async (trx) => {+ await trx+ .insertInto('node_tombstones')+ .values(nodeTombstonesToCreate)+ .onConflict((b) => b.columns(['id']).doNothing()).execute();- if (descendants.length === 0) {- debug(`No descendants found for ${parentIds}`);- hasMore = false;- break;- }-- const nodeIds: string[] = descendants.map((d) => d.id);- const nodeTombstonesToCreate: CreateNodeTombstone[] = descendants.map(- (node) => ({- id: node.id,- root_id: node.root_id,- workspace_id: node.workspace_id,- deleted_at: new Date(),- deleted_by: userId,- })- );-- await database.transaction().execute(async (trx) => {- await trx.deleteFrom('nodes').where('id', 'in', nodeIds).execute();-- await trx- .deleteFrom('node_interactions')- .where('node_id', 'in', nodeIds)- .execute();-- await trx- .deleteFrom('node_reactions')- .where('node_id', 'in', nodeIds)- .execute();-- await trx- .deleteFrom('node_embeddings')- .where('node_id', 'in', nodeIds)- .execute();-- await trx- .deleteFrom('document_embeddings')- .where('document_id', 'in', nodeIds)- .execute();-- await trx- .updateTable('collaborations')- .set({- deleted_at: new Date(),- deleted_by: userId,- })- .where('node_id', 'in', nodeIds)- .execute();-- await trx- .insertInto('node_tombstones')- .values(nodeTombstonesToCreate)- .execute();+ await trx.deleteFrom('nodes').where('id', 'in', nodeIds).execute();+ });++ for (const node of nodes) {+ eventBus.publish({+ type: 'node_deleted',+ nodeId: node.id,+ rootId: node.root_id,+ workspaceId: node.workspace_id,});+ }+ }+};- const uploads = await database- .selectFrom('uploads')- .selectAll()- .where('file_id', 'in', nodeIds)- .execute();+const cleanNodeRelations = async (nodeIds: string[]) => {+ await database+ .deleteFrom('node_updates')+ .where('node_id', 'in', nodeIds)+ .execute();++ await database+ .deleteFrom('node_reactions')+ .where('node_id', 'in', nodeIds)+ .execute();++ await database+ .deleteFrom('node_interactions')+ .where('node_id', 'in', nodeIds)+ .execute();++ await database+ .deleteFrom('node_embeddings')+ .where('node_id', 'in', nodeIds)+ .execute();++ await database+ .deleteFrom('collaborations')+ .where('node_id', 'in', nodeIds)+ .execute();++ await database+ .deleteFrom('document_embeddings')+ .where('document_id', 'in', nodeIds)+ .execute();++ await database+ .deleteFrom('document_updates')+ .where('document_id', 'in', nodeIds)+ .execute();++ await database+ .deleteFrom('document_embeddings')+ .where('document_id', 'in', nodeIds)+ .execute();+};- if (uploads.length > 0) {- for (const upload of uploads) {- await deleteFile(upload.path);- }-- await database- .deleteFrom('uploads')- .where('file_id', 'in', nodeIds)- .execute();- }-- for (const node of descendants) {- eventBus.publish({- type: 'node_deleted',- nodeId: node.id,- rootId: node.root_id,- workspaceId: node.workspace_id,- });-- deletedNodeIds.push(node.id);- }-- hasMore = descendants.length === BATCH_SIZE;- } catch (error) {- debug(`Error cleaning node data for ${parentIds}: ${error}`);- hasMore = false;+const cleanNodeFiles = async (nodeIds: string[]) => {+ const uploads = await database+ .selectFrom('uploads')+ .selectAll()+ .where('file_id', 'in', nodeIds)+ .execute();++ if (uploads.length > 0) {+ for (const upload of uploads) {+ await deleteFile(upload.path);}- }- return deletedNodeIds;+ await database+ .deleteFrom('uploads')+ .where('file_id', 'in', nodeIds)+ .execute();+ }};diff --git a/apps/server/src/jobs/clean-workspace-data.ts b/apps/server/src/jobs/clean-workspace-data.tsindex bd7177e5..6cc64936 100644--- a/apps/server/src/jobs/clean-workspace-data.ts+++ b/apps/server/src/jobs/clean-workspace-data.ts@@ -1,7 +1,12 @@+import { createDebugger } from '@colanode/core';+import { JobHandler } from '@/types/jobs';import { database } from '@/data/database';import { deleteFile } from '@/lib/files';+const BATCH_SIZE = 500;+const debug = createDebugger('server:job:clean-workspace-data');+export type CleanWorkspaceDataInput = {type: 'clean_workspace_data';workspaceId: string;@@ -15,17 +20,17 @@ declare module '@/types/jobs' {}}-const BATCH_SIZE = 500;-export const cleanWorkspaceDataHandler: JobHandler<CleanWorkspaceDataInput> = async (input) => {+ debug(`Cleaning workspace data for ${input.workspaceId}`);+try {await deleteWorkspaceUsers(input.workspaceId);await deleteWorkspaceNodes(input.workspaceId);await deleteWorkspaceUploads(input.workspaceId);} catch (error) {- console.error('Error cleaning workspace data:', error);+ debug(`Error cleaning workspace data: ${error}`);throw error;}};diff --git a/apps/server/src/lib/nodes.ts b/apps/server/src/lib/nodes.tsindex 583f22a9..379400fc 100644--- a/apps/server/src/lib/nodes.ts+++ b/apps/server/src/lib/nodes.ts@@ -673,65 +673,37 @@ export const deleteNode = async (return null;}- const { deletedNode, updatedCollaborations } = await database- .transaction()- .execute(async (trx) => {- const deletedNode = await trx- .deleteFrom('nodes')- .returningAll()- .where('id', '=', input.nodeId)- .executeTakeFirst();-- if (!deletedNode) {- throw new Error('Failed to delete node');- }-- const createdTombstone = await trx- .insertInto('node_tombstones')- .returningAll()- .values({- id: node.id,- root_id: node.root_id,- workspace_id: node.workspace_id,- deleted_at: new Date(input.deletedAt),- deleted_by: user.id,- })- .executeTakeFirst();-- if (!createdTombstone) {- throw new Error('Failed to create tombstone');- }-- await trx- .deleteFrom('node_updates')- .where('node_id', '=', input.nodeId)- .execute();+ const { deletedNode } = await database.transaction().execute(async (trx) => {+ const deletedNode = await trx+ .deleteFrom('nodes')+ .returningAll()+ .where('id', '=', input.nodeId)+ .executeTakeFirst();- await trx- .deleteFrom('node_reactions')- .where('node_id', '=', input.nodeId)- .execute();+ if (!deletedNode) {+ throw new Error('Failed to delete node');+ }- await trx- .deleteFrom('node_interactions')- .where('node_id', '=', input.nodeId)- .execute();+ const createdTombstone = await trx+ .insertInto('node_tombstones')+ .returningAll()+ .values({+ id: node.id,+ root_id: node.root_id,+ workspace_id: node.workspace_id,+ deleted_at: new Date(input.deletedAt),+ deleted_by: user.id,+ })+ .executeTakeFirst();- const updatedCollaborations = await trx- .updateTable('collaborations')- .set({- deleted_at: new Date(),- deleted_by: user.id,- })- .returningAll()- .where('node_id', '=', input.nodeId)- .execute();+ if (!createdTombstone) {+ throw new Error('Failed to create tombstone');+ }- return {- deletedNode,- updatedCollaborations,- };- });+ return {+ deletedNode,+ };+ });if (deletedNode.type === 'file') {const upload = await database@@ -757,15 +729,6 @@ export const deleteNode = async (workspaceId: user.workspace_id,});- for (const updatedCollaboration of updatedCollaborations) {- eventBus.publish({- type: 'collaboration_updated',- collaboratorId: updatedCollaboration.collaborator_id,- nodeId: input.nodeId,- workspaceId: user.workspace_id,- });- }-await jobService.addJob({type: 'clean_node_data',nodeId: input.nodeId,