Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Add extra checks when changes fail to sync
Hakan Shehu14f0c0ab19f9
diff --git a/apps/desktop/src/main/bootstrapper.ts b/apps/desktop/src/main/bootstrapper.tsindex 2fc220a4..c6dd9b7f 100644--- a/apps/desktop/src/main/bootstrapper.ts+++ b/apps/desktop/src/main/bootstrapper.ts@@ -4,7 +4,7 @@ import { radarService } from '@/main/services/radar-service';import { socketService } from '@/main/services/socket-service';import { serverService } from '@/main/services/server-service';import { accountService } from '@/main/services/account-service';-import { workspaceService } from '@/main/services/workspace-service';+import { syncService } from '@/main/services/sync-service';const EVENT_LOOP_INTERVAL = 1000 * 60;@@ -42,7 +42,7 @@ class Bootstrapper {await accountService.syncAccounts();await socketService.checkConnections();await accountService.syncDeletedTokens();- await workspaceService.syncAllWorkspaces();+ await syncService.syncAllWorkspaces();} catch (error) {console.log('error', error);}diff --git a/apps/desktop/src/main/mutations/server-node-delete.ts b/apps/desktop/src/main/mutations/server-node-delete.tsindex f61c3e2d..a4e815a5 100644--- a/apps/desktop/src/main/mutations/server-node-delete.ts+++ b/apps/desktop/src/main/mutations/server-node-delete.ts@@ -1,12 +1,11 @@-import { eventBus } from '@/shared/lib/event-bus';import { databaseService } from '@/main/data/database-service';import { socketService } from '@/main/services/socket-service';import { MutationHandler } from '@/main/types';-import { mapNode } from '@/main/utils';import {ServerNodeDeleteMutationInput,ServerNodeDeleteMutationOutput,} from '@/shared/mutations/server-node-delete';+import { nodeService } from '@/main/services/node-service';export class ServerNodeDeleteMutationHandlerimplements MutationHandler<ServerNodeDeleteMutationInput>@@ -32,26 +31,13 @@ export class ServerNodeDeleteMutationHandler}const userId = workspace.user_id;- const workspaceDatabase =- await databaseService.getWorkspaceDatabase(userId);+ const deleted = await nodeService.serverDelete(userId, input.id);- const deletedNode = await workspaceDatabase- .deleteFrom('nodes')- .returningAll()- .where('id', '=', input.id)- .executeTakeFirst();-- socketService.sendMessage(workspace.account_id, {- type: 'local_node_delete',- nodeId: input.id,- workspaceId: input.workspaceId,- });-- if (deletedNode) {- await eventBus.publish({- type: 'node_deleted',- userId,- node: mapNode(deletedNode),+ if (deleted) {+ socketService.sendMessage(workspace.account_id, {+ type: 'local_node_delete',+ nodeId: input.id,+ workspaceId: input.workspaceId,});}diff --git a/apps/desktop/src/main/mutations/server-node-sync.ts b/apps/desktop/src/main/mutations/server-node-sync.tsindex d208676e..58b1db8a 100644--- a/apps/desktop/src/main/mutations/server-node-sync.ts+++ b/apps/desktop/src/main/mutations/server-node-sync.ts@@ -1,13 +1,11 @@import { databaseService } from '@/main/data/database-service';import { socketService } from '@/main/services/socket-service';-import { mapNode } from '@/main/utils';import { MutationHandler } from '@/main/types';import {ServerNodeSyncMutationInput,ServerNodeSyncMutationOutput,} from '@/shared/mutations/server-node-sync';-import { YDoc } from '@colanode/crdt';-import { eventBus } from '@/shared/lib/event-bus';+import { nodeService } from '@/main/services/node-service';export class ServerNodeSyncMutationHandlerimplements MutationHandler<ServerNodeSyncMutationInput>@@ -22,7 +20,7 @@ export class ServerNodeSyncMutationHandler.where((eb) =>eb.and([eb('account_id', '=', input.accountId),- eb('workspace_id', '=', input.workspaceId),+ eb('workspace_id', '=', input.node.workspaceId),])).executeTakeFirst();@@ -34,101 +32,19 @@ export class ServerNodeSyncMutationHandler}const userId = workspace.user_id;- const workspaceDatabase =- await databaseService.getWorkspaceDatabase(userId);+ const synced = await nodeService.serverSync(userId, input.node);+ if (synced) {+ socketService.sendMessage(workspace.account_id, {+ type: 'local_node_sync',+ nodeId: input.node.id,+ userId: userId,+ versionId: input.node.versionId,+ workspaceId: input.node.workspaceId,+ });- let count = 0;- while (count++ < 10) {- const existingNode = await workspaceDatabase- .selectFrom('nodes')- .where('id', '=', input.id)- .selectAll()- .executeTakeFirst();-- if (!existingNode) {- const ydoc = new YDoc(input.id, input.state);- const attributes = ydoc.getAttributes();- const state = ydoc.getState();-- const result = await workspaceDatabase- .insertInto('nodes')- .returningAll()- .values({- id: input.id,- attributes: JSON.stringify(attributes),- state: state,- created_at: input.createdAt,- created_by: input.createdBy,- version_id: input.versionId,- server_created_at: input.serverCreatedAt,- server_version_id: input.versionId,- })- .onConflict((cb) => cb.doNothing())- .executeTakeFirst();-- if (result) {- socketService.sendMessage(workspace.account_id, {- type: 'local_node_sync',- nodeId: input.id,- userId: userId,- versionId: input.versionId,- workspaceId: input.workspaceId,- });-- await eventBus.publish({- type: 'node_created',- userId: userId,- node: mapNode(result),- });-- return {- success: true,- };- }- } else {- const ydoc = new YDoc(input.id, existingNode.state);- ydoc.applyUpdate(input.state);-- const attributes = ydoc.getAttributes();- const state = ydoc.getState();-- const updatedNode = await workspaceDatabase- .updateTable('nodes')- .returningAll()- .set({- state: state,- attributes: JSON.stringify(attributes),- server_created_at: input.serverCreatedAt,- server_updated_at: input.serverUpdatedAt,- server_version_id: input.versionId,- updated_at: input.updatedAt,- updated_by: input.updatedBy,- version_id: input.versionId,- })- .where('id', '=', input.id)- .where('version_id', '=', existingNode.version_id)- .executeTakeFirst();-- if (updatedNode) {- socketService.sendMessage(workspace.account_id, {- type: 'local_node_sync',- nodeId: input.id,- userId: userId,- versionId: input.versionId,- workspaceId: input.workspaceId,- });-- await eventBus.publish({- type: 'node_updated',- userId: userId,- node: mapNode(updatedNode),- });-- return {- success: true,- };- }- }+ return {+ success: true,+ };}} catch (error) {// console.error('Error syncing node', error);diff --git a/apps/desktop/src/main/mutations/server-user-node-sync.ts b/apps/desktop/src/main/mutations/server-user-node-sync.tsindex 6cae340d..4252e9bb 100644--- a/apps/desktop/src/main/mutations/server-user-node-sync.ts+++ b/apps/desktop/src/main/mutations/server-user-node-sync.ts@@ -1,4 +1,3 @@-import { eventBus } from '@/shared/lib/event-bus';import { databaseService } from '@/main/data/database-service';import { socketService } from '@/main/services/socket-service';import { MutationHandler } from '@/main/types';@@ -6,6 +5,7 @@ import {ServerUserNodeSyncMutationInput,ServerUserNodeSyncMutationOutput,} from '@/shared/mutations/server-user-node-sync';+import { nodeService } from '@/main/services/node-service';export class ServerUserNodeSyncMutationHandlerimplements MutationHandler<ServerUserNodeSyncMutationInput>@@ -20,7 +20,7 @@ export class ServerUserNodeSyncMutationHandler.where((eb) =>eb.and([eb('account_id', '=', input.accountId),- eb('workspace_id', '=', input.workspaceId),+ eb('workspace_id', '=', input.userNode.workspaceId),])).executeTakeFirst();@@ -32,56 +32,18 @@ export class ServerUserNodeSyncMutationHandler}const userId = workspace.user_id;- const workspaceDatabase =- await databaseService.getWorkspaceDatabase(userId);+ const synced = await nodeService.serverUserNodeSync(+ userId,+ input.userNode+ );- const createdUserNode = await workspaceDatabase- .insertInto('user_nodes')- .returningAll()- .values({- user_id: input.userId,- node_id: input.nodeId,- version_id: input.versionId,- last_seen_at: input.lastSeenAt,- last_seen_version_id: input.lastSeenVersionId,- mentions_count: input.mentionsCount,- created_at: input.createdAt,- updated_at: input.updatedAt,- })- .onConflict((cb) =>- cb.columns(['node_id', 'user_id']).doUpdateSet({- last_seen_at: input.lastSeenAt,- last_seen_version_id: input.lastSeenVersionId,- mentions_count: input.mentionsCount,- updated_at: input.updatedAt,- version_id: input.versionId,- })- )- .executeTakeFirst();-- socketService.sendMessage(workspace.account_id, {- type: 'local_user_node_sync',- userId: input.userId,- nodeId: input.nodeId,- versionId: input.versionId,- workspaceId: input.workspaceId,- });-- if (createdUserNode) {- await eventBus.publish({- type: 'user_node_created',- userId: userId,- userNode: {- userId: createdUserNode.user_id,- nodeId: createdUserNode.node_id,- lastSeenAt: createdUserNode.last_seen_at,- lastSeenVersionId: createdUserNode.last_seen_version_id,- mentionsCount: createdUserNode.mentions_count,- attributes: createdUserNode.attributes,- versionId: createdUserNode.version_id,- createdAt: createdUserNode.created_at,- updatedAt: createdUserNode.updated_at,- },+ if (synced) {+ socketService.sendMessage(workspace.account_id, {+ type: 'local_user_node_sync',+ userId: input.userNode.userId,+ nodeId: input.userNode.nodeId,+ versionId: input.userNode.versionId,+ workspaceId: input.userNode.workspaceId,});}} catch (error) {diff --git a/apps/desktop/src/main/services/account-service.ts b/apps/desktop/src/main/services/account-service.tsindex 722a7463..a671c46d 100644--- a/apps/desktop/src/main/services/account-service.ts+++ b/apps/desktop/src/main/services/account-service.ts@@ -3,10 +3,13 @@ import { databaseService } from '@/main/data/database-service';import { SelectAccount } from '@/main/data/app/schema';import { AccountSyncOutput } from '@colanode/core';import { httpClient } from '@/shared/lib/http-client';-import { mapAccount, mapWorkspace } from '@/main/utils';+import {+ getWorkspaceDirectoryPath,+ mapAccount,+ mapWorkspace,+} from '@/main/utils';import { getAccountAvatarsDirectoryPath } from '@/main/utils';import { eventBus } from '@/shared/lib/event-bus';-import { workspaceService } from '@/main/services/workspace-service';import { serverService } from '@/main/services/server-service';class AccountService {@@ -140,7 +143,7 @@ class AccountService {);if (!updatedWorkspace) {- await workspaceService.deleteWorkspace(workspace.user_id);+ await this.deleteWorkspace(workspace.user_id);}}}@@ -153,7 +156,7 @@ class AccountService {.execute();for (const workspace of workspaces) {- await workspaceService.deleteWorkspace(workspace.user_id);+ await this.deleteWorkspace(workspace.user_id);}const avatarsDir = getAccountAvatarsDirectoryPath(account.id);@@ -230,6 +233,31 @@ class AccountService {}}}++ private async deleteWorkspace(userId: string): Promise<boolean> {+ const deletedWorkspace = await databaseService.appDatabase+ .deleteFrom('workspaces')+ .returningAll()+ .where('user_id', '=', userId)+ .executeTakeFirst();++ if (!deletedWorkspace) {+ return false;+ }++ await databaseService.deleteWorkspaceDatabase(userId);+ const workspaceDir = getWorkspaceDirectoryPath(userId);+ if (fs.existsSync(workspaceDir)) {+ fs.rmSync(workspaceDir, { recursive: true });+ }++ await eventBus.publish({+ type: 'workspace_deleted',+ workspace: mapWorkspace(deletedWorkspace),+ });++ return true;+ }}export const accountService = new AccountService();diff --git a/apps/desktop/src/main/services/node-service.ts b/apps/desktop/src/main/services/node-service.tsindex 1b8fc3ff..2ca8c127 100644--- a/apps/desktop/src/main/services/node-service.ts+++ b/apps/desktop/src/main/services/node-service.ts@@ -6,6 +6,8 @@ import {LocalCreateNodeChangeData,LocalDeleteNodeChangeData,LocalUpdateNodeChangeData,+ ServerNodeState,+ ServerUserNodeState,} from '@colanode/core';import { YDoc } from '@colanode/crdt';import { generateId, IdType } from '@colanode/core';@@ -15,6 +17,7 @@ import { CreateDownload, CreateUpload } from '@/main/data/workspace/schema';import { eventBus } from '@/shared/lib/event-bus';import { Download } from '@/shared/types/nodes';import { Upload } from '@/shared/types/nodes';+import { SelectWorkspace } from '@/main/data/app/schema';export type CreateNodeInput = {id: string;@@ -24,16 +27,11 @@ export type CreateNodeInput = {};class NodeService {- async createNode(userId: string, input: CreateNodeInput | CreateNodeInput[]) {- const workspace = await databaseService.appDatabase- .selectFrom('workspaces')- .selectAll()- .where('user_id', '=', userId)- .executeTakeFirst();-- if (!workspace) {- throw new Error('Workspace not found');- }+ public async createNode(+ userId: string,+ input: CreateNodeInput | CreateNodeInput[]+ ) {+ const workspace = await this.fetchWorkspace(userId);const inputs = Array.isArray(input) ? input : [input];const createdNodes: Node[] = [];@@ -190,7 +188,7 @@ class NodeService {}}- async updateNode(+ public async updateNode(nodeId: string,userId: string,updater: (attributes: NodeAttributes) => NodeAttributes@@ -211,15 +209,7 @@ class NodeService {userId: string,updater: (attributes: NodeAttributes) => NodeAttributes): Promise<boolean> {- const workspace = await databaseService.appDatabase- .selectFrom('workspaces')- .selectAll()- .where('user_id', '=', userId)- .executeTakeFirst();-- if (!workspace) {- throw new Error('Workspace not found');- }+ const workspace = await this.fetchWorkspace(userId);const workspaceDatabase =await databaseService.getWorkspaceDatabase(userId);@@ -333,16 +323,8 @@ class NodeService {return result;}- async deleteNode(nodeId: string, userId: string) {- const workspace = await databaseService.appDatabase- .selectFrom('workspaces')- .selectAll()- .where('user_id', '=', userId)- .executeTakeFirst();-- if (!workspace) {- throw new Error('Workspace not found');- }+ public async deleteNode(nodeId: string, userId: string) {+ const workspace = await this.fetchWorkspace(userId);const workspaceDatabase =await databaseService.getWorkspaceDatabase(userId);@@ -414,6 +396,188 @@ class NodeService {});}}++ public async serverSync(+ userId: string,+ node: ServerNodeState+ ): Promise<boolean> {+ let count = 0;+ while (count++ < 20) {+ const synced = await this.tryServerSync(userId, node);+ if (synced) {+ return true;+ }+ }++ return false;+ }++ private async tryServerSync(userId: string, node: ServerNodeState) {+ const workspaceDatabase =+ await databaseService.getWorkspaceDatabase(userId);++ const existingNode = await workspaceDatabase+ .selectFrom('nodes')+ .where('id', '=', node.id)+ .selectAll()+ .executeTakeFirst();++ if (!existingNode) {+ const ydoc = new YDoc(node.id, node.state);+ const attributes = ydoc.getAttributes();+ const state = ydoc.getState();++ const result = await workspaceDatabase+ .insertInto('nodes')+ .returningAll()+ .values({+ id: node.id,+ attributes: JSON.stringify(attributes),+ state: state,+ created_at: node.createdAt,+ created_by: node.createdBy,+ version_id: node.versionId,+ server_created_at: node.serverCreatedAt,+ server_version_id: node.versionId,+ })+ .onConflict((cb) => cb.doNothing())+ .executeTakeFirst();++ if (result) {+ await eventBus.publish({+ type: 'node_created',+ userId: userId,+ node: mapNode(result),+ });++ return true;+ }+ } else {+ const ydoc = new YDoc(node.id, existingNode.state);+ ydoc.applyUpdate(node.state);++ const attributes = ydoc.getAttributes();+ const state = ydoc.getState();++ const updatedNode = await workspaceDatabase+ .updateTable('nodes')+ .returningAll()+ .set({+ state: state,+ attributes: JSON.stringify(attributes),+ server_created_at: node.serverCreatedAt,+ server_updated_at: node.serverUpdatedAt,+ server_version_id: node.versionId,+ updated_at: node.updatedAt,+ updated_by: node.updatedBy,+ version_id: node.versionId,+ })+ .where('id', '=', node.id)+ .where('version_id', '=', existingNode.version_id)+ .executeTakeFirst();++ if (updatedNode) {+ await eventBus.publish({+ type: 'node_updated',+ userId: userId,+ node: mapNode(updatedNode),+ });++ return true;+ }+ }++ return false;+ }++ public async serverDelete(userId: string, nodeId: string) {+ const workspaceDatabase =+ await databaseService.getWorkspaceDatabase(userId);++ const deletedNode = await workspaceDatabase+ .deleteFrom('nodes')+ .returningAll()+ .where('id', '=', nodeId)+ .executeTakeFirst();++ if (deletedNode) {+ await eventBus.publish({+ type: 'node_deleted',+ userId,+ node: mapNode(deletedNode),+ });+ }++ return false;+ }++ public async serverUserNodeSync(+ userId: string,+ userNode: ServerUserNodeState+ ): Promise<boolean> {+ const workspaceDatabase =+ await databaseService.getWorkspaceDatabase(userId);++ const createdUserNode = await workspaceDatabase+ .insertInto('user_nodes')+ .returningAll()+ .values({+ user_id: userNode.userId,+ node_id: userNode.nodeId,+ version_id: userNode.versionId,+ last_seen_at: userNode.lastSeenAt,+ last_seen_version_id: userNode.lastSeenVersionId,+ mentions_count: userNode.mentionsCount,+ created_at: userNode.createdAt,+ updated_at: userNode.updatedAt,+ })+ .onConflict((cb) =>+ cb.columns(['node_id', 'user_id']).doUpdateSet({+ last_seen_at: userNode.lastSeenAt,+ last_seen_version_id: userNode.lastSeenVersionId,+ mentions_count: userNode.mentionsCount,+ updated_at: userNode.updatedAt,+ version_id: userNode.versionId,+ })+ )+ .executeTakeFirst();++ if (createdUserNode) {+ await eventBus.publish({+ type: 'user_node_created',+ userId: userId,+ userNode: {+ userId: createdUserNode.user_id,+ nodeId: createdUserNode.node_id,+ lastSeenAt: createdUserNode.last_seen_at,+ lastSeenVersionId: createdUserNode.last_seen_version_id,+ mentionsCount: createdUserNode.mentions_count,+ attributes: createdUserNode.attributes,+ versionId: createdUserNode.version_id,+ createdAt: createdUserNode.created_at,+ updatedAt: createdUserNode.updated_at,+ },+ });++ return true;+ }++ return false;+ }++ async fetchWorkspace(userId: string): Promise<SelectWorkspace> {+ const workspace = await databaseService.appDatabase+ .selectFrom('workspaces')+ .selectAll()+ .where('user_id', '=', userId)+ .executeTakeFirst();++ if (!workspace) {+ throw new Error('Workspace not found');+ }++ return workspace;+ }}export const nodeService = new NodeService();diff --git a/apps/desktop/src/main/services/socket-connection.ts b/apps/desktop/src/main/services/socket-connection.tsindex faf6c5de..d1f9c662 100644--- a/apps/desktop/src/main/services/socket-connection.ts+++ b/apps/desktop/src/main/services/socket-connection.ts@@ -57,30 +57,13 @@ export class SocketConnection {mutationService.executeMutation({type: 'server_node_sync',accountId: this.account.id,- id: message.id,- workspaceId: message.workspaceId,- state: message.state,- createdAt: message.createdAt,- createdBy: message.createdBy,- updatedAt: message.updatedAt,- updatedBy: message.updatedBy,- serverCreatedAt: message.serverCreatedAt,- serverUpdatedAt: message.serverUpdatedAt,- versionId: message.versionId,+ node: message.node,});} else if (message.type === 'server_user_node_sync') {mutationService.executeMutation({type: 'server_user_node_sync',accountId: this.account.id,- nodeId: message.nodeId,- userId: message.userId,- workspaceId: message.workspaceId,- versionId: message.versionId,- lastSeenAt: message.lastSeenAt,- lastSeenVersionId: message.lastSeenVersionId,- mentionsCount: message.mentionsCount,- createdAt: message.createdAt,- updatedAt: message.updatedAt,+ userNode: message.userNode,});}};diff --git a/apps/desktop/src/main/services/workspace-service.ts b/apps/desktop/src/main/services/sync-service.tssimilarity index 62%rename from apps/desktop/src/main/services/workspace-service.tsrename to apps/desktop/src/main/services/sync-service.tsindex f83a7d88..b97375a3 100644--- a/apps/desktop/src/main/services/workspace-service.ts+++ b/apps/desktop/src/main/services/sync-service.ts@@ -1,23 +1,24 @@-import fs from 'fs';import { databaseService } from '@/main/data/database-service';-import {- getWorkspaceDirectoryPath,- mapChange,- mapWorkspace,-} from '@/main/utils';+import { mapChange } from '@/main/utils';import { eventBus } from '@/shared/lib/event-bus';-import { LocalChange, SyncChangesOutput } from '@colanode/core';+import {+ LocalChange,+ SyncChangesOutput,+ SyncNodeStatesOutput,+} from '@colanode/core';import { WorkspaceDatabaseSchema } from '@/main/data/workspace/schema';import { Kysely } from 'kysely';import { httpClient } from '@/shared/lib/http-client';import { serverService } from '@/main/services/server-service';+import { nodeService } from '@/main/services/node-service';+import { socketService } from '@/main/services/socket-service';type WorkspaceSyncState = {isSyncing: boolean;scheduledSync: boolean;};-class WorkspaceService {+class SyncService {private syncStates: Map<string, WorkspaceSyncState> = new Map();constructor() {@@ -28,31 +29,6 @@ class WorkspaceService {});}- public async deleteWorkspace(userId: string): Promise<boolean> {- const deletedWorkspace = await databaseService.appDatabase- .deleteFrom('workspaces')- .returningAll()- .where('user_id', '=', userId)- .executeTakeFirst();-- if (!deletedWorkspace) {- return false;- }-- await databaseService.deleteWorkspaceDatabase(userId);- const workspaceDir = getWorkspaceDirectoryPath(userId);- if (fs.existsSync(workspaceDir)) {- fs.rmSync(workspaceDir, { recursive: true });- }-- await eventBus.publish({- type: 'workspace_deleted',- workspace: mapWorkspace(deletedWorkspace),- });-- return true;- }-public async syncAllWorkspaces() {const workspaces = await databaseService.appDatabase.selectFrom('workspaces')@@ -162,15 +138,111 @@ class WorkspaceService {.set((eb) => ({ retry_count: eb('retry_count', '+', 1) })).where('id', 'in', unsyncedChangeIds).execute();+ }+ }- //we just delete changes that have failed to sync for more than 5 times.- //in the future we might need to revert the change locally.- await workspaceDatabase- .deleteFrom('changes')- .where('retry_count', '>=', 5)- .execute();+ const invalidChanges = await this.fetchInvalidChanges(workspaceDatabase);+ if (invalidChanges.length === 0) {+ return;+ }++ const nodeChanges: Record<string, number[]> = {};+ const changesToDelete: number[] = [];+ for (const change of invalidChanges) {+ if (change.retryCount >= 100) {+ // if the change has been retried 100 times, we delete it (it should never happen)+ changesToDelete.push(change.id);+ continue;+ }++ let nodeId: string | null = null;+ if (+ change.data.type === 'node_create' ||+ change.data.type === 'node_update' ||+ change.data.type === 'node_delete'+ ) {+ nodeId = change.data.id;+ } else if (change.data.type === 'user_node_update') {+ nodeId = change.data.nodeId;+ }++ if (nodeId) {+ if (!nodeChanges[nodeId]) {+ nodeChanges[nodeId] = [];+ }++ nodeChanges[nodeId].push(change.id);}}++ const nodeIds = Object.keys(nodeChanges);+ const { data, status } = await httpClient.post<SyncNodeStatesOutput>(+ `/v1/sync/states/${workspace.workspace_id}`,+ {+ nodeIds,+ },+ { domain: workspace.domain, token: workspace.token }+ );++ if (status !== 200) {+ return;+ }++ for (const nodeId of nodeIds) {+ const changeIds = nodeChanges[nodeId];+ const states = data.nodes[nodeId];++ if (!states) {+ const deleted = await nodeService.serverDelete(userId, nodeId);+ if (deleted) {+ changesToDelete.push(...changeIds);++ socketService.sendMessage(workspace.account_id, {+ type: 'local_node_delete',+ nodeId,+ workspaceId: workspace.workspace_id,+ });+ }++ continue;+ }++ const nodeSynced = await nodeService.serverSync(userId, states.node);+ if (nodeSynced) {+ changesToDelete.push(...changeIds);+ socketService.sendMessage(workspace.account_id, {+ type: 'local_node_sync',+ nodeId,+ userId,+ versionId: states.node.versionId,+ workspaceId: workspace.workspace_id,+ });++ const userNodeSynced = await nodeService.serverUserNodeSync(+ userId,+ states.userNode+ );++ if (userNodeSynced) {+ socketService.sendMessage(workspace.account_id, {+ type: 'local_user_node_sync',+ nodeId,+ userId,+ versionId: states.userNode.versionId,+ workspaceId: workspace.workspace_id,+ });+ }++ changesToDelete.push(...changeIds);+ }+ }++ if (changesToDelete.length > 0) {+ await workspaceDatabase+ .deleteFrom('changes')+ .where('id', 'in', changesToDelete)+ .execute();+ }}private async fetchAndCompactWorkspaceChanges(@@ -239,6 +311,20 @@ class WorkspaceService {return changes.filter((change) => !changesToDelete.has(change.id));}++ private async fetchInvalidChanges(database: Kysely<WorkspaceDatabaseSchema>) {+ const rows = await database+ .selectFrom('changes')+ .selectAll()+ .where('retry_count', '>=', 5)+ .execute();++ if (rows.length === 0) {+ return [];+ }++ return rows.map(mapChange);+ }}-export const workspaceService = new WorkspaceService();+export const syncService = new SyncService();diff --git a/apps/desktop/src/main/utils.ts b/apps/desktop/src/main/utils.tsindex 3477593d..20ff3934 100644--- a/apps/desktop/src/main/utils.ts+++ b/apps/desktop/src/main/utils.ts@@ -130,6 +130,7 @@ export const mapChange = (row: SelectChange): LocalChange => {id: row.id,data: JSON.parse(row.data),createdAt: row.created_at,+ retryCount: row.retry_count,};};diff --git a/apps/desktop/src/shared/mutations/server-node-sync.ts b/apps/desktop/src/shared/mutations/server-node-sync.tsindex 37daa899..78d0c993 100644--- a/apps/desktop/src/shared/mutations/server-node-sync.ts+++ b/apps/desktop/src/shared/mutations/server-node-sync.ts@@ -1,16 +1,9 @@+import { ServerNodeState } from '@colanode/core';+export type ServerNodeSyncMutationInput = {type: 'server_node_sync';accountId: string;- id: string;- workspaceId: string;- state: string;- createdAt: string;- createdBy: string;- updatedAt: string | null;- updatedBy: string | null;- serverCreatedAt: string;- serverUpdatedAt: string | null;- versionId: string;+ node: ServerNodeState;};export type ServerNodeSyncMutationOutput = {diff --git a/apps/desktop/src/shared/mutations/server-user-node-sync.ts b/apps/desktop/src/shared/mutations/server-user-node-sync.tsindex 5265bace..ce89033b 100644--- a/apps/desktop/src/shared/mutations/server-user-node-sync.ts+++ b/apps/desktop/src/shared/mutations/server-user-node-sync.ts@@ -1,15 +1,9 @@+import { ServerUserNodeState } from '@colanode/core';+export type ServerUserNodeSyncMutationInput = {type: 'server_user_node_sync';accountId: string;- nodeId: string;- userId: string;- workspaceId: string;- versionId: string;- lastSeenAt: string | null;- lastSeenVersionId: string | null;- mentionsCount: number;- createdAt: string;- updatedAt: string | null;+ userNode: ServerUserNodeState;};export type ServerUserNodeSyncMutationOutput = {diff --git a/apps/server/src/routes/sync.ts b/apps/server/src/routes/sync.tsindex e295d412..d3546ff6 100644--- a/apps/server/src/routes/sync.ts+++ b/apps/server/src/routes/sync.ts@@ -21,16 +21,18 @@ import {SyncChangeResult,SyncChangesInput,SyncChangeStatus,+ SyncNodeStatesInput,+ SyncNodeStatesOutput,+ ServerNodeState,+ ServerUserNodeState,} from '@colanode/core';-import { YDoc } from '@colanode/crdt';+import { encodeState, YDoc } from '@colanode/crdt';export const syncRouter = Router();syncRouter.post('/:workspaceId',async (req: ColanodeRequest, res: ColanodeResponse) => {- const workspaceId = req.params.workspaceId as string;- const input = req.body as SyncChangesInput;if (!req.account) {return res.status(401).json({code: ApiError.Unauthorized,@@ -38,6 +40,9 @@ syncRouter.post(});}+ const workspaceId = req.params.workspaceId as string;+ const input = req.body as SyncChangesInput;+const workspaceUser = await database.selectFrom('workspace_users').selectAll()@@ -74,6 +79,97 @@ syncRouter.post(});+syncRouter.post(+ '/:workspaceId/states',+ async (req: ColanodeRequest, res: ColanodeResponse) => {+ if (!req.account) {+ return res.status(401).json({+ code: ApiError.Unauthorized,+ message: 'Unauthorized.',+ });+ }++ const workspaceId = req.params.workspaceId as string;+ const input = req.body as SyncNodeStatesInput;+ const ids = input.ids;++ const workspaceUser = await database+ .selectFrom('workspace_users')+ .selectAll()+ .where('workspace_id', '=', workspaceId)+ .where('account_id', '=', req.account.id)+ .executeTakeFirst();++ if (!workspaceUser) {+ return res.status(403).json({+ code: ApiError.Forbidden,+ message: 'Forbidden.',+ });+ }++ const rows = await database+ .selectFrom('nodes as n')+ .innerJoin('user_nodes as un', 'un.node_id', 'n.id')+ .select([+ 'n.id as node_id',+ 'n.state as node_state',+ 'n.created_at as node_created_at',+ 'n.created_by as node_created_by',+ 'n.updated_at as node_updated_at',+ 'n.updated_by as node_updated_by',+ 'n.version_id as node_version_id',+ 'un.version_id as user_node_version_id',+ 'un.last_seen_at as user_node_last_seen_at',+ 'un.last_seen_version_id as user_node_last_seen_version_id',+ 'un.mentions_count as user_node_mentions_count',+ 'un.created_at as user_node_created_at',+ 'un.updated_at as user_node_updated_at',+ ])+ .where('un.user_id', '=', workspaceUser.id)+ .where('n.workspace_id', '=', workspaceId)+ .where('n.id', 'in', ids)+ .execute();++ const states: SyncNodeStatesOutput = {+ nodes: {},+ };++ for (const row of rows) {+ const nodeState: ServerNodeState = {+ id: row.node_id,+ workspaceId: workspaceId,+ state: encodeState(row.node_state),+ createdAt: row.node_created_at.toISOString(),+ createdBy: row.node_created_by,+ updatedAt: row.node_updated_at?.toISOString() ?? null,+ updatedBy: row.node_updated_by,+ serverCreatedAt: new Date().toISOString(),+ serverUpdatedAt: row.user_node_updated_at?.toISOString() ?? null,+ versionId: row.node_version_id,+ };++ const userNodeState: ServerUserNodeState = {+ nodeId: row.node_id,+ userId: workspaceUser.id,+ workspaceId: workspaceId,+ versionId: row.user_node_version_id,+ lastSeenAt: row.user_node_last_seen_at?.toISOString() ?? null,+ lastSeenVersionId: row.user_node_last_seen_version_id,+ mentionsCount: row.user_node_mentions_count,+ createdAt: row.user_node_created_at.toISOString(),+ updatedAt: row.user_node_updated_at?.toISOString() ?? null,+ };++ states.nodes[row.node_id] = {+ node: nodeState,+ userNode: userNodeState,+ };+ }++ res.status(200).json({ nodes: states });+ }+);+const handleLocalChange = async (workspaceUser: SelectWorkspaceUser,change: LocalChangediff --git a/apps/server/src/services/synapse.ts b/apps/server/src/services/synapse.tsindex 8e70ef9e..f4b71bef 100644--- a/apps/server/src/services/synapse.ts+++ b/apps/server/src/services/synapse.ts@@ -300,16 +300,18 @@ class SynapseService {} else {this.sendSocketMessage(socketConnection, {type: 'server_node_sync',- id: node.id,- workspaceId: data.workspaceId,- state: fromUint8Array(node.state),- createdAt: node.created_at.toISOString(),- createdBy: node.created_by,- updatedAt: node.updated_at?.toISOString() ?? null,- updatedBy: node.updated_by ?? null,- serverCreatedAt: node.server_created_at.toISOString(),- serverUpdatedAt: node.server_updated_at?.toISOString() ?? null,- versionId: node.version_id,+ node: {+ id: node.id,+ workspaceId: data.workspaceId,+ state: fromUint8Array(node.state),+ createdAt: node.created_at.toISOString(),+ createdBy: node.created_by,+ updatedAt: node.updated_at?.toISOString() ?? null,+ updatedBy: node.updated_by ?? null,+ serverCreatedAt: node.server_created_at.toISOString(),+ serverUpdatedAt: node.server_updated_at?.toISOString() ?? null,+ versionId: node.version_id,+ },});}}@@ -344,15 +346,17 @@ class SynapseService {this.sendSocketMessage(socketConnection, {type: 'server_user_node_sync',- userId: data.userId,- nodeId: data.nodeId,- lastSeenVersionId: userNode.last_seen_version_id,- workspaceId: data.workspaceId,- versionId: userNode.version_id,- lastSeenAt: userNode.last_seen_at?.toISOString() ?? null,- createdAt: userNode.created_at.toISOString(),- updatedAt: userNode.updated_at?.toISOString() ?? null,- mentionsCount: userNode.mentions_count,+ userNode: {+ userId: data.userId,+ nodeId: data.nodeId,+ lastSeenVersionId: userNode.last_seen_version_id,+ workspaceId: data.workspaceId,+ versionId: userNode.version_id,+ lastSeenAt: userNode.last_seen_at?.toISOString() ?? null,+ createdAt: userNode.created_at.toISOString(),+ updatedAt: userNode.updated_at?.toISOString() ?? null,+ mentionsCount: userNode.mentions_count,+ },});}}@@ -444,31 +448,35 @@ class SynapseService {if (row.node_version_id !== row.device_node_version_id) {this.sendSocketMessage(connection, {type: 'server_node_sync',- id: row.id,- workspaceId: row.workspace_id,- state: fromUint8Array(row.state!),- createdAt: row.created_at!.toISOString(),- createdBy: row.created_by!,- updatedAt: row.updated_at?.toISOString() ?? null,- updatedBy: row.updated_by ?? null,- serverCreatedAt: row.server_created_at!.toISOString(),- serverUpdatedAt: row.server_updated_at?.toISOString() ?? null,- versionId: row.node_version_id!,+ node: {+ id: row.id,+ workspaceId: row.workspace_id,+ state: fromUint8Array(row.state!),+ createdAt: row.created_at!.toISOString(),+ createdBy: row.created_by!,+ updatedAt: row.updated_at?.toISOString() ?? null,+ updatedBy: row.updated_by ?? null,+ serverCreatedAt: row.server_created_at!.toISOString(),+ serverUpdatedAt: row.server_updated_at?.toISOString() ?? null,+ versionId: row.node_version_id!,+ },});}if (row.user_node_version_id !== row.device_user_node_version_id) {this.sendSocketMessage(connection, {type: 'server_user_node_sync',- nodeId: row.node_id,- userId: row.user_id,- workspaceId: row.workspace_id,- versionId: row.user_node_version_id!,- lastSeenAt: row.last_seen_at?.toISOString() ?? null,- lastSeenVersionId: row.last_seen_version_id ?? null,- mentionsCount: row.mentions_count,- createdAt: row.created_at!.toISOString(),- updatedAt: row.updated_at?.toISOString() ?? null,+ userNode: {+ nodeId: row.node_id,+ userId: row.user_id,+ workspaceId: row.workspace_id,+ versionId: row.user_node_version_id!,+ lastSeenAt: row.last_seen_at?.toISOString() ?? null,+ lastSeenVersionId: row.last_seen_version_id ?? null,+ mentionsCount: row.mentions_count,+ createdAt: row.created_at!.toISOString(),+ updatedAt: row.updated_at?.toISOString() ?? null,+ },});}}diff --git a/packages/core/src/types/messages.ts b/packages/core/src/types/messages.tsindex 6ac2280b..d7880740 100644--- a/packages/core/src/types/messages.ts+++ b/packages/core/src/types/messages.ts@@ -1,3 +1,5 @@+import { ServerNodeState, ServerUserNodeState } from './sync';+export type LocalNodeDeleteMessage = {type: 'local_node_delete';nodeId: string;@@ -28,29 +30,12 @@ export type ServerNodeDeleteMessage = {export type ServerNodeSyncMessage = {type: 'server_node_sync';- id: string;- workspaceId: string;- state: string;- createdAt: string;- createdBy: string;- updatedAt: string | null;- updatedBy: string | null;- serverCreatedAt: string;- serverUpdatedAt: string | null;- versionId: string;+ node: ServerNodeState;};export type ServerUserNodeSyncMessage = {type: 'server_user_node_sync';- nodeId: string;- userId: string;- workspaceId: string;- versionId: string;- lastSeenAt: string | null;- lastSeenVersionId: string | null;- mentionsCount: number;- createdAt: string;- updatedAt: string | null;+ userNode: ServerUserNodeState;};export type Message =diff --git a/packages/core/src/types/sync.ts b/packages/core/src/types/sync.tsindex 9b54f5ac..685524a7 100644--- a/packages/core/src/types/sync.ts+++ b/packages/core/src/types/sync.ts@@ -2,6 +2,19 @@ export type SyncChangesInput = {changes: LocalChange[];};+export type SyncNodeStatesInput = {+ ids: string[];+};++export type SyncNodeStatesOutput = {+ nodes: Record<string, SyncNodeStateResult | null>;+};++export type SyncNodeStateResult = {+ node: ServerNodeState;+ userNode: ServerUserNodeState;+};+export type SyncChangesOutput = {results: SyncChangeResult[];};@@ -17,6 +30,7 @@ export type LocalChange = {id: number;data: LocalChangeData;createdAt: string;+ retryCount: number;};export type LocalCreateNodeChangeData = {@@ -59,3 +73,28 @@ export type LocalChangeData =| LocalUpdateNodeChangeData| LocalDeleteNodeChangeData| LocalUserNodeChangeData;++export type ServerNodeState = {+ id: string;+ workspaceId: string;+ state: string;+ createdAt: string;+ createdBy: string;+ updatedAt: string | null;+ updatedBy: string | null;+ serverCreatedAt: string;+ serverUpdatedAt: string | null;+ versionId: string;+};++export type ServerUserNodeState = {+ nodeId: string;+ userId: string;+ workspaceId: string;+ versionId: string;+ lastSeenAt: string | null;+ lastSeenVersionId: string | null;+ mentionsCount: number;+ createdAt: string;+ updatedAt: string | null;+};diff --git a/packages/crdt/src/index.ts b/packages/crdt/src/index.tsindex 0ce407c1..c7642eef 100644--- a/packages/crdt/src/index.ts+++ b/packages/crdt/src/index.ts@@ -5,6 +5,14 @@ import { isEqual } from 'lodash-es';import { diffChars } from 'diff';import { fromUint8Array, toUint8Array } from 'js-base64';+export const encodeState = (state: Uint8Array) => {+ return fromUint8Array(state);+};++export const decodeState = (state: string) => {+ return toUint8Array(state);+};+export class YDoc {private doc: Y.Doc;private updates: Uint8Array[] = [];