Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Add extra checks when changes fail to sync

Hakan Shehu14f0c0ab19f9
diff --git a/apps/desktop/src/main/bootstrapper.ts b/apps/desktop/src/main/bootstrapper.ts
index 2fc220a4..c6dd9b7f 100644
--- a/apps/desktop/src/main/bootstrapper.ts
+++ b/apps/desktop/src/main/bootstrapper.ts
@@ -4,7 +4,7 @@ import { radarService } from '@/main/services/radar-service';
import { socketService } from '@/main/services/socket-service';
import { serverService } from '@/main/services/server-service';
import { accountService } from '@/main/services/account-service';
-import { workspaceService } from '@/main/services/workspace-service';
+import { syncService } from '@/main/services/sync-service';
const EVENT_LOOP_INTERVAL = 1000 * 60;
@@ -42,7 +42,7 @@ class Bootstrapper {
await accountService.syncAccounts();
await socketService.checkConnections();
await accountService.syncDeletedTokens();
- await workspaceService.syncAllWorkspaces();
+ await syncService.syncAllWorkspaces();
} catch (error) {
console.log('error', error);
}
diff --git a/apps/desktop/src/main/mutations/server-node-delete.ts b/apps/desktop/src/main/mutations/server-node-delete.ts
index f61c3e2d..a4e815a5 100644
--- a/apps/desktop/src/main/mutations/server-node-delete.ts
+++ b/apps/desktop/src/main/mutations/server-node-delete.ts
@@ -1,12 +1,11 @@
-import { eventBus } from '@/shared/lib/event-bus';
import { databaseService } from '@/main/data/database-service';
import { socketService } from '@/main/services/socket-service';
import { MutationHandler } from '@/main/types';
-import { mapNode } from '@/main/utils';
import {
ServerNodeDeleteMutationInput,
ServerNodeDeleteMutationOutput,
} from '@/shared/mutations/server-node-delete';
+import { nodeService } from '@/main/services/node-service';
export class ServerNodeDeleteMutationHandler
implements MutationHandler<ServerNodeDeleteMutationInput>
@@ -32,26 +31,13 @@ export class ServerNodeDeleteMutationHandler
}
const userId = workspace.user_id;
- const workspaceDatabase =
- await databaseService.getWorkspaceDatabase(userId);
+ const deleted = await nodeService.serverDelete(userId, input.id);
- const deletedNode = await workspaceDatabase
- .deleteFrom('nodes')
- .returningAll()
- .where('id', '=', input.id)
- .executeTakeFirst();
-
- socketService.sendMessage(workspace.account_id, {
- type: 'local_node_delete',
- nodeId: input.id,
- workspaceId: input.workspaceId,
- });
-
- if (deletedNode) {
- await eventBus.publish({
- type: 'node_deleted',
- userId,
- node: mapNode(deletedNode),
+ if (deleted) {
+ socketService.sendMessage(workspace.account_id, {
+ type: 'local_node_delete',
+ nodeId: input.id,
+ workspaceId: input.workspaceId,
});
}
diff --git a/apps/desktop/src/main/mutations/server-node-sync.ts b/apps/desktop/src/main/mutations/server-node-sync.ts
index d208676e..58b1db8a 100644
--- a/apps/desktop/src/main/mutations/server-node-sync.ts
+++ b/apps/desktop/src/main/mutations/server-node-sync.ts
@@ -1,13 +1,11 @@
import { databaseService } from '@/main/data/database-service';
import { socketService } from '@/main/services/socket-service';
-import { mapNode } from '@/main/utils';
import { MutationHandler } from '@/main/types';
import {
ServerNodeSyncMutationInput,
ServerNodeSyncMutationOutput,
} from '@/shared/mutations/server-node-sync';
-import { YDoc } from '@colanode/crdt';
-import { eventBus } from '@/shared/lib/event-bus';
+import { nodeService } from '@/main/services/node-service';
export class ServerNodeSyncMutationHandler
implements MutationHandler<ServerNodeSyncMutationInput>
@@ -22,7 +20,7 @@ export class ServerNodeSyncMutationHandler
.where((eb) =>
eb.and([
eb('account_id', '=', input.accountId),
- eb('workspace_id', '=', input.workspaceId),
+ eb('workspace_id', '=', input.node.workspaceId),
])
)
.executeTakeFirst();
@@ -34,101 +32,19 @@ export class ServerNodeSyncMutationHandler
}
const userId = workspace.user_id;
- const workspaceDatabase =
- await databaseService.getWorkspaceDatabase(userId);
+ const synced = await nodeService.serverSync(userId, input.node);
+ if (synced) {
+ socketService.sendMessage(workspace.account_id, {
+ type: 'local_node_sync',
+ nodeId: input.node.id,
+ userId: userId,
+ versionId: input.node.versionId,
+ workspaceId: input.node.workspaceId,
+ });
- let count = 0;
- while (count++ < 10) {
- const existingNode = await workspaceDatabase
- .selectFrom('nodes')
- .where('id', '=', input.id)
- .selectAll()
- .executeTakeFirst();
-
- if (!existingNode) {
- const ydoc = new YDoc(input.id, input.state);
- const attributes = ydoc.getAttributes();
- const state = ydoc.getState();
-
- const result = await workspaceDatabase
- .insertInto('nodes')
- .returningAll()
- .values({
- id: input.id,
- attributes: JSON.stringify(attributes),
- state: state,
- created_at: input.createdAt,
- created_by: input.createdBy,
- version_id: input.versionId,
- server_created_at: input.serverCreatedAt,
- server_version_id: input.versionId,
- })
- .onConflict((cb) => cb.doNothing())
- .executeTakeFirst();
-
- if (result) {
- socketService.sendMessage(workspace.account_id, {
- type: 'local_node_sync',
- nodeId: input.id,
- userId: userId,
- versionId: input.versionId,
- workspaceId: input.workspaceId,
- });
-
- await eventBus.publish({
- type: 'node_created',
- userId: userId,
- node: mapNode(result),
- });
-
- return {
- success: true,
- };
- }
- } else {
- const ydoc = new YDoc(input.id, existingNode.state);
- ydoc.applyUpdate(input.state);
-
- const attributes = ydoc.getAttributes();
- const state = ydoc.getState();
-
- const updatedNode = await workspaceDatabase
- .updateTable('nodes')
- .returningAll()
- .set({
- state: state,
- attributes: JSON.stringify(attributes),
- server_created_at: input.serverCreatedAt,
- server_updated_at: input.serverUpdatedAt,
- server_version_id: input.versionId,
- updated_at: input.updatedAt,
- updated_by: input.updatedBy,
- version_id: input.versionId,
- })
- .where('id', '=', input.id)
- .where('version_id', '=', existingNode.version_id)
- .executeTakeFirst();
-
- if (updatedNode) {
- socketService.sendMessage(workspace.account_id, {
- type: 'local_node_sync',
- nodeId: input.id,
- userId: userId,
- versionId: input.versionId,
- workspaceId: input.workspaceId,
- });
-
- await eventBus.publish({
- type: 'node_updated',
- userId: userId,
- node: mapNode(updatedNode),
- });
-
- return {
- success: true,
- };
- }
- }
+ return {
+ success: true,
+ };
}
} catch (error) {
// console.error('Error syncing node', error);
diff --git a/apps/desktop/src/main/mutations/server-user-node-sync.ts b/apps/desktop/src/main/mutations/server-user-node-sync.ts
index 6cae340d..4252e9bb 100644
--- a/apps/desktop/src/main/mutations/server-user-node-sync.ts
+++ b/apps/desktop/src/main/mutations/server-user-node-sync.ts
@@ -1,4 +1,3 @@
-import { eventBus } from '@/shared/lib/event-bus';
import { databaseService } from '@/main/data/database-service';
import { socketService } from '@/main/services/socket-service';
import { MutationHandler } from '@/main/types';
@@ -6,6 +5,7 @@ import {
ServerUserNodeSyncMutationInput,
ServerUserNodeSyncMutationOutput,
} from '@/shared/mutations/server-user-node-sync';
+import { nodeService } from '@/main/services/node-service';
export class ServerUserNodeSyncMutationHandler
implements MutationHandler<ServerUserNodeSyncMutationInput>
@@ -20,7 +20,7 @@ export class ServerUserNodeSyncMutationHandler
.where((eb) =>
eb.and([
eb('account_id', '=', input.accountId),
- eb('workspace_id', '=', input.workspaceId),
+ eb('workspace_id', '=', input.userNode.workspaceId),
])
)
.executeTakeFirst();
@@ -32,56 +32,18 @@ export class ServerUserNodeSyncMutationHandler
}
const userId = workspace.user_id;
- const workspaceDatabase =
- await databaseService.getWorkspaceDatabase(userId);
+ const synced = await nodeService.serverUserNodeSync(
+ userId,
+ input.userNode
+ );
- const createdUserNode = await workspaceDatabase
- .insertInto('user_nodes')
- .returningAll()
- .values({
- user_id: input.userId,
- node_id: input.nodeId,
- version_id: input.versionId,
- last_seen_at: input.lastSeenAt,
- last_seen_version_id: input.lastSeenVersionId,
- mentions_count: input.mentionsCount,
- created_at: input.createdAt,
- updated_at: input.updatedAt,
- })
- .onConflict((cb) =>
- cb.columns(['node_id', 'user_id']).doUpdateSet({
- last_seen_at: input.lastSeenAt,
- last_seen_version_id: input.lastSeenVersionId,
- mentions_count: input.mentionsCount,
- updated_at: input.updatedAt,
- version_id: input.versionId,
- })
- )
- .executeTakeFirst();
-
- socketService.sendMessage(workspace.account_id, {
- type: 'local_user_node_sync',
- userId: input.userId,
- nodeId: input.nodeId,
- versionId: input.versionId,
- workspaceId: input.workspaceId,
- });
-
- if (createdUserNode) {
- await eventBus.publish({
- type: 'user_node_created',
- userId: userId,
- userNode: {
- userId: createdUserNode.user_id,
- nodeId: createdUserNode.node_id,
- lastSeenAt: createdUserNode.last_seen_at,
- lastSeenVersionId: createdUserNode.last_seen_version_id,
- mentionsCount: createdUserNode.mentions_count,
- attributes: createdUserNode.attributes,
- versionId: createdUserNode.version_id,
- createdAt: createdUserNode.created_at,
- updatedAt: createdUserNode.updated_at,
- },
+ if (synced) {
+ socketService.sendMessage(workspace.account_id, {
+ type: 'local_user_node_sync',
+ userId: input.userNode.userId,
+ nodeId: input.userNode.nodeId,
+ versionId: input.userNode.versionId,
+ workspaceId: input.userNode.workspaceId,
});
}
} catch (error) {
diff --git a/apps/desktop/src/main/services/account-service.ts b/apps/desktop/src/main/services/account-service.ts
index 722a7463..a671c46d 100644
--- a/apps/desktop/src/main/services/account-service.ts
+++ b/apps/desktop/src/main/services/account-service.ts
@@ -3,10 +3,13 @@ import { databaseService } from '@/main/data/database-service';
import { SelectAccount } from '@/main/data/app/schema';
import { AccountSyncOutput } from '@colanode/core';
import { httpClient } from '@/shared/lib/http-client';
-import { mapAccount, mapWorkspace } from '@/main/utils';
+import {
+ getWorkspaceDirectoryPath,
+ mapAccount,
+ mapWorkspace,
+} from '@/main/utils';
import { getAccountAvatarsDirectoryPath } from '@/main/utils';
import { eventBus } from '@/shared/lib/event-bus';
-import { workspaceService } from '@/main/services/workspace-service';
import { serverService } from '@/main/services/server-service';
class AccountService {
@@ -140,7 +143,7 @@ class AccountService {
);
if (!updatedWorkspace) {
- await workspaceService.deleteWorkspace(workspace.user_id);
+ await this.deleteWorkspace(workspace.user_id);
}
}
}
@@ -153,7 +156,7 @@ class AccountService {
.execute();
for (const workspace of workspaces) {
- await workspaceService.deleteWorkspace(workspace.user_id);
+ await this.deleteWorkspace(workspace.user_id);
}
const avatarsDir = getAccountAvatarsDirectoryPath(account.id);
@@ -230,6 +233,31 @@ class AccountService {
}
}
}
+
+ private async deleteWorkspace(userId: string): Promise<boolean> {
+ const deletedWorkspace = await databaseService.appDatabase
+ .deleteFrom('workspaces')
+ .returningAll()
+ .where('user_id', '=', userId)
+ .executeTakeFirst();
+
+ if (!deletedWorkspace) {
+ return false;
+ }
+
+ await databaseService.deleteWorkspaceDatabase(userId);
+ const workspaceDir = getWorkspaceDirectoryPath(userId);
+ if (fs.existsSync(workspaceDir)) {
+ fs.rmSync(workspaceDir, { recursive: true });
+ }
+
+ await eventBus.publish({
+ type: 'workspace_deleted',
+ workspace: mapWorkspace(deletedWorkspace),
+ });
+
+ return true;
+ }
}
export const accountService = new AccountService();
diff --git a/apps/desktop/src/main/services/node-service.ts b/apps/desktop/src/main/services/node-service.ts
index 1b8fc3ff..2ca8c127 100644
--- a/apps/desktop/src/main/services/node-service.ts
+++ b/apps/desktop/src/main/services/node-service.ts
@@ -6,6 +6,8 @@ import {
LocalCreateNodeChangeData,
LocalDeleteNodeChangeData,
LocalUpdateNodeChangeData,
+ ServerNodeState,
+ ServerUserNodeState,
} from '@colanode/core';
import { YDoc } from '@colanode/crdt';
import { generateId, IdType } from '@colanode/core';
@@ -15,6 +17,7 @@ import { CreateDownload, CreateUpload } from '@/main/data/workspace/schema';
import { eventBus } from '@/shared/lib/event-bus';
import { Download } from '@/shared/types/nodes';
import { Upload } from '@/shared/types/nodes';
+import { SelectWorkspace } from '@/main/data/app/schema';
export type CreateNodeInput = {
id: string;
@@ -24,16 +27,11 @@ export type CreateNodeInput = {
};
class NodeService {
- async createNode(userId: string, input: CreateNodeInput | CreateNodeInput[]) {
- const workspace = await databaseService.appDatabase
- .selectFrom('workspaces')
- .selectAll()
- .where('user_id', '=', userId)
- .executeTakeFirst();
-
- if (!workspace) {
- throw new Error('Workspace not found');
- }
+ public async createNode(
+ userId: string,
+ input: CreateNodeInput | CreateNodeInput[]
+ ) {
+ const workspace = await this.fetchWorkspace(userId);
const inputs = Array.isArray(input) ? input : [input];
const createdNodes: Node[] = [];
@@ -190,7 +188,7 @@ class NodeService {
}
}
- async updateNode(
+ public async updateNode(
nodeId: string,
userId: string,
updater: (attributes: NodeAttributes) => NodeAttributes
@@ -211,15 +209,7 @@ class NodeService {
userId: string,
updater: (attributes: NodeAttributes) => NodeAttributes
): Promise<boolean> {
- const workspace = await databaseService.appDatabase
- .selectFrom('workspaces')
- .selectAll()
- .where('user_id', '=', userId)
- .executeTakeFirst();
-
- if (!workspace) {
- throw new Error('Workspace not found');
- }
+ const workspace = await this.fetchWorkspace(userId);
const workspaceDatabase =
await databaseService.getWorkspaceDatabase(userId);
@@ -333,16 +323,8 @@ class NodeService {
return result;
}
- async deleteNode(nodeId: string, userId: string) {
- const workspace = await databaseService.appDatabase
- .selectFrom('workspaces')
- .selectAll()
- .where('user_id', '=', userId)
- .executeTakeFirst();
-
- if (!workspace) {
- throw new Error('Workspace not found');
- }
+ public async deleteNode(nodeId: string, userId: string) {
+ const workspace = await this.fetchWorkspace(userId);
const workspaceDatabase =
await databaseService.getWorkspaceDatabase(userId);
@@ -414,6 +396,188 @@ class NodeService {
});
}
}
+
+ public async serverSync(
+ userId: string,
+ node: ServerNodeState
+ ): Promise<boolean> {
+ let count = 0;
+ while (count++ < 20) {
+ const synced = await this.tryServerSync(userId, node);
+ if (synced) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private async tryServerSync(userId: string, node: ServerNodeState) {
+ const workspaceDatabase =
+ await databaseService.getWorkspaceDatabase(userId);
+
+ const existingNode = await workspaceDatabase
+ .selectFrom('nodes')
+ .where('id', '=', node.id)
+ .selectAll()
+ .executeTakeFirst();
+
+ if (!existingNode) {
+ const ydoc = new YDoc(node.id, node.state);
+ const attributes = ydoc.getAttributes();
+ const state = ydoc.getState();
+
+ const result = await workspaceDatabase
+ .insertInto('nodes')
+ .returningAll()
+ .values({
+ id: node.id,
+ attributes: JSON.stringify(attributes),
+ state: state,
+ created_at: node.createdAt,
+ created_by: node.createdBy,
+ version_id: node.versionId,
+ server_created_at: node.serverCreatedAt,
+ server_version_id: node.versionId,
+ })
+ .onConflict((cb) => cb.doNothing())
+ .executeTakeFirst();
+
+ if (result) {
+ await eventBus.publish({
+ type: 'node_created',
+ userId: userId,
+ node: mapNode(result),
+ });
+
+ return true;
+ }
+ } else {
+ const ydoc = new YDoc(node.id, existingNode.state);
+ ydoc.applyUpdate(node.state);
+
+ const attributes = ydoc.getAttributes();
+ const state = ydoc.getState();
+
+ const updatedNode = await workspaceDatabase
+ .updateTable('nodes')
+ .returningAll()
+ .set({
+ state: state,
+ attributes: JSON.stringify(attributes),
+ server_created_at: node.serverCreatedAt,
+ server_updated_at: node.serverUpdatedAt,
+ server_version_id: node.versionId,
+ updated_at: node.updatedAt,
+ updated_by: node.updatedBy,
+ version_id: node.versionId,
+ })
+ .where('id', '=', node.id)
+ .where('version_id', '=', existingNode.version_id)
+ .executeTakeFirst();
+
+ if (updatedNode) {
+ await eventBus.publish({
+ type: 'node_updated',
+ userId: userId,
+ node: mapNode(updatedNode),
+ });
+
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ public async serverDelete(userId: string, nodeId: string) {
+ const workspaceDatabase =
+ await databaseService.getWorkspaceDatabase(userId);
+
+ const deletedNode = await workspaceDatabase
+ .deleteFrom('nodes')
+ .returningAll()
+ .where('id', '=', nodeId)
+ .executeTakeFirst();
+
+ if (deletedNode) {
+ await eventBus.publish({
+ type: 'node_deleted',
+ userId,
+ node: mapNode(deletedNode),
+ });
+ }
+
+ return false;
+ }
+
+ public async serverUserNodeSync(
+ userId: string,
+ userNode: ServerUserNodeState
+ ): Promise<boolean> {
+ const workspaceDatabase =
+ await databaseService.getWorkspaceDatabase(userId);
+
+ const createdUserNode = await workspaceDatabase
+ .insertInto('user_nodes')
+ .returningAll()
+ .values({
+ user_id: userNode.userId,
+ node_id: userNode.nodeId,
+ version_id: userNode.versionId,
+ last_seen_at: userNode.lastSeenAt,
+ last_seen_version_id: userNode.lastSeenVersionId,
+ mentions_count: userNode.mentionsCount,
+ created_at: userNode.createdAt,
+ updated_at: userNode.updatedAt,
+ })
+ .onConflict((cb) =>
+ cb.columns(['node_id', 'user_id']).doUpdateSet({
+ last_seen_at: userNode.lastSeenAt,
+ last_seen_version_id: userNode.lastSeenVersionId,
+ mentions_count: userNode.mentionsCount,
+ updated_at: userNode.updatedAt,
+ version_id: userNode.versionId,
+ })
+ )
+ .executeTakeFirst();
+
+ if (createdUserNode) {
+ await eventBus.publish({
+ type: 'user_node_created',
+ userId: userId,
+ userNode: {
+ userId: createdUserNode.user_id,
+ nodeId: createdUserNode.node_id,
+ lastSeenAt: createdUserNode.last_seen_at,
+ lastSeenVersionId: createdUserNode.last_seen_version_id,
+ mentionsCount: createdUserNode.mentions_count,
+ attributes: createdUserNode.attributes,
+ versionId: createdUserNode.version_id,
+ createdAt: createdUserNode.created_at,
+ updatedAt: createdUserNode.updated_at,
+ },
+ });
+
+ return true;
+ }
+
+ return false;
+ }
+
+ async fetchWorkspace(userId: string): Promise<SelectWorkspace> {
+ const workspace = await databaseService.appDatabase
+ .selectFrom('workspaces')
+ .selectAll()
+ .where('user_id', '=', userId)
+ .executeTakeFirst();
+
+ if (!workspace) {
+ throw new Error('Workspace not found');
+ }
+
+ return workspace;
+ }
}
export const nodeService = new NodeService();
diff --git a/apps/desktop/src/main/services/socket-connection.ts b/apps/desktop/src/main/services/socket-connection.ts
index faf6c5de..d1f9c662 100644
--- a/apps/desktop/src/main/services/socket-connection.ts
+++ b/apps/desktop/src/main/services/socket-connection.ts
@@ -57,30 +57,13 @@ export class SocketConnection {
mutationService.executeMutation({
type: 'server_node_sync',
accountId: this.account.id,
- id: message.id,
- workspaceId: message.workspaceId,
- state: message.state,
- createdAt: message.createdAt,
- createdBy: message.createdBy,
- updatedAt: message.updatedAt,
- updatedBy: message.updatedBy,
- serverCreatedAt: message.serverCreatedAt,
- serverUpdatedAt: message.serverUpdatedAt,
- versionId: message.versionId,
+ node: message.node,
});
} else if (message.type === 'server_user_node_sync') {
mutationService.executeMutation({
type: 'server_user_node_sync',
accountId: this.account.id,
- nodeId: message.nodeId,
- userId: message.userId,
- workspaceId: message.workspaceId,
- versionId: message.versionId,
- lastSeenAt: message.lastSeenAt,
- lastSeenVersionId: message.lastSeenVersionId,
- mentionsCount: message.mentionsCount,
- createdAt: message.createdAt,
- updatedAt: message.updatedAt,
+ userNode: message.userNode,
});
}
};
diff --git a/apps/desktop/src/main/services/workspace-service.ts b/apps/desktop/src/main/services/sync-service.ts
similarity index 62%
rename from apps/desktop/src/main/services/workspace-service.ts
rename to apps/desktop/src/main/services/sync-service.ts
index f83a7d88..b97375a3 100644
--- a/apps/desktop/src/main/services/workspace-service.ts
+++ b/apps/desktop/src/main/services/sync-service.ts
@@ -1,23 +1,24 @@
-import fs from 'fs';
import { databaseService } from '@/main/data/database-service';
-import {
- getWorkspaceDirectoryPath,
- mapChange,
- mapWorkspace,
-} from '@/main/utils';
+import { mapChange } from '@/main/utils';
import { eventBus } from '@/shared/lib/event-bus';
-import { LocalChange, SyncChangesOutput } from '@colanode/core';
+import {
+ LocalChange,
+ SyncChangesOutput,
+ SyncNodeStatesOutput,
+} from '@colanode/core';
import { WorkspaceDatabaseSchema } from '@/main/data/workspace/schema';
import { Kysely } from 'kysely';
import { httpClient } from '@/shared/lib/http-client';
import { serverService } from '@/main/services/server-service';
+import { nodeService } from '@/main/services/node-service';
+import { socketService } from '@/main/services/socket-service';
type WorkspaceSyncState = {
isSyncing: boolean;
scheduledSync: boolean;
};
-class WorkspaceService {
+class SyncService {
private syncStates: Map<string, WorkspaceSyncState> = new Map();
constructor() {
@@ -28,31 +29,6 @@ class WorkspaceService {
});
}
- public async deleteWorkspace(userId: string): Promise<boolean> {
- const deletedWorkspace = await databaseService.appDatabase
- .deleteFrom('workspaces')
- .returningAll()
- .where('user_id', '=', userId)
- .executeTakeFirst();
-
- if (!deletedWorkspace) {
- return false;
- }
-
- await databaseService.deleteWorkspaceDatabase(userId);
- const workspaceDir = getWorkspaceDirectoryPath(userId);
- if (fs.existsSync(workspaceDir)) {
- fs.rmSync(workspaceDir, { recursive: true });
- }
-
- await eventBus.publish({
- type: 'workspace_deleted',
- workspace: mapWorkspace(deletedWorkspace),
- });
-
- return true;
- }
-
public async syncAllWorkspaces() {
const workspaces = await databaseService.appDatabase
.selectFrom('workspaces')
@@ -162,15 +138,111 @@ class WorkspaceService {
.set((eb) => ({ retry_count: eb('retry_count', '+', 1) }))
.where('id', 'in', unsyncedChangeIds)
.execute();
+ }
+ }
- //we just delete changes that have failed to sync for more than 5 times.
- //in the future we might need to revert the change locally.
- await workspaceDatabase
- .deleteFrom('changes')
- .where('retry_count', '>=', 5)
- .execute();
+ const invalidChanges = await this.fetchInvalidChanges(workspaceDatabase);
+ if (invalidChanges.length === 0) {
+ return;
+ }
+
+ const nodeChanges: Record<string, number[]> = {};
+ const changesToDelete: number[] = [];
+ for (const change of invalidChanges) {
+ if (change.retryCount >= 100) {
+ // if the change has been retried 100 times, we delete it (it should never happen)
+ changesToDelete.push(change.id);
+ continue;
+ }
+
+ let nodeId: string | null = null;
+ if (
+ change.data.type === 'node_create' ||
+ change.data.type === 'node_update' ||
+ change.data.type === 'node_delete'
+ ) {
+ nodeId = change.data.id;
+ } else if (change.data.type === 'user_node_update') {
+ nodeId = change.data.nodeId;
+ }
+
+ if (nodeId) {
+ if (!nodeChanges[nodeId]) {
+ nodeChanges[nodeId] = [];
+ }
+
+ nodeChanges[nodeId].push(change.id);
}
}
+
+ const nodeIds = Object.keys(nodeChanges);
+ const { data, status } = await httpClient.post<SyncNodeStatesOutput>(
+ `/v1/sync/states/${workspace.workspace_id}`,
+ {
+ nodeIds,
+ },
+ { domain: workspace.domain, token: workspace.token }
+ );
+
+ if (status !== 200) {
+ return;
+ }
+
+ for (const nodeId of nodeIds) {
+ const changeIds = nodeChanges[nodeId];
+ const states = data.nodes[nodeId];
+
+ if (!states) {
+ const deleted = await nodeService.serverDelete(userId, nodeId);
+ if (deleted) {
+ changesToDelete.push(...changeIds);
+
+ socketService.sendMessage(workspace.account_id, {
+ type: 'local_node_delete',
+ nodeId,
+ workspaceId: workspace.workspace_id,
+ });
+ }
+
+ continue;
+ }
+
+ const nodeSynced = await nodeService.serverSync(userId, states.node);
+ if (nodeSynced) {
+ changesToDelete.push(...changeIds);
+ socketService.sendMessage(workspace.account_id, {
+ type: 'local_node_sync',
+ nodeId,
+ userId,
+ versionId: states.node.versionId,
+ workspaceId: workspace.workspace_id,
+ });
+
+ const userNodeSynced = await nodeService.serverUserNodeSync(
+ userId,
+ states.userNode
+ );
+
+ if (userNodeSynced) {
+ socketService.sendMessage(workspace.account_id, {
+ type: 'local_user_node_sync',
+ nodeId,
+ userId,
+ versionId: states.userNode.versionId,
+ workspaceId: workspace.workspace_id,
+ });
+ }
+
+ changesToDelete.push(...changeIds);
+ }
+ }
+
+ if (changesToDelete.length > 0) {
+ await workspaceDatabase
+ .deleteFrom('changes')
+ .where('id', 'in', changesToDelete)
+ .execute();
+ }
}
private async fetchAndCompactWorkspaceChanges(
@@ -239,6 +311,20 @@ class WorkspaceService {
return changes.filter((change) => !changesToDelete.has(change.id));
}
+
+ private async fetchInvalidChanges(database: Kysely<WorkspaceDatabaseSchema>) {
+ const rows = await database
+ .selectFrom('changes')
+ .selectAll()
+ .where('retry_count', '>=', 5)
+ .execute();
+
+ if (rows.length === 0) {
+ return [];
+ }
+
+ return rows.map(mapChange);
+ }
}
-export const workspaceService = new WorkspaceService();
+export const syncService = new SyncService();
diff --git a/apps/desktop/src/main/utils.ts b/apps/desktop/src/main/utils.ts
index 3477593d..20ff3934 100644
--- a/apps/desktop/src/main/utils.ts
+++ b/apps/desktop/src/main/utils.ts
@@ -130,6 +130,7 @@ export const mapChange = (row: SelectChange): LocalChange => {
id: row.id,
data: JSON.parse(row.data),
createdAt: row.created_at,
+ retryCount: row.retry_count,
};
};
diff --git a/apps/desktop/src/shared/mutations/server-node-sync.ts b/apps/desktop/src/shared/mutations/server-node-sync.ts
index 37daa899..78d0c993 100644
--- a/apps/desktop/src/shared/mutations/server-node-sync.ts
+++ b/apps/desktop/src/shared/mutations/server-node-sync.ts
@@ -1,16 +1,9 @@
+import { ServerNodeState } from '@colanode/core';
+
export type ServerNodeSyncMutationInput = {
type: 'server_node_sync';
accountId: string;
- id: string;
- workspaceId: string;
- state: string;
- createdAt: string;
- createdBy: string;
- updatedAt: string | null;
- updatedBy: string | null;
- serverCreatedAt: string;
- serverUpdatedAt: string | null;
- versionId: string;
+ node: ServerNodeState;
};
export type ServerNodeSyncMutationOutput = {
diff --git a/apps/desktop/src/shared/mutations/server-user-node-sync.ts b/apps/desktop/src/shared/mutations/server-user-node-sync.ts
index 5265bace..ce89033b 100644
--- a/apps/desktop/src/shared/mutations/server-user-node-sync.ts
+++ b/apps/desktop/src/shared/mutations/server-user-node-sync.ts
@@ -1,15 +1,9 @@
+import { ServerUserNodeState } from '@colanode/core';
+
export type ServerUserNodeSyncMutationInput = {
type: 'server_user_node_sync';
accountId: string;
- nodeId: string;
- userId: string;
- workspaceId: string;
- versionId: string;
- lastSeenAt: string | null;
- lastSeenVersionId: string | null;
- mentionsCount: number;
- createdAt: string;
- updatedAt: string | null;
+ userNode: ServerUserNodeState;
};
export type ServerUserNodeSyncMutationOutput = {
diff --git a/apps/server/src/routes/sync.ts b/apps/server/src/routes/sync.ts
index e295d412..d3546ff6 100644
--- a/apps/server/src/routes/sync.ts
+++ b/apps/server/src/routes/sync.ts
@@ -21,16 +21,18 @@ import {
SyncChangeResult,
SyncChangesInput,
SyncChangeStatus,
+ SyncNodeStatesInput,
+ SyncNodeStatesOutput,
+ ServerNodeState,
+ ServerUserNodeState,
} from '@colanode/core';
-import { YDoc } from '@colanode/crdt';
+import { encodeState, YDoc } from '@colanode/crdt';
export const syncRouter = Router();
syncRouter.post(
'/:workspaceId',
async (req: ColanodeRequest, res: ColanodeResponse) => {
- const workspaceId = req.params.workspaceId as string;
- const input = req.body as SyncChangesInput;
if (!req.account) {
return res.status(401).json({
code: ApiError.Unauthorized,
@@ -38,6 +40,9 @@ syncRouter.post(
});
}
+ const workspaceId = req.params.workspaceId as string;
+ const input = req.body as SyncChangesInput;
+
const workspaceUser = await database
.selectFrom('workspace_users')
.selectAll()
@@ -74,6 +79,97 @@ syncRouter.post(
}
);
+syncRouter.post(
+ '/:workspaceId/states',
+ async (req: ColanodeRequest, res: ColanodeResponse) => {
+ if (!req.account) {
+ return res.status(401).json({
+ code: ApiError.Unauthorized,
+ message: 'Unauthorized.',
+ });
+ }
+
+ const workspaceId = req.params.workspaceId as string;
+ const input = req.body as SyncNodeStatesInput;
+ const ids = input.ids;
+
+ const workspaceUser = await database
+ .selectFrom('workspace_users')
+ .selectAll()
+ .where('workspace_id', '=', workspaceId)
+ .where('account_id', '=', req.account.id)
+ .executeTakeFirst();
+
+ if (!workspaceUser) {
+ return res.status(403).json({
+ code: ApiError.Forbidden,
+ message: 'Forbidden.',
+ });
+ }
+
+ const rows = await database
+ .selectFrom('nodes as n')
+ .innerJoin('user_nodes as un', 'un.node_id', 'n.id')
+ .select([
+ 'n.id as node_id',
+ 'n.state as node_state',
+ 'n.created_at as node_created_at',
+ 'n.created_by as node_created_by',
+ 'n.updated_at as node_updated_at',
+ 'n.updated_by as node_updated_by',
+ 'n.version_id as node_version_id',
+ 'un.version_id as user_node_version_id',
+ 'un.last_seen_at as user_node_last_seen_at',
+ 'un.last_seen_version_id as user_node_last_seen_version_id',
+ 'un.mentions_count as user_node_mentions_count',
+ 'un.created_at as user_node_created_at',
+ 'un.updated_at as user_node_updated_at',
+ ])
+ .where('un.user_id', '=', workspaceUser.id)
+ .where('n.workspace_id', '=', workspaceId)
+ .where('n.id', 'in', ids)
+ .execute();
+
+ const states: SyncNodeStatesOutput = {
+ nodes: {},
+ };
+
+ for (const row of rows) {
+ const nodeState: ServerNodeState = {
+ id: row.node_id,
+ workspaceId: workspaceId,
+ state: encodeState(row.node_state),
+ createdAt: row.node_created_at.toISOString(),
+ createdBy: row.node_created_by,
+ updatedAt: row.node_updated_at?.toISOString() ?? null,
+ updatedBy: row.node_updated_by,
+ serverCreatedAt: new Date().toISOString(),
+ serverUpdatedAt: row.user_node_updated_at?.toISOString() ?? null,
+ versionId: row.node_version_id,
+ };
+
+ const userNodeState: ServerUserNodeState = {
+ nodeId: row.node_id,
+ userId: workspaceUser.id,
+ workspaceId: workspaceId,
+ versionId: row.user_node_version_id,
+ lastSeenAt: row.user_node_last_seen_at?.toISOString() ?? null,
+ lastSeenVersionId: row.user_node_last_seen_version_id,
+ mentionsCount: row.user_node_mentions_count,
+ createdAt: row.user_node_created_at.toISOString(),
+ updatedAt: row.user_node_updated_at?.toISOString() ?? null,
+ };
+
+ states.nodes[row.node_id] = {
+ node: nodeState,
+ userNode: userNodeState,
+ };
+ }
+
+ res.status(200).json({ nodes: states });
+ }
+);
+
const handleLocalChange = async (
workspaceUser: SelectWorkspaceUser,
change: LocalChange
diff --git a/apps/server/src/services/synapse.ts b/apps/server/src/services/synapse.ts
index 8e70ef9e..f4b71bef 100644
--- a/apps/server/src/services/synapse.ts
+++ b/apps/server/src/services/synapse.ts
@@ -300,16 +300,18 @@ class SynapseService {
} else {
this.sendSocketMessage(socketConnection, {
type: 'server_node_sync',
- id: node.id,
- workspaceId: data.workspaceId,
- state: fromUint8Array(node.state),
- createdAt: node.created_at.toISOString(),
- createdBy: node.created_by,
- updatedAt: node.updated_at?.toISOString() ?? null,
- updatedBy: node.updated_by ?? null,
- serverCreatedAt: node.server_created_at.toISOString(),
- serverUpdatedAt: node.server_updated_at?.toISOString() ?? null,
- versionId: node.version_id,
+ node: {
+ id: node.id,
+ workspaceId: data.workspaceId,
+ state: fromUint8Array(node.state),
+ createdAt: node.created_at.toISOString(),
+ createdBy: node.created_by,
+ updatedAt: node.updated_at?.toISOString() ?? null,
+ updatedBy: node.updated_by ?? null,
+ serverCreatedAt: node.server_created_at.toISOString(),
+ serverUpdatedAt: node.server_updated_at?.toISOString() ?? null,
+ versionId: node.version_id,
+ },
});
}
}
@@ -344,15 +346,17 @@ class SynapseService {
this.sendSocketMessage(socketConnection, {
type: 'server_user_node_sync',
- userId: data.userId,
- nodeId: data.nodeId,
- lastSeenVersionId: userNode.last_seen_version_id,
- workspaceId: data.workspaceId,
- versionId: userNode.version_id,
- lastSeenAt: userNode.last_seen_at?.toISOString() ?? null,
- createdAt: userNode.created_at.toISOString(),
- updatedAt: userNode.updated_at?.toISOString() ?? null,
- mentionsCount: userNode.mentions_count,
+ userNode: {
+ userId: data.userId,
+ nodeId: data.nodeId,
+ lastSeenVersionId: userNode.last_seen_version_id,
+ workspaceId: data.workspaceId,
+ versionId: userNode.version_id,
+ lastSeenAt: userNode.last_seen_at?.toISOString() ?? null,
+ createdAt: userNode.created_at.toISOString(),
+ updatedAt: userNode.updated_at?.toISOString() ?? null,
+ mentionsCount: userNode.mentions_count,
+ },
});
}
}
@@ -444,31 +448,35 @@ class SynapseService {
if (row.node_version_id !== row.device_node_version_id) {
this.sendSocketMessage(connection, {
type: 'server_node_sync',
- id: row.id,
- workspaceId: row.workspace_id,
- state: fromUint8Array(row.state!),
- createdAt: row.created_at!.toISOString(),
- createdBy: row.created_by!,
- updatedAt: row.updated_at?.toISOString() ?? null,
- updatedBy: row.updated_by ?? null,
- serverCreatedAt: row.server_created_at!.toISOString(),
- serverUpdatedAt: row.server_updated_at?.toISOString() ?? null,
- versionId: row.node_version_id!,
+ node: {
+ id: row.id,
+ workspaceId: row.workspace_id,
+ state: fromUint8Array(row.state!),
+ createdAt: row.created_at!.toISOString(),
+ createdBy: row.created_by!,
+ updatedAt: row.updated_at?.toISOString() ?? null,
+ updatedBy: row.updated_by ?? null,
+ serverCreatedAt: row.server_created_at!.toISOString(),
+ serverUpdatedAt: row.server_updated_at?.toISOString() ?? null,
+ versionId: row.node_version_id!,
+ },
});
}
if (row.user_node_version_id !== row.device_user_node_version_id) {
this.sendSocketMessage(connection, {
type: 'server_user_node_sync',
- nodeId: row.node_id,
- userId: row.user_id,
- workspaceId: row.workspace_id,
- versionId: row.user_node_version_id!,
- lastSeenAt: row.last_seen_at?.toISOString() ?? null,
- lastSeenVersionId: row.last_seen_version_id ?? null,
- mentionsCount: row.mentions_count,
- createdAt: row.created_at!.toISOString(),
- updatedAt: row.updated_at?.toISOString() ?? null,
+ userNode: {
+ nodeId: row.node_id,
+ userId: row.user_id,
+ workspaceId: row.workspace_id,
+ versionId: row.user_node_version_id!,
+ lastSeenAt: row.last_seen_at?.toISOString() ?? null,
+ lastSeenVersionId: row.last_seen_version_id ?? null,
+ mentionsCount: row.mentions_count,
+ createdAt: row.created_at!.toISOString(),
+ updatedAt: row.updated_at?.toISOString() ?? null,
+ },
});
}
}
diff --git a/packages/core/src/types/messages.ts b/packages/core/src/types/messages.ts
index 6ac2280b..d7880740 100644
--- a/packages/core/src/types/messages.ts
+++ b/packages/core/src/types/messages.ts
@@ -1,3 +1,5 @@
+import { ServerNodeState, ServerUserNodeState } from './sync';
+
export type LocalNodeDeleteMessage = {
type: 'local_node_delete';
nodeId: string;
@@ -28,29 +30,12 @@ export type ServerNodeDeleteMessage = {
export type ServerNodeSyncMessage = {
type: 'server_node_sync';
- id: string;
- workspaceId: string;
- state: string;
- createdAt: string;
- createdBy: string;
- updatedAt: string | null;
- updatedBy: string | null;
- serverCreatedAt: string;
- serverUpdatedAt: string | null;
- versionId: string;
+ node: ServerNodeState;
};
export type ServerUserNodeSyncMessage = {
type: 'server_user_node_sync';
- nodeId: string;
- userId: string;
- workspaceId: string;
- versionId: string;
- lastSeenAt: string | null;
- lastSeenVersionId: string | null;
- mentionsCount: number;
- createdAt: string;
- updatedAt: string | null;
+ userNode: ServerUserNodeState;
};
export type Message =
diff --git a/packages/core/src/types/sync.ts b/packages/core/src/types/sync.ts
index 9b54f5ac..685524a7 100644
--- a/packages/core/src/types/sync.ts
+++ b/packages/core/src/types/sync.ts
@@ -2,6 +2,19 @@ export type SyncChangesInput = {
changes: LocalChange[];
};
+export type SyncNodeStatesInput = {
+ ids: string[];
+};
+
+export type SyncNodeStatesOutput = {
+ nodes: Record<string, SyncNodeStateResult | null>;
+};
+
+export type SyncNodeStateResult = {
+ node: ServerNodeState;
+ userNode: ServerUserNodeState;
+};
+
export type SyncChangesOutput = {
results: SyncChangeResult[];
};
@@ -17,6 +30,7 @@ export type LocalChange = {
id: number;
data: LocalChangeData;
createdAt: string;
+ retryCount: number;
};
export type LocalCreateNodeChangeData = {
@@ -59,3 +73,28 @@ export type LocalChangeData =
| LocalUpdateNodeChangeData
| LocalDeleteNodeChangeData
| LocalUserNodeChangeData;
+
+export type ServerNodeState = {
+ id: string;
+ workspaceId: string;
+ state: string;
+ createdAt: string;
+ createdBy: string;
+ updatedAt: string | null;
+ updatedBy: string | null;
+ serverCreatedAt: string;
+ serverUpdatedAt: string | null;
+ versionId: string;
+};
+
+export type ServerUserNodeState = {
+ nodeId: string;
+ userId: string;
+ workspaceId: string;
+ versionId: string;
+ lastSeenAt: string | null;
+ lastSeenVersionId: string | null;
+ mentionsCount: number;
+ createdAt: string;
+ updatedAt: string | null;
+};
diff --git a/packages/crdt/src/index.ts b/packages/crdt/src/index.ts
index 0ce407c1..c7642eef 100644
--- a/packages/crdt/src/index.ts
+++ b/packages/crdt/src/index.ts
@@ -5,6 +5,14 @@ import { isEqual } from 'lodash-es';
import { diffChars } from 'diff';
import { fromUint8Array, toUint8Array } from 'js-base64';
+export const encodeState = (state: Uint8Array) => {
+ return fromUint8Array(state);
+};
+
+export const decodeState = (state: string) => {
+ return toUint8Array(state);
+};
+
export class YDoc {
private doc: Y.Doc;
private updates: Uint8Array[] = [];