Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Refactor sync consumers

Hakan Shehu6286147f2ec3
diff --git a/apps/desktop/src/main/consumers/base.ts b/apps/desktop/src/main/consumers/base.ts
new file mode 100644
index 00000000..c22530cd
--- /dev/null
+++ b/apps/desktop/src/main/consumers/base.ts
@@ -0,0 +1,64 @@
+import { Kysely } from 'kysely';
+import { Message } from '@colanode/core';
+
+import { socketService } from '@/main/services/socket-service';
+import { WorkspaceDatabaseSchema } from '@/main/data/workspace/schema';
+
+export type ConsumerStatus = 'idle' | 'fetching' | 'syncing';
+
+export abstract class BaseConsumer {
+ protected status: ConsumerStatus = 'idle';
+ protected userId: string;
+ protected accountId: string;
+ protected workspaceDatabase: Kysely<WorkspaceDatabaseSchema>;
+
+ private cursorValue: bigint = 0n;
+
+ constructor(
+ userId: string,
+ accountId: string,
+ workspaceDatabase: Kysely<WorkspaceDatabaseSchema>
+ ) {
+ this.userId = userId;
+ this.accountId = accountId;
+ this.workspaceDatabase = workspaceDatabase;
+ }
+
+ protected abstract get cursorKey(): string;
+
+ protected get cursor() {
+ return this.cursorValue;
+ }
+
+ protected async initCursor() {
+ const cursor = await this.workspaceDatabase
+ .selectFrom('cursors')
+ .select('value')
+ .where('key', '=', this.cursorKey)
+ .executeTakeFirst();
+
+ this.cursorValue = cursor?.value ?? 0n;
+ }
+
+ protected async setCursor(cursor: bigint) {
+ this.cursorValue = cursor;
+ await this.workspaceDatabase
+ .insertInto('cursors')
+ .values({
+ key: this.cursorKey,
+ value: cursor,
+ created_at: new Date().toISOString(),
+ })
+ .onConflict((eb) =>
+ eb.column('key').doUpdateSet({
+ value: cursor,
+ updated_at: new Date().toISOString(),
+ })
+ )
+ .execute();
+ }
+
+ protected sendMessage(message: Message) {
+ return socketService.sendMessage(this.accountId, message);
+ }
+}
diff --git a/apps/desktop/src/main/consumers/collaborations.ts b/apps/desktop/src/main/consumers/collaborations.ts
new file mode 100644
index 00000000..728dd55b
--- /dev/null
+++ b/apps/desktop/src/main/consumers/collaborations.ts
@@ -0,0 +1,66 @@
+import {
+ CollaborationsBatchMessage,
+ ConsumeCollaborationsMessage,
+} from '@colanode/core';
+
+import { collaborationService } from '@/main/services/collaboration-service';
+import { BaseConsumer } from '@/main/consumers/base';
+import { createDebugger } from '@/main/debugger';
+
+export class CollaborationsConsumer extends BaseConsumer {
+ private readonly debug = createDebugger('consumer:collaborations');
+
+ protected get cursorKey(): string {
+ return 'collaborations';
+ }
+
+ public async init() {
+ await super.initCursor();
+ this.requestCollaborations();
+ }
+
+ public async processCollaborations(message: CollaborationsBatchMessage) {
+ if (this.status === 'syncing') {
+ this.debug(
+ `Syncing of server collaborations already in progress for user ${message.userId}, skipping`
+ );
+ return;
+ }
+
+ this.debug(`Syncing server collaborators for user ${message.userId}`);
+
+ this.status = 'syncing';
+ let cursor: bigint | null = null;
+ try {
+ for (const collaboration of message.collaborations) {
+ await collaborationService.syncServerCollaboration(
+ message.userId,
+ collaboration
+ );
+ cursor = BigInt(collaboration.version);
+ }
+ } catch (error) {
+ this.debug(
+ error,
+ `Error syncing server collaborators for user ${message.userId}`
+ );
+ } finally {
+ if (cursor) {
+ await this.setCursor(cursor);
+ }
+
+ this.status = 'idle';
+ this.requestCollaborations();
+ }
+ }
+
+ private async requestCollaborations() {
+ const message: ConsumeCollaborationsMessage = {
+ type: 'consume_collaborations',
+ userId: this.userId,
+ cursor: this.cursor.toString(),
+ };
+
+ this.sendMessage(message);
+ }
+}
diff --git a/apps/desktop/src/main/consumers/interactions.ts b/apps/desktop/src/main/consumers/interactions.ts
new file mode 100644
index 00000000..cefa734d
--- /dev/null
+++ b/apps/desktop/src/main/consumers/interactions.ts
@@ -0,0 +1,84 @@
+import { Kysely } from 'kysely';
+import {
+ ConsumeInteractionsMessage,
+ InteractionsBatchMessage,
+} from '@colanode/core';
+
+import { interactionService } from '@/main/services/interaction-service';
+import { createDebugger } from '@/main/debugger';
+import { WorkspaceDatabaseSchema } from '@/main/data/workspace/schema';
+import { BaseConsumer } from '@/main/consumers/base';
+
+export class InteractionsConsumer extends BaseConsumer {
+ private readonly debug = createDebugger('consumer:interactions');
+ private readonly rootId: string;
+
+ constructor(
+ userId: string,
+ accountId: string,
+ rootId: string,
+ workspaceDatabase: Kysely<WorkspaceDatabaseSchema>
+ ) {
+ super(userId, accountId, workspaceDatabase);
+ this.rootId = rootId;
+ }
+
+ protected get cursorKey(): string {
+ return `interactions:${this.rootId}`;
+ }
+
+ public async init() {
+ await super.initCursor();
+ this.requestInteractions();
+ }
+
+ public async processInteractions(message: InteractionsBatchMessage) {
+ if (this.status === 'syncing') {
+ this.debug(
+ `Syncing of server interactions already in progress for user ${message.userId}, skipping`
+ );
+ return;
+ }
+
+ this.debug(`Syncing server interactions for user ${message.userId}`);
+
+ this.status = 'syncing';
+ let cursor: bigint | null = null;
+ try {
+ for (const interaction of message.interactions) {
+ await interactionService.applyServerInteraction(
+ message.userId,
+ interaction
+ );
+ cursor = BigInt(interaction.version);
+ }
+ } catch (error) {
+ this.debug(
+ error,
+ `Error syncing server interactions for user ${message.userId}`
+ );
+ } finally {
+ this.debug(
+ `Syncing of server interactions completed for user ${message.userId}`
+ );
+
+ if (cursor) {
+ await this.setCursor(cursor);
+ }
+
+ this.status = 'idle';
+ this.requestInteractions();
+ }
+ }
+
+ private async requestInteractions() {
+ const message: ConsumeInteractionsMessage = {
+ type: 'consume_interactions',
+ userId: this.userId,
+ rootId: this.rootId,
+ cursor: this.cursor.toString(),
+ };
+
+ this.sendMessage(message);
+ }
+}
diff --git a/apps/desktop/src/main/consumers/transactions.ts b/apps/desktop/src/main/consumers/transactions.ts
new file mode 100644
index 00000000..6427872c
--- /dev/null
+++ b/apps/desktop/src/main/consumers/transactions.ts
@@ -0,0 +1,81 @@
+import { Kysely } from 'kysely';
+import {
+ ConsumeTransactionsMessage,
+ TransactionsBatchMessage,
+} from '@colanode/core';
+
+import { nodeService } from '@/main/services/node-service';
+import { createDebugger } from '@/main/debugger';
+import { WorkspaceDatabaseSchema } from '@/main/data/workspace/schema';
+import { BaseConsumer } from '@/main/consumers/base';
+
+export class TransactionsConsumer extends BaseConsumer {
+ private readonly debug = createDebugger('consumer:transactions');
+ private readonly rootId: string;
+
+ constructor(
+ userId: string,
+ accountId: string,
+ rootId: string,
+ workspaceDatabase: Kysely<WorkspaceDatabaseSchema>
+ ) {
+ super(userId, accountId, workspaceDatabase);
+ this.rootId = rootId;
+ }
+
+ protected get cursorKey(): string {
+ return `transactions:${this.rootId}`;
+ }
+
+ public async init() {
+ await super.initCursor();
+ this.requestTransactions();
+ }
+
+ public async processTransactions(message: TransactionsBatchMessage) {
+ if (this.status === 'syncing') {
+ this.debug(
+ `Syncing of server transactions already in progress for user ${message.userId}, skipping`
+ );
+ return;
+ }
+
+ this.debug(`Syncing server transactions for user ${message.userId}`);
+
+ this.status = 'syncing';
+ let cursor: bigint | null = null;
+ try {
+ for (const transaction of message.transactions) {
+ await nodeService.applyServerTransaction(message.userId, transaction);
+ cursor = BigInt(transaction.version);
+ }
+ } catch (error) {
+ this.debug(
+ error,
+ `Error syncing server transactions for user ${message.userId}`
+ );
+ } finally {
+ this.debug(
+ `Syncing of server transactions completed for user ${message.userId}`
+ );
+
+ if (cursor) {
+ await this.setCursor(cursor);
+ }
+
+ this.status = 'idle';
+ this.requestTransactions();
+ }
+ }
+
+ private async requestTransactions() {
+ const message: ConsumeTransactionsMessage = {
+ type: 'consume_transactions',
+ userId: this.userId,
+ rootId: this.rootId,
+ cursor: this.cursor.toString(),
+ };
+
+ this.sendMessage(message);
+ }
+}
diff --git a/apps/desktop/src/main/consumers/users.ts b/apps/desktop/src/main/consumers/users.ts
new file mode 100644
index 00000000..173441dd
--- /dev/null
+++ b/apps/desktop/src/main/consumers/users.ts
@@ -0,0 +1,55 @@
+import { ConsumeUsersMessage, UsersBatchMessage } from '@colanode/core';
+
+import { userService } from '@/main/services/user-service';
+import { BaseConsumer } from '@/main/consumers/base';
+import { createDebugger } from '@/main/debugger';
+
+export class UsersConsumer extends BaseConsumer {
+ private readonly debug = createDebugger('consumer:users');
+
+ protected get cursorKey(): string {
+ return 'users';
+ }
+
+ public async init() {
+ await super.initCursor();
+ this.requestUsers();
+ }
+
+ public async processUsers(message: UsersBatchMessage) {
+ this.status = 'syncing';
+ let cursor: bigint | null = null;
+ try {
+ for (const user of message.users) {
+ await userService.syncServerUser(message.userId, user);
+ cursor = BigInt(user.version);
+ }
+ } catch (error) {
+ this.debug(
+ error,
+ `Error syncing server users for user ${message.userId}`
+ );
+ } finally {
+ this.debug(
+ `Syncing of server users completed for user ${message.userId}`
+ );
+
+ if (cursor) {
+ await this.setCursor(cursor);
+ }
+
+ this.status = 'idle';
+ this.requestUsers();
+ }
+ }
+
+ private async requestUsers() {
+ const message: ConsumeUsersMessage = {
+ type: 'consume_users',
+ userId: this.userId,
+ cursor: this.cursor.toString(),
+ };
+
+ this.sendMessage(message);
+ }
+}
diff --git a/apps/desktop/src/main/data/workspace/migrations.ts b/apps/desktop/src/main/data/workspace/migrations.ts
index 55a67fa4..d9211ab0 100644
--- a/apps/desktop/src/main/data/workspace/migrations.ts
+++ b/apps/desktop/src/main/data/workspace/migrations.ts
@@ -279,7 +279,7 @@ const createCursorsTable: Migration = {
up: async (db) => {
await db.schema
.createTable('cursors')
- .addColumn('type', 'text', (col) => col.notNull().primaryKey())
+ .addColumn('key', 'text', (col) => col.notNull().primaryKey())
.addColumn('value', 'integer', (col) => col.notNull().defaultTo(0))
.addColumn('created_at', 'text', (col) => col.notNull())
.addColumn('updated_at', 'text')
diff --git a/apps/desktop/src/main/data/workspace/schema.ts b/apps/desktop/src/main/data/workspace/schema.ts
index 961360a0..8cd12c12 100644
--- a/apps/desktop/src/main/data/workspace/schema.ts
+++ b/apps/desktop/src/main/data/workspace/schema.ts
@@ -1,9 +1,4 @@
-import {
- InteractionAttribute,
- NodeType,
- SyncConsumerType,
- WorkspaceRole,
-} from '@colanode/core';
+import { InteractionAttribute, NodeType, WorkspaceRole } from '@colanode/core';
import { ColumnType, Insertable, Selectable, Updateable } from 'kysely';
interface UserTable {
@@ -157,7 +152,7 @@ export type CreateNodeText = Insertable<NodeTextTable>;
export type UpdateNodeText = Updateable<NodeTextTable>;
interface CursorTable {
- type: ColumnType<SyncConsumerType, SyncConsumerType, never>;
+ key: ColumnType<string, string, never>;
value: ColumnType<bigint, bigint, bigint>;
created_at: ColumnType<string, string, never>;
updated_at: ColumnType<string | null, string | null, string | null>;
diff --git a/apps/desktop/src/main/jobs/sync-incomplete-transactions.ts b/apps/desktop/src/main/jobs/sync-incomplete-transactions.ts
deleted file mode 100644
index f46ed07e..00000000
--- a/apps/desktop/src/main/jobs/sync-incomplete-transactions.ts
+++ /dev/null
@@ -1,155 +0,0 @@
-import { GetTransactionsOutput } from '@colanode/core';
-import { sql } from 'kysely';
-
-import { SelectTransaction } from '@/main/data/workspace/schema';
-import { nodeService } from '@/main/services/node-service';
-import { fetchCursor, fetchWorkspaceCredentials } from '@/main/utils';
-import { createDebugger } from '@/main/debugger';
-import { databaseService } from '@/main/data/database-service';
-import { serverService } from '@/main/services/server-service';
-import { JobHandler } from '@/main/jobs';
-import { httpClient } from '@/shared/lib/http-client';
-
-export type SyncIncompleteTransactionsInput = {
- type: 'sync_incomplete_transactions';
- userId: string;
-};
-
-declare module '@/main/jobs' {
- interface JobMap {
- sync_incomplete_transactions: {
- input: SyncIncompleteTransactionsInput;
- };
- }
-}
-
-export class SyncIncompleteTransactionsJobHandler
- implements JobHandler<SyncIncompleteTransactionsInput>
-{
- public triggerDebounce = 100;
- public interval = 1000 * 60;
-
- private readonly debug = createDebugger('job:sync-incomplete-transactions');
-
- public async handleJob(input: SyncIncompleteTransactionsInput) {
- this.debug(`Syncing incomplete transactions for user ${input.userId}`);
-
- const workspaceDatabase = await databaseService.getWorkspaceDatabase(
- input.userId
- );
-
- const incompleteTransactions = await workspaceDatabase
- .selectFrom('transactions')
- .selectAll()
- .where('status', '=', 'incomplete')
- .execute();
-
- if (incompleteTransactions.length === 0) {
- this.debug(
- `No incomplete transactions found for user ${input.userId}, skipping`
- );
- return;
- }
-
- const credentials = await fetchWorkspaceCredentials(input.userId);
- if (!credentials) {
- this.debug(
- `No workspace credentials found for user ${input.userId}, skipping`
- );
- return;
- }
-
- if (!serverService.isAvailable(credentials.serverDomain)) {
- this.debug(
- `Server ${credentials.serverDomain} is not available, skipping`
- );
- return;
- }
-
- const groupedByNodeId = incompleteTransactions.reduce<
- Record<string, SelectTransaction[]>
- >((acc, transaction) => {
- acc[transaction.node_id] = [
- ...(acc[transaction.node_id] ?? []),
- transaction,
- ];
- return acc;
- }, {});
-
- for (const [nodeId, transactions] of Object.entries(groupedByNodeId)) {
- try {
- this.debug(
- `Syncing incomplete transactions for node ${nodeId} for user ${input.userId}`
- );
-
- const { data } = await httpClient.get<GetTransactionsOutput>(
- `/v1/workspaces/${credentials.workspaceId}/transactions/${nodeId}`,
- {
- domain: credentials.serverDomain,
- token: credentials.token,
- }
- );
-
- if (data.transactions.length === 0) {
- this.debug(
- `No transactions found for node ${nodeId} for user ${input.userId}, deleting`
- );
-
- await workspaceDatabase
- .deleteFrom('transactions')
- .where(
- 'id',
- 'in',
- transactions.map((t) => t.id)
- )
- .execute();
- continue;
- }
-
- const cursor = await fetchCursor(input.userId, 'transactions');
- const synced = await nodeService.replaceTransactions(
- input.userId,
- nodeId,
- data.transactions,
- cursor
- );
-
- if (!synced) {
- this.debug(
- `Failed to sync transactions for node ${nodeId} for user ${input.userId}, incrementing retry count`
- );
-
- await workspaceDatabase
- .updateTable('transactions')
- .set({ retry_count: sql`retry_count + 1` })
- .where(
- 'id',
- 'in',
- transactions.map((t) => t.id)
- )
- .execute();
- } else {
- this.debug(
- `Successfully synced transactions for node ${nodeId} for user ${input.userId}, resetting retry count`
- );
-
- await workspaceDatabase
- .updateTable('transactions')
- .set({ retry_count: sql`retry_count + 1` })
- .where(
- 'id',
- 'in',
- transactions.map((t) => t.id)
- )
- .where('status', '=', 'incomplete')
- .execute();
- }
- } catch (error) {
- this.debug(
- error,
- `Error syncing incomplete transactions for node ${nodeId} for user ${input.userId}`
- );
- }
- }
- }
-}
diff --git a/apps/desktop/src/main/jobs/sync-pending-interactions.ts b/apps/desktop/src/main/jobs/sync-pending-interactions.ts
index dea30a44..affc1410 100644
--- a/apps/desktop/src/main/jobs/sync-pending-interactions.ts
+++ b/apps/desktop/src/main/jobs/sync-pending-interactions.ts
@@ -31,6 +31,10 @@ export class SyncPendingInteractionsJobHandler
private readonly debug = createDebugger('job:sync-pending-interactions');
public async handleJob(input: SyncPendingInteractionsInput) {
+ if (this.triggerDebounce === 100) {
+ return;
+ }
+
this.debug(`Sending local pending interactions for user ${input.userId}`);
const workspaceDatabase = await databaseService.getWorkspaceDatabase(
@@ -103,6 +107,7 @@ export class SyncPendingInteractionsJobHandler
const message: SyncInteractionsMessage = {
type: 'sync_interactions',
nodeId,
+ rootId: firstEvent.node_id,
userId: credentials.userId,
events: events.map((e) => ({
attribute: e.attribute,
diff --git a/apps/desktop/src/main/scheduler.ts b/apps/desktop/src/main/scheduler.ts
index 568d8895..48f0592d 100644
--- a/apps/desktop/src/main/scheduler.ts
+++ b/apps/desktop/src/main/scheduler.ts
@@ -10,7 +10,6 @@ import { SyncAccountJobHandler } from '@/main/jobs/sync-account';
import { InitSyncConsumersJobHandler } from '@/main/jobs/init-sync-consumers';
import { RevertInvalidTransactionsJobHandler } from '@/main/jobs/revert-invalid-transactions';
import { SyncPendingTransactionsJobHandler } from '@/main/jobs/sync-pending-transactions';
-import { SyncIncompleteTransactionsJobHandler } from '@/main/jobs/sync-incomplete-transactions';
import { SyncPendingInteractionsJobHandler } from '@/main/jobs/sync-pending-interactions';
import { SyncDeletedTokensJobHandler } from '@/main/jobs/sync-deleted-tokens';
import { ConnectSocketJobHandler } from '@/main/jobs/connect-socket';
@@ -30,7 +29,6 @@ export const jobHandlerMap: JobHandlerMap = {
init_sync_consumers: new InitSyncConsumersJobHandler(),
revert_invalid_transactions: new RevertInvalidTransactionsJobHandler(),
sync_pending_transactions: new SyncPendingTransactionsJobHandler(),
- sync_incomplete_transactions: new SyncIncompleteTransactionsJobHandler(),
sync_pending_interactions: new SyncPendingInteractionsJobHandler(),
sync_deleted_tokens: new SyncDeletedTokensJobHandler(),
connect_socket: new ConnectSocketJobHandler(),
@@ -234,11 +232,6 @@ class Scheduler {
userId,
});
- this.schedule({
- type: 'sync_incomplete_transactions',
- userId,
- });
-
this.schedule({
type: 'sync_pending_interactions',
userId,
@@ -300,13 +293,6 @@ class Scheduler {
return true;
}
- if (
- state.input.type === 'sync_incomplete_transactions' &&
- state.input.userId === userId
- ) {
- return true;
- }
-
if (
state.input.type === 'sync_pending_interactions' &&
state.input.userId === userId
@@ -372,11 +358,6 @@ class Scheduler {
type: 'sync_pending_transactions',
userId: event.userId,
});
- } else if (event.type === 'transaction_incomplete') {
- this.trigger({
- type: 'sync_incomplete_transactions',
- userId: event.userId,
- });
} else if (event.type === 'interaction_event_created') {
this.trigger({
type: 'sync_pending_interactions',
diff --git a/apps/desktop/src/main/services/collaboration-service.ts b/apps/desktop/src/main/services/collaboration-service.ts
index 951789da..df7afd41 100644
--- a/apps/desktop/src/main/services/collaboration-service.ts
+++ b/apps/desktop/src/main/services/collaboration-service.ts
@@ -63,14 +63,19 @@ class CollaborationService {
.where('node_id', '=', collaboration.nodeId)
.execute();
});
- }
- eventBus.publish({
- type: 'collaboration_synced',
- userId,
- nodeId: collaboration.nodeId,
- createdAt: new Date(collaboration.createdAt),
- });
+ eventBus.publish({
+ type: 'collaboration_deleted',
+ userId,
+ nodeId: collaboration.nodeId,
+ });
+ } else {
+ eventBus.publish({
+ type: 'collaboration_created',
+ userId,
+ nodeId: collaboration.nodeId,
+ });
+ }
}
}
diff --git a/apps/desktop/src/main/services/node-service.ts b/apps/desktop/src/main/services/node-service.ts
index 6941c69e..af676776 100644
--- a/apps/desktop/src/main/services/node-service.ts
+++ b/apps/desktop/src/main/services/node-service.ts
@@ -932,32 +932,27 @@ class NodeService {
return { createdNode };
});
- if (createdNode) {
+ if (!createdNode) {
this.debug(
- `Created node ${createdNode.id} with type ${createdNode.type} with transaction ${transaction.id}`
+ `Server create transaction ${transaction.id} for node ${transaction.nodeId} is incomplete`
);
+ return;
+ }
- eventBus.publish({
- type: 'node_created',
- userId,
- node: mapNode(createdNode),
- });
+ this.debug(
+ `Created node ${createdNode.id} with type ${createdNode.type} with transaction ${transaction.id}`
+ );
- await interactionService.setInteraction(userId, createdNode.id, {
- attribute: 'lastReceivedTransactionId',
- value: transaction.id,
- });
- } else {
- this.debug(
- `Server create transaction ${transaction.id} for node ${transaction.nodeId} is incomplete`
- );
+ eventBus.publish({
+ type: 'node_created',
+ userId,
+ node: mapNode(createdNode),
+ });
- eventBus.publish({
- type: 'transaction_incomplete',
- userId,
- transactionId: transaction.id,
- });
- }
+ await interactionService.setInteraction(userId, createdNode.id, {
+ attribute: 'lastReceivedTransactionId',
+ value: transaction.id,
+ });
}
private async applyServerUpdateTransaction(
@@ -1091,32 +1086,27 @@ class NodeService {
return { updatedNode };
});
- if (updatedNode) {
+ if (!updatedNode) {
this.debug(
- `Updated node ${updatedNode.id} with type ${updatedNode.type} with transaction ${transaction.id}`
+ `Server update transaction ${transaction.id} for node ${transaction.nodeId} is incomplete`
);
+ return;
+ }
- eventBus.publish({
- type: 'node_updated',
- userId,
- node: mapNode(updatedNode),
- });
+ this.debug(
+ `Updated node ${updatedNode.id} with type ${updatedNode.type} with transaction ${transaction.id}`
+ );
- await interactionService.setInteraction(userId, updatedNode.id, {
- attribute: 'lastReceivedTransactionId',
- value: transaction.id,
- });
- } else {
- this.debug(
- `Server update transaction ${transaction.id} for node ${transaction.nodeId} is incomplete`
- );
+ eventBus.publish({
+ type: 'node_updated',
+ userId,
+ node: mapNode(updatedNode),
+ });
- eventBus.publish({
- type: 'transaction_incomplete',
- userId,
- transactionId: transaction.id,
- });
- }
+ await interactionService.setInteraction(userId, updatedNode.id, {
+ attribute: 'lastReceivedTransactionId',
+ value: transaction.id,
+ });
}
private async applyServerDeleteTransaction(
diff --git a/apps/desktop/src/main/services/socket-connection.ts b/apps/desktop/src/main/services/socket-connection.ts
index 7f30e4e3..4df9a159 100644
--- a/apps/desktop/src/main/services/socket-connection.ts
+++ b/apps/desktop/src/main/services/socket-connection.ts
@@ -59,13 +59,25 @@ export class SocketConnection {
);
if (message.type === 'transactions_batch') {
- syncService.syncServerTransactions(message);
+ const transactionsConsumer = syncService.getTransactionsConsumer(
+ message.userId,
+ message.rootId
+ );
+ transactionsConsumer?.processTransactions(message);
} else if (message.type === 'collaborations_batch') {
- syncService.syncServerCollaborations(message);
+ const collaborationsConsumer = syncService.getCollaborationsConsumer(
+ message.userId
+ );
+ collaborationsConsumer?.processCollaborations(message);
} else if (message.type === 'interactions_batch') {
- syncService.syncServerInteractions(message);
+ const interactionsConsumer = syncService.getInteractionsConsumer(
+ message.userId,
+ message.rootId
+ );
+ interactionsConsumer?.processInteractions(message);
} else if (message.type === 'users_batch') {
- syncService.syncServerUsers(message);
+ const usersConsumer = syncService.getUsersConsumer(message.userId);
+ usersConsumer?.processUsers(message);
} else if (message.type === 'account_updated') {
scheduler.trigger({
type: 'sync_account',
diff --git a/apps/desktop/src/main/services/sync-consumer.ts b/apps/desktop/src/main/services/sync-consumer.ts
deleted file mode 100644
index b15ffe06..00000000
--- a/apps/desktop/src/main/services/sync-consumer.ts
+++ /dev/null
@@ -1,254 +0,0 @@
-import {
- CollaborationsBatchMessage,
- InitSyncConsumerMessage,
- InteractionsBatchMessage,
- SyncConsumerType,
- TransactionsBatchMessage,
- UsersBatchMessage,
-} from '@colanode/core';
-
-import { userService } from './user-service';
-
-import { interactionService } from '@/main/services/interaction-service';
-import { nodeService } from '@/main/services/node-service';
-import { collaborationService } from '@/main/services/collaboration-service';
-import { socketService } from '@/main/services/socket-service';
-import { fetchCursor, updateCursor } from '@/main/utils';
-import { createDebugger } from '@/main/debugger';
-
-type SyncConsumerStatus = 'idle' | 'fetching' | 'syncing';
-
-export class SyncConsumer {
- private readonly debug = createDebugger('service:sync-consumer');
- public readonly accountId: string;
- public readonly userId: string;
- public readonly type: SyncConsumerType;
-
- private cursor: bigint | null = null;
- private status: SyncConsumerStatus = 'idle';
-
- constructor(accountId: string, userId: string, type: SyncConsumerType) {
- this.accountId = accountId;
- this.userId = userId;
- this.type = type;
- }
-
- public async init() {
- if (this.status === 'fetching') {
- return;
- }
-
- const cursor = await fetchCursor(this.userId, this.type);
- this.initConsumer(cursor);
- }
-
- public async ping() {
- if (this.status === 'idle' || this.cursor === null) {
- await this.init();
- return;
- }
-
- if (this.status === 'syncing') {
- return;
- }
-
- const cursor = await fetchCursor(this.userId, this.type);
- if (cursor !== this.cursor) {
- this.cursor = cursor;
- this.initConsumer(cursor);
- }
- }
-
- public async syncTransactions(message: TransactionsBatchMessage) {
- if (this.type !== 'transactions') {
- this.debug(
- `Syncing of server transactions not supported for consumer type ${this.type}, skipping`
- );
- return;
- }
-
- if (this.status === 'syncing') {
- this.debug(
- `Syncing of server transactions already in progress for user ${message.userId}, skipping`
- );
- return;
- }
-
- this.debug(`Syncing server transactions for user ${message.userId}`);
-
- this.status = 'syncing';
- let cursor: bigint | null = null;
- try {
- for (const transaction of message.transactions) {
- await nodeService.applyServerTransaction(message.userId, transaction);
- cursor = BigInt(transaction.version);
- }
- } catch (error) {
- this.debug(
- error,
- `Error syncing server transactions for user ${message.userId}`
- );
- } finally {
- this.debug(
- `Syncing of server transactions completed for user ${message.userId}`
- );
-
- if (cursor) {
- this.cursor = cursor;
- this.initConsumer(cursor);
- await updateCursor(message.userId, 'transactions', cursor);
- this.status = 'idle';
- } else {
- this.status = 'idle';
- this.ping();
- }
- }
- }
-
- public async syncCollaborations(message: CollaborationsBatchMessage) {
- if (this.type !== 'collaborations') {
- this.debug(
- `Syncing of server collaborations not supported for consumer type ${this.type}, skipping`
- );
- return;
- }
-
- if (this.status === 'syncing') {
- this.debug(
- `Syncing of server collaborations already in progress for user ${message.userId}, skipping`
- );
- return;
- }
-
- this.debug(`Syncing server collaborators for user ${message.userId}`);
-
- this.status = 'syncing';
- let cursor: bigint | null = null;
- try {
- for (const collaboration of message.collaborations) {
- await collaborationService.syncServerCollaboration(
- message.userId,
- collaboration
- );
- cursor = BigInt(collaboration.version);
- }
- } catch (error) {
- this.debug(
- error,
- `Error syncing server collaborators for user ${message.userId}`
- );
- } finally {
- if (cursor) {
- await updateCursor(message.userId, 'collaborations', cursor);
- this.cursor = cursor;
- this.status = 'idle';
- this.initConsumer(cursor);
- } else {
- this.status = 'idle';
- this.ping();
- }
- }
- }
-
- public async syncInteractions(message: InteractionsBatchMessage) {
- if (this.type !== 'interactions') {
- this.debug(
- `Syncing of server interactions not supported for consumer type ${this.type}, skipping`
- );
- return;
- }
-
- if (this.status === 'syncing') {
- this.debug(
- `Syncing of server interactions already in progress for user ${message.userId}, skipping`
- );
- return;
- }
-
- this.debug(`Syncing server interactions for user ${message.userId}`);
-
- this.status = 'syncing';
- let cursor: bigint | null = null;
- try {
- for (const interaction of message.interactions) {
- await interactionService.applyServerInteraction(
- message.userId,
- interaction
- );
- cursor = BigInt(interaction.version);
- }
- } catch (error) {
- this.debug(
- error,
- `Error syncing server interactions for user ${message.userId}`
- );
- } finally {
- this.debug(
- `Syncing of server interactions completed for user ${message.userId}`
- );
-
- if (cursor) {
- await updateCursor(message.userId, 'interactions', cursor);
- this.cursor = cursor;
- this.status = 'idle';
- this.initConsumer(cursor);
- } else {
- this.status = 'idle';
- this.ping();
- }
- }
- }
-
- public async syncUsers(message: UsersBatchMessage) {
- if (this.type !== 'users') {
- this.debug(
- `Syncing of server users not supported for consumer type ${this.type}, skipping`
- );
- return;
- }
-
- this.debug(`Syncing server users for user ${message.userId}`);
-
- this.status = 'syncing';
- let cursor: bigint | null = null;
- try {
- for (const user of message.users) {
- await userService.syncServerUser(message.userId, user);
- cursor = BigInt(user.version);
- }
- } catch (error) {
- this.debug(
- error,
- `Error syncing server users for user ${message.userId}`
- );
- } finally {
- this.debug(
- `Syncing of server users completed for user ${message.userId}`
- );
-
- if (cursor) {
- await updateCursor(message.userId, 'users', cursor);
- this.cursor = cursor;
- this.status = 'idle';
- this.initConsumer(cursor);
- } else {
- this.status = 'idle';
- this.ping();
- }
- }
- }
-
- private initConsumer(cursor: bigint) {
- const message: InitSyncConsumerMessage = {
- type: 'init_sync_consumer',
- userId: this.userId,
- consumerType: this.type,
- cursor: cursor.toString(),
- };
-
- const sent = socketService.sendMessage(this.accountId, message);
- if (sent) {
- this.status = 'fetching';
- }
- }
-}
diff --git a/apps/desktop/src/main/services/sync-service.ts b/apps/desktop/src/main/services/sync-service.ts
index 460d359e..925de278 100644
--- a/apps/desktop/src/main/services/sync-service.ts
+++ b/apps/desktop/src/main/services/sync-service.ts
@@ -1,92 +1,118 @@
-import {
- CollaborationsBatchMessage,
- InteractionsBatchMessage,
- SyncConsumerType,
- TransactionsBatchMessage,
- UsersBatchMessage,
-} from '@colanode/core';
-
-import { SyncConsumer } from '@/main/services/sync-consumer';
+import { databaseService } from '@/main/data/database-service';
+import { UsersConsumer } from '@/main/consumers/users';
+import { TransactionsConsumer } from '@/main/consumers/transactions';
+import { CollaborationsConsumer } from '@/main/consumers/collaborations';
+import { InteractionsConsumer } from '@/main/consumers/interactions';
import { createDebugger } from '@/main/debugger';
+export type NodeConsumersWrapper = {
+ transactions: TransactionsConsumer;
+ interactions: InteractionsConsumer;
+};
+
+export type ConsumersWrapper = {
+ users: UsersConsumer;
+ collaborations: CollaborationsConsumer;
+ nodes: Record<string, NodeConsumersWrapper>;
+};
+
class SyncService {
private readonly debug = createDebugger('service:sync');
- private readonly consumers: Map<string, SyncConsumer> = new Map();
+ private readonly users: Map<string, ConsumersWrapper> = new Map();
public async initUserConsumers(accountId: string, userId: string) {
- await this.createConsumer(accountId, userId, 'transactions');
- await this.createConsumer(accountId, userId, 'collaborations');
- await this.createConsumer(accountId, userId, 'interactions');
- await this.createConsumer(accountId, userId, 'users');
- }
+ const workspaceDatabase =
+ await databaseService.getWorkspaceDatabase(userId);
- public async deleteUserConsumers(userId: string) {
- this.consumers.forEach((consumer, key) => {
- if (consumer.userId === userId) {
- this.consumers.delete(key);
- }
- });
- }
+ let consumers = this.users.get(userId);
+ if (!consumers) {
+ const usersConsumer = new UsersConsumer(
+ userId,
+ accountId,
+ workspaceDatabase
+ );
+ await usersConsumer.init();
- private async createConsumer(
- accountId: string,
- userId: string,
- type: SyncConsumerType
- ) {
- const key = this.getConsumerKey(userId, type);
- const consumer = this.consumers.get(key);
- if (consumer) {
- consumer.ping();
- return;
+ const collaborationsConsumer = new CollaborationsConsumer(
+ userId,
+ accountId,
+ workspaceDatabase
+ );
+ await collaborationsConsumer.init();
+
+ consumers = {
+ users: usersConsumer,
+ collaborations: collaborationsConsumer,
+ nodes: {},
+ };
+
+ this.users.set(userId, consumers);
}
- this.debug(
- `Creating new sync consumer for account ${accountId} and user ${userId} of type ${type}`
- );
- const newConsumer = new SyncConsumer(accountId, userId, type);
- this.consumers.set(key, newConsumer);
+ const collaborations = await workspaceDatabase
+ .selectFrom('collaborations')
+ .selectAll()
+ .execute();
- await newConsumer.init();
- }
+ for (const collaboration of collaborations) {
+ const rootId = collaboration.node_id;
+
+ if (consumers.nodes[rootId]) {
+ continue;
+ }
- public async syncServerTransactions(message: TransactionsBatchMessage) {
- const consumer = this.consumers.get(
- this.getConsumerKey(message.userId, 'transactions')
- );
- if (consumer) {
- consumer.syncTransactions(message);
+ const transactionsConsumer = new TransactionsConsumer(
+ userId,
+ accountId,
+ rootId,
+ workspaceDatabase
+ );
+ await transactionsConsumer.init();
+
+ const interactionsConsumer = new InteractionsConsumer(
+ userId,
+ accountId,
+ rootId,
+ workspaceDatabase
+ );
+ await interactionsConsumer.init();
+
+ consumers.nodes[rootId] = {
+ transactions: transactionsConsumer,
+ interactions: interactionsConsumer,
+ };
}
- }
- public async syncServerCollaborations(message: CollaborationsBatchMessage) {
- const consumer = this.consumers.get(
- this.getConsumerKey(message.userId, 'collaborations')
- );
- if (consumer) {
- consumer.syncCollaborations(message);
+ // check for deleted collaborations and remove them from the consumers
+ for (const nodeId of Object.keys(consumers.nodes)) {
+ if (!collaborations.some((c) => c.node_id === nodeId)) {
+ delete consumers.nodes[nodeId];
+ }
}
}
- public async syncServerInteractions(message: InteractionsBatchMessage) {
- const consumer = this.consumers.get(
- this.getConsumerKey(message.userId, 'interactions')
- );
- if (consumer) {
- consumer.syncInteractions(message);
- }
+ public getUsersConsumer(userId: string): UsersConsumer | undefined {
+ return this.users.get(userId)?.users;
}
- public async syncServerUsers(message: UsersBatchMessage) {
- const consumer = this.consumers.get(
- this.getConsumerKey(message.userId, 'users')
- );
- if (consumer) {
- consumer.syncUsers(message);
- }
+ public getCollaborationsConsumer(
+ userId: string
+ ): CollaborationsConsumer | undefined {
+ return this.users.get(userId)?.collaborations;
}
- private getConsumerKey(userId: string, type: SyncConsumerType) {
- return `${userId}-${type}`;
+ public getTransactionsConsumer(
+ userId: string,
+ rootId: string
+ ): TransactionsConsumer | undefined {
+ return this.users.get(userId)?.nodes[rootId]?.transactions;
+ }
+
+ public getInteractionsConsumer(
+ userId: string,
+ rootId: string
+ ): InteractionsConsumer | undefined {
+ return this.users.get(userId)?.nodes[rootId]?.interactions;
}
}
diff --git a/apps/desktop/src/main/utils.ts b/apps/desktop/src/main/utils.ts
index 54f17afb..c1b1c460 100644
--- a/apps/desktop/src/main/utils.ts
+++ b/apps/desktop/src/main/utils.ts
@@ -1,4 +1,4 @@
-import { LocalTransaction, Node, SyncConsumerType } from '@colanode/core';
+import { LocalTransaction, Node } from '@colanode/core';
import { encodeState } from '@colanode/crdt';
import {
DeleteResult,
@@ -136,44 +136,6 @@ export const fetchWorkspaceCredentials = async (
};
};
-export const updateCursor = async (
- userId: string,
- type: SyncConsumerType,
- cursor: bigint
-) => {
- const workspaceDatabase = await databaseService.getWorkspaceDatabase(userId);
-
- await workspaceDatabase
- .insertInto('cursors')
- .values({
- type,
- value: cursor,
- created_at: new Date().toISOString(),
- })
- .onConflict((eb) =>
- eb.column('type').doUpdateSet({
- value: cursor,
- updated_at: new Date().toISOString(),
- })
- )
- .execute();
-};
-
-export const fetchCursor = async (
- userId: string,
- type: SyncConsumerType
-): Promise<bigint> => {
- const workspaceDatabase = await databaseService.getWorkspaceDatabase(userId);
-
- const cursor = await workspaceDatabase
- .selectFrom('cursors')
- .select('value')
- .where('type', '=', type)
- .executeTakeFirst();
-
- return cursor?.value ?? 0n;
-};
-
export const mapUser = (row: SelectUser): User => {
return {
id: row.id,
diff --git a/apps/desktop/src/shared/types/events.ts b/apps/desktop/src/shared/types/events.ts
index 25d5f322..6dbe20b1 100644
--- a/apps/desktop/src/shared/types/events.ts
+++ b/apps/desktop/src/shared/types/events.ts
@@ -135,17 +135,16 @@ export type TransactionCreatedEvent = {
transaction: LocalTransaction;
};
-export type TransactionIncompleteEvent = {
- type: 'transaction_incomplete';
+export type CollaborationCreatedEvent = {
+ type: 'collaboration_created';
userId: string;
- transactionId: string;
+ nodeId: string;
};
-export type CollaborationSyncedEvent = {
- type: 'collaboration_synced';
+export type CollaborationDeletedEvent = {
+ type: 'collaboration_deleted';
userId: string;
nodeId: string;
- createdAt: Date;
};
export type ServerAvailabilityChangedEvent = {
@@ -195,9 +194,9 @@ export type Event =
| QueryResultUpdatedEvent
| RadarDataUpdatedEvent
| TransactionCreatedEvent
- | TransactionIncompleteEvent
| ServerAvailabilityChangedEvent
| SocketConnectionOpenedEvent
- | CollaborationSyncedEvent
+ | CollaborationCreatedEvent
+ | CollaborationDeletedEvent
| InteractionEventCreatedEvent
| InteractionUpdatedEvent;
diff --git a/apps/server/src/consumers/base.ts b/apps/server/src/consumers/base.ts
new file mode 100644
index 00000000..c9ca15fb
--- /dev/null
+++ b/apps/server/src/consumers/base.ts
@@ -0,0 +1,16 @@
+import { ConnectedUser } from '@/types/users';
+import { Event } from '@/types/events';
+
+export type ConsumerStatus = 'idle' | 'pending' | 'fetching';
+
+export abstract class BaseConsumer {
+ protected status: ConsumerStatus = 'idle';
+ protected user: ConnectedUser;
+ protected cursor: bigint | null = null;
+
+ constructor(user: ConnectedUser) {
+ this.user = user;
+ }
+
+ public abstract processEvent(event: Event): void;
+}
diff --git a/apps/server/src/consumers/collaborations.ts b/apps/server/src/consumers/collaborations.ts
new file mode 100644
index 00000000..ccb275fb
--- /dev/null
+++ b/apps/server/src/consumers/collaborations.ts
@@ -0,0 +1,76 @@
+import {
+ CollaborationsBatchMessage,
+ ConsumeCollaborationsMessage,
+} from '@colanode/core';
+
+import { BaseConsumer } from '@/consumers/base';
+import { Event } from '@/types/events';
+import { database } from '@/data/database';
+import { mapCollaboration } from '@/lib/nodes';
+
+export class CollaborationsConsumer extends BaseConsumer {
+ public processEvent(event: Event): void {
+ if (
+ event.type === 'collaboration_created' &&
+ event.collaboratorId === this.user.userId
+ ) {
+ this.fetchCollaborations();
+ } else if (
+ event.type === 'collaboration_updated' &&
+ event.collaboratorId === this.user.userId
+ ) {
+ this.fetchCollaborations();
+ }
+ }
+
+ public async consume(message: ConsumeCollaborationsMessage) {
+ if (this.status === 'fetching') {
+ return;
+ }
+
+ const cursor = BigInt(message.cursor);
+ if (this.status === 'idle') {
+ this.cursor = cursor;
+ this.status = 'pending';
+ await this.fetchCollaborations();
+ } else if (this.status === 'pending' && this.cursor !== cursor) {
+ this.cursor = cursor;
+ await this.fetchCollaborations();
+ }
+ }
+
+ private async fetchCollaborations() {
+ if (this.cursor === null) {
+ return;
+ }
+
+ if (this.status !== 'pending') {
+ return;
+ }
+
+ this.status = 'fetching';
+ const unsyncedCollaborations = await database
+ .selectFrom('collaborations')
+ .selectAll()
+ .where('workspace_id', '=', this.user.workspaceId)
+ .where('version', '>', this.cursor)
+ .orderBy('version', 'asc')
+ .limit(50)
+ .execute();
+
+ if (unsyncedCollaborations.length === 0) {
+ this.status = 'pending';
+ return;
+ }
+
+ const collaborations = unsyncedCollaborations.map(mapCollaboration);
+ const message: CollaborationsBatchMessage = {
+ type: 'collaborations_batch',
+ userId: this.user.userId,
+ collaborations,
+ };
+
+ this.status = 'idle';
+ this.user.sendMessage(message);
+ }
+}
diff --git a/apps/server/src/consumers/interactions.ts b/apps/server/src/consumers/interactions.ts
new file mode 100644
index 00000000..d08ab0c8
--- /dev/null
+++ b/apps/server/src/consumers/interactions.ts
@@ -0,0 +1,77 @@
+import {
+ ConsumeInteractionsMessage,
+ InteractionsBatchMessage,
+} from '@colanode/core';
+
+import { BaseConsumer } from '@/consumers/base';
+import { Event } from '@/types/events';
+import { database } from '@/data/database';
+import { mapInteraction } from '@/lib/nodes';
+import { ConnectedUser } from '@/types/users';
+
+export class InteractionsConsumer extends BaseConsumer {
+ private readonly rootId: string;
+
+ constructor(user: ConnectedUser, rootId: string) {
+ super(user);
+ this.rootId = rootId;
+ }
+
+ public processEvent(event: Event): void {
+ if (event.type === 'interaction_updated' && event.rootId === this.rootId) {
+ this.fetchInteractions();
+ }
+ }
+
+ public async consume(message: ConsumeInteractionsMessage) {
+ if (this.status === 'fetching') {
+ return;
+ }
+
+ const cursor = BigInt(message.cursor);
+ if (this.status === 'idle') {
+ this.cursor = cursor;
+ this.status = 'pending';
+ await this.fetchInteractions();
+ } else if (this.status === 'pending' && this.cursor !== cursor) {
+ this.cursor = cursor;
+ await this.fetchInteractions();
+ }
+ }
+
+ private async fetchInteractions() {
+ if (this.cursor === null) {
+ return;
+ }
+
+ if (this.status !== 'pending') {
+ return;
+ }
+
+ this.status = 'fetching';
+ const unsyncedInteractions = await database
+ .selectFrom('interactions')
+ .selectAll()
+ .where('node_id', '=', this.rootId)
+ .where('version', '>', this.cursor)
+ .orderBy('version', 'asc')
+ .limit(20)
+ .execute();
+
+ if (unsyncedInteractions.length === 0) {
+ this.status = 'pending';
+ return;
+ }
+
+ const interactions = unsyncedInteractions.map(mapInteraction);
+ const message: InteractionsBatchMessage = {
+ type: 'interactions_batch',
+ userId: this.user.userId,
+ rootId: this.rootId,
+ interactions,
+ };
+
+ this.status = 'idle';
+ this.user.sendMessage(message);
+ }
+}
diff --git a/apps/server/src/consumers/transactions.ts b/apps/server/src/consumers/transactions.ts
new file mode 100644
index 00000000..c3f8783f
--- /dev/null
+++ b/apps/server/src/consumers/transactions.ts
@@ -0,0 +1,81 @@
+import {
+ ConsumeTransactionsMessage,
+ TransactionsBatchMessage,
+} from '@colanode/core';
+
+import { BaseConsumer } from '@/consumers/base';
+import { Event } from '@/types/events';
+import { database } from '@/data/database';
+import { mapTransaction } from '@/lib/nodes';
+import { ConnectedUser } from '@/types/users';
+
+export class TransactionsConsumer extends BaseConsumer {
+ private readonly rootId: string;
+
+ constructor(user: ConnectedUser, rootId: string) {
+ super(user);
+ this.rootId = rootId;
+ }
+
+ public processEvent(event: Event): void {
+ if (event.type === 'node_created' && event.rootId === this.rootId) {
+ this.fetchTransactions();
+ } else if (event.type === 'node_updated' && event.rootId === this.rootId) {
+ this.fetchTransactions();
+ } else if (event.type === 'node_deleted' && event.rootId === this.rootId) {
+ this.fetchTransactions();
+ }
+ }
+
+ public async consume(message: ConsumeTransactionsMessage) {
+ if (this.status === 'fetching') {
+ return;
+ }
+
+ const cursor = BigInt(message.cursor);
+ if (this.status === 'idle') {
+ this.cursor = cursor;
+ this.status = 'pending';
+ await this.fetchTransactions();
+ } else if (this.status === 'pending' && this.cursor !== cursor) {
+ this.cursor = cursor;
+ await this.fetchTransactions();
+ }
+ }
+
+ private async fetchTransactions() {
+ if (this.cursor === null) {
+ return;
+ }
+
+ if (this.status !== 'pending') {
+ return;
+ }
+
+ this.status = 'fetching';
+ const unsyncedTransactions = await database
+ .selectFrom('transactions')
+ .selectAll()
+ .where('root_id', '=', this.rootId)
+ .where('version', '>', this.cursor)
+ .orderBy('version', 'asc')
+ .limit(20)
+ .execute();
+
+ if (unsyncedTransactions.length === 0) {
+ this.status = 'pending';
+ return;
+ }
+
+ const transactions = unsyncedTransactions.map(mapTransaction);
+ const message: TransactionsBatchMessage = {
+ type: 'transactions_batch',
+ userId: this.user.userId,
+ rootId: this.rootId,
+ transactions,
+ };
+
+ this.status = 'idle';
+ this.user.sendMessage(message);
+ }
+}
diff --git a/apps/server/src/consumers/users.ts b/apps/server/src/consumers/users.ts
new file mode 100644
index 00000000..361de3d9
--- /dev/null
+++ b/apps/server/src/consumers/users.ts
@@ -0,0 +1,73 @@
+import { ConsumeUsersMessage, UsersBatchMessage } from '@colanode/core';
+
+import { BaseConsumer } from '@/consumers/base';
+import { Event } from '@/types/events';
+import { database } from '@/data/database';
+import { mapUser } from '@/lib/nodes';
+
+export class UsersConsumer extends BaseConsumer {
+ public processEvent(event: Event): void {
+ if (
+ event.type === 'user_created' &&
+ event.workspaceId === this.user.workspaceId
+ ) {
+ this.fetchUsers();
+ } else if (
+ event.type === 'user_updated' &&
+ event.workspaceId === this.user.workspaceId
+ ) {
+ this.fetchUsers();
+ }
+ }
+
+ public async consume(message: ConsumeUsersMessage) {
+ if (this.status === 'fetching') {
+ return;
+ }
+
+ const cursor = BigInt(message.cursor);
+ if (this.status === 'idle') {
+ this.cursor = cursor;
+ this.status = 'pending';
+ await this.fetchUsers();
+ } else if (this.status === 'pending' && this.cursor !== cursor) {
+ this.cursor = cursor;
+ await this.fetchUsers();
+ }
+ }
+
+ private async fetchUsers() {
+ if (this.cursor === null) {
+ return;
+ }
+
+ if (this.status !== 'pending') {
+ return;
+ }
+
+ this.status = 'fetching';
+ const unsyncedUsers = await database
+ .selectFrom('users')
+ .selectAll()
+ .where('workspace_id', '=', this.user.workspaceId)
+ .where('version', '>', this.cursor)
+ .orderBy('version', 'asc')
+ .limit(50)
+ .execute();
+
+ if (unsyncedUsers.length === 0) {
+ this.status = 'pending';
+ return;
+ }
+
+ const users = unsyncedUsers.map(mapUser);
+ const message: UsersBatchMessage = {
+ type: 'users_batch',
+ userId: this.user.userId,
+ users,
+ };
+
+ this.status = 'idle';
+ this.user.sendMessage(message);
+ }
+}
diff --git a/apps/server/src/services/interaction-service.ts b/apps/server/src/services/interaction-service.ts
index 2e07347a..a50f95c0 100644
--- a/apps/server/src/services/interaction-service.ts
+++ b/apps/server/src/services/interaction-service.ts
@@ -89,6 +89,7 @@ class InteractionService {
type: 'interaction_updated',
userId: message.userId,
nodeId: message.nodeId,
+ rootId: message.rootId,
workspaceId,
});
@@ -115,6 +116,7 @@ class InteractionService {
type: 'interaction_updated',
userId: message.userId,
nodeId: message.nodeId,
+ rootId: message.rootId,
workspaceId,
});
diff --git a/apps/server/src/services/socket-connection.ts b/apps/server/src/services/socket-connection.ts
index 410f4a2d..aa39211d 100644
--- a/apps/server/src/services/socket-connection.ts
+++ b/apps/server/src/services/socket-connection.ts
@@ -1,51 +1,43 @@
import { WebSocket } from 'ws';
import {
- InitSyncConsumerMessage,
- InteractionsBatchMessage,
Message,
- TransactionsBatchMessage,
- SyncConsumerType,
WorkspaceStatus,
- UsersBatchMessage,
- CollaborationsBatchMessage,
+ ConsumeUsersMessage,
+ ConsumeCollaborationsMessage,
+ ConsumeTransactionsMessage,
+ ConsumeInteractionsMessage,
} from '@colanode/core';
import { interactionService } from '@/services/interaction-service';
import { createLogger } from '@/lib/logger';
import { RequestAccount } from '@/types/api';
import { database } from '@/data/database';
-import {
- mapCollaboration,
- mapInteraction,
- mapTransaction,
- mapUser,
-} from '@/lib/nodes';
import {
AccountUpdatedEvent,
- CollaborationCreatedEvent,
- CollaborationUpdatedEvent,
Event,
- InteractionUpdatedEvent,
- NodeCreatedEvent,
- NodeDeletedEvent,
- NodeUpdatedEvent,
- UserCreatedEvent,
- UserUpdatedEvent,
WorkspaceDeletedEvent,
WorkspaceUpdatedEvent,
} from '@/types/events';
+import { ConnectedUser } from '@/types/users';
+import { UsersConsumer } from '@/consumers/users';
+import { CollaborationsConsumer } from '@/consumers/collaborations';
+import { TransactionsConsumer } from '@/consumers/transactions';
+import { InteractionsConsumer } from '@/consumers/interactions';
+
+type NodeConsumersWrapper = {
+ transactions: TransactionsConsumer;
+ interactions: InteractionsConsumer;
+};
-type SocketSyncConsumer = {
- type: SyncConsumerType;
- cursor: string;
- fetching: boolean;
+type ConsumersWrapper = {
+ users: UsersConsumer;
+ collaborations: CollaborationsConsumer;
+ nodes: Map<string, NodeConsumersWrapper>;
};
type SocketUser = {
- userId: string;
- workspaceId: string;
- rootIds: string[];
- consumers: Map<SyncConsumerType, SocketSyncConsumer>;
+ user: ConnectedUser;
+ consumers: ConsumersWrapper;
};
export class SocketConnection {
@@ -84,44 +76,43 @@ export class SocketConnection {
}
public handleEvent(event: Event) {
- if (event.type === 'node_created') {
- this.handleNodeCreatedEvent(event);
- } else if (event.type === 'node_updated') {
- this.handleNodeUpdatedEvent(event);
- } else if (event.type === 'node_deleted') {
- this.handleNodeDeletedEvent(event);
- } else if (event.type === 'collaboration_created') {
- this.handleCollaborationCreatedEvent(event);
- } else if (event.type === 'collaboration_updated') {
- this.handleCollaborationUpdatedEvent(event);
- } else if (event.type === 'interaction_updated') {
- this.handleInteractionUpdatedEvent(event);
- } else if (event.type === 'account_updated') {
+ if (event.type === 'account_updated') {
this.handleAccountUpdatedEvent(event);
- } else if (event.type === 'user_created') {
- this.handleUserCreatedEvent(event);
- } else if (event.type === 'user_updated') {
- this.handleUserUpdatedEvent(event);
} else if (event.type === 'workspace_updated') {
this.handleWorkspaceUpdatedEvent(event);
} else if (event.type === 'workspace_deleted') {
this.handleWorkspaceDeletedEvent(event);
+ } else {
+ for (const user of this.users.values()) {
+ user.consumers.users.processEvent(event);
+ user.consumers.collaborations.processEvent(event);
+ for (const node of user.consumers.nodes.values()) {
+ node.transactions.processEvent(event);
+ node.interactions.processEvent(event);
+ }
+ }
}
}
private async handleMessage(message: Message) {
this.logger.trace(message, `Socket message from ${this.account.id}`);
- if (message.type === 'init_sync_consumer') {
- this.handleInitSyncConsumer(message);
+ if (message.type === 'consume_users') {
+ this.handleConsumeUsers(message);
+ } else if (message.type === 'consume_collaborations') {
+ this.handleConsumeCollaborations(message);
+ } else if (message.type === 'consume_transactions') {
+ this.handleConsumeTransactions(message);
+ } else if (message.type === 'consume_interactions') {
+ this.handleConsumeInteractions(message);
} else if (message.type === 'sync_interactions') {
interactionService.syncLocalInteractions(this.account.id, message);
}
}
- private async handleInitSyncConsumer(message: InitSyncConsumerMessage) {
+ private async handleConsumeUsers(message: ConsumeUsersMessage) {
this.logger.info(
- `Init sync consumer from ${this.account.id} and user ${message.userId} for ${message.consumerType}`
+ `Consume users from ${this.account.id} and user ${message.userId}`
);
const user = await this.getOrCreateUser(message.userId);
@@ -129,329 +120,68 @@ export class SocketConnection {
return;
}
- const consumer = user.consumers.get(message.consumerType);
- if (!consumer) {
- const newConsumer: SocketSyncConsumer = {
- type: message.consumerType,
- cursor: message.cursor,
- fetching: false,
- };
- user.consumers.set(message.consumerType, newConsumer);
- this.consumePendingSync(user, newConsumer);
- } else if (!consumer.fetching && consumer.cursor !== message.cursor) {
- consumer.cursor = message.cursor;
- this.consumePendingSync(user, consumer);
- }
- }
-
- private async consumePendingSync(
- user: SocketUser,
- consumer: SocketSyncConsumer
- ) {
- if (consumer.type === 'transactions') {
- this.consumeTransactions(user, consumer);
- } else if (consumer.type === 'collaborations') {
- this.consumeCollaborations(user, consumer);
- } else if (consumer.type === 'interactions') {
- this.consumeInteractions(user, consumer);
- } else if (consumer.type === 'users') {
- this.consumeUsers(user, consumer);
- }
+ await user.consumers.users.consume(message);
}
- private async consumeTransactions(
- user: SocketUser,
- consumer: SocketSyncConsumer
+ private async handleConsumeCollaborations(
+ message: ConsumeCollaborationsMessage
) {
- if (consumer.fetching) {
- return;
- }
-
- if (user.rootIds.length === 0) {
- return;
- }
-
- consumer.fetching = true;
- this.logger.trace(
- `Checking for pending node transactions for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`
+ this.logger.info(
+ `Consume collaborations from ${this.account.id} and user ${message.userId}`
);
- const unsyncedTransactions = await database
- .selectFrom('transactions')
- .selectAll()
- .where('root_id', 'in', user.rootIds)
- .where('version', '>', BigInt(consumer.cursor))
- .orderBy('version', 'asc')
- .limit(20)
- .execute();
-
- if (unsyncedTransactions.length === 0) {
- this.logger.trace(
- `No pending node transactions found for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`
- );
- consumer.fetching = false;
+ const user = await this.getOrCreateUser(message.userId);
+ if (user === null) {
return;
}
- const transactions = unsyncedTransactions.map(mapTransaction);
- const message: TransactionsBatchMessage = {
- type: 'transactions_batch',
- userId: user.userId,
- transactions,
- };
-
- user.consumers.delete(consumer.type);
- this.sendMessage(message);
+ await user.consumers.collaborations.consume(message);
}
- private async consumeCollaborations(
- user: SocketUser,
- consumer: SocketSyncConsumer
- ) {
- if (consumer.fetching) {
- return;
- }
-
- consumer.fetching = true;
- this.logger.trace(
- `Checking for pending collaborations for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`
+ private async handleConsumeTransactions(message: ConsumeTransactionsMessage) {
+ this.logger.info(
+ `Consume transactions from ${this.account.id} and user ${message.userId}`
);
- const unsyncedCollaborations = await database
- .selectFrom('collaborations')
- .selectAll()
- .where('collaborator_id', '=', user.userId)
- .where('version', '>', BigInt(consumer.cursor))
- .orderBy('version', 'asc')
- .limit(50)
- .execute();
-
- if (unsyncedCollaborations.length === 0) {
- this.logger.trace(
- `No pending collaborations found for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`
- );
- consumer.fetching = false;
- return;
- }
-
- const collaborations = unsyncedCollaborations.map(mapCollaboration);
- const message: CollaborationsBatchMessage = {
- type: 'collaborations_batch',
- userId: user.userId,
- collaborations,
- };
-
- user.consumers.delete(consumer.type);
- this.sendMessage(message);
- }
-
- private async consumeInteractions(
- user: SocketUser,
- consumer: SocketSyncConsumer
- ) {
- if (consumer.fetching) {
+ const user = await this.getOrCreateUser(message.userId);
+ if (user === null) {
return;
}
- consumer.fetching = true;
- this.logger.trace(
- `Checking for pending interactions for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`
- );
+ let nodeConsumers = user.consumers.nodes.get(message.rootId);
+ if (!nodeConsumers) {
+ nodeConsumers = {
+ transactions: new TransactionsConsumer(user.user, message.rootId),
+ interactions: new InteractionsConsumer(user.user, message.rootId),
+ };
- const unsyncedInteractions = await database
- .selectFrom('interactions')
- .selectAll()
- .where('node_id', 'in', user.rootIds)
- .where('version', '>', BigInt(consumer.cursor))
- .orderBy('version', 'asc')
- .limit(20)
- .execute();
-
- if (unsyncedInteractions.length === 0) {
- this.logger.trace(
- `No pending interactions found for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`
- );
- consumer.fetching = false;
- return;
+ user.consumers.nodes.set(message.rootId, nodeConsumers);
}
- const interactions = unsyncedInteractions.map(mapInteraction);
- const message: InteractionsBatchMessage = {
- type: 'interactions_batch',
- userId: user.userId,
- interactions,
- };
-
- user.consumers.delete(consumer.type);
- this.sendMessage(message);
+ await nodeConsumers.transactions.consume(message);
}
- private async consumeUsers(user: SocketUser, consumer: SocketSyncConsumer) {
- if (consumer.fetching) {
- return;
- }
-
- consumer.fetching = true;
- this.logger.trace(
- `Checking for pending users for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`
+ private async handleConsumeInteractions(message: ConsumeInteractionsMessage) {
+ this.logger.info(
+ `Consume interactions from ${this.account.id} and user ${message.userId}`
);
- const unsyncedUsers = await database
- .selectFrom('users')
- .selectAll()
- .where('workspace_id', '=', user.workspaceId)
- .where('version', '>', BigInt(consumer.cursor))
- .orderBy('version', 'asc')
- .limit(50)
- .execute();
-
- if (unsyncedUsers.length === 0) {
- consumer.fetching = false;
+ const user = await this.getOrCreateUser(message.userId);
+ if (user === null) {
return;
}
- const users = unsyncedUsers.map(mapUser);
- const message: UsersBatchMessage = {
- type: 'users_batch',
- userId: user.userId,
- users,
- };
-
- user.consumers.delete(consumer.type);
- this.sendMessage(message);
- }
-
- private async handleNodeCreatedEvent(event: NodeCreatedEvent) {
- for (const user of this.users.values()) {
- if (user.workspaceId !== event.workspaceId) {
- continue;
- }
-
- if (!user.rootIds.includes(event.rootId)) {
- continue;
- }
-
- const transactionsConsumer = user.consumers.get('transactions');
- if (transactionsConsumer) {
- this.consumePendingSync(user, transactionsConsumer);
- }
-
- const interactionsConsumer = user.consumers.get('interactions');
- if (interactionsConsumer) {
- this.consumePendingSync(user, interactionsConsumer);
- }
- }
- }
-
- private async handleNodeUpdatedEvent(event: NodeUpdatedEvent) {
- for (const user of this.users.values()) {
- if (user.workspaceId !== event.workspaceId) {
- continue;
- }
-
- if (!user.rootIds.includes(event.rootId)) {
- continue;
- }
-
- const transactionsConsumer = user.consumers.get('transactions');
- if (transactionsConsumer) {
- this.consumePendingSync(user, transactionsConsumer);
- }
- }
- }
-
- private async handleNodeDeletedEvent(event: NodeDeletedEvent) {
- for (const user of this.users.values()) {
- if (user.workspaceId !== event.workspaceId) {
- continue;
- }
-
- if (!user.rootIds.includes(event.rootId)) {
- continue;
- }
-
- const transactionsConsumer = user.consumers.get('transactions');
- if (transactionsConsumer) {
- this.consumePendingSync(user, transactionsConsumer);
- }
- }
- }
-
- private async handleCollaborationCreatedEvent(
- event: CollaborationCreatedEvent
- ) {
- for (const user of this.users.values()) {
- if (user.userId !== event.collaboratorId) {
- continue;
- }
-
- if (!user.rootIds.includes(event.nodeId)) {
- user.rootIds.push(event.nodeId);
- }
-
- const collaborationsConsumer = user.consumers.get('collaborations');
- if (collaborationsConsumer) {
- this.consumePendingSync(user, collaborationsConsumer);
- }
- }
- }
-
- private async handleCollaborationUpdatedEvent(
- event: CollaborationUpdatedEvent
- ) {
- for (const user of this.users.values()) {
- if (user.userId !== event.collaboratorId) {
- continue;
- }
-
- if (!user.rootIds.includes(event.nodeId)) {
- user.rootIds.push(event.nodeId);
- }
-
- const collaborationsConsumer = user.consumers.get('collaborations');
- if (collaborationsConsumer) {
- this.consumePendingSync(user, collaborationsConsumer);
- }
- }
- }
-
- private async handleInteractionUpdatedEvent(event: InteractionUpdatedEvent) {
- for (const user of this.users.values()) {
- if (user.userId !== event.userId) {
- continue;
- }
-
- const interactionsConsumer = user.consumers.get('interactions');
- if (interactionsConsumer) {
- this.consumePendingSync(user, interactionsConsumer);
- }
- }
- }
-
- private handleUserCreatedEvent(event: UserCreatedEvent) {
- for (const user of this.users.values()) {
- if (user.userId !== event.userId) {
- continue;
- }
+ let nodeConsumers = user.consumers.nodes.get(message.rootId);
+ if (!nodeConsumers) {
+ nodeConsumers = {
+ transactions: new TransactionsConsumer(user.user, message.rootId),
+ interactions: new InteractionsConsumer(user.user, message.rootId),
+ };
- const usersConsumer = user.consumers.get('users');
- if (usersConsumer) {
- this.consumePendingSync(user, usersConsumer);
- }
+ user.consumers.nodes.set(message.rootId, nodeConsumers);
}
- }
-
- private handleUserUpdatedEvent(event: UserUpdatedEvent) {
- for (const user of this.users.values()) {
- if (user.userId !== event.userId) {
- continue;
- }
- const usersConsumer = user.consumers.get('users');
- if (usersConsumer) {
- this.consumePendingSync(user, usersConsumer);
- }
- }
+ await nodeConsumers.interactions.consume(message);
}
private async getOrCreateUser(userId: string): Promise<SocketUser | null> {
@@ -491,28 +221,31 @@ export class SocketConnection {
return null;
}
- const collaborations = await database
- .selectFrom('collaborations')
- .select(['node_id'])
- .where('collaborator_id', '=', userId)
- .where('deleted_at', 'is', null)
- .execute();
-
- const rootIds = collaborations.map(
- (collaboration) => collaboration.node_id
- );
-
const addedSocketUser = this.users.get(userId);
if (addedSocketUser) {
return addedSocketUser;
}
// Create and store the new SocketUser
- const newSocketUser: SocketUser = {
- userId,
+ const connectedUser: ConnectedUser = {
+ userId: user.id,
workspaceId: user.workspace_id,
- rootIds,
- consumers: new Map(),
+ accountId: this.account.id,
+ deviceId: this.account.deviceId,
+ sendMessage: this.sendMessage.bind(this),
+ };
+
+ const usersConsumer = new UsersConsumer(connectedUser);
+ const collaborationsConsumer = new CollaborationsConsumer(connectedUser);
+ const nodesConsumers = new Map<string, NodeConsumersWrapper>();
+
+ const newSocketUser: SocketUser = {
+ user: connectedUser,
+ consumers: {
+ users: usersConsumer,
+ collaborations: collaborationsConsumer,
+ nodes: nodesConsumers,
+ },
};
this.users.set(userId, newSocketUser);
@@ -531,11 +264,11 @@ export class SocketConnection {
}
private handleWorkspaceUpdatedEvent(event: WorkspaceUpdatedEvent) {
- const users = Array.from(this.users.values()).filter(
- (user) => user.workspaceId === event.workspaceId
+ const socketUsers = Array.from(this.users.values()).filter(
+ (user) => user.user.workspaceId === event.workspaceId
);
- if (users.length === 0) {
+ if (socketUsers.length === 0) {
return;
}
@@ -546,11 +279,11 @@ export class SocketConnection {
}
private handleWorkspaceDeletedEvent(event: WorkspaceDeletedEvent) {
- const users = Array.from(this.users.values()).filter(
- (user) => user.workspaceId === event.workspaceId
+ const socketUsers = Array.from(this.users.values()).filter(
+ (user) => user.user.workspaceId === event.workspaceId
);
- if (users.length === 0) {
+ if (socketUsers.length === 0) {
return;
}
diff --git a/apps/server/src/types/events.ts b/apps/server/src/types/events.ts
index e71725f1..d3b5c34f 100644
--- a/apps/server/src/types/events.ts
+++ b/apps/server/src/types/events.ts
@@ -42,6 +42,7 @@ export type InteractionUpdatedEvent = {
type: 'interaction_updated';
userId: string;
nodeId: string;
+ rootId: string;
workspaceId: string;
};
diff --git a/apps/server/src/types/users.ts b/apps/server/src/types/users.ts
new file mode 100644
index 00000000..44ae49d2
--- /dev/null
+++ b/apps/server/src/types/users.ts
@@ -0,0 +1,9 @@
+import { Message } from '@colanode/core';
+
+export type ConnectedUser = {
+ userId: string;
+ workspaceId: string;
+ accountId: string;
+ deviceId: string;
+ sendMessage: (message: Message) => void;
+};
diff --git a/packages/core/src/types/messages.ts b/packages/core/src/types/messages.ts
index 3cf3ed34..568b0d21 100644
--- a/packages/core/src/types/messages.ts
+++ b/packages/core/src/types/messages.ts
@@ -4,12 +4,12 @@ import {
ServerInteraction,
ServerTransaction,
ServerUser,
- SyncConsumerType,
} from './sync';
export type TransactionsBatchMessage = {
type: 'transactions_batch';
userId: string;
+ rootId: string;
transactions: ServerTransaction[];
};
@@ -22,6 +22,7 @@ export type CollaborationsBatchMessage = {
export type InteractionsBatchMessage = {
type: 'interactions_batch';
userId: string;
+ rootId: string;
interactions: ServerInteraction[];
};
@@ -35,13 +36,33 @@ export type SyncInteractionsMessage = {
type: 'sync_interactions';
userId: string;
nodeId: string;
+ rootId: string;
events: InteractionEvent[];
};
-export type InitSyncConsumerMessage = {
- type: 'init_sync_consumer';
+export type ConsumeUsersMessage = {
+ type: 'consume_users';
userId: string;
- consumerType: SyncConsumerType;
+ cursor: string;
+};
+
+export type ConsumeCollaborationsMessage = {
+ type: 'consume_collaborations';
+ userId: string;
+ cursor: string;
+};
+
+export type ConsumeInteractionsMessage = {
+ type: 'consume_interactions';
+ userId: string;
+ rootId: string;
+ cursor: string;
+};
+
+export type ConsumeTransactionsMessage = {
+ type: 'consume_transactions';
+ userId: string;
+ rootId: string;
cursor: string;
};
@@ -66,7 +87,10 @@ export type Message =
| InteractionsBatchMessage
| UsersBatchMessage
| SyncInteractionsMessage
- | InitSyncConsumerMessage
+ | ConsumeUsersMessage
+ | ConsumeCollaborationsMessage
+ | ConsumeInteractionsMessage
+ | ConsumeTransactionsMessage
| AccountUpdatedMessage
| WorkspaceUpdatedMessage
| WorkspaceDeletedMessage;
diff --git a/packages/core/src/types/sync.ts b/packages/core/src/types/sync.ts
index f5d921ab..3fcdb1ed 100644
--- a/packages/core/src/types/sync.ts
+++ b/packages/core/src/types/sync.ts
@@ -143,9 +143,3 @@ export type ServerUser = {
updatedAt: string | null;
version: string;
};
-
-export type SyncConsumerType =
- | 'transactions'
- | 'collaborations'
- | 'interactions'
- | 'users';