Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Refactor sync consumers
Hakan Shehu6286147f2ec3
diff --git a/apps/desktop/src/main/consumers/base.ts b/apps/desktop/src/main/consumers/base.tsnew file mode 100644index 00000000..c22530cd--- /dev/null+++ b/apps/desktop/src/main/consumers/base.ts@@ -0,0 +1,64 @@+import { Kysely } from 'kysely';+import { Message } from '@colanode/core';++import { socketService } from '@/main/services/socket-service';+import { WorkspaceDatabaseSchema } from '@/main/data/workspace/schema';++export type ConsumerStatus = 'idle' | 'fetching' | 'syncing';++export abstract class BaseConsumer {+ protected status: ConsumerStatus = 'idle';+ protected userId: string;+ protected accountId: string;+ protected workspaceDatabase: Kysely<WorkspaceDatabaseSchema>;++ private cursorValue: bigint = 0n;++ constructor(+ userId: string,+ accountId: string,+ workspaceDatabase: Kysely<WorkspaceDatabaseSchema>+ ) {+ this.userId = userId;+ this.accountId = accountId;+ this.workspaceDatabase = workspaceDatabase;+ }++ protected abstract get cursorKey(): string;++ protected get cursor() {+ return this.cursorValue;+ }++ protected async initCursor() {+ const cursor = await this.workspaceDatabase+ .selectFrom('cursors')+ .select('value')+ .where('key', '=', this.cursorKey)+ .executeTakeFirst();++ this.cursorValue = cursor?.value ?? 0n;+ }++ protected async setCursor(cursor: bigint) {+ this.cursorValue = cursor;+ await this.workspaceDatabase+ .insertInto('cursors')+ .values({+ key: this.cursorKey,+ value: cursor,+ created_at: new Date().toISOString(),+ })+ .onConflict((eb) =>+ eb.column('key').doUpdateSet({+ value: cursor,+ updated_at: new Date().toISOString(),+ })+ )+ .execute();+ }++ protected sendMessage(message: Message) {+ return socketService.sendMessage(this.accountId, message);+ }+}diff --git a/apps/desktop/src/main/consumers/collaborations.ts b/apps/desktop/src/main/consumers/collaborations.tsnew file mode 100644index 00000000..728dd55b--- /dev/null+++ b/apps/desktop/src/main/consumers/collaborations.ts@@ -0,0 +1,66 @@+import {+ CollaborationsBatchMessage,+ ConsumeCollaborationsMessage,+} from '@colanode/core';++import { collaborationService } from '@/main/services/collaboration-service';+import { BaseConsumer } from '@/main/consumers/base';+import { createDebugger } from '@/main/debugger';++export class CollaborationsConsumer extends BaseConsumer {+ private readonly debug = createDebugger('consumer:collaborations');++ protected get cursorKey(): string {+ return 'collaborations';+ }++ public async init() {+ await super.initCursor();+ this.requestCollaborations();+ }++ public async processCollaborations(message: CollaborationsBatchMessage) {+ if (this.status === 'syncing') {+ this.debug(+ `Syncing of server collaborations already in progress for user ${message.userId}, skipping`+ );+ return;+ }++ this.debug(`Syncing server collaborators for user ${message.userId}`);++ this.status = 'syncing';+ let cursor: bigint | null = null;+ try {+ for (const collaboration of message.collaborations) {+ await collaborationService.syncServerCollaboration(+ message.userId,+ collaboration+ );+ cursor = BigInt(collaboration.version);+ }+ } catch (error) {+ this.debug(+ error,+ `Error syncing server collaborators for user ${message.userId}`+ );+ } finally {+ if (cursor) {+ await this.setCursor(cursor);+ }++ this.status = 'idle';+ this.requestCollaborations();+ }+ }++ private async requestCollaborations() {+ const message: ConsumeCollaborationsMessage = {+ type: 'consume_collaborations',+ userId: this.userId,+ cursor: this.cursor.toString(),+ };++ this.sendMessage(message);+ }+}diff --git a/apps/desktop/src/main/consumers/interactions.ts b/apps/desktop/src/main/consumers/interactions.tsnew file mode 100644index 00000000..cefa734d--- /dev/null+++ b/apps/desktop/src/main/consumers/interactions.ts@@ -0,0 +1,84 @@+import { Kysely } from 'kysely';+import {+ ConsumeInteractionsMessage,+ InteractionsBatchMessage,+} from '@colanode/core';++import { interactionService } from '@/main/services/interaction-service';+import { createDebugger } from '@/main/debugger';+import { WorkspaceDatabaseSchema } from '@/main/data/workspace/schema';+import { BaseConsumer } from '@/main/consumers/base';++export class InteractionsConsumer extends BaseConsumer {+ private readonly debug = createDebugger('consumer:interactions');+ private readonly rootId: string;++ constructor(+ userId: string,+ accountId: string,+ rootId: string,+ workspaceDatabase: Kysely<WorkspaceDatabaseSchema>+ ) {+ super(userId, accountId, workspaceDatabase);+ this.rootId = rootId;+ }++ protected get cursorKey(): string {+ return `interactions:${this.rootId}`;+ }++ public async init() {+ await super.initCursor();+ this.requestInteractions();+ }++ public async processInteractions(message: InteractionsBatchMessage) {+ if (this.status === 'syncing') {+ this.debug(+ `Syncing of server interactions already in progress for user ${message.userId}, skipping`+ );+ return;+ }++ this.debug(`Syncing server interactions for user ${message.userId}`);++ this.status = 'syncing';+ let cursor: bigint | null = null;+ try {+ for (const interaction of message.interactions) {+ await interactionService.applyServerInteraction(+ message.userId,+ interaction+ );+ cursor = BigInt(interaction.version);+ }+ } catch (error) {+ this.debug(+ error,+ `Error syncing server interactions for user ${message.userId}`+ );+ } finally {+ this.debug(+ `Syncing of server interactions completed for user ${message.userId}`+ );++ if (cursor) {+ await this.setCursor(cursor);+ }++ this.status = 'idle';+ this.requestInteractions();+ }+ }++ private async requestInteractions() {+ const message: ConsumeInteractionsMessage = {+ type: 'consume_interactions',+ userId: this.userId,+ rootId: this.rootId,+ cursor: this.cursor.toString(),+ };++ this.sendMessage(message);+ }+}diff --git a/apps/desktop/src/main/consumers/transactions.ts b/apps/desktop/src/main/consumers/transactions.tsnew file mode 100644index 00000000..6427872c--- /dev/null+++ b/apps/desktop/src/main/consumers/transactions.ts@@ -0,0 +1,81 @@+import { Kysely } from 'kysely';+import {+ ConsumeTransactionsMessage,+ TransactionsBatchMessage,+} from '@colanode/core';++import { nodeService } from '@/main/services/node-service';+import { createDebugger } from '@/main/debugger';+import { WorkspaceDatabaseSchema } from '@/main/data/workspace/schema';+import { BaseConsumer } from '@/main/consumers/base';++export class TransactionsConsumer extends BaseConsumer {+ private readonly debug = createDebugger('consumer:transactions');+ private readonly rootId: string;++ constructor(+ userId: string,+ accountId: string,+ rootId: string,+ workspaceDatabase: Kysely<WorkspaceDatabaseSchema>+ ) {+ super(userId, accountId, workspaceDatabase);+ this.rootId = rootId;+ }++ protected get cursorKey(): string {+ return `transactions:${this.rootId}`;+ }++ public async init() {+ await super.initCursor();+ this.requestTransactions();+ }++ public async processTransactions(message: TransactionsBatchMessage) {+ if (this.status === 'syncing') {+ this.debug(+ `Syncing of server transactions already in progress for user ${message.userId}, skipping`+ );+ return;+ }++ this.debug(`Syncing server transactions for user ${message.userId}`);++ this.status = 'syncing';+ let cursor: bigint | null = null;+ try {+ for (const transaction of message.transactions) {+ await nodeService.applyServerTransaction(message.userId, transaction);+ cursor = BigInt(transaction.version);+ }+ } catch (error) {+ this.debug(+ error,+ `Error syncing server transactions for user ${message.userId}`+ );+ } finally {+ this.debug(+ `Syncing of server transactions completed for user ${message.userId}`+ );++ if (cursor) {+ await this.setCursor(cursor);+ }++ this.status = 'idle';+ this.requestTransactions();+ }+ }++ private async requestTransactions() {+ const message: ConsumeTransactionsMessage = {+ type: 'consume_transactions',+ userId: this.userId,+ rootId: this.rootId,+ cursor: this.cursor.toString(),+ };++ this.sendMessage(message);+ }+}diff --git a/apps/desktop/src/main/consumers/users.ts b/apps/desktop/src/main/consumers/users.tsnew file mode 100644index 00000000..173441dd--- /dev/null+++ b/apps/desktop/src/main/consumers/users.ts@@ -0,0 +1,55 @@+import { ConsumeUsersMessage, UsersBatchMessage } from '@colanode/core';++import { userService } from '@/main/services/user-service';+import { BaseConsumer } from '@/main/consumers/base';+import { createDebugger } from '@/main/debugger';++export class UsersConsumer extends BaseConsumer {+ private readonly debug = createDebugger('consumer:users');++ protected get cursorKey(): string {+ return 'users';+ }++ public async init() {+ await super.initCursor();+ this.requestUsers();+ }++ public async processUsers(message: UsersBatchMessage) {+ this.status = 'syncing';+ let cursor: bigint | null = null;+ try {+ for (const user of message.users) {+ await userService.syncServerUser(message.userId, user);+ cursor = BigInt(user.version);+ }+ } catch (error) {+ this.debug(+ error,+ `Error syncing server users for user ${message.userId}`+ );+ } finally {+ this.debug(+ `Syncing of server users completed for user ${message.userId}`+ );++ if (cursor) {+ await this.setCursor(cursor);+ }++ this.status = 'idle';+ this.requestUsers();+ }+ }++ private async requestUsers() {+ const message: ConsumeUsersMessage = {+ type: 'consume_users',+ userId: this.userId,+ cursor: this.cursor.toString(),+ };++ this.sendMessage(message);+ }+}diff --git a/apps/desktop/src/main/data/workspace/migrations.ts b/apps/desktop/src/main/data/workspace/migrations.tsindex 55a67fa4..d9211ab0 100644--- a/apps/desktop/src/main/data/workspace/migrations.ts+++ b/apps/desktop/src/main/data/workspace/migrations.ts@@ -279,7 +279,7 @@ const createCursorsTable: Migration = {up: async (db) => {await db.schema.createTable('cursors')- .addColumn('type', 'text', (col) => col.notNull().primaryKey())+ .addColumn('key', 'text', (col) => col.notNull().primaryKey()).addColumn('value', 'integer', (col) => col.notNull().defaultTo(0)).addColumn('created_at', 'text', (col) => col.notNull()).addColumn('updated_at', 'text')diff --git a/apps/desktop/src/main/data/workspace/schema.ts b/apps/desktop/src/main/data/workspace/schema.tsindex 961360a0..8cd12c12 100644--- a/apps/desktop/src/main/data/workspace/schema.ts+++ b/apps/desktop/src/main/data/workspace/schema.ts@@ -1,9 +1,4 @@-import {- InteractionAttribute,- NodeType,- SyncConsumerType,- WorkspaceRole,-} from '@colanode/core';+import { InteractionAttribute, NodeType, WorkspaceRole } from '@colanode/core';import { ColumnType, Insertable, Selectable, Updateable } from 'kysely';interface UserTable {@@ -157,7 +152,7 @@ export type CreateNodeText = Insertable<NodeTextTable>;export type UpdateNodeText = Updateable<NodeTextTable>;interface CursorTable {- type: ColumnType<SyncConsumerType, SyncConsumerType, never>;+ key: ColumnType<string, string, never>;value: ColumnType<bigint, bigint, bigint>;created_at: ColumnType<string, string, never>;updated_at: ColumnType<string | null, string | null, string | null>;diff --git a/apps/desktop/src/main/jobs/sync-incomplete-transactions.ts b/apps/desktop/src/main/jobs/sync-incomplete-transactions.tsdeleted file mode 100644index f46ed07e..00000000--- a/apps/desktop/src/main/jobs/sync-incomplete-transactions.ts+++ /dev/null@@ -1,155 +0,0 @@-import { GetTransactionsOutput } from '@colanode/core';-import { sql } from 'kysely';--import { SelectTransaction } from '@/main/data/workspace/schema';-import { nodeService } from '@/main/services/node-service';-import { fetchCursor, fetchWorkspaceCredentials } from '@/main/utils';-import { createDebugger } from '@/main/debugger';-import { databaseService } from '@/main/data/database-service';-import { serverService } from '@/main/services/server-service';-import { JobHandler } from '@/main/jobs';-import { httpClient } from '@/shared/lib/http-client';--export type SyncIncompleteTransactionsInput = {- type: 'sync_incomplete_transactions';- userId: string;-};--declare module '@/main/jobs' {- interface JobMap {- sync_incomplete_transactions: {- input: SyncIncompleteTransactionsInput;- };- }-}--export class SyncIncompleteTransactionsJobHandler- implements JobHandler<SyncIncompleteTransactionsInput>-{- public triggerDebounce = 100;- public interval = 1000 * 60;-- private readonly debug = createDebugger('job:sync-incomplete-transactions');-- public async handleJob(input: SyncIncompleteTransactionsInput) {- this.debug(`Syncing incomplete transactions for user ${input.userId}`);-- const workspaceDatabase = await databaseService.getWorkspaceDatabase(- input.userId- );-- const incompleteTransactions = await workspaceDatabase- .selectFrom('transactions')- .selectAll()- .where('status', '=', 'incomplete')- .execute();-- if (incompleteTransactions.length === 0) {- this.debug(- `No incomplete transactions found for user ${input.userId}, skipping`- );- return;- }-- const credentials = await fetchWorkspaceCredentials(input.userId);- if (!credentials) {- this.debug(- `No workspace credentials found for user ${input.userId}, skipping`- );- return;- }-- if (!serverService.isAvailable(credentials.serverDomain)) {- this.debug(- `Server ${credentials.serverDomain} is not available, skipping`- );- return;- }-- const groupedByNodeId = incompleteTransactions.reduce<- Record<string, SelectTransaction[]>- >((acc, transaction) => {- acc[transaction.node_id] = [- ...(acc[transaction.node_id] ?? []),- transaction,- ];- return acc;- }, {});-- for (const [nodeId, transactions] of Object.entries(groupedByNodeId)) {- try {- this.debug(- `Syncing incomplete transactions for node ${nodeId} for user ${input.userId}`- );-- const { data } = await httpClient.get<GetTransactionsOutput>(- `/v1/workspaces/${credentials.workspaceId}/transactions/${nodeId}`,- {- domain: credentials.serverDomain,- token: credentials.token,- }- );-- if (data.transactions.length === 0) {- this.debug(- `No transactions found for node ${nodeId} for user ${input.userId}, deleting`- );-- await workspaceDatabase- .deleteFrom('transactions')- .where(- 'id',- 'in',- transactions.map((t) => t.id)- )- .execute();- continue;- }-- const cursor = await fetchCursor(input.userId, 'transactions');- const synced = await nodeService.replaceTransactions(- input.userId,- nodeId,- data.transactions,- cursor- );-- if (!synced) {- this.debug(- `Failed to sync transactions for node ${nodeId} for user ${input.userId}, incrementing retry count`- );-- await workspaceDatabase- .updateTable('transactions')- .set({ retry_count: sql`retry_count + 1` })- .where(- 'id',- 'in',- transactions.map((t) => t.id)- )- .execute();- } else {- this.debug(- `Successfully synced transactions for node ${nodeId} for user ${input.userId}, resetting retry count`- );-- await workspaceDatabase- .updateTable('transactions')- .set({ retry_count: sql`retry_count + 1` })- .where(- 'id',- 'in',- transactions.map((t) => t.id)- )- .where('status', '=', 'incomplete')- .execute();- }- } catch (error) {- this.debug(- error,- `Error syncing incomplete transactions for node ${nodeId} for user ${input.userId}`- );- }- }- }-}diff --git a/apps/desktop/src/main/jobs/sync-pending-interactions.ts b/apps/desktop/src/main/jobs/sync-pending-interactions.tsindex dea30a44..affc1410 100644--- a/apps/desktop/src/main/jobs/sync-pending-interactions.ts+++ b/apps/desktop/src/main/jobs/sync-pending-interactions.ts@@ -31,6 +31,10 @@ export class SyncPendingInteractionsJobHandlerprivate readonly debug = createDebugger('job:sync-pending-interactions');public async handleJob(input: SyncPendingInteractionsInput) {+ if (this.triggerDebounce === 100) {+ return;+ }+this.debug(`Sending local pending interactions for user ${input.userId}`);const workspaceDatabase = await databaseService.getWorkspaceDatabase(@@ -103,6 +107,7 @@ export class SyncPendingInteractionsJobHandlerconst message: SyncInteractionsMessage = {type: 'sync_interactions',nodeId,+ rootId: firstEvent.node_id,userId: credentials.userId,events: events.map((e) => ({attribute: e.attribute,diff --git a/apps/desktop/src/main/scheduler.ts b/apps/desktop/src/main/scheduler.tsindex 568d8895..48f0592d 100644--- a/apps/desktop/src/main/scheduler.ts+++ b/apps/desktop/src/main/scheduler.ts@@ -10,7 +10,6 @@ import { SyncAccountJobHandler } from '@/main/jobs/sync-account';import { InitSyncConsumersJobHandler } from '@/main/jobs/init-sync-consumers';import { RevertInvalidTransactionsJobHandler } from '@/main/jobs/revert-invalid-transactions';import { SyncPendingTransactionsJobHandler } from '@/main/jobs/sync-pending-transactions';-import { SyncIncompleteTransactionsJobHandler } from '@/main/jobs/sync-incomplete-transactions';import { SyncPendingInteractionsJobHandler } from '@/main/jobs/sync-pending-interactions';import { SyncDeletedTokensJobHandler } from '@/main/jobs/sync-deleted-tokens';import { ConnectSocketJobHandler } from '@/main/jobs/connect-socket';@@ -30,7 +29,6 @@ export const jobHandlerMap: JobHandlerMap = {init_sync_consumers: new InitSyncConsumersJobHandler(),revert_invalid_transactions: new RevertInvalidTransactionsJobHandler(),sync_pending_transactions: new SyncPendingTransactionsJobHandler(),- sync_incomplete_transactions: new SyncIncompleteTransactionsJobHandler(),sync_pending_interactions: new SyncPendingInteractionsJobHandler(),sync_deleted_tokens: new SyncDeletedTokensJobHandler(),connect_socket: new ConnectSocketJobHandler(),@@ -234,11 +232,6 @@ class Scheduler {userId,});- this.schedule({- type: 'sync_incomplete_transactions',- userId,- });-this.schedule({type: 'sync_pending_interactions',userId,@@ -300,13 +293,6 @@ class Scheduler {return true;}- if (- state.input.type === 'sync_incomplete_transactions' &&- state.input.userId === userId- ) {- return true;- }-if (state.input.type === 'sync_pending_interactions' &&state.input.userId === userId@@ -372,11 +358,6 @@ class Scheduler {type: 'sync_pending_transactions',userId: event.userId,});- } else if (event.type === 'transaction_incomplete') {- this.trigger({- type: 'sync_incomplete_transactions',- userId: event.userId,- });} else if (event.type === 'interaction_event_created') {this.trigger({type: 'sync_pending_interactions',diff --git a/apps/desktop/src/main/services/collaboration-service.ts b/apps/desktop/src/main/services/collaboration-service.tsindex 951789da..df7afd41 100644--- a/apps/desktop/src/main/services/collaboration-service.ts+++ b/apps/desktop/src/main/services/collaboration-service.ts@@ -63,14 +63,19 @@ class CollaborationService {.where('node_id', '=', collaboration.nodeId).execute();});- }- eventBus.publish({- type: 'collaboration_synced',- userId,- nodeId: collaboration.nodeId,- createdAt: new Date(collaboration.createdAt),- });+ eventBus.publish({+ type: 'collaboration_deleted',+ userId,+ nodeId: collaboration.nodeId,+ });+ } else {+ eventBus.publish({+ type: 'collaboration_created',+ userId,+ nodeId: collaboration.nodeId,+ });+ }}}diff --git a/apps/desktop/src/main/services/node-service.ts b/apps/desktop/src/main/services/node-service.tsindex 6941c69e..af676776 100644--- a/apps/desktop/src/main/services/node-service.ts+++ b/apps/desktop/src/main/services/node-service.ts@@ -932,32 +932,27 @@ class NodeService {return { createdNode };});- if (createdNode) {+ if (!createdNode) {this.debug(- `Created node ${createdNode.id} with type ${createdNode.type} with transaction ${transaction.id}`+ `Server create transaction ${transaction.id} for node ${transaction.nodeId} is incomplete`);+ return;+ }- eventBus.publish({- type: 'node_created',- userId,- node: mapNode(createdNode),- });+ this.debug(+ `Created node ${createdNode.id} with type ${createdNode.type} with transaction ${transaction.id}`+ );- await interactionService.setInteraction(userId, createdNode.id, {- attribute: 'lastReceivedTransactionId',- value: transaction.id,- });- } else {- this.debug(- `Server create transaction ${transaction.id} for node ${transaction.nodeId} is incomplete`- );+ eventBus.publish({+ type: 'node_created',+ userId,+ node: mapNode(createdNode),+ });- eventBus.publish({- type: 'transaction_incomplete',- userId,- transactionId: transaction.id,- });- }+ await interactionService.setInteraction(userId, createdNode.id, {+ attribute: 'lastReceivedTransactionId',+ value: transaction.id,+ });}private async applyServerUpdateTransaction(@@ -1091,32 +1086,27 @@ class NodeService {return { updatedNode };});- if (updatedNode) {+ if (!updatedNode) {this.debug(- `Updated node ${updatedNode.id} with type ${updatedNode.type} with transaction ${transaction.id}`+ `Server update transaction ${transaction.id} for node ${transaction.nodeId} is incomplete`);+ return;+ }- eventBus.publish({- type: 'node_updated',- userId,- node: mapNode(updatedNode),- });+ this.debug(+ `Updated node ${updatedNode.id} with type ${updatedNode.type} with transaction ${transaction.id}`+ );- await interactionService.setInteraction(userId, updatedNode.id, {- attribute: 'lastReceivedTransactionId',- value: transaction.id,- });- } else {- this.debug(- `Server update transaction ${transaction.id} for node ${transaction.nodeId} is incomplete`- );+ eventBus.publish({+ type: 'node_updated',+ userId,+ node: mapNode(updatedNode),+ });- eventBus.publish({- type: 'transaction_incomplete',- userId,- transactionId: transaction.id,- });- }+ await interactionService.setInteraction(userId, updatedNode.id, {+ attribute: 'lastReceivedTransactionId',+ value: transaction.id,+ });}private async applyServerDeleteTransaction(diff --git a/apps/desktop/src/main/services/socket-connection.ts b/apps/desktop/src/main/services/socket-connection.tsindex 7f30e4e3..4df9a159 100644--- a/apps/desktop/src/main/services/socket-connection.ts+++ b/apps/desktop/src/main/services/socket-connection.ts@@ -59,13 +59,25 @@ export class SocketConnection {);if (message.type === 'transactions_batch') {- syncService.syncServerTransactions(message);+ const transactionsConsumer = syncService.getTransactionsConsumer(+ message.userId,+ message.rootId+ );+ transactionsConsumer?.processTransactions(message);} else if (message.type === 'collaborations_batch') {- syncService.syncServerCollaborations(message);+ const collaborationsConsumer = syncService.getCollaborationsConsumer(+ message.userId+ );+ collaborationsConsumer?.processCollaborations(message);} else if (message.type === 'interactions_batch') {- syncService.syncServerInteractions(message);+ const interactionsConsumer = syncService.getInteractionsConsumer(+ message.userId,+ message.rootId+ );+ interactionsConsumer?.processInteractions(message);} else if (message.type === 'users_batch') {- syncService.syncServerUsers(message);+ const usersConsumer = syncService.getUsersConsumer(message.userId);+ usersConsumer?.processUsers(message);} else if (message.type === 'account_updated') {scheduler.trigger({type: 'sync_account',diff --git a/apps/desktop/src/main/services/sync-consumer.ts b/apps/desktop/src/main/services/sync-consumer.tsdeleted file mode 100644index b15ffe06..00000000--- a/apps/desktop/src/main/services/sync-consumer.ts+++ /dev/null@@ -1,254 +0,0 @@-import {- CollaborationsBatchMessage,- InitSyncConsumerMessage,- InteractionsBatchMessage,- SyncConsumerType,- TransactionsBatchMessage,- UsersBatchMessage,-} from '@colanode/core';--import { userService } from './user-service';--import { interactionService } from '@/main/services/interaction-service';-import { nodeService } from '@/main/services/node-service';-import { collaborationService } from '@/main/services/collaboration-service';-import { socketService } from '@/main/services/socket-service';-import { fetchCursor, updateCursor } from '@/main/utils';-import { createDebugger } from '@/main/debugger';--type SyncConsumerStatus = 'idle' | 'fetching' | 'syncing';--export class SyncConsumer {- private readonly debug = createDebugger('service:sync-consumer');- public readonly accountId: string;- public readonly userId: string;- public readonly type: SyncConsumerType;-- private cursor: bigint | null = null;- private status: SyncConsumerStatus = 'idle';-- constructor(accountId: string, userId: string, type: SyncConsumerType) {- this.accountId = accountId;- this.userId = userId;- this.type = type;- }-- public async init() {- if (this.status === 'fetching') {- return;- }-- const cursor = await fetchCursor(this.userId, this.type);- this.initConsumer(cursor);- }-- public async ping() {- if (this.status === 'idle' || this.cursor === null) {- await this.init();- return;- }-- if (this.status === 'syncing') {- return;- }-- const cursor = await fetchCursor(this.userId, this.type);- if (cursor !== this.cursor) {- this.cursor = cursor;- this.initConsumer(cursor);- }- }-- public async syncTransactions(message: TransactionsBatchMessage) {- if (this.type !== 'transactions') {- this.debug(- `Syncing of server transactions not supported for consumer type ${this.type}, skipping`- );- return;- }-- if (this.status === 'syncing') {- this.debug(- `Syncing of server transactions already in progress for user ${message.userId}, skipping`- );- return;- }-- this.debug(`Syncing server transactions for user ${message.userId}`);-- this.status = 'syncing';- let cursor: bigint | null = null;- try {- for (const transaction of message.transactions) {- await nodeService.applyServerTransaction(message.userId, transaction);- cursor = BigInt(transaction.version);- }- } catch (error) {- this.debug(- error,- `Error syncing server transactions for user ${message.userId}`- );- } finally {- this.debug(- `Syncing of server transactions completed for user ${message.userId}`- );-- if (cursor) {- this.cursor = cursor;- this.initConsumer(cursor);- await updateCursor(message.userId, 'transactions', cursor);- this.status = 'idle';- } else {- this.status = 'idle';- this.ping();- }- }- }-- public async syncCollaborations(message: CollaborationsBatchMessage) {- if (this.type !== 'collaborations') {- this.debug(- `Syncing of server collaborations not supported for consumer type ${this.type}, skipping`- );- return;- }-- if (this.status === 'syncing') {- this.debug(- `Syncing of server collaborations already in progress for user ${message.userId}, skipping`- );- return;- }-- this.debug(`Syncing server collaborators for user ${message.userId}`);-- this.status = 'syncing';- let cursor: bigint | null = null;- try {- for (const collaboration of message.collaborations) {- await collaborationService.syncServerCollaboration(- message.userId,- collaboration- );- cursor = BigInt(collaboration.version);- }- } catch (error) {- this.debug(- error,- `Error syncing server collaborators for user ${message.userId}`- );- } finally {- if (cursor) {- await updateCursor(message.userId, 'collaborations', cursor);- this.cursor = cursor;- this.status = 'idle';- this.initConsumer(cursor);- } else {- this.status = 'idle';- this.ping();- }- }- }-- public async syncInteractions(message: InteractionsBatchMessage) {- if (this.type !== 'interactions') {- this.debug(- `Syncing of server interactions not supported for consumer type ${this.type}, skipping`- );- return;- }-- if (this.status === 'syncing') {- this.debug(- `Syncing of server interactions already in progress for user ${message.userId}, skipping`- );- return;- }-- this.debug(`Syncing server interactions for user ${message.userId}`);-- this.status = 'syncing';- let cursor: bigint | null = null;- try {- for (const interaction of message.interactions) {- await interactionService.applyServerInteraction(- message.userId,- interaction- );- cursor = BigInt(interaction.version);- }- } catch (error) {- this.debug(- error,- `Error syncing server interactions for user ${message.userId}`- );- } finally {- this.debug(- `Syncing of server interactions completed for user ${message.userId}`- );-- if (cursor) {- await updateCursor(message.userId, 'interactions', cursor);- this.cursor = cursor;- this.status = 'idle';- this.initConsumer(cursor);- } else {- this.status = 'idle';- this.ping();- }- }- }-- public async syncUsers(message: UsersBatchMessage) {- if (this.type !== 'users') {- this.debug(- `Syncing of server users not supported for consumer type ${this.type}, skipping`- );- return;- }-- this.debug(`Syncing server users for user ${message.userId}`);-- this.status = 'syncing';- let cursor: bigint | null = null;- try {- for (const user of message.users) {- await userService.syncServerUser(message.userId, user);- cursor = BigInt(user.version);- }- } catch (error) {- this.debug(- error,- `Error syncing server users for user ${message.userId}`- );- } finally {- this.debug(- `Syncing of server users completed for user ${message.userId}`- );-- if (cursor) {- await updateCursor(message.userId, 'users', cursor);- this.cursor = cursor;- this.status = 'idle';- this.initConsumer(cursor);- } else {- this.status = 'idle';- this.ping();- }- }- }-- private initConsumer(cursor: bigint) {- const message: InitSyncConsumerMessage = {- type: 'init_sync_consumer',- userId: this.userId,- consumerType: this.type,- cursor: cursor.toString(),- };-- const sent = socketService.sendMessage(this.accountId, message);- if (sent) {- this.status = 'fetching';- }- }-}diff --git a/apps/desktop/src/main/services/sync-service.ts b/apps/desktop/src/main/services/sync-service.tsindex 460d359e..925de278 100644--- a/apps/desktop/src/main/services/sync-service.ts+++ b/apps/desktop/src/main/services/sync-service.ts@@ -1,92 +1,118 @@-import {- CollaborationsBatchMessage,- InteractionsBatchMessage,- SyncConsumerType,- TransactionsBatchMessage,- UsersBatchMessage,-} from '@colanode/core';--import { SyncConsumer } from '@/main/services/sync-consumer';+import { databaseService } from '@/main/data/database-service';+import { UsersConsumer } from '@/main/consumers/users';+import { TransactionsConsumer } from '@/main/consumers/transactions';+import { CollaborationsConsumer } from '@/main/consumers/collaborations';+import { InteractionsConsumer } from '@/main/consumers/interactions';import { createDebugger } from '@/main/debugger';+export type NodeConsumersWrapper = {+ transactions: TransactionsConsumer;+ interactions: InteractionsConsumer;+};++export type ConsumersWrapper = {+ users: UsersConsumer;+ collaborations: CollaborationsConsumer;+ nodes: Record<string, NodeConsumersWrapper>;+};+class SyncService {private readonly debug = createDebugger('service:sync');- private readonly consumers: Map<string, SyncConsumer> = new Map();+ private readonly users: Map<string, ConsumersWrapper> = new Map();public async initUserConsumers(accountId: string, userId: string) {- await this.createConsumer(accountId, userId, 'transactions');- await this.createConsumer(accountId, userId, 'collaborations');- await this.createConsumer(accountId, userId, 'interactions');- await this.createConsumer(accountId, userId, 'users');- }+ const workspaceDatabase =+ await databaseService.getWorkspaceDatabase(userId);- public async deleteUserConsumers(userId: string) {- this.consumers.forEach((consumer, key) => {- if (consumer.userId === userId) {- this.consumers.delete(key);- }- });- }+ let consumers = this.users.get(userId);+ if (!consumers) {+ const usersConsumer = new UsersConsumer(+ userId,+ accountId,+ workspaceDatabase+ );+ await usersConsumer.init();- private async createConsumer(- accountId: string,- userId: string,- type: SyncConsumerType- ) {- const key = this.getConsumerKey(userId, type);- const consumer = this.consumers.get(key);- if (consumer) {- consumer.ping();- return;+ const collaborationsConsumer = new CollaborationsConsumer(+ userId,+ accountId,+ workspaceDatabase+ );+ await collaborationsConsumer.init();++ consumers = {+ users: usersConsumer,+ collaborations: collaborationsConsumer,+ nodes: {},+ };++ this.users.set(userId, consumers);}- this.debug(- `Creating new sync consumer for account ${accountId} and user ${userId} of type ${type}`- );- const newConsumer = new SyncConsumer(accountId, userId, type);- this.consumers.set(key, newConsumer);+ const collaborations = await workspaceDatabase+ .selectFrom('collaborations')+ .selectAll()+ .execute();- await newConsumer.init();- }+ for (const collaboration of collaborations) {+ const rootId = collaboration.node_id;++ if (consumers.nodes[rootId]) {+ continue;+ }- public async syncServerTransactions(message: TransactionsBatchMessage) {- const consumer = this.consumers.get(- this.getConsumerKey(message.userId, 'transactions')- );- if (consumer) {- consumer.syncTransactions(message);+ const transactionsConsumer = new TransactionsConsumer(+ userId,+ accountId,+ rootId,+ workspaceDatabase+ );+ await transactionsConsumer.init();++ const interactionsConsumer = new InteractionsConsumer(+ userId,+ accountId,+ rootId,+ workspaceDatabase+ );+ await interactionsConsumer.init();++ consumers.nodes[rootId] = {+ transactions: transactionsConsumer,+ interactions: interactionsConsumer,+ };}- }- public async syncServerCollaborations(message: CollaborationsBatchMessage) {- const consumer = this.consumers.get(- this.getConsumerKey(message.userId, 'collaborations')- );- if (consumer) {- consumer.syncCollaborations(message);+ // check for deleted collaborations and remove them from the consumers+ for (const nodeId of Object.keys(consumers.nodes)) {+ if (!collaborations.some((c) => c.node_id === nodeId)) {+ delete consumers.nodes[nodeId];+ }}}- public async syncServerInteractions(message: InteractionsBatchMessage) {- const consumer = this.consumers.get(- this.getConsumerKey(message.userId, 'interactions')- );- if (consumer) {- consumer.syncInteractions(message);- }+ public getUsersConsumer(userId: string): UsersConsumer | undefined {+ return this.users.get(userId)?.users;}- public async syncServerUsers(message: UsersBatchMessage) {- const consumer = this.consumers.get(- this.getConsumerKey(message.userId, 'users')- );- if (consumer) {- consumer.syncUsers(message);- }+ public getCollaborationsConsumer(+ userId: string+ ): CollaborationsConsumer | undefined {+ return this.users.get(userId)?.collaborations;}- private getConsumerKey(userId: string, type: SyncConsumerType) {- return `${userId}-${type}`;+ public getTransactionsConsumer(+ userId: string,+ rootId: string+ ): TransactionsConsumer | undefined {+ return this.users.get(userId)?.nodes[rootId]?.transactions;+ }++ public getInteractionsConsumer(+ userId: string,+ rootId: string+ ): InteractionsConsumer | undefined {+ return this.users.get(userId)?.nodes[rootId]?.interactions;}}diff --git a/apps/desktop/src/main/utils.ts b/apps/desktop/src/main/utils.tsindex 54f17afb..c1b1c460 100644--- a/apps/desktop/src/main/utils.ts+++ b/apps/desktop/src/main/utils.ts@@ -1,4 +1,4 @@-import { LocalTransaction, Node, SyncConsumerType } from '@colanode/core';+import { LocalTransaction, Node } from '@colanode/core';import { encodeState } from '@colanode/crdt';import {DeleteResult,@@ -136,44 +136,6 @@ export const fetchWorkspaceCredentials = async (};};-export const updateCursor = async (- userId: string,- type: SyncConsumerType,- cursor: bigint-) => {- const workspaceDatabase = await databaseService.getWorkspaceDatabase(userId);-- await workspaceDatabase- .insertInto('cursors')- .values({- type,- value: cursor,- created_at: new Date().toISOString(),- })- .onConflict((eb) =>- eb.column('type').doUpdateSet({- value: cursor,- updated_at: new Date().toISOString(),- })- )- .execute();-};--export const fetchCursor = async (- userId: string,- type: SyncConsumerType-): Promise<bigint> => {- const workspaceDatabase = await databaseService.getWorkspaceDatabase(userId);-- const cursor = await workspaceDatabase- .selectFrom('cursors')- .select('value')- .where('type', '=', type)- .executeTakeFirst();-- return cursor?.value ?? 0n;-};-export const mapUser = (row: SelectUser): User => {return {id: row.id,diff --git a/apps/desktop/src/shared/types/events.ts b/apps/desktop/src/shared/types/events.tsindex 25d5f322..6dbe20b1 100644--- a/apps/desktop/src/shared/types/events.ts+++ b/apps/desktop/src/shared/types/events.ts@@ -135,17 +135,16 @@ export type TransactionCreatedEvent = {transaction: LocalTransaction;};-export type TransactionIncompleteEvent = {- type: 'transaction_incomplete';+export type CollaborationCreatedEvent = {+ type: 'collaboration_created';userId: string;- transactionId: string;+ nodeId: string;};-export type CollaborationSyncedEvent = {- type: 'collaboration_synced';+export type CollaborationDeletedEvent = {+ type: 'collaboration_deleted';userId: string;nodeId: string;- createdAt: Date;};export type ServerAvailabilityChangedEvent = {@@ -195,9 +194,9 @@ export type Event =| QueryResultUpdatedEvent| RadarDataUpdatedEvent| TransactionCreatedEvent- | TransactionIncompleteEvent| ServerAvailabilityChangedEvent| SocketConnectionOpenedEvent- | CollaborationSyncedEvent+ | CollaborationCreatedEvent+ | CollaborationDeletedEvent| InteractionEventCreatedEvent| InteractionUpdatedEvent;diff --git a/apps/server/src/consumers/base.ts b/apps/server/src/consumers/base.tsnew file mode 100644index 00000000..c9ca15fb--- /dev/null+++ b/apps/server/src/consumers/base.ts@@ -0,0 +1,16 @@+import { ConnectedUser } from '@/types/users';+import { Event } from '@/types/events';++export type ConsumerStatus = 'idle' | 'pending' | 'fetching';++export abstract class BaseConsumer {+ protected status: ConsumerStatus = 'idle';+ protected user: ConnectedUser;+ protected cursor: bigint | null = null;++ constructor(user: ConnectedUser) {+ this.user = user;+ }++ public abstract processEvent(event: Event): void;+}diff --git a/apps/server/src/consumers/collaborations.ts b/apps/server/src/consumers/collaborations.tsnew file mode 100644index 00000000..ccb275fb--- /dev/null+++ b/apps/server/src/consumers/collaborations.ts@@ -0,0 +1,76 @@+import {+ CollaborationsBatchMessage,+ ConsumeCollaborationsMessage,+} from '@colanode/core';++import { BaseConsumer } from '@/consumers/base';+import { Event } from '@/types/events';+import { database } from '@/data/database';+import { mapCollaboration } from '@/lib/nodes';++export class CollaborationsConsumer extends BaseConsumer {+ public processEvent(event: Event): void {+ if (+ event.type === 'collaboration_created' &&+ event.collaboratorId === this.user.userId+ ) {+ this.fetchCollaborations();+ } else if (+ event.type === 'collaboration_updated' &&+ event.collaboratorId === this.user.userId+ ) {+ this.fetchCollaborations();+ }+ }++ public async consume(message: ConsumeCollaborationsMessage) {+ if (this.status === 'fetching') {+ return;+ }++ const cursor = BigInt(message.cursor);+ if (this.status === 'idle') {+ this.cursor = cursor;+ this.status = 'pending';+ await this.fetchCollaborations();+ } else if (this.status === 'pending' && this.cursor !== cursor) {+ this.cursor = cursor;+ await this.fetchCollaborations();+ }+ }++ private async fetchCollaborations() {+ if (this.cursor === null) {+ return;+ }++ if (this.status !== 'pending') {+ return;+ }++ this.status = 'fetching';+ const unsyncedCollaborations = await database+ .selectFrom('collaborations')+ .selectAll()+ .where('workspace_id', '=', this.user.workspaceId)+ .where('version', '>', this.cursor)+ .orderBy('version', 'asc')+ .limit(50)+ .execute();++ if (unsyncedCollaborations.length === 0) {+ this.status = 'pending';+ return;+ }++ const collaborations = unsyncedCollaborations.map(mapCollaboration);+ const message: CollaborationsBatchMessage = {+ type: 'collaborations_batch',+ userId: this.user.userId,+ collaborations,+ };++ this.status = 'idle';+ this.user.sendMessage(message);+ }+}diff --git a/apps/server/src/consumers/interactions.ts b/apps/server/src/consumers/interactions.tsnew file mode 100644index 00000000..d08ab0c8--- /dev/null+++ b/apps/server/src/consumers/interactions.ts@@ -0,0 +1,77 @@+import {+ ConsumeInteractionsMessage,+ InteractionsBatchMessage,+} from '@colanode/core';++import { BaseConsumer } from '@/consumers/base';+import { Event } from '@/types/events';+import { database } from '@/data/database';+import { mapInteraction } from '@/lib/nodes';+import { ConnectedUser } from '@/types/users';++export class InteractionsConsumer extends BaseConsumer {+ private readonly rootId: string;++ constructor(user: ConnectedUser, rootId: string) {+ super(user);+ this.rootId = rootId;+ }++ public processEvent(event: Event): void {+ if (event.type === 'interaction_updated' && event.rootId === this.rootId) {+ this.fetchInteractions();+ }+ }++ public async consume(message: ConsumeInteractionsMessage) {+ if (this.status === 'fetching') {+ return;+ }++ const cursor = BigInt(message.cursor);+ if (this.status === 'idle') {+ this.cursor = cursor;+ this.status = 'pending';+ await this.fetchInteractions();+ } else if (this.status === 'pending' && this.cursor !== cursor) {+ this.cursor = cursor;+ await this.fetchInteractions();+ }+ }++ private async fetchInteractions() {+ if (this.cursor === null) {+ return;+ }++ if (this.status !== 'pending') {+ return;+ }++ this.status = 'fetching';+ const unsyncedInteractions = await database+ .selectFrom('interactions')+ .selectAll()+ .where('node_id', '=', this.rootId)+ .where('version', '>', this.cursor)+ .orderBy('version', 'asc')+ .limit(20)+ .execute();++ if (unsyncedInteractions.length === 0) {+ this.status = 'pending';+ return;+ }++ const interactions = unsyncedInteractions.map(mapInteraction);+ const message: InteractionsBatchMessage = {+ type: 'interactions_batch',+ userId: this.user.userId,+ rootId: this.rootId,+ interactions,+ };++ this.status = 'idle';+ this.user.sendMessage(message);+ }+}diff --git a/apps/server/src/consumers/transactions.ts b/apps/server/src/consumers/transactions.tsnew file mode 100644index 00000000..c3f8783f--- /dev/null+++ b/apps/server/src/consumers/transactions.ts@@ -0,0 +1,81 @@+import {+ ConsumeTransactionsMessage,+ TransactionsBatchMessage,+} from '@colanode/core';++import { BaseConsumer } from '@/consumers/base';+import { Event } from '@/types/events';+import { database } from '@/data/database';+import { mapTransaction } from '@/lib/nodes';+import { ConnectedUser } from '@/types/users';++export class TransactionsConsumer extends BaseConsumer {+ private readonly rootId: string;++ constructor(user: ConnectedUser, rootId: string) {+ super(user);+ this.rootId = rootId;+ }++ public processEvent(event: Event): void {+ if (event.type === 'node_created' && event.rootId === this.rootId) {+ this.fetchTransactions();+ } else if (event.type === 'node_updated' && event.rootId === this.rootId) {+ this.fetchTransactions();+ } else if (event.type === 'node_deleted' && event.rootId === this.rootId) {+ this.fetchTransactions();+ }+ }++ public async consume(message: ConsumeTransactionsMessage) {+ if (this.status === 'fetching') {+ return;+ }++ const cursor = BigInt(message.cursor);+ if (this.status === 'idle') {+ this.cursor = cursor;+ this.status = 'pending';+ await this.fetchTransactions();+ } else if (this.status === 'pending' && this.cursor !== cursor) {+ this.cursor = cursor;+ await this.fetchTransactions();+ }+ }++ private async fetchTransactions() {+ if (this.cursor === null) {+ return;+ }++ if (this.status !== 'pending') {+ return;+ }++ this.status = 'fetching';+ const unsyncedTransactions = await database+ .selectFrom('transactions')+ .selectAll()+ .where('root_id', '=', this.rootId)+ .where('version', '>', this.cursor)+ .orderBy('version', 'asc')+ .limit(20)+ .execute();++ if (unsyncedTransactions.length === 0) {+ this.status = 'pending';+ return;+ }++ const transactions = unsyncedTransactions.map(mapTransaction);+ const message: TransactionsBatchMessage = {+ type: 'transactions_batch',+ userId: this.user.userId,+ rootId: this.rootId,+ transactions,+ };++ this.status = 'idle';+ this.user.sendMessage(message);+ }+}diff --git a/apps/server/src/consumers/users.ts b/apps/server/src/consumers/users.tsnew file mode 100644index 00000000..361de3d9--- /dev/null+++ b/apps/server/src/consumers/users.ts@@ -0,0 +1,73 @@+import { ConsumeUsersMessage, UsersBatchMessage } from '@colanode/core';++import { BaseConsumer } from '@/consumers/base';+import { Event } from '@/types/events';+import { database } from '@/data/database';+import { mapUser } from '@/lib/nodes';++export class UsersConsumer extends BaseConsumer {+ public processEvent(event: Event): void {+ if (+ event.type === 'user_created' &&+ event.workspaceId === this.user.workspaceId+ ) {+ this.fetchUsers();+ } else if (+ event.type === 'user_updated' &&+ event.workspaceId === this.user.workspaceId+ ) {+ this.fetchUsers();+ }+ }++ public async consume(message: ConsumeUsersMessage) {+ if (this.status === 'fetching') {+ return;+ }++ const cursor = BigInt(message.cursor);+ if (this.status === 'idle') {+ this.cursor = cursor;+ this.status = 'pending';+ await this.fetchUsers();+ } else if (this.status === 'pending' && this.cursor !== cursor) {+ this.cursor = cursor;+ await this.fetchUsers();+ }+ }++ private async fetchUsers() {+ if (this.cursor === null) {+ return;+ }++ if (this.status !== 'pending') {+ return;+ }++ this.status = 'fetching';+ const unsyncedUsers = await database+ .selectFrom('users')+ .selectAll()+ .where('workspace_id', '=', this.user.workspaceId)+ .where('version', '>', this.cursor)+ .orderBy('version', 'asc')+ .limit(50)+ .execute();++ if (unsyncedUsers.length === 0) {+ this.status = 'pending';+ return;+ }++ const users = unsyncedUsers.map(mapUser);+ const message: UsersBatchMessage = {+ type: 'users_batch',+ userId: this.user.userId,+ users,+ };++ this.status = 'idle';+ this.user.sendMessage(message);+ }+}diff --git a/apps/server/src/services/interaction-service.ts b/apps/server/src/services/interaction-service.tsindex 2e07347a..a50f95c0 100644--- a/apps/server/src/services/interaction-service.ts+++ b/apps/server/src/services/interaction-service.ts@@ -89,6 +89,7 @@ class InteractionService {type: 'interaction_updated',userId: message.userId,nodeId: message.nodeId,+ rootId: message.rootId,workspaceId,});@@ -115,6 +116,7 @@ class InteractionService {type: 'interaction_updated',userId: message.userId,nodeId: message.nodeId,+ rootId: message.rootId,workspaceId,});diff --git a/apps/server/src/services/socket-connection.ts b/apps/server/src/services/socket-connection.tsindex 410f4a2d..aa39211d 100644--- a/apps/server/src/services/socket-connection.ts+++ b/apps/server/src/services/socket-connection.ts@@ -1,51 +1,43 @@import { WebSocket } from 'ws';import {- InitSyncConsumerMessage,- InteractionsBatchMessage,Message,- TransactionsBatchMessage,- SyncConsumerType,WorkspaceStatus,- UsersBatchMessage,- CollaborationsBatchMessage,+ ConsumeUsersMessage,+ ConsumeCollaborationsMessage,+ ConsumeTransactionsMessage,+ ConsumeInteractionsMessage,} from '@colanode/core';import { interactionService } from '@/services/interaction-service';import { createLogger } from '@/lib/logger';import { RequestAccount } from '@/types/api';import { database } from '@/data/database';-import {- mapCollaboration,- mapInteraction,- mapTransaction,- mapUser,-} from '@/lib/nodes';import {AccountUpdatedEvent,- CollaborationCreatedEvent,- CollaborationUpdatedEvent,Event,- InteractionUpdatedEvent,- NodeCreatedEvent,- NodeDeletedEvent,- NodeUpdatedEvent,- UserCreatedEvent,- UserUpdatedEvent,WorkspaceDeletedEvent,WorkspaceUpdatedEvent,} from '@/types/events';+import { ConnectedUser } from '@/types/users';+import { UsersConsumer } from '@/consumers/users';+import { CollaborationsConsumer } from '@/consumers/collaborations';+import { TransactionsConsumer } from '@/consumers/transactions';+import { InteractionsConsumer } from '@/consumers/interactions';++type NodeConsumersWrapper = {+ transactions: TransactionsConsumer;+ interactions: InteractionsConsumer;+};-type SocketSyncConsumer = {- type: SyncConsumerType;- cursor: string;- fetching: boolean;+type ConsumersWrapper = {+ users: UsersConsumer;+ collaborations: CollaborationsConsumer;+ nodes: Map<string, NodeConsumersWrapper>;};type SocketUser = {- userId: string;- workspaceId: string;- rootIds: string[];- consumers: Map<SyncConsumerType, SocketSyncConsumer>;+ user: ConnectedUser;+ consumers: ConsumersWrapper;};export class SocketConnection {@@ -84,44 +76,43 @@ export class SocketConnection {}public handleEvent(event: Event) {- if (event.type === 'node_created') {- this.handleNodeCreatedEvent(event);- } else if (event.type === 'node_updated') {- this.handleNodeUpdatedEvent(event);- } else if (event.type === 'node_deleted') {- this.handleNodeDeletedEvent(event);- } else if (event.type === 'collaboration_created') {- this.handleCollaborationCreatedEvent(event);- } else if (event.type === 'collaboration_updated') {- this.handleCollaborationUpdatedEvent(event);- } else if (event.type === 'interaction_updated') {- this.handleInteractionUpdatedEvent(event);- } else if (event.type === 'account_updated') {+ if (event.type === 'account_updated') {this.handleAccountUpdatedEvent(event);- } else if (event.type === 'user_created') {- this.handleUserCreatedEvent(event);- } else if (event.type === 'user_updated') {- this.handleUserUpdatedEvent(event);} else if (event.type === 'workspace_updated') {this.handleWorkspaceUpdatedEvent(event);} else if (event.type === 'workspace_deleted') {this.handleWorkspaceDeletedEvent(event);+ } else {+ for (const user of this.users.values()) {+ user.consumers.users.processEvent(event);+ user.consumers.collaborations.processEvent(event);+ for (const node of user.consumers.nodes.values()) {+ node.transactions.processEvent(event);+ node.interactions.processEvent(event);+ }+ }}}private async handleMessage(message: Message) {this.logger.trace(message, `Socket message from ${this.account.id}`);- if (message.type === 'init_sync_consumer') {- this.handleInitSyncConsumer(message);+ if (message.type === 'consume_users') {+ this.handleConsumeUsers(message);+ } else if (message.type === 'consume_collaborations') {+ this.handleConsumeCollaborations(message);+ } else if (message.type === 'consume_transactions') {+ this.handleConsumeTransactions(message);+ } else if (message.type === 'consume_interactions') {+ this.handleConsumeInteractions(message);} else if (message.type === 'sync_interactions') {interactionService.syncLocalInteractions(this.account.id, message);}}- private async handleInitSyncConsumer(message: InitSyncConsumerMessage) {+ private async handleConsumeUsers(message: ConsumeUsersMessage) {this.logger.info(- `Init sync consumer from ${this.account.id} and user ${message.userId} for ${message.consumerType}`+ `Consume users from ${this.account.id} and user ${message.userId}`);const user = await this.getOrCreateUser(message.userId);@@ -129,329 +120,68 @@ export class SocketConnection {return;}- const consumer = user.consumers.get(message.consumerType);- if (!consumer) {- const newConsumer: SocketSyncConsumer = {- type: message.consumerType,- cursor: message.cursor,- fetching: false,- };- user.consumers.set(message.consumerType, newConsumer);- this.consumePendingSync(user, newConsumer);- } else if (!consumer.fetching && consumer.cursor !== message.cursor) {- consumer.cursor = message.cursor;- this.consumePendingSync(user, consumer);- }- }-- private async consumePendingSync(- user: SocketUser,- consumer: SocketSyncConsumer- ) {- if (consumer.type === 'transactions') {- this.consumeTransactions(user, consumer);- } else if (consumer.type === 'collaborations') {- this.consumeCollaborations(user, consumer);- } else if (consumer.type === 'interactions') {- this.consumeInteractions(user, consumer);- } else if (consumer.type === 'users') {- this.consumeUsers(user, consumer);- }+ await user.consumers.users.consume(message);}- private async consumeTransactions(- user: SocketUser,- consumer: SocketSyncConsumer+ private async handleConsumeCollaborations(+ message: ConsumeCollaborationsMessage) {- if (consumer.fetching) {- return;- }-- if (user.rootIds.length === 0) {- return;- }-- consumer.fetching = true;- this.logger.trace(- `Checking for pending node transactions for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`+ this.logger.info(+ `Consume collaborations from ${this.account.id} and user ${message.userId}`);- const unsyncedTransactions = await database- .selectFrom('transactions')- .selectAll()- .where('root_id', 'in', user.rootIds)- .where('version', '>', BigInt(consumer.cursor))- .orderBy('version', 'asc')- .limit(20)- .execute();-- if (unsyncedTransactions.length === 0) {- this.logger.trace(- `No pending node transactions found for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`- );- consumer.fetching = false;+ const user = await this.getOrCreateUser(message.userId);+ if (user === null) {return;}- const transactions = unsyncedTransactions.map(mapTransaction);- const message: TransactionsBatchMessage = {- type: 'transactions_batch',- userId: user.userId,- transactions,- };-- user.consumers.delete(consumer.type);- this.sendMessage(message);+ await user.consumers.collaborations.consume(message);}- private async consumeCollaborations(- user: SocketUser,- consumer: SocketSyncConsumer- ) {- if (consumer.fetching) {- return;- }-- consumer.fetching = true;- this.logger.trace(- `Checking for pending collaborations for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`+ private async handleConsumeTransactions(message: ConsumeTransactionsMessage) {+ this.logger.info(+ `Consume transactions from ${this.account.id} and user ${message.userId}`);- const unsyncedCollaborations = await database- .selectFrom('collaborations')- .selectAll()- .where('collaborator_id', '=', user.userId)- .where('version', '>', BigInt(consumer.cursor))- .orderBy('version', 'asc')- .limit(50)- .execute();-- if (unsyncedCollaborations.length === 0) {- this.logger.trace(- `No pending collaborations found for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`- );- consumer.fetching = false;- return;- }-- const collaborations = unsyncedCollaborations.map(mapCollaboration);- const message: CollaborationsBatchMessage = {- type: 'collaborations_batch',- userId: user.userId,- collaborations,- };-- user.consumers.delete(consumer.type);- this.sendMessage(message);- }-- private async consumeInteractions(- user: SocketUser,- consumer: SocketSyncConsumer- ) {- if (consumer.fetching) {+ const user = await this.getOrCreateUser(message.userId);+ if (user === null) {return;}- consumer.fetching = true;- this.logger.trace(- `Checking for pending interactions for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`- );+ let nodeConsumers = user.consumers.nodes.get(message.rootId);+ if (!nodeConsumers) {+ nodeConsumers = {+ transactions: new TransactionsConsumer(user.user, message.rootId),+ interactions: new InteractionsConsumer(user.user, message.rootId),+ };- const unsyncedInteractions = await database- .selectFrom('interactions')- .selectAll()- .where('node_id', 'in', user.rootIds)- .where('version', '>', BigInt(consumer.cursor))- .orderBy('version', 'asc')- .limit(20)- .execute();-- if (unsyncedInteractions.length === 0) {- this.logger.trace(- `No pending interactions found for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`- );- consumer.fetching = false;- return;+ user.consumers.nodes.set(message.rootId, nodeConsumers);}- const interactions = unsyncedInteractions.map(mapInteraction);- const message: InteractionsBatchMessage = {- type: 'interactions_batch',- userId: user.userId,- interactions,- };-- user.consumers.delete(consumer.type);- this.sendMessage(message);+ await nodeConsumers.transactions.consume(message);}- private async consumeUsers(user: SocketUser, consumer: SocketSyncConsumer) {- if (consumer.fetching) {- return;- }-- consumer.fetching = true;- this.logger.trace(- `Checking for pending users for ${this.account.id} and user ${user.userId} with cursor ${consumer.cursor}`+ private async handleConsumeInteractions(message: ConsumeInteractionsMessage) {+ this.logger.info(+ `Consume interactions from ${this.account.id} and user ${message.userId}`);- const unsyncedUsers = await database- .selectFrom('users')- .selectAll()- .where('workspace_id', '=', user.workspaceId)- .where('version', '>', BigInt(consumer.cursor))- .orderBy('version', 'asc')- .limit(50)- .execute();-- if (unsyncedUsers.length === 0) {- consumer.fetching = false;+ const user = await this.getOrCreateUser(message.userId);+ if (user === null) {return;}- const users = unsyncedUsers.map(mapUser);- const message: UsersBatchMessage = {- type: 'users_batch',- userId: user.userId,- users,- };-- user.consumers.delete(consumer.type);- this.sendMessage(message);- }-- private async handleNodeCreatedEvent(event: NodeCreatedEvent) {- for (const user of this.users.values()) {- if (user.workspaceId !== event.workspaceId) {- continue;- }-- if (!user.rootIds.includes(event.rootId)) {- continue;- }-- const transactionsConsumer = user.consumers.get('transactions');- if (transactionsConsumer) {- this.consumePendingSync(user, transactionsConsumer);- }-- const interactionsConsumer = user.consumers.get('interactions');- if (interactionsConsumer) {- this.consumePendingSync(user, interactionsConsumer);- }- }- }-- private async handleNodeUpdatedEvent(event: NodeUpdatedEvent) {- for (const user of this.users.values()) {- if (user.workspaceId !== event.workspaceId) {- continue;- }-- if (!user.rootIds.includes(event.rootId)) {- continue;- }-- const transactionsConsumer = user.consumers.get('transactions');- if (transactionsConsumer) {- this.consumePendingSync(user, transactionsConsumer);- }- }- }-- private async handleNodeDeletedEvent(event: NodeDeletedEvent) {- for (const user of this.users.values()) {- if (user.workspaceId !== event.workspaceId) {- continue;- }-- if (!user.rootIds.includes(event.rootId)) {- continue;- }-- const transactionsConsumer = user.consumers.get('transactions');- if (transactionsConsumer) {- this.consumePendingSync(user, transactionsConsumer);- }- }- }-- private async handleCollaborationCreatedEvent(- event: CollaborationCreatedEvent- ) {- for (const user of this.users.values()) {- if (user.userId !== event.collaboratorId) {- continue;- }-- if (!user.rootIds.includes(event.nodeId)) {- user.rootIds.push(event.nodeId);- }-- const collaborationsConsumer = user.consumers.get('collaborations');- if (collaborationsConsumer) {- this.consumePendingSync(user, collaborationsConsumer);- }- }- }-- private async handleCollaborationUpdatedEvent(- event: CollaborationUpdatedEvent- ) {- for (const user of this.users.values()) {- if (user.userId !== event.collaboratorId) {- continue;- }-- if (!user.rootIds.includes(event.nodeId)) {- user.rootIds.push(event.nodeId);- }-- const collaborationsConsumer = user.consumers.get('collaborations');- if (collaborationsConsumer) {- this.consumePendingSync(user, collaborationsConsumer);- }- }- }-- private async handleInteractionUpdatedEvent(event: InteractionUpdatedEvent) {- for (const user of this.users.values()) {- if (user.userId !== event.userId) {- continue;- }-- const interactionsConsumer = user.consumers.get('interactions');- if (interactionsConsumer) {- this.consumePendingSync(user, interactionsConsumer);- }- }- }-- private handleUserCreatedEvent(event: UserCreatedEvent) {- for (const user of this.users.values()) {- if (user.userId !== event.userId) {- continue;- }+ let nodeConsumers = user.consumers.nodes.get(message.rootId);+ if (!nodeConsumers) {+ nodeConsumers = {+ transactions: new TransactionsConsumer(user.user, message.rootId),+ interactions: new InteractionsConsumer(user.user, message.rootId),+ };- const usersConsumer = user.consumers.get('users');- if (usersConsumer) {- this.consumePendingSync(user, usersConsumer);- }+ user.consumers.nodes.set(message.rootId, nodeConsumers);}- }-- private handleUserUpdatedEvent(event: UserUpdatedEvent) {- for (const user of this.users.values()) {- if (user.userId !== event.userId) {- continue;- }- const usersConsumer = user.consumers.get('users');- if (usersConsumer) {- this.consumePendingSync(user, usersConsumer);- }- }+ await nodeConsumers.interactions.consume(message);}private async getOrCreateUser(userId: string): Promise<SocketUser | null> {@@ -491,28 +221,31 @@ export class SocketConnection {return null;}- const collaborations = await database- .selectFrom('collaborations')- .select(['node_id'])- .where('collaborator_id', '=', userId)- .where('deleted_at', 'is', null)- .execute();-- const rootIds = collaborations.map(- (collaboration) => collaboration.node_id- );-const addedSocketUser = this.users.get(userId);if (addedSocketUser) {return addedSocketUser;}// Create and store the new SocketUser- const newSocketUser: SocketUser = {- userId,+ const connectedUser: ConnectedUser = {+ userId: user.id,workspaceId: user.workspace_id,- rootIds,- consumers: new Map(),+ accountId: this.account.id,+ deviceId: this.account.deviceId,+ sendMessage: this.sendMessage.bind(this),+ };++ const usersConsumer = new UsersConsumer(connectedUser);+ const collaborationsConsumer = new CollaborationsConsumer(connectedUser);+ const nodesConsumers = new Map<string, NodeConsumersWrapper>();++ const newSocketUser: SocketUser = {+ user: connectedUser,+ consumers: {+ users: usersConsumer,+ collaborations: collaborationsConsumer,+ nodes: nodesConsumers,+ },};this.users.set(userId, newSocketUser);@@ -531,11 +264,11 @@ export class SocketConnection {}private handleWorkspaceUpdatedEvent(event: WorkspaceUpdatedEvent) {- const users = Array.from(this.users.values()).filter(- (user) => user.workspaceId === event.workspaceId+ const socketUsers = Array.from(this.users.values()).filter(+ (user) => user.user.workspaceId === event.workspaceId);- if (users.length === 0) {+ if (socketUsers.length === 0) {return;}@@ -546,11 +279,11 @@ export class SocketConnection {}private handleWorkspaceDeletedEvent(event: WorkspaceDeletedEvent) {- const users = Array.from(this.users.values()).filter(- (user) => user.workspaceId === event.workspaceId+ const socketUsers = Array.from(this.users.values()).filter(+ (user) => user.user.workspaceId === event.workspaceId);- if (users.length === 0) {+ if (socketUsers.length === 0) {return;}diff --git a/apps/server/src/types/events.ts b/apps/server/src/types/events.tsindex e71725f1..d3b5c34f 100644--- a/apps/server/src/types/events.ts+++ b/apps/server/src/types/events.ts@@ -42,6 +42,7 @@ export type InteractionUpdatedEvent = {type: 'interaction_updated';userId: string;nodeId: string;+ rootId: string;workspaceId: string;};diff --git a/apps/server/src/types/users.ts b/apps/server/src/types/users.tsnew file mode 100644index 00000000..44ae49d2--- /dev/null+++ b/apps/server/src/types/users.ts@@ -0,0 +1,9 @@+import { Message } from '@colanode/core';++export type ConnectedUser = {+ userId: string;+ workspaceId: string;+ accountId: string;+ deviceId: string;+ sendMessage: (message: Message) => void;+};diff --git a/packages/core/src/types/messages.ts b/packages/core/src/types/messages.tsindex 3cf3ed34..568b0d21 100644--- a/packages/core/src/types/messages.ts+++ b/packages/core/src/types/messages.ts@@ -4,12 +4,12 @@ import {ServerInteraction,ServerTransaction,ServerUser,- SyncConsumerType,} from './sync';export type TransactionsBatchMessage = {type: 'transactions_batch';userId: string;+ rootId: string;transactions: ServerTransaction[];};@@ -22,6 +22,7 @@ export type CollaborationsBatchMessage = {export type InteractionsBatchMessage = {type: 'interactions_batch';userId: string;+ rootId: string;interactions: ServerInteraction[];};@@ -35,13 +36,33 @@ export type SyncInteractionsMessage = {type: 'sync_interactions';userId: string;nodeId: string;+ rootId: string;events: InteractionEvent[];};-export type InitSyncConsumerMessage = {- type: 'init_sync_consumer';+export type ConsumeUsersMessage = {+ type: 'consume_users';userId: string;- consumerType: SyncConsumerType;+ cursor: string;+};++export type ConsumeCollaborationsMessage = {+ type: 'consume_collaborations';+ userId: string;+ cursor: string;+};++export type ConsumeInteractionsMessage = {+ type: 'consume_interactions';+ userId: string;+ rootId: string;+ cursor: string;+};++export type ConsumeTransactionsMessage = {+ type: 'consume_transactions';+ userId: string;+ rootId: string;cursor: string;};@@ -66,7 +87,10 @@ export type Message =| InteractionsBatchMessage| UsersBatchMessage| SyncInteractionsMessage- | InitSyncConsumerMessage+ | ConsumeUsersMessage+ | ConsumeCollaborationsMessage+ | ConsumeInteractionsMessage+ | ConsumeTransactionsMessage| AccountUpdatedMessage| WorkspaceUpdatedMessage| WorkspaceDeletedMessage;diff --git a/packages/core/src/types/sync.ts b/packages/core/src/types/sync.tsindex f5d921ab..3fcdb1ed 100644--- a/packages/core/src/types/sync.ts+++ b/packages/core/src/types/sync.ts@@ -143,9 +143,3 @@ export type ServerUser = {updatedAt: string | null;version: string;};--export type SyncConsumerType =- | 'transactions'- | 'collaborations'- | 'interactions'- | 'users';