Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Refactor socket and sync communications
Hakan Shehu1fcf0b135c86
diff --git a/apps/desktop/src/main/data/workspace/migrations.ts b/apps/desktop/src/main/data/workspace/migrations.tsindex 43714ecc..b507f7ec 100644--- a/apps/desktop/src/main/data/workspace/migrations.ts+++ b/apps/desktop/src/main/data/workspace/migrations.ts@@ -304,7 +304,7 @@ const createCursorsTable: Migration = {up: async (db) => {await db.schema.createTable('cursors')- .addColumn('name', 'text', (col) => col.notNull().primaryKey())+ .addColumn('type', 'text', (col) => col.notNull().primaryKey()).addColumn('value', 'integer', (col) => col.notNull().defaultTo(0)).addColumn('created_at', 'text', (col) => col.notNull()).addColumn('updated_at', 'text')diff --git a/apps/desktop/src/main/data/workspace/schema.ts b/apps/desktop/src/main/data/workspace/schema.tsindex bdacce6e..c3b7c706 100644--- a/apps/desktop/src/main/data/workspace/schema.ts+++ b/apps/desktop/src/main/data/workspace/schema.ts@@ -1,4 +1,8 @@-import { InteractionAttribute, NodeType } from '@colanode/core';+import {+ InteractionAttribute,+ NodeType,+ SyncConsumerType,+} from '@colanode/core';import { ColumnType, Insertable, Selectable, Updateable } from 'kysely';interface NodeTable {@@ -116,7 +120,7 @@ export type CreateInteractionEvent = Insertable<InteractionEventTable>;export type UpdateInteractionEvent = Updateable<InteractionEventTable>;interface CursorTable {- name: ColumnType<string, string, never>;+ type: ColumnType<SyncConsumerType, SyncConsumerType, never>;value: ColumnType<bigint, bigint, bigint>;created_at: ColumnType<string, string, never>;updated_at: ColumnType<string | null, string | null, string | null>;diff --git a/apps/desktop/src/main/services/sync-service.ts b/apps/desktop/src/main/services/sync-service.tsindex 9c3b24fd..514bc5e6 100644--- a/apps/desktop/src/main/services/sync-service.ts+++ b/apps/desktop/src/main/services/sync-service.ts@@ -1,16 +1,14 @@import {CollaborationRevocationsBatchMessage,CollaborationsBatchMessage,- FetchCollaborationRevocationsMessage,- FetchCollaborationsMessage,- FetchInteractionsMessage,- FetchNodeTransactionsMessage,+ InitSyncConsumerMessage,GetNodeTransactionsOutput,InteractionsBatchMessage,LocalNodeTransaction,NodeTransactionsBatchMessage,SyncInteractionsMessage,SyncNodeTransactionsOutput,+ SyncConsumerType,} from '@colanode/core';import { sql } from 'kysely';@@ -92,10 +90,10 @@ class SyncService {this.syncLocalIncompleteTransactions(userId);this.syncLocalPendingInteractions(userId);- this.requireNodeTransactions(userId);- this.requireCollaborations(userId);- this.requireCollaborationRevocations(userId);- this.requireInteractions(userId);+ this.initSyncConsumer(userId, 'transactions');+ this.initSyncConsumer(userId, 'collaborations');+ this.initSyncConsumer(userId, 'revocations');+ this.initSyncConsumer(userId, 'interactions');this.syncMissingNodes(userId);}@@ -246,7 +244,7 @@ class SyncService {);this.syncingTransactions.delete(message.userId);- this.requireNodeTransactions(message.userId);+ this.initSyncConsumer(message.userId, 'transactions');}}@@ -281,7 +279,7 @@ class SyncService {);} finally {this.syncingCollaborations.delete(message.userId);- this.requireCollaborations(message.userId);+ this.initSyncConsumer(message.userId, 'collaborations');}}@@ -322,7 +320,7 @@ class SyncService {);this.syncingRevocations.delete(message.userId);- this.requireCollaborationRevocations(message.userId);+ this.initSyncConsumer(message.userId, 'revocations');}}@@ -361,7 +359,7 @@ class SyncService {);this.syncingInteractions.delete(message.userId);- this.requireInteractions(message.userId);+ this.initSyncConsumer(message.userId, 'interactions');}}@@ -776,89 +774,10 @@ class SyncService {}}- private async requireNodeTransactions(userId: string) {- this.debug(`Requiring node transactions for user ${userId}`);-- const workspace = await databaseService.appDatabase- .selectFrom('workspaces')- .select(['user_id', 'workspace_id', 'account_id'])- .where('user_id', '=', userId)- .executeTakeFirst();-- if (!workspace) {- this.debug(- `No workspace found for user ${userId}, skipping requiring node transactions`- );- return;- }-- const cursor = await this.fetchCursor(userId, 'transactions');- const message: FetchNodeTransactionsMessage = {- type: 'fetch_node_transactions',- userId,- workspaceId: workspace.workspace_id,- cursor: cursor.toString(),- };-- socketService.sendMessage(workspace.account_id, message);- }-- private async requireCollaborations(userId: string) {- this.debug(`Requiring collaborations for user ${userId}`);-- const workspace = await databaseService.appDatabase- .selectFrom('workspaces')- .select(['user_id', 'workspace_id', 'account_id'])- .where('user_id', '=', userId)- .executeTakeFirst();-- if (!workspace) {- this.debug(- `No workspace found for user ${userId}, skipping requiring collaborations`- );- return;- }-- const cursor = await this.fetchCursor(userId, 'collaborations');- const message: FetchCollaborationsMessage = {- type: 'fetch_collaborations',- userId: workspace.user_id,- workspaceId: workspace.workspace_id,- cursor: cursor.toString(),- };-- socketService.sendMessage(workspace.account_id, message);- }-- private async requireCollaborationRevocations(userId: string) {- this.debug(`Requiring collaboration revocations for user ${userId}`);-- const workspace = await databaseService.appDatabase- .selectFrom('workspaces')- .select(['user_id', 'workspace_id', 'account_id'])- .where('user_id', '=', userId)- .executeTakeFirst();-- if (!workspace) {- this.debug(- `No workspace found for user ${userId}, skipping requiring collaboration revocations`- );- return;- }-- const cursor = await this.fetchCursor(userId, 'revocations');- const message: FetchCollaborationRevocationsMessage = {- type: 'fetch_collaboration_revocations',- userId: workspace.user_id,- workspaceId: workspace.workspace_id,- cursor: cursor.toString(),- };-- socketService.sendMessage(workspace.account_id, message);- }-- private async requireInteractions(userId: string) {- this.debug(`Requiring interactions for user ${userId}`);+ private async initSyncConsumer(userId: string, type: SyncConsumerType) {+ this.debug(+ `Initializing sync consumer for user ${userId} with type ${type}`+ );const workspace = await databaseService.appDatabase.selectFrom('workspaces')@@ -873,19 +792,23 @@ class SyncService {return;}- const cursor = await this.fetchCursor(userId, 'interactions');- const message: FetchInteractionsMessage = {- type: 'fetch_interactions',- userId: workspace.user_id,- workspaceId: workspace.workspace_id,+ const cursor = await this.fetchCursor(userId, type);+ const message: InitSyncConsumerMessage = {+ type: 'init_sync_consumer',+ userId,+ consumerType: type,cursor: cursor.toString(),};socketService.sendMessage(workspace.account_id, message);}- private async updateCursor(userId: string, name: string, cursor: bigint) {- this.debug(`Updating cursor ${name} for user ${userId} to ${cursor}`);+ private async updateCursor(+ userId: string,+ type: SyncConsumerType,+ cursor: bigint+ ) {+ this.debug(`Updating cursor ${type} for user ${userId} to ${cursor}`);const workspaceDatabase =await databaseService.getWorkspaceDatabase(userId);@@ -893,12 +816,12 @@ class SyncService {await workspaceDatabase.insertInto('cursors').values({- name,+ type,value: cursor,created_at: new Date().toISOString(),}).onConflict((eb) =>- eb.column('name').doUpdateSet({+ eb.column('type').doUpdateSet({value: cursor,updated_at: new Date().toISOString(),})@@ -906,14 +829,17 @@ class SyncService {.execute();}- private async fetchCursor(userId: string, name: string): Promise<bigint> {+ private async fetchCursor(+ userId: string,+ type: SyncConsumerType+ ): Promise<bigint> {const workspaceDatabase =await databaseService.getWorkspaceDatabase(userId);const cursor = await workspaceDatabase.selectFrom('cursors').select('value')- .where('name', '=', name)+ .where('type', '=', type).executeTakeFirst();return cursor?.value ?? 0n;diff --git a/apps/server/src/api.ts b/apps/server/src/api.tsindex 29ff6d37..989c8b2a 100644--- a/apps/server/src/api.ts+++ b/apps/server/src/api.ts@@ -11,7 +11,7 @@ import { filesRouter } from '@/routes/files';import { nodesRouter } from '@/routes/nodes';import { syncRouter } from '@/routes/sync';import { workspacesRouter } from '@/routes/workspaces';-import { synapse } from '@/services/synapse-service';+import { socketService } from '@/services/socket-service';import { createLogger } from '@/lib/logger';const logger = createLogger('api');@@ -40,7 +40,7 @@ export const initApi = async () => {app.use('/v1/nodes', authMiddleware, nodesRouter);const server = http.createServer(app);- await synapse.init(server);+ await socketService.init(server);server.listen(port, () => {logger.info(`Server is running at http://localhost:${port}`);diff --git a/apps/server/src/lib/tokens.ts b/apps/server/src/lib/tokens.tsindex f1801881..67bd1e7c 100644--- a/apps/server/src/lib/tokens.ts+++ b/apps/server/src/lib/tokens.ts@@ -2,7 +2,7 @@ import { sha256 } from 'js-sha256';import { database } from '@/data/database';import { uuid } from '@/lib/utils';-import { ColanodeRequestAccount } from '@/types/api';+import { RequestAccount } from '@/types/api';interface GenerateTokenResult {token: string;@@ -16,7 +16,7 @@ type VerifyTokenResult =}| {authenticated: true;- account: ColanodeRequestAccount;+ account: RequestAccount;};export const generateToken = (id: string): GenerateTokenResult => {diff --git a/apps/server/src/services/socket-connection.ts b/apps/server/src/services/socket-connection.tsnew file mode 100644index 00000000..387dd7df--- /dev/null+++ b/apps/server/src/services/socket-connection.ts@@ -0,0 +1,452 @@+import { WebSocket } from 'ws';+import {+ CollaborationRevocationsBatchMessage,+ CollaborationsBatchMessage,+ InitSyncConsumerMessage,+ InteractionsBatchMessage,+ Message,+ NodeTransactionsBatchMessage,+ NodeType,+ SyncConsumerType,+ WorkspaceStatus,+} from '@colanode/core';++import { interactionService } from '@/services/interaction-service';+import { createLogger } from '@/lib/logger';+import { RequestAccount } from '@/types/api';+import { database } from '@/data/database';+import {+ mapCollaboration,+ mapCollaborationRevocation,+ mapInteraction,+ mapNodeTransaction,+} from '@/lib/nodes';+import {+ CollaboratorAddedEvent,+ CollaboratorRemovedEvent,+ Event,+ InteractionUpdatedEvent,+ NodeCreatedEvent,+ NodeDeletedEvent,+ NodeUpdatedEvent,+} from '@/types/events';++const PUBLIC_NODES: NodeType[] = ['workspace', 'user'];++type SocketSyncConsumer = {+ type: SyncConsumerType;+ cursor: string;+ fetching: boolean;+};++type SocketUser = {+ userId: string;+ workspaceId: string;+ consumers: Map<SyncConsumerType, SocketSyncConsumer>;+};++export class SocketConnection {+ private readonly logger = createLogger('socket-connection');+ private readonly account: RequestAccount;+ private readonly socket: WebSocket;++ private readonly users: Map<string, SocketUser> = new Map();++ constructor(account: RequestAccount, socket: WebSocket) {+ this.account = account;+ this.socket = socket;++ this.socket.on('message', (data) => {+ const message = JSON.parse(data.toString()) as Message;+ this.handleMessage(message);+ });+ }++ public getDeviceId() {+ return this.account.deviceId;+ }++ public getAccountId() {+ return this.account.id;+ }++ public sendMessage(message: Message) {+ this.socket.send(JSON.stringify(message));+ }++ public handleEvent(event: Event) {+ if (event.type === 'node_created') {+ this.handleNodeCreatedEvent(event);+ } else if (event.type === 'node_updated') {+ this.handleNodeUpdatedEvent(event);+ } else if (event.type === 'node_deleted') {+ this.handleNodeDeletedEvent(event);+ } else if (event.type === 'collaborator_added') {+ this.handleCollaboratorAddedEvent(event);+ } else if (event.type === 'collaborator_removed') {+ this.handleCollaboratorRemovedEvent(event);+ } else if (event.type === 'interaction_updated') {+ this.handleInteractionUpdatedEvent(event);+ }+ }++ private async handleMessage(message: Message) {+ this.logger.trace(message, `Socket message from ${this.account.id}`);++ if (message.type === 'init_sync_consumer') {+ this.handleInitSyncConsumer(message);+ } else if (message.type === 'sync_interactions') {+ interactionService.syncLocalInteractions(this.account.id, message);+ }+ }++ private async handleInitSyncConsumer(message: InitSyncConsumerMessage) {+ this.logger.info(+ `Init sync consumer from ${this.account.id} for ${message.consumerType}`+ );++ const user = await this.getOrCreateUser(message.userId);+ if (user === null) {+ return;+ }++ const consumer = user.consumers.get(message.consumerType);+ if (!consumer) {+ const newConsumer: SocketSyncConsumer = {+ type: message.consumerType,+ cursor: message.cursor,+ fetching: false,+ };+ user.consumers.set(message.consumerType, newConsumer);+ this.consumePendingSync(user, newConsumer);+ } else if (!consumer.fetching && consumer.cursor !== message.cursor) {+ consumer.cursor = message.cursor;+ this.consumePendingSync(user, consumer);+ }+ }++ private async consumePendingSync(+ user: SocketUser,+ consumer: SocketSyncConsumer+ ) {+ if (consumer.type === 'transactions') {+ this.consumeTransactions(user, consumer);+ } else if (consumer.type === 'revocations') {+ this.consumeRevocations(user, consumer);+ } else if (consumer.type === 'collaborations') {+ this.consumeCollaborations(user, consumer);+ } else if (consumer.type === 'interactions') {+ this.consumeInteractions(user, consumer);+ }+ }++ private async consumeTransactions(+ user: SocketUser,+ consumer: SocketSyncConsumer+ ) {+ if (consumer.fetching) {+ return;+ }++ consumer.fetching = true;+ this.logger.trace(+ `Sending pending node transactions for ${this.account.id} with ${consumer.type}`+ );++ const unsyncedTransactions = await database+ .selectFrom('node_transactions as nt')+ .leftJoin('collaborations as c', (join) =>+ join+ .on('c.user_id', '=', user.userId)+ .onRef('c.node_id', '=', 'nt.node_id')+ )+ .selectAll('nt')+ .where((eb) =>+ eb.or([+ eb.and([+ eb('nt.workspace_id', '=', user.workspaceId),+ eb('nt.node_type', 'in', PUBLIC_NODES),+ ]),+ eb('c.node_id', '=', eb.ref('nt.node_id')),+ ])+ )+ .where('nt.version', '>', BigInt(consumer.cursor))+ .orderBy('nt.version', 'asc')+ .limit(20)+ .execute();++ if (unsyncedTransactions.length === 0) {+ consumer.fetching = false;+ return;+ }++ const transactions = unsyncedTransactions.map(mapNodeTransaction);+ const message: NodeTransactionsBatchMessage = {+ type: 'node_transactions_batch',+ userId: user.userId,+ transactions,+ };++ user.consumers.delete(consumer.type);+ this.sendMessage(message);+ }++ private async consumeRevocations(+ user: SocketUser,+ consumer: SocketSyncConsumer+ ) {+ if (consumer.fetching) {+ return;+ }++ consumer.fetching = true;+ this.logger.trace(+ consumer,+ `Sending pending collaboration revocations for ${this.account.id} with ${consumer.type}`+ );++ const unsyncedRevocations = await database+ .selectFrom('collaboration_revocations as cr')+ .selectAll()+ .where('cr.user_id', '=', user.userId)+ .where('cr.version', '>', BigInt(consumer.cursor))+ .orderBy('cr.version', 'asc')+ .limit(50)+ .execute();++ if (unsyncedRevocations.length === 0) {+ consumer.fetching = false;+ return;+ }++ const revocations = unsyncedRevocations.map(mapCollaborationRevocation);+ const message: CollaborationRevocationsBatchMessage = {+ type: 'collaboration_revocations_batch',+ userId: user.userId,+ revocations,+ };++ user.consumers.delete(consumer.type);+ this.sendMessage(message);+ }++ private async consumeCollaborations(+ user: SocketUser,+ consumer: SocketSyncConsumer+ ) {+ if (consumer.fetching) {+ return;+ }++ consumer.fetching = true;+ this.logger.trace(+ consumer,+ `Sending pending collaborations for ${this.account.id} with ${consumer.type}`+ );++ const unsyncedCollaborations = await database+ .selectFrom('collaborations as c')+ .selectAll()+ .where('c.user_id', '=', user.userId)+ .where('c.version', '>', BigInt(consumer.cursor))+ .orderBy('c.version', 'asc')+ .limit(50)+ .execute();++ if (unsyncedCollaborations.length === 0) {+ consumer.fetching = false;+ return;+ }++ const collaborations = unsyncedCollaborations.map(mapCollaboration);+ const message: CollaborationsBatchMessage = {+ type: 'collaborations_batch',+ userId: user.userId,+ collaborations,+ };++ user.consumers.delete(consumer.type);+ this.sendMessage(message);+ }++ private async consumeInteractions(+ user: SocketUser,+ consumer: SocketSyncConsumer+ ) {+ if (consumer.fetching) {+ return;+ }++ consumer.fetching = true;+ this.logger.trace(+ consumer,+ `Sending pending interactions for ${this.account.id} with ${consumer.type}`+ );++ const unsyncedInteractions = await database+ .selectFrom('interactions as i')+ .leftJoin('collaborations as c', (join) =>+ join+ .on('c.user_id', '=', user.userId)+ .onRef('c.node_id', '=', 'i.node_id')+ )+ .where((eb) =>+ eb.or([+ eb.and([+ eb('i.workspace_id', '=', user.workspaceId),+ eb('i.node_type', 'in', PUBLIC_NODES),+ ]),+ eb('c.node_id', '=', eb.ref('i.node_id')),+ ])+ )+ .selectAll('i')+ .where('i.version', '>', BigInt(consumer.cursor))+ .orderBy('i.version', 'asc')+ .limit(20)+ .execute();++ if (unsyncedInteractions.length === 0) {+ consumer.fetching = false;+ return;+ }++ const interactions = unsyncedInteractions.map(mapInteraction);+ const message: InteractionsBatchMessage = {+ type: 'interactions_batch',+ userId: user.userId,+ interactions,+ };++ user.consumers.delete(consumer.type);+ this.sendMessage(message);+ }++ private async handleNodeCreatedEvent(event: NodeCreatedEvent) {+ for (const user of this.users.values()) {+ if (user.workspaceId !== event.workspaceId) {+ continue;+ }++ const transactionsConsumer = user.consumers.get('transactions');+ if (transactionsConsumer) {+ this.consumePendingSync(user, transactionsConsumer);+ }++ const collaborationsConsumer = user.consumers.get('collaborations');+ if (collaborationsConsumer) {+ this.consumePendingSync(user, collaborationsConsumer);+ }++ const interactionsConsumer = user.consumers.get('interactions');+ if (interactionsConsumer) {+ this.consumePendingSync(user, interactionsConsumer);+ }+ }+ }++ private async handleNodeUpdatedEvent(event: NodeUpdatedEvent) {+ for (const user of this.users.values()) {+ if (user.workspaceId !== event.workspaceId) {+ continue;+ }++ const transactionsConsumer = user.consumers.get('transactions');+ if (transactionsConsumer) {+ this.consumePendingSync(user, transactionsConsumer);+ }+ }+ }++ private async handleNodeDeletedEvent(event: NodeDeletedEvent) {+ for (const user of this.users.values()) {+ if (user.workspaceId !== event.workspaceId) {+ continue;+ }++ const transactionsConsumer = user.consumers.get('transactions');+ if (transactionsConsumer) {+ this.consumePendingSync(user, transactionsConsumer);+ }++ const revocationsConsumer = user.consumers.get('revocations');+ if (revocationsConsumer) {+ this.consumePendingSync(user, revocationsConsumer);+ }+ }+ }++ private async handleCollaboratorAddedEvent(event: CollaboratorAddedEvent) {+ for (const user of this.users.values()) {+ if (user.userId !== event.userId) {+ continue;+ }++ const collaborationsConsumer = user.consumers.get('collaborations');+ if (collaborationsConsumer) {+ this.consumePendingSync(user, collaborationsConsumer);+ }+ }+ }++ private async handleCollaboratorRemovedEvent(+ event: CollaboratorRemovedEvent+ ) {+ for (const user of this.users.values()) {+ if (user.userId !== event.userId) {+ continue;+ }++ const revocationsConsumer = user.consumers.get('revocations');+ if (revocationsConsumer) {+ this.consumePendingSync(user, revocationsConsumer);+ }+ }+ }++ private async handleInteractionUpdatedEvent(event: InteractionUpdatedEvent) {+ for (const user of this.users.values()) {+ if (user.userId !== event.userId) {+ continue;+ }++ const interactionsConsumer = user.consumers.get('interactions');+ if (interactionsConsumer) {+ this.consumePendingSync(user, interactionsConsumer);+ }+ }+ }++ private async getOrCreateUser(userId: string): Promise<SocketUser | null> {+ const socketUser = this.users.get(userId);+ if (socketUser) {+ return socketUser;+ }++ const workspaceUser = await database+ .selectFrom('workspace_users')+ .where('id', '=', userId)+ .selectAll()+ .executeTakeFirst();++ if (!workspaceUser) {+ return null;+ }++ if (workspaceUser.status !== WorkspaceStatus.Active) {+ return null;+ }++ if (workspaceUser.account_id !== this.account.id) {+ return null;+ }++ const newSocketUser: SocketUser = {+ userId,+ workspaceId: workspaceUser.workspace_id,+ consumers: new Map(),+ };++ this.users.set(userId, newSocketUser);+ return newSocketUser;+ }+}diff --git a/apps/server/src/services/socket-service.ts b/apps/server/src/services/socket-service.tsnew file mode 100644index 00000000..734369c7--- /dev/null+++ b/apps/server/src/services/socket-service.ts@@ -0,0 +1,75 @@+import { WebSocketServer, WebSocket } from 'ws';++import { IncomingMessage, Server } from 'http';++import { SocketConnection } from '@/services/socket-connection';+import { eventBus } from '@/lib/event-bus';+import { verifyToken } from '@/lib/tokens';+import { createLogger } from '@/lib/logger';+import { RequestAccount } from '@/types/api';++class SocketService {+ private readonly logger = createLogger('socket-service');+ private readonly connections: Map<string, SocketConnection> = new Map();++ constructor() {+ eventBus.subscribe((event) => {+ for (const connection of this.connections.values()) {+ connection.handleEvent(event);+ }+ });+ }++ public async init(server: Server) {+ this.logger.info('Initializing socket service');++ const wss = new WebSocketServer({+ noServer: true,+ });++ server.on('upgrade', async (request, socket, head) => {+ if (request.url !== '/v1/synapse') {+ socket.destroy();+ return;+ }++ const token = request.headers['authorization'];+ if (!token) {+ socket.destroy();+ return;+ }++ const result = await verifyToken(token);+ if (!result.authenticated) {+ socket.destroy();+ return;+ }++ const account = result.account;+ wss.handleUpgrade(request, socket, head, (ws) => {+ wss.emit('connection', ws, request, account);+ });+ });++ wss.on(+ 'connection',+ async (+ socket: WebSocket,+ _: IncomingMessage,+ account: RequestAccount+ ) => {+ socket.on('close', () => {+ const connection = this.connections.get(account.deviceId);+ if (connection) {+ this.connections.delete(account.deviceId);+ }+ });++ const connection = new SocketConnection(account, socket);+ this.connections.set(account.deviceId, connection);+ }+ );+ }+}++export const socketService = new SocketService();diff --git a/apps/server/src/services/synapse-service.ts b/apps/server/src/services/synapse-service.tsdeleted file mode 100644index 032943d2..00000000--- a/apps/server/src/services/synapse-service.ts+++ /dev/null@@ -1,688 +0,0 @@-import {- CollaborationRevocationsBatchMessage,- CollaborationsBatchMessage,- InteractionsBatchMessage,- Message,- NodeTransactionsBatchMessage,- NodeType,-} from '@colanode/core';-import { WebSocket, WebSocketServer } from 'ws';--import { Server } from 'http';--import { database } from '@/data/database';-import { eventBus } from '@/lib/event-bus';-import {- mapCollaboration,- mapCollaborationRevocation,- mapInteraction,- mapNodeTransaction,-} from '@/lib/nodes';-import { verifyToken } from '@/lib/tokens';-import { interactionService } from '@/services/interaction-service';-import {- CollaboratorAddedEvent,- CollaboratorRemovedEvent,- InteractionUpdatedEvent,- NodeCreatedEvent,- NodeDeletedEvent,- NodeUpdatedEvent,-} from '@/types/events';-import { createLogger } from '@/lib/logger';--interface SynapseUserCursor {- workspaceId: string;- userId: string;- cursor: string;- syncing: boolean;-}--interface SynapseConnection {- accountId: string;- deviceId: string;- socket: WebSocket;- transactions: Map<string, SynapseUserCursor>;- revocations: Map<string, SynapseUserCursor>;- collaborations: Map<string, SynapseUserCursor>;- interactions: Map<string, SynapseUserCursor>;-}--const PUBLIC_NODES: NodeType[] = ['workspace', 'user'];--class SynapseService {- private readonly logger = createLogger('synapse-service');- private readonly connections: Map<string, SynapseConnection> = new Map();-- constructor() {- eventBus.subscribe((event) => {- if (event.type === 'node_created') {- this.handleNodeCreatedEvent(event);- } else if (event.type === 'node_updated') {- this.handleNodeUpdatedEvent(event);- } else if (event.type === 'node_deleted') {- this.handleNodeDeletedEvent(event);- } else if (event.type === 'collaborator_added') {- this.handleCollaboratorAddedEvent(event);- } else if (event.type === 'collaborator_removed') {- this.handleCollaboratorRemovedEvent(event);- } else if (event.type === 'interaction_updated') {- this.handleInteractionUpdatedEvent(event);- }- });- }-- public async init(server: Server) {- this.logger.info('Initializing synapse service');-- const wss = new WebSocketServer({- server,- path: '/v1/synapse',- verifyClient: async (info, callback) => {- const req = info.req;- const token = req.headers['authorization'];-- if (!token) {- return callback(false, 401, 'Unauthorized');- }-- callback(true);- },- });-- wss.on('connection', async (socket, req) => {- let verified = false;- const messageBuffer: Message[] = [];- socket.onmessage = async (event) => {- let data: string;-- if (typeof event.data === 'string') {- data = event.data;- } else if (event.data instanceof ArrayBuffer) {- data = new TextDecoder().decode(event.data);- } else {- console.error('Unsupported message data type:', typeof event.data);- return;- }-- const message: Message = JSON.parse(data);- if (!verified) {- messageBuffer.push(message);- } else {- this.handleSocketMessage(connection, message);- }- };-- const token = req.headers['authorization'];- if (!token) {- socket.close();- return;- }-- const result = await verifyToken(token);- if (!result.authenticated) {- socket.close();- return;- }-- const account = result.account;- this.logger.info(`New synapse connection from ${account.id}`);-- socket.onclose = () => {- const connection = this.connections.get(account.deviceId);- if (connection) {- this.connections.delete(account.deviceId);- }- };-- const connection: SynapseConnection = {- accountId: account.id,- deviceId: account.deviceId,- socket,- transactions: new Map(),- revocations: new Map(),- collaborations: new Map(),- interactions: new Map(),- };-- this.connections.set(account.deviceId, connection);-- verified = true;- for (const message of messageBuffer) {- this.handleSocketMessage(connection, message);- }- messageBuffer.splice(0, messageBuffer.length);- });- }-- private sendMessage(connection: SynapseConnection, message: Message) {- connection.socket.send(JSON.stringify(message));- }-- private async handleSocketMessage(- connection: SynapseConnection,- message: Message- ) {- this.logger.trace(message, `Socket message from ${connection.deviceId}`);-- if (message.type === 'fetch_node_transactions') {- const state = connection.transactions.get(message.userId);- if (!state) {- connection.transactions.set(message.userId, {- userId: message.userId,- workspaceId: message.workspaceId,- cursor: message.cursor,- syncing: false,- });-- this.sendPendingTransactions(connection, message.userId);- } else if (!state.syncing && state.cursor !== message.cursor) {- state.cursor = message.cursor;- this.sendPendingTransactions(connection, message.userId);- }- } else if (message.type === 'fetch_collaboration_revocations') {- const state = connection.revocations.get(message.userId);- if (!state) {- connection.revocations.set(message.userId, {- userId: message.userId,- workspaceId: message.workspaceId,- cursor: message.cursor,- syncing: false,- });-- this.sendPendingRevocations(connection, message.userId);- } else if (!state.syncing && state.cursor !== message.cursor) {- state.cursor = message.cursor;- this.sendPendingRevocations(connection, message.userId);- }- } else if (message.type === 'fetch_collaborations') {- const state = connection.collaborations.get(message.userId);- if (!state) {- connection.collaborations.set(message.userId, {- userId: message.userId,- workspaceId: message.workspaceId,- cursor: message.cursor,- syncing: false,- });-- this.sendPendingCollaborations(connection, message.userId);- } else if (!state.syncing && state.cursor !== message.cursor) {- state.cursor = message.cursor;- this.sendPendingCollaborations(connection, message.userId);- }- } else if (message.type === 'fetch_interactions') {- const state = connection.interactions.get(message.userId);- if (!state) {- connection.interactions.set(message.userId, {- userId: message.userId,- workspaceId: message.workspaceId,- cursor: message.cursor,- syncing: false,- });-- this.sendPendingInteractions(connection, message.userId);- } else if (!state.syncing && state.cursor !== message.cursor) {- state.cursor = message.cursor;- this.sendPendingInteractions(connection, message.userId);- }- } else if (message.type === 'sync_interactions') {- interactionService.syncLocalInteractions(connection.accountId, message);- }- }-- private async sendPendingTransactions(- connection: SynapseConnection,- userId: string- ) {- const state = connection.transactions.get(userId);- if (!state || state.syncing) {- return;- }-- state.syncing = true;- this.logger.trace(- state,- `Sending pending node transactions for ${connection.deviceId} with ${userId}`- );-- const unsyncedTransactions = await database- .selectFrom('node_transactions as nt')- .leftJoin('collaborations as c', (join) =>- join.on('c.user_id', '=', userId).onRef('c.node_id', '=', 'nt.node_id')- )- .selectAll('nt')- .where((eb) =>- eb.or([- eb.and([- eb('nt.workspace_id', '=', state.workspaceId),- eb('nt.node_type', 'in', PUBLIC_NODES),- ]),- eb('c.node_id', '=', eb.ref('nt.node_id')),- ])- )- .where('nt.version', '>', BigInt(state.cursor))- .orderBy('nt.version', 'asc')- .limit(20)- .execute();-- if (unsyncedTransactions.length === 0) {- state.syncing = false;- return;- }-- const transactions = unsyncedTransactions.map(mapNodeTransaction);- const message: NodeTransactionsBatchMessage = {- type: 'node_transactions_batch',- userId,- transactions,- };-- connection.transactions.delete(userId);- this.sendMessage(connection, message);- }-- private async sendPendingRevocations(- connection: SynapseConnection,- userId: string- ) {- const state = connection.revocations.get(userId);- if (!state || state.syncing) {- return;- }-- state.syncing = true;- this.logger.trace(- state,- `Sending pending collaboration revocations for ${connection.deviceId} with ${userId}`- );-- const unsyncedRevocations = await database- .selectFrom('collaboration_revocations as cr')- .selectAll()- .where('cr.user_id', '=', userId)- .where('cr.version', '>', BigInt(state.cursor))- .orderBy('cr.version', 'asc')- .limit(50)- .execute();-- if (unsyncedRevocations.length === 0) {- state.syncing = false;- return;- }-- const revocations = unsyncedRevocations.map(mapCollaborationRevocation);- const message: CollaborationRevocationsBatchMessage = {- type: 'collaboration_revocations_batch',- userId,- revocations,- };-- connection.revocations.delete(userId);- this.sendMessage(connection, message);- }-- private async sendPendingCollaborations(- connection: SynapseConnection,- userId: string- ) {- const state = connection.collaborations.get(userId);- if (!state || state.syncing) {- return;- }-- state.syncing = true;- this.logger.trace(- state,- `Sending pending collaborations for ${connection.deviceId} with ${userId}`- );-- const unsyncedCollaborations = await database- .selectFrom('collaborations as c')- .selectAll()- .where('c.user_id', '=', userId)- .where('c.version', '>', BigInt(state.cursor))- .orderBy('c.version', 'asc')- .limit(50)- .execute();-- if (unsyncedCollaborations.length === 0) {- state.syncing = false;- return;- }-- const collaborations = unsyncedCollaborations.map(mapCollaboration);- const message: CollaborationsBatchMessage = {- type: 'collaborations_batch',- userId,- collaborations,- };-- connection.collaborations.delete(userId);- this.sendMessage(connection, message);- }-- private async sendPendingInteractions(- connection: SynapseConnection,- userId: string- ) {- const state = connection.interactions.get(userId);- if (!state || state.syncing) {- return;- }-- state.syncing = true;- this.logger.trace(- state,- `Sending pending interactions for ${connection.deviceId} with ${userId}`- );-- const unsyncedInteractions = await database- .selectFrom('interactions as i')- .leftJoin('collaborations as c', (join) =>- join.on('c.user_id', '=', userId).onRef('c.node_id', '=', 'i.node_id')- )- .where((eb) =>- eb.or([- eb.and([- eb('i.workspace_id', '=', state.workspaceId),- eb('i.node_type', 'in', PUBLIC_NODES),- ]),- eb('c.node_id', '=', eb.ref('i.node_id')),- ])- )- .selectAll('i')- .where('i.version', '>', BigInt(state.cursor))- .orderBy('i.version', 'asc')- .limit(20)- .execute();-- if (unsyncedInteractions.length === 0) {- state.syncing = false;- return;- }-- const interactions = unsyncedInteractions.map(mapInteraction);- const message: InteractionsBatchMessage = {- type: 'interactions_batch',- userId,- interactions,- };-- connection.interactions.delete(userId);- this.sendMessage(connection, message);- }-- private async handleNodeCreatedEvent(event: NodeCreatedEvent) {- const userDevices = this.getPendingNodeTransactionCursors(- event.workspaceId- );- const userIds = Array.from(userDevices.keys());- if (userIds.length === 0) {- return;- }-- let usersToSend: string[] = [];- if (PUBLIC_NODES.includes(event.nodeType)) {- usersToSend = userIds;- } else {- const collaborations = await database- .selectFrom('collaborations')- .selectAll()- .where((eb) =>- eb.and([- eb('user_id', 'in', userIds),- eb('node_id', '=', event.nodeId),- ])- )- .execute();-- usersToSend = collaborations.map((c) => c.user_id);- }-- if (usersToSend.length === 0) {- return;- }-- for (const userId of usersToSend) {- const deviceIds = userDevices.get(userId) ?? [];- for (const deviceId of deviceIds) {- const socketConnection = this.connections.get(deviceId);- if (socketConnection === undefined) {- continue;- }-- this.sendPendingTransactions(socketConnection, userId);- this.sendPendingCollaborations(socketConnection, userId);- this.sendPendingInteractions(socketConnection, userId);- }- }- }-- private async handleNodeUpdatedEvent(event: NodeUpdatedEvent) {- const userDevices = this.getPendingNodeTransactionCursors(- event.workspaceId- );- const userIds = Array.from(userDevices.keys());- if (userIds.length === 0) {- return;- }-- let usersToSend: string[] = [];- if (PUBLIC_NODES.includes(event.nodeType)) {- usersToSend = userIds;- } else {- const collaborations = await database- .selectFrom('collaborations')- .selectAll()- .where((eb) =>- eb.and([- eb('user_id', 'in', userIds),- eb('node_id', '=', event.nodeId),- ])- )- .execute();-- usersToSend = collaborations.map((c) => c.user_id);- }-- if (usersToSend.length === 0) {- return;- }-- for (const userId of usersToSend) {- const deviceIds = userDevices.get(userId) ?? [];- for (const deviceId of deviceIds) {- const socketConnection = this.connections.get(deviceId);- if (socketConnection === undefined) {- continue;- }-- this.sendPendingTransactions(socketConnection, userId);- }- }- }-- private async handleNodeDeletedEvent(event: NodeDeletedEvent) {- const userDevices = this.getPendingNodeTransactionCursors(- event.workspaceId- );- const userIds = Array.from(userDevices.keys());- if (userIds.length === 0) {- return;- }-- let usersToSend: string[] = [];- if (PUBLIC_NODES.includes(event.nodeType)) {- usersToSend = userIds;- } else {- const revocations = await database- .selectFrom('collaboration_revocations')- .selectAll()- .where((eb) =>- eb.and([- eb('user_id', 'in', userIds),- eb('node_id', '=', event.nodeId),- ])- )- .execute();-- usersToSend = revocations.map((r) => r.user_id);- }-- if (usersToSend.length === 0) {- return;- }-- for (const userId of usersToSend) {- const deviceIds = userDevices.get(userId) ?? [];- for (const deviceId of deviceIds) {- const socketConnection = this.connections.get(deviceId);- if (socketConnection === undefined) {- continue;- }-- this.sendPendingTransactions(socketConnection, userId);- this.sendPendingRevocations(socketConnection, userId);- }- }- }-- private async handleInteractionUpdatedEvent(event: InteractionUpdatedEvent) {- const userDevices = this.getPendingInteractionsCursors(event.workspaceId);- const userIds = Array.from(userDevices.keys());- if (userIds.length === 0) {- return;- }-- let usersToSend: string[] = [];- if (PUBLIC_NODES.includes(event.nodeType)) {- usersToSend = userIds;- } else {- const collaborations = await database- .selectFrom('collaborations')- .selectAll()- .where((eb) =>- eb.and([- eb('user_id', 'in', userIds),- eb('node_id', '=', event.nodeId),- ])- )- .execute();-- usersToSend = collaborations.map((c) => c.user_id);- }-- if (userIds.includes(event.userId) && !usersToSend.includes(event.userId)) {- usersToSend.push(event.userId);- }-- for (const userId of usersToSend) {- const deviceIds = userDevices.get(userId) ?? [];- for (const deviceId of deviceIds) {- const socketConnection = this.connections.get(deviceId);- if (socketConnection === undefined) {- continue;- }-- this.sendPendingInteractions(socketConnection, userId);- }- }- }-- private handleCollaboratorAddedEvent(event: CollaboratorAddedEvent) {- const deviceIds = this.getPendingCollaborationsCursors(event.userId);- for (const deviceId of deviceIds) {- const socketConnection = this.connections.get(deviceId);- if (socketConnection === undefined) {- continue;- }-- this.sendPendingCollaborations(socketConnection, event.userId);- }- }-- private handleCollaboratorRemovedEvent(event: CollaboratorRemovedEvent) {- const deviceIds = this.getPendingRevocationsCursors(event.userId);- for (const deviceId of deviceIds) {- const socketConnection = this.connections.get(deviceId);- if (socketConnection === undefined) {- continue;- }-- this.sendPendingRevocations(socketConnection, event.userId);- }- }-- private getPendingNodeTransactionCursors(- workspaceId: string- ): Map<string, string[]> {- const userDevices = new Map<string, string[]>();- for (const connection of this.connections.values()) {- const connectionUsers = connection.transactions.values();- for (const user of connectionUsers) {- if (user.workspaceId !== workspaceId || user.syncing) {- continue;- }-- const userIds = userDevices.get(user.userId) ?? [];- userIds.push(connection.deviceId);- userDevices.set(user.userId, userIds);- }- }-- return userDevices;- }-- private getPendingInteractionsCursors(- workspaceId: string- ): Map<string, string[]> {- const userDevices = new Map<string, string[]>();- for (const connection of this.connections.values()) {- const connectionUsers = connection.interactions.values();- for (const user of connectionUsers) {- if (user.workspaceId !== workspaceId || user.syncing) {- continue;- }-- const userIds = userDevices.get(user.userId) ?? [];- userIds.push(connection.deviceId);- userDevices.set(user.userId, userIds);- }- }-- return userDevices;- }-- private getPendingCollaborationsCursors(userId: string): string[] {- const userDevices: string[] = [];- for (const connection of this.connections.values()) {- const connectionUsers = connection.collaborations.values();- for (const user of connectionUsers) {- if (user.userId !== userId || user.syncing) {- continue;- }-- userDevices.push(connection.deviceId);- }- }-- return userDevices;- }-- private getPendingRevocationsCursors(userId: string): string[] {- const userDevices: string[] = [];- for (const connection of this.connections.values()) {- const connectionUsers = connection.revocations.values();- for (const user of connectionUsers) {- if (user.userId !== userId || user.syncing) {- continue;- }-- userDevices.push(connection.deviceId);- }- }-- return userDevices;- }-}--export const synapse = new SynapseService();diff --git a/apps/server/src/types/api.ts b/apps/server/src/types/api.tsindex 4160fe4d..f9aa12b6 100644--- a/apps/server/src/types/api.ts+++ b/apps/server/src/types/api.ts@@ -11,7 +11,7 @@ export enum ApiError {BadRequest = 'BadRequest',}-export type ColanodeRequestAccount = {+export type RequestAccount = {id: string;deviceId: string;};diff --git a/packages/core/src/types/messages.ts b/packages/core/src/types/messages.tsindex ea7e6054..2c6e6e0f 100644--- a/packages/core/src/types/messages.ts+++ b/packages/core/src/types/messages.ts@@ -4,38 +4,11 @@ import {ServerCollaborationRevocation,ServerInteraction,ServerNodeTransaction,+ SyncConsumerType,} from './sync';import { NodeType } from '../registry';-export type FetchNodeTransactionsMessage = {- type: 'fetch_node_transactions';- userId: string;- workspaceId: string;- cursor: string;-};--export type FetchCollaborationRevocationsMessage = {- type: 'fetch_collaboration_revocations';- userId: string;- workspaceId: string;- cursor: string;-};--export type FetchCollaborationsMessage = {- type: 'fetch_collaborations';- userId: string;- workspaceId: string;- cursor: string;-};--export type FetchInteractionsMessage = {- type: 'fetch_interactions';- userId: string;- workspaceId: string;- cursor: string;-};-export type NodeTransactionsBatchMessage = {type: 'node_transactions_batch';userId: string;@@ -68,13 +41,17 @@ export type SyncInteractionsMessage = {events: InteractionEvent[];};+export type InitSyncConsumerMessage = {+ type: 'init_sync_consumer';+ userId: string;+ consumerType: SyncConsumerType;+ cursor: string;+};+export type Message =- | FetchNodeTransactionsMessage| NodeTransactionsBatchMessage- | FetchCollaborationRevocationsMessage| CollaborationRevocationsBatchMessage- | FetchCollaborationsMessage| CollaborationsBatchMessage- | FetchInteractionsMessage| InteractionsBatchMessage- | SyncInteractionsMessage;+ | SyncInteractionsMessage+ | InitSyncConsumerMessage;diff --git a/packages/core/src/types/sync.ts b/packages/core/src/types/sync.tsindex 62e1807f..6ff8f7fc 100644--- a/packages/core/src/types/sync.ts+++ b/packages/core/src/types/sync.ts@@ -125,3 +125,9 @@ export type ServerInteraction = {serverUpdatedAt: string | null;version: string;};++export type SyncConsumerType =+ | 'transactions'+ | 'collaborations'+ | 'revocations'+ | 'interactions';