Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Refactor and improve synchronization
Hakan Shehuff15b24d0d36
diff --git a/desktop/src/lib/servers.ts b/desktop/src/lib/servers.tsindex d86e18ee..4145f657 100644--- a/desktop/src/lib/servers.ts+++ b/desktop/src/lib/servers.ts@@ -1,14 +1,27 @@import { SelectServer } from '@/main/data/app/schema';-import { Server, ServerAttributes } from '@/types/servers';--export const buildApiBaseUrl = (server: SelectServer): string => {- const attributes = JSON.parse(server.attributes) as ServerAttributes;- const protocol = attributes?.insecure ? 'http' : 'https';- return `${protocol}://${server.domain}`;-};+import { ServerAttributes } from '@/types/servers';+import axios, { AxiosInstance } from 'axios';export const buildSynapseUrl = (server: SelectServer, deviceId: string) => {const attributes = JSON.parse(server.attributes) as ServerAttributes;const protocol = attributes?.insecure ? 'ws' : 'wss';return `${protocol}://${server.domain}/v1/synapse?device_id=${deviceId}`;};++export const buildAxiosInstance = (+ domain: string,+ attributes: string | ServerAttributes,+ token?: string,+): AxiosInstance => {+ const parsedAttributes: ServerAttributes =+ typeof attributes === 'string' ? JSON.parse(attributes) : attributes;+ const protocol = parsedAttributes?.insecure ? 'http' : 'https';+ const baseURL = `${protocol}://${domain}`;+ const instance = axios.create({ baseURL });++ if (token) {+ instance.defaults.headers.common['Authorization'] = `Bearer ${token}`;+ }++ return instance;+};diff --git a/desktop/src/main/data/workspace/schema.ts b/desktop/src/main/data/workspace/schema.tsindex 8614344d..847052bf 100644--- a/desktop/src/main/data/workspace/schema.ts+++ b/desktop/src/main/data/workspace/schema.ts@@ -51,7 +51,7 @@ export type SelectNodeReaction = Selectable<NodeReactionTable>;export type CreateNodeReaction = Insertable<NodeReactionTable>;export type UpdateNodeReaction = Updateable<NodeReactionTable>;-interface MutationTable {+interface ChangeTable {id: ColumnType<number, never, never>;table: ColumnType<string, string, never>;action: ColumnType<string, string, never>;@@ -61,13 +61,13 @@ interface MutationTable {retry_count: ColumnType<number, number, number>;}-export type SelectMutation = Selectable<MutationTable>;-export type CreateMutation = Insertable<MutationTable>;-export type UpdateMutation = Updateable<MutationTable>;+export type SelectChange = Selectable<ChangeTable>;+export type CreateChange = Insertable<ChangeTable>;+export type UpdateChange = Updateable<ChangeTable>;export interface WorkspaceDatabaseSchema {nodes: NodeTable;- mutations: MutationTable;+ changes: ChangeTable;node_collaborators: NodeCollaboratorTable;node_reactions: NodeReactionTable;}diff --git a/desktop/src/main/mutations/email-login.ts b/desktop/src/main/mutations/email-login.tsindex 9c4e492a..821d98f9 100644--- a/desktop/src/main/mutations/email-login.ts+++ b/desktop/src/main/mutations/email-login.ts@@ -1,7 +1,6 @@-import axios from 'axios';import { LoginOutput } from '@/types/accounts';import { databaseContext } from '@/main/data/database-context';-import { buildApiBaseUrl } from '@/lib/servers';+import { buildAxiosInstance } from '@/lib/servers';import { EmailLoginMutationInput } from '@/types/mutations/email-login';import {MutationChange,@@ -29,13 +28,12 @@ export class EmailLoginMutationHandler};}- const { data } = await axios.post<LoginOutput>(- `${buildApiBaseUrl(server)}/v1/accounts/login/email`,- {- email: input.email,- password: input.password,- },- );+ const axios = buildAxiosInstance(server.domain, server.attributes);+ const { data } = await axios.post<LoginOutput>('/v1/accounts/login/email', {+ email: input.email,+ password: input.password,+ });+const changedTables: MutationChange[] = [];await databaseContext.appDatabase.transaction().execute(async (trx) => {await trxdiff --git a/desktop/src/main/mutations/email-register.ts b/desktop/src/main/mutations/email-register.tsindex 0cb27879..7560298b 100644--- a/desktop/src/main/mutations/email-register.ts+++ b/desktop/src/main/mutations/email-register.ts@@ -1,7 +1,6 @@-import axios from 'axios';import { LoginOutput } from '@/types/accounts';import { databaseContext } from '@/main/data/database-context';-import { buildApiBaseUrl } from '@/lib/servers';+import { buildAxiosInstance } from '@/lib/servers';import { EmailRegisterMutationInput } from '@/types/mutations/email-register';import {MutationChange,@@ -29,8 +28,9 @@ export class EmailRegisterMutationHandler};}+ const axios = buildAxiosInstance(server.domain, server.attributes);const { data } = await axios.post<LoginOutput>(- `${buildApiBaseUrl(server)}/v1/accounts/register/email`,+ '/v1/accounts/register/email',{name: input.name,email: input.email,diff --git a/desktop/src/main/mutations/space-create.ts b/desktop/src/main/mutations/space-create.tsindex 98048492..97080240 100644--- a/desktop/src/main/mutations/space-create.ts+++ b/desktop/src/main/mutations/space-create.ts@@ -22,46 +22,60 @@ export class SpaceCreateMutationHandlerconst pageIndex = generateNodeIndex(null, null);const channelIndex = generateNodeIndex(pageIndex, null);- await workspaceDatabase- .insertInto('nodes')- .values([- buildCreateNode(- {- id: spaceId,- attributes: {- type: NodeTypes.Space,- name: input.name,- description: input.description,+ await workspaceDatabase.transaction().execute(async (trx) => {+ await trx+ .insertInto('nodes')+ .values([+ buildCreateNode(+ {+ id: spaceId,+ attributes: {+ type: NodeTypes.Space,+ name: input.name,+ description: input.description,+ },},- },- input.userId,- ),- buildCreateNode(- {- id: pageId,- attributes: {- type: NodeTypes.Page,- parentId: spaceId,- index: pageIndex,- name: 'Home',+ input.userId,+ ),+ buildCreateNode(+ {+ id: pageId,+ attributes: {+ type: NodeTypes.Page,+ parentId: spaceId,+ index: pageIndex,+ name: 'Home',+ },},- },- input.userId,- ),- buildCreateNode(- {- id: channelId,- attributes: {- type: NodeTypes.Channel,- parentId: spaceId,- index: channelIndex,- name: 'Discussions',+ input.userId,+ ),+ buildCreateNode(+ {+ id: channelId,+ attributes: {+ type: NodeTypes.Channel,+ parentId: spaceId,+ index: channelIndex,+ name: 'Discussions',+ },},- },- input.userId,- ),- ])- .execute();+ input.userId,+ ),+ ])+ .execute();++ await trx+ .insertInto('node_collaborators')+ .values({+ node_id: spaceId,+ collaborator_id: input.userId,+ role: 'owner',+ created_at: new Date().toISOString(),+ created_by: input.userId,+ version_id: NeuronId.generate(NeuronId.Type.Version),+ })+ .execute();+ });return {output: {diff --git a/desktop/src/main/mutations/workspace-account-role-update.ts b/desktop/src/main/mutations/workspace-account-role-update.tsindex 9b1db4e2..41df130c 100644--- a/desktop/src/main/mutations/workspace-account-role-update.ts+++ b/desktop/src/main/mutations/workspace-account-role-update.ts@@ -1,6 +1,5 @@-import axios from 'axios';import { databaseContext } from '@/main/data/database-context';-import { buildApiBaseUrl } from '@/lib/servers';+import { buildAxiosInstance } from '@/lib/servers';import { WorkspaceAccountRoleUpdateMutationInput } from '@/types/mutations/workspace-account-role-update';import {MutationChange,@@ -57,16 +56,16 @@ export class WorkspaceAccountRoleUpdateMutationHandler};}+ const axios = buildAxiosInstance(+ server.domain,+ server.attributes,+ account.token,+ );const { data } = await axios.post<WorkspaceAccountRoleUpdateOutput>(- `${buildApiBaseUrl(server)}/v1/workspaces/${workspace.workspace_id}/accounts/${input.accountId}`,+ `/v1/workspaces/${workspace.workspace_id}/users/${input.accountId}`,{role: input.role,},- {- headers: {- Authorization: `Bearer ${account.token}`,- },- },);const workspaceDatabase = await databaseContext.getWorkspaceDatabase(diff --git a/desktop/src/main/mutations/workspace-accounts-invite.ts b/desktop/src/main/mutations/workspace-accounts-invite.tsindex 5ff01959..4f3a600d 100644--- a/desktop/src/main/mutations/workspace-accounts-invite.ts+++ b/desktop/src/main/mutations/workspace-accounts-invite.ts@@ -1,6 +1,5 @@-import axios from 'axios';import { databaseContext } from '@/main/data/database-context';-import { buildApiBaseUrl } from '@/lib/servers';+import { buildAxiosInstance } from '@/lib/servers';import { WorkspaceAccountsInviteMutationInput } from '@/types/mutations/workspace-accounts-invite';import {MutationChange,@@ -58,16 +57,16 @@ export class WorkspaceAccountsInviteMutationHandler};}+ const axios = buildAxiosInstance(+ server.domain,+ server.attributes,+ account.token,+ );const { data } = await axios.post<WorkspaceAccountsInviteOutput>(- `${buildApiBaseUrl(server)}/v1/workspaces/${workspace.workspace_id}/accounts`,+ `/v1/workspaces/${workspace.workspace_id}/users`,{emails: input.emails,},- {- headers: {- Authorization: `Bearer ${account.token}`,- },- },);const workspaceDatabase = await databaseContext.getWorkspaceDatabase(diff --git a/desktop/src/main/mutations/workspace-create.ts b/desktop/src/main/mutations/workspace-create.tsindex 42095e52..ff775478 100644--- a/desktop/src/main/mutations/workspace-create.ts+++ b/desktop/src/main/mutations/workspace-create.ts@@ -1,6 +1,5 @@-import axios from 'axios';import { databaseContext } from '@/main/data/database-context';-import { buildApiBaseUrl } from '@/lib/servers';+import { buildAxiosInstance } from '@/lib/servers';import { WorkspaceCreateMutationInput } from '@/types/mutations/workspace-create';import {MutationChange,@@ -35,18 +34,15 @@ export class WorkspaceCreateMutationHandlerthrow new Error('Account not found');}- const { data } = await axios.post<Workspace>(- `${buildApiBaseUrl(server)}/v1/workspaces`,- {- name: input.name,- description: input.description,- },- {- headers: {- Authorization: `Bearer ${account.token}`,- },- },+ const axios = buildAxiosInstance(+ server.domain,+ server.attributes,+ account.token,);+ const { data } = await axios.post<Workspace>(`/v1/workspaces`, {+ name: input.name,+ description: input.description,+ });await databaseContext.appDatabase.insertInto('workspaces')diff --git a/desktop/src/main/mutations/workspace-update.ts b/desktop/src/main/mutations/workspace-update.tsindex 3448b751..d5a3a90c 100644--- a/desktop/src/main/mutations/workspace-update.ts+++ b/desktop/src/main/mutations/workspace-update.ts@@ -1,6 +1,5 @@-import axios from 'axios';import { databaseContext } from '@/main/data/database-context';-import { buildApiBaseUrl } from '@/lib/servers';+import { buildAxiosInstance } from '@/lib/servers';import { WorkspaceUpdateMutationInput } from '@/types/mutations/workspace-update';import {MutationChange,@@ -35,13 +34,15 @@ export class WorkspaceUpdateMutationHandlerthrow new Error('Account not found');}- const { data } = await axios.post<Workspace>(- `${buildApiBaseUrl(server)}/v1/workspaces/${input.id}`,- {- name: input.name,- description: input.description,- },+ const axios = buildAxiosInstance(+ server.domain,+ server.attributes,+ account.token,);+ const { data } = await axios.post<Workspace>(`/v1/workspaces/${input.id}`, {+ name: input.name,+ description: input.description,+ });await databaseContext.appDatabase.updateTable('workspaces')diff --git a/desktop/src/main/queries/node-collaborator-search.ts b/desktop/src/main/queries/node-collaborator-search.tsindex 498538a8..ed09d59d 100644--- a/desktop/src/main/queries/node-collaborator-search.ts+++ b/desktop/src/main/queries/node-collaborator-search.ts@@ -14,6 +14,15 @@ export class NodeCollaboratorSearchQueryHandlerpublic async handleQuery(input: NodeCollaboratorSearchQueryInput,): Promise<QueryResult<NodeCollaboratorSearchQueryInput>> {+ if (input.searchQuery.length === 0) {+ return {+ output: [],+ state: {+ rows: [],+ },+ };+ }+const rows = await this.fetchNodes(input);return {output: this.buildCollaboratorNodes(rows),diff --git a/desktop/src/main/synchronizer.ts b/desktop/src/main/synchronizer.tsindex eb9f439b..cb945b45 100644--- a/desktop/src/main/synchronizer.ts+++ b/desktop/src/main/synchronizer.ts@@ -1,8 +1,7 @@-import axios from 'axios';-import { buildApiBaseUrl } from '@/lib/servers';+import { buildAxiosInstance } from '@/lib/servers';import { SelectWorkspace } from '@/main/data/app/schema';import { databaseContext } from '@/main/data/database-context';-import { WorkspaceSyncData } from '@/types/sync';+import { ServerSyncResponse, WorkspaceSyncData } from '@/types/sync';const EVENT_LOOP_INTERVAL = 100;@@ -23,7 +22,12 @@ class Synchronizer {}private async executeEventLoop() {+ // try {await this.checkForWorkspaceSyncs();+ await this.checkForWorkspaceChanges();+ // } catch (error) {+ // console.log('error', error);+ // }setTimeout(this.executeEventLoop, EVENT_LOOP_INTERVAL);}@@ -44,34 +48,103 @@ class Synchronizer {}}- private async syncWorkspace(workspace: SelectWorkspace): Promise<void> {- const account = await databaseContext.appDatabase- .selectFrom('accounts')- .selectAll()- .where('id', '=', workspace.account_id)- .executeTakeFirst();+ private async checkForWorkspaceChanges() {+ const workspaces = await databaseContext.appDatabase+ .selectFrom('workspaces')+ .innerJoin('accounts', 'workspaces.account_id', 'accounts.id')+ .innerJoin('servers', 'accounts.server', 'servers.domain')+ .select([+ 'workspaces.workspace_id',+ 'workspaces.user_id',+ 'accounts.token',+ 'servers.domain',+ 'servers.attributes',+ ])+ .where('workspaces.synced', '=', 1)+ .execute();- if (!account) {- return;+ for (const workspace of workspaces) {+ const workspaceDatabase = await databaseContext.getWorkspaceDatabase(+ workspace.user_id,+ );++ const changes = await workspaceDatabase+ .selectFrom('changes')+ .selectAll()+ .orderBy('id asc')+ .limit(20)+ .execute();++ if (changes.length === 0) {+ return;+ }++ const axios = buildAxiosInstance(+ workspace.domain,+ workspace.attributes,+ workspace.token,+ );++ const { data } = await axios.post<ServerSyncResponse>(+ `/v1/sync/${workspace.workspace_id}`,+ {+ changes: changes,+ },+ );++ const syncedChangeIds: number[] = [];+ const unsyncedChangeIds: number[] = [];+ for (const result of data.results) {+ if (result.status === 'success') {+ syncedChangeIds.push(result.id);+ } else {+ unsyncedChangeIds.push(result.id);+ }+ }++ if (syncedChangeIds.length > 0) {+ await workspaceDatabase+ .deleteFrom('changes')+ .where('id', 'in', syncedChangeIds)+ .execute();+ }++ if (unsyncedChangeIds.length > 0) {+ await workspaceDatabase+ .updateTable('changes')+ .set((eb) => ({ retry_count: eb('retry_count', '+', 1) }))+ .where('id', 'in', unsyncedChangeIds)+ .execute();++ //we just delete changes that have failed to sync for more than 5 times.+ //in the future we might need to revert the change locally.+ await workspaceDatabase+ .deleteFrom('changes')+ .where('retry_count', '>=', 5)+ .execute();+ }}+ }- const server = await databaseContext.appDatabase- .selectFrom('servers')- .selectAll()- .where('domain', '=', account.server)+ private async syncWorkspace(workspace: SelectWorkspace): Promise<void> {+ const credentials = await databaseContext.appDatabase+ .selectFrom('accounts')+ .innerJoin('servers', 'accounts.server', 'servers.domain')+ .select(['domain', 'attributes', 'token'])+ .where('id', '=', workspace.account_id).executeTakeFirst();- if (!server) {+ if (!credentials) {return;}+ const axios = buildAxiosInstance(+ credentials.domain,+ credentials.attributes,+ credentials.token,+ );const { data } = await axios.get<WorkspaceSyncData>(- `${buildApiBaseUrl(server)}/v1/sync/${workspace.workspace_id}`,- {- headers: {- Authorization: `Bearer ${account.token}`,- },- },+ `/v1/sync/${workspace.workspace_id}`,);const workspaceDatabase = await databaseContext.getWorkspaceDatabase(diff --git a/server/src/routes/sync.ts b/server/src/routes/sync.tsindex 597caefe..f2041f98 100644--- a/server/src/routes/sync.ts+++ b/server/src/routes/sync.ts@@ -88,6 +88,7 @@ syncRouter.get(syncRouter.post('/:workspaceId',async (req: NeuronRequest, res: NeuronResponse) => {+ const workspaceId = req.params.workspaceId as string;const input = req.body as SyncLocalChangesInput;if (!req.accountId) {return res.status(401).json({@@ -99,7 +100,7 @@ syncRouter.post(const workspace = await database.selectFrom('workspaces').selectAll()- .where('id', '=', input.workspaceId)+ .where('id', '=', workspaceId).executeTakeFirst();if (!workspace) {diff --git a/server/src/types/sync.ts b/server/src/types/sync.tsindex 39122122..13eb99cf 100644--- a/server/src/types/sync.ts+++ b/server/src/types/sync.ts@@ -1,5 +1,4 @@export type SyncLocalChangesInput = {- workspaceId: string;changes: LocalChange[];};