Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Refactor and improve synchronization

Hakan Shehuff15b24d0d36
diff --git a/desktop/src/lib/servers.ts b/desktop/src/lib/servers.ts
index d86e18ee..4145f657 100644
--- a/desktop/src/lib/servers.ts
+++ b/desktop/src/lib/servers.ts
@@ -1,14 +1,27 @@
import { SelectServer } from '@/main/data/app/schema';
-import { Server, ServerAttributes } from '@/types/servers';
-
-export const buildApiBaseUrl = (server: SelectServer): string => {
- const attributes = JSON.parse(server.attributes) as ServerAttributes;
- const protocol = attributes?.insecure ? 'http' : 'https';
- return `${protocol}://${server.domain}`;
-};
+import { ServerAttributes } from '@/types/servers';
+import axios, { AxiosInstance } from 'axios';
export const buildSynapseUrl = (server: SelectServer, deviceId: string) => {
const attributes = JSON.parse(server.attributes) as ServerAttributes;
const protocol = attributes?.insecure ? 'ws' : 'wss';
return `${protocol}://${server.domain}/v1/synapse?device_id=${deviceId}`;
};
+
+export const buildAxiosInstance = (
+ domain: string,
+ attributes: string | ServerAttributes,
+ token?: string,
+): AxiosInstance => {
+ const parsedAttributes: ServerAttributes =
+ typeof attributes === 'string' ? JSON.parse(attributes) : attributes;
+ const protocol = parsedAttributes?.insecure ? 'http' : 'https';
+ const baseURL = `${protocol}://${domain}`;
+ const instance = axios.create({ baseURL });
+
+ if (token) {
+ instance.defaults.headers.common['Authorization'] = `Bearer ${token}`;
+ }
+
+ return instance;
+};
diff --git a/desktop/src/main/data/workspace/schema.ts b/desktop/src/main/data/workspace/schema.ts
index 8614344d..847052bf 100644
--- a/desktop/src/main/data/workspace/schema.ts
+++ b/desktop/src/main/data/workspace/schema.ts
@@ -51,7 +51,7 @@ export type SelectNodeReaction = Selectable<NodeReactionTable>;
export type CreateNodeReaction = Insertable<NodeReactionTable>;
export type UpdateNodeReaction = Updateable<NodeReactionTable>;
-interface MutationTable {
+interface ChangeTable {
id: ColumnType<number, never, never>;
table: ColumnType<string, string, never>;
action: ColumnType<string, string, never>;
@@ -61,13 +61,13 @@ interface MutationTable {
retry_count: ColumnType<number, number, number>;
}
-export type SelectMutation = Selectable<MutationTable>;
-export type CreateMutation = Insertable<MutationTable>;
-export type UpdateMutation = Updateable<MutationTable>;
+export type SelectChange = Selectable<ChangeTable>;
+export type CreateChange = Insertable<ChangeTable>;
+export type UpdateChange = Updateable<ChangeTable>;
export interface WorkspaceDatabaseSchema {
nodes: NodeTable;
- mutations: MutationTable;
+ changes: ChangeTable;
node_collaborators: NodeCollaboratorTable;
node_reactions: NodeReactionTable;
}
diff --git a/desktop/src/main/mutations/email-login.ts b/desktop/src/main/mutations/email-login.ts
index 9c4e492a..821d98f9 100644
--- a/desktop/src/main/mutations/email-login.ts
+++ b/desktop/src/main/mutations/email-login.ts
@@ -1,7 +1,6 @@
-import axios from 'axios';
import { LoginOutput } from '@/types/accounts';
import { databaseContext } from '@/main/data/database-context';
-import { buildApiBaseUrl } from '@/lib/servers';
+import { buildAxiosInstance } from '@/lib/servers';
import { EmailLoginMutationInput } from '@/types/mutations/email-login';
import {
MutationChange,
@@ -29,13 +28,12 @@ export class EmailLoginMutationHandler
};
}
- const { data } = await axios.post<LoginOutput>(
- `${buildApiBaseUrl(server)}/v1/accounts/login/email`,
- {
- email: input.email,
- password: input.password,
- },
- );
+ const axios = buildAxiosInstance(server.domain, server.attributes);
+ const { data } = await axios.post<LoginOutput>('/v1/accounts/login/email', {
+ email: input.email,
+ password: input.password,
+ });
+
const changedTables: MutationChange[] = [];
await databaseContext.appDatabase.transaction().execute(async (trx) => {
await trx
diff --git a/desktop/src/main/mutations/email-register.ts b/desktop/src/main/mutations/email-register.ts
index 0cb27879..7560298b 100644
--- a/desktop/src/main/mutations/email-register.ts
+++ b/desktop/src/main/mutations/email-register.ts
@@ -1,7 +1,6 @@
-import axios from 'axios';
import { LoginOutput } from '@/types/accounts';
import { databaseContext } from '@/main/data/database-context';
-import { buildApiBaseUrl } from '@/lib/servers';
+import { buildAxiosInstance } from '@/lib/servers';
import { EmailRegisterMutationInput } from '@/types/mutations/email-register';
import {
MutationChange,
@@ -29,8 +28,9 @@ export class EmailRegisterMutationHandler
};
}
+ const axios = buildAxiosInstance(server.domain, server.attributes);
const { data } = await axios.post<LoginOutput>(
- `${buildApiBaseUrl(server)}/v1/accounts/register/email`,
+ '/v1/accounts/register/email',
{
name: input.name,
email: input.email,
diff --git a/desktop/src/main/mutations/space-create.ts b/desktop/src/main/mutations/space-create.ts
index 98048492..97080240 100644
--- a/desktop/src/main/mutations/space-create.ts
+++ b/desktop/src/main/mutations/space-create.ts
@@ -22,46 +22,60 @@ export class SpaceCreateMutationHandler
const pageIndex = generateNodeIndex(null, null);
const channelIndex = generateNodeIndex(pageIndex, null);
- await workspaceDatabase
- .insertInto('nodes')
- .values([
- buildCreateNode(
- {
- id: spaceId,
- attributes: {
- type: NodeTypes.Space,
- name: input.name,
- description: input.description,
+ await workspaceDatabase.transaction().execute(async (trx) => {
+ await trx
+ .insertInto('nodes')
+ .values([
+ buildCreateNode(
+ {
+ id: spaceId,
+ attributes: {
+ type: NodeTypes.Space,
+ name: input.name,
+ description: input.description,
+ },
},
- },
- input.userId,
- ),
- buildCreateNode(
- {
- id: pageId,
- attributes: {
- type: NodeTypes.Page,
- parentId: spaceId,
- index: pageIndex,
- name: 'Home',
+ input.userId,
+ ),
+ buildCreateNode(
+ {
+ id: pageId,
+ attributes: {
+ type: NodeTypes.Page,
+ parentId: spaceId,
+ index: pageIndex,
+ name: 'Home',
+ },
},
- },
- input.userId,
- ),
- buildCreateNode(
- {
- id: channelId,
- attributes: {
- type: NodeTypes.Channel,
- parentId: spaceId,
- index: channelIndex,
- name: 'Discussions',
+ input.userId,
+ ),
+ buildCreateNode(
+ {
+ id: channelId,
+ attributes: {
+ type: NodeTypes.Channel,
+ parentId: spaceId,
+ index: channelIndex,
+ name: 'Discussions',
+ },
},
- },
- input.userId,
- ),
- ])
- .execute();
+ input.userId,
+ ),
+ ])
+ .execute();
+
+ await trx
+ .insertInto('node_collaborators')
+ .values({
+ node_id: spaceId,
+ collaborator_id: input.userId,
+ role: 'owner',
+ created_at: new Date().toISOString(),
+ created_by: input.userId,
+ version_id: NeuronId.generate(NeuronId.Type.Version),
+ })
+ .execute();
+ });
return {
output: {
diff --git a/desktop/src/main/mutations/workspace-account-role-update.ts b/desktop/src/main/mutations/workspace-account-role-update.ts
index 9b1db4e2..41df130c 100644
--- a/desktop/src/main/mutations/workspace-account-role-update.ts
+++ b/desktop/src/main/mutations/workspace-account-role-update.ts
@@ -1,6 +1,5 @@
-import axios from 'axios';
import { databaseContext } from '@/main/data/database-context';
-import { buildApiBaseUrl } from '@/lib/servers';
+import { buildAxiosInstance } from '@/lib/servers';
import { WorkspaceAccountRoleUpdateMutationInput } from '@/types/mutations/workspace-account-role-update';
import {
MutationChange,
@@ -57,16 +56,16 @@ export class WorkspaceAccountRoleUpdateMutationHandler
};
}
+ const axios = buildAxiosInstance(
+ server.domain,
+ server.attributes,
+ account.token,
+ );
const { data } = await axios.post<WorkspaceAccountRoleUpdateOutput>(
- `${buildApiBaseUrl(server)}/v1/workspaces/${workspace.workspace_id}/accounts/${input.accountId}`,
+ `/v1/workspaces/${workspace.workspace_id}/users/${input.accountId}`,
{
role: input.role,
},
- {
- headers: {
- Authorization: `Bearer ${account.token}`,
- },
- },
);
const workspaceDatabase = await databaseContext.getWorkspaceDatabase(
diff --git a/desktop/src/main/mutations/workspace-accounts-invite.ts b/desktop/src/main/mutations/workspace-accounts-invite.ts
index 5ff01959..4f3a600d 100644
--- a/desktop/src/main/mutations/workspace-accounts-invite.ts
+++ b/desktop/src/main/mutations/workspace-accounts-invite.ts
@@ -1,6 +1,5 @@
-import axios from 'axios';
import { databaseContext } from '@/main/data/database-context';
-import { buildApiBaseUrl } from '@/lib/servers';
+import { buildAxiosInstance } from '@/lib/servers';
import { WorkspaceAccountsInviteMutationInput } from '@/types/mutations/workspace-accounts-invite';
import {
MutationChange,
@@ -58,16 +57,16 @@ export class WorkspaceAccountsInviteMutationHandler
};
}
+ const axios = buildAxiosInstance(
+ server.domain,
+ server.attributes,
+ account.token,
+ );
const { data } = await axios.post<WorkspaceAccountsInviteOutput>(
- `${buildApiBaseUrl(server)}/v1/workspaces/${workspace.workspace_id}/accounts`,
+ `/v1/workspaces/${workspace.workspace_id}/users`,
{
emails: input.emails,
},
- {
- headers: {
- Authorization: `Bearer ${account.token}`,
- },
- },
);
const workspaceDatabase = await databaseContext.getWorkspaceDatabase(
diff --git a/desktop/src/main/mutations/workspace-create.ts b/desktop/src/main/mutations/workspace-create.ts
index 42095e52..ff775478 100644
--- a/desktop/src/main/mutations/workspace-create.ts
+++ b/desktop/src/main/mutations/workspace-create.ts
@@ -1,6 +1,5 @@
-import axios from 'axios';
import { databaseContext } from '@/main/data/database-context';
-import { buildApiBaseUrl } from '@/lib/servers';
+import { buildAxiosInstance } from '@/lib/servers';
import { WorkspaceCreateMutationInput } from '@/types/mutations/workspace-create';
import {
MutationChange,
@@ -35,18 +34,15 @@ export class WorkspaceCreateMutationHandler
throw new Error('Account not found');
}
- const { data } = await axios.post<Workspace>(
- `${buildApiBaseUrl(server)}/v1/workspaces`,
- {
- name: input.name,
- description: input.description,
- },
- {
- headers: {
- Authorization: `Bearer ${account.token}`,
- },
- },
+ const axios = buildAxiosInstance(
+ server.domain,
+ server.attributes,
+ account.token,
);
+ const { data } = await axios.post<Workspace>(`/v1/workspaces`, {
+ name: input.name,
+ description: input.description,
+ });
await databaseContext.appDatabase
.insertInto('workspaces')
diff --git a/desktop/src/main/mutations/workspace-update.ts b/desktop/src/main/mutations/workspace-update.ts
index 3448b751..d5a3a90c 100644
--- a/desktop/src/main/mutations/workspace-update.ts
+++ b/desktop/src/main/mutations/workspace-update.ts
@@ -1,6 +1,5 @@
-import axios from 'axios';
import { databaseContext } from '@/main/data/database-context';
-import { buildApiBaseUrl } from '@/lib/servers';
+import { buildAxiosInstance } from '@/lib/servers';
import { WorkspaceUpdateMutationInput } from '@/types/mutations/workspace-update';
import {
MutationChange,
@@ -35,13 +34,15 @@ export class WorkspaceUpdateMutationHandler
throw new Error('Account not found');
}
- const { data } = await axios.post<Workspace>(
- `${buildApiBaseUrl(server)}/v1/workspaces/${input.id}`,
- {
- name: input.name,
- description: input.description,
- },
+ const axios = buildAxiosInstance(
+ server.domain,
+ server.attributes,
+ account.token,
);
+ const { data } = await axios.post<Workspace>(`/v1/workspaces/${input.id}`, {
+ name: input.name,
+ description: input.description,
+ });
await databaseContext.appDatabase
.updateTable('workspaces')
diff --git a/desktop/src/main/queries/node-collaborator-search.ts b/desktop/src/main/queries/node-collaborator-search.ts
index 498538a8..ed09d59d 100644
--- a/desktop/src/main/queries/node-collaborator-search.ts
+++ b/desktop/src/main/queries/node-collaborator-search.ts
@@ -14,6 +14,15 @@ export class NodeCollaboratorSearchQueryHandler
public async handleQuery(
input: NodeCollaboratorSearchQueryInput,
): Promise<QueryResult<NodeCollaboratorSearchQueryInput>> {
+ if (input.searchQuery.length === 0) {
+ return {
+ output: [],
+ state: {
+ rows: [],
+ },
+ };
+ }
+
const rows = await this.fetchNodes(input);
return {
output: this.buildCollaboratorNodes(rows),
diff --git a/desktop/src/main/synchronizer.ts b/desktop/src/main/synchronizer.ts
index eb9f439b..cb945b45 100644
--- a/desktop/src/main/synchronizer.ts
+++ b/desktop/src/main/synchronizer.ts
@@ -1,8 +1,7 @@
-import axios from 'axios';
-import { buildApiBaseUrl } from '@/lib/servers';
+import { buildAxiosInstance } from '@/lib/servers';
import { SelectWorkspace } from '@/main/data/app/schema';
import { databaseContext } from '@/main/data/database-context';
-import { WorkspaceSyncData } from '@/types/sync';
+import { ServerSyncResponse, WorkspaceSyncData } from '@/types/sync';
const EVENT_LOOP_INTERVAL = 100;
@@ -23,7 +22,12 @@ class Synchronizer {
}
private async executeEventLoop() {
+ // try {
await this.checkForWorkspaceSyncs();
+ await this.checkForWorkspaceChanges();
+ // } catch (error) {
+ // console.log('error', error);
+ // }
setTimeout(this.executeEventLoop, EVENT_LOOP_INTERVAL);
}
@@ -44,34 +48,103 @@ class Synchronizer {
}
}
- private async syncWorkspace(workspace: SelectWorkspace): Promise<void> {
- const account = await databaseContext.appDatabase
- .selectFrom('accounts')
- .selectAll()
- .where('id', '=', workspace.account_id)
- .executeTakeFirst();
+ private async checkForWorkspaceChanges() {
+ const workspaces = await databaseContext.appDatabase
+ .selectFrom('workspaces')
+ .innerJoin('accounts', 'workspaces.account_id', 'accounts.id')
+ .innerJoin('servers', 'accounts.server', 'servers.domain')
+ .select([
+ 'workspaces.workspace_id',
+ 'workspaces.user_id',
+ 'accounts.token',
+ 'servers.domain',
+ 'servers.attributes',
+ ])
+ .where('workspaces.synced', '=', 1)
+ .execute();
- if (!account) {
- return;
+ for (const workspace of workspaces) {
+ const workspaceDatabase = await databaseContext.getWorkspaceDatabase(
+ workspace.user_id,
+ );
+
+ const changes = await workspaceDatabase
+ .selectFrom('changes')
+ .selectAll()
+ .orderBy('id asc')
+ .limit(20)
+ .execute();
+
+ if (changes.length === 0) {
+ return;
+ }
+
+ const axios = buildAxiosInstance(
+ workspace.domain,
+ workspace.attributes,
+ workspace.token,
+ );
+
+ const { data } = await axios.post<ServerSyncResponse>(
+ `/v1/sync/${workspace.workspace_id}`,
+ {
+ changes: changes,
+ },
+ );
+
+ const syncedChangeIds: number[] = [];
+ const unsyncedChangeIds: number[] = [];
+ for (const result of data.results) {
+ if (result.status === 'success') {
+ syncedChangeIds.push(result.id);
+ } else {
+ unsyncedChangeIds.push(result.id);
+ }
+ }
+
+ if (syncedChangeIds.length > 0) {
+ await workspaceDatabase
+ .deleteFrom('changes')
+ .where('id', 'in', syncedChangeIds)
+ .execute();
+ }
+
+ if (unsyncedChangeIds.length > 0) {
+ await workspaceDatabase
+ .updateTable('changes')
+ .set((eb) => ({ retry_count: eb('retry_count', '+', 1) }))
+ .where('id', 'in', unsyncedChangeIds)
+ .execute();
+
+ //we just delete changes that have failed to sync for more than 5 times.
+ //in the future we might need to revert the change locally.
+ await workspaceDatabase
+ .deleteFrom('changes')
+ .where('retry_count', '>=', 5)
+ .execute();
+ }
}
+ }
- const server = await databaseContext.appDatabase
- .selectFrom('servers')
- .selectAll()
- .where('domain', '=', account.server)
+ private async syncWorkspace(workspace: SelectWorkspace): Promise<void> {
+ const credentials = await databaseContext.appDatabase
+ .selectFrom('accounts')
+ .innerJoin('servers', 'accounts.server', 'servers.domain')
+ .select(['domain', 'attributes', 'token'])
+ .where('id', '=', workspace.account_id)
.executeTakeFirst();
- if (!server) {
+ if (!credentials) {
return;
}
+ const axios = buildAxiosInstance(
+ credentials.domain,
+ credentials.attributes,
+ credentials.token,
+ );
const { data } = await axios.get<WorkspaceSyncData>(
- `${buildApiBaseUrl(server)}/v1/sync/${workspace.workspace_id}`,
- {
- headers: {
- Authorization: `Bearer ${account.token}`,
- },
- },
+ `/v1/sync/${workspace.workspace_id}`,
);
const workspaceDatabase = await databaseContext.getWorkspaceDatabase(
diff --git a/server/src/routes/sync.ts b/server/src/routes/sync.ts
index 597caefe..f2041f98 100644
--- a/server/src/routes/sync.ts
+++ b/server/src/routes/sync.ts
@@ -88,6 +88,7 @@ syncRouter.get(
syncRouter.post(
'/:workspaceId',
async (req: NeuronRequest, res: NeuronResponse) => {
+ const workspaceId = req.params.workspaceId as string;
const input = req.body as SyncLocalChangesInput;
if (!req.accountId) {
return res.status(401).json({
@@ -99,7 +100,7 @@ syncRouter.post(
const workspace = await database
.selectFrom('workspaces')
.selectAll()
- .where('id', '=', input.workspaceId)
+ .where('id', '=', workspaceId)
.executeTakeFirst();
if (!workspace) {
diff --git a/server/src/types/sync.ts b/server/src/types/sync.ts
index 39122122..13eb99cf 100644
--- a/server/src/types/sync.ts
+++ b/server/src/types/sync.ts
@@ -1,5 +1,4 @@
export type SyncLocalChangesInput = {
- workspaceId: string;
changes: LocalChange[];
};