Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Initial node sync event loop

Hakan Shehuba4d8b315df3
diff --git a/desktop/src/electron/database/global/index.ts b/desktop/src/electron/database/global/index.ts
index f93cde17..93cba192 100644
--- a/desktop/src/electron/database/global/index.ts
+++ b/desktop/src/electron/database/global/index.ts
@@ -4,7 +4,7 @@ import { Kysely, Migration, Migrator, SqliteDialect } from 'kysely';
import { GlobalDatabaseSchema } from '@/electron/database/global/schema';
import { Account } from '@/types/accounts';
import { Workspace } from '@/types/workspaces';
-import { AccountTransactions, Transaction } from '@/types/transactions';
+import { Transaction } from '@/types/transactions';
import { globalDatabaseMigrations } from '@/electron/database/global/migrations';
import { GlobalDatabaseData, WorkspaceDatabaseData } from '@/types/global';
import { WorkspaceDatabase } from '@/electron/database/workspace';
@@ -123,6 +123,7 @@ export class GlobalDatabase {
accountId: workspace.account_id,
role: workspace.role,
userNodeId: workspace.user_node_id,
+ syncedAt: workspace.synced_at ? new Date(workspace.synced_at) : null,
}));
};
@@ -138,6 +139,7 @@ export class GlobalDatabase {
account_id: workspace.accountId,
role: workspace.role,
user_node_id: workspace.userNodeId,
+ synced_at: workspace.syncedAt?.toISOString(),
})
.execute();
@@ -159,56 +161,21 @@ export class GlobalDatabase {
.execute();
};
- getGroupedAccountTransactions = async (
- count: number,
- ): Promise<AccountTransactions[]> => {
+ getTransactions = async (count: number): Promise<Transaction[]> => {
const transactions = await this.database
.selectFrom('transactions')
.selectAll()
.limit(count)
.execute();
- if (transactions.length === 0) {
- return [];
- }
-
- const accountIds = transactions.map(
- (transaction) => transaction.account_id,
- );
- const accounts = await this.database
- .selectFrom('accounts')
- .selectAll()
- .where('id', 'in', accountIds)
- .execute();
-
- const transactionsMap = new Map<string, Transaction[]>();
- transactions.forEach((transaction) => {
- const transactionObj = {
- id: transaction.id,
- workspaceId: transaction.workspace_id,
- accountId: transaction.account_id,
- type: transaction.type,
- nodeId: transaction.node_id,
- input: transaction.input,
- createdAt: new Date(transaction.created_at),
- };
-
- if (transactionsMap.has(transaction.account_id)) {
- transactionsMap.get(transaction.account_id)?.push(transactionObj);
- } else {
- transactionsMap.set(transaction.account_id, [transactionObj]);
- }
- });
-
- return accounts.map((account) => ({
- account: {
- id: account.id,
- name: account.name,
- email: account.email,
- avatar: account.avatar,
- token: account.token,
- },
- transactions: transactionsMap.get(account.id) || [],
+ return transactions.map((transaction) => ({
+ id: transaction.id,
+ workspaceId: transaction.workspace_id,
+ accountId: transaction.account_id,
+ type: transaction.type,
+ nodeId: transaction.node_id,
+ input: transaction.input,
+ createdAt: new Date(transaction.created_at),
}));
};
@@ -218,6 +185,14 @@ export class GlobalDatabase {
.where('id', 'in', ids)
.execute();
};
+
+ updateWorkspaceSyncedAt = async (workspaceId: string, date: Date) => {
+ await this.database
+ .updateTable('workspaces')
+ .set('synced_at', date.toISOString())
+ .where('id', '=', workspaceId)
+ .execute();
+ };
}
export const globalDatabase = new GlobalDatabase();
diff --git a/desktop/src/electron/database/global/migrations.ts b/desktop/src/electron/database/global/migrations.ts
index 4bb2fcf4..a93664e0 100644
--- a/desktop/src/electron/database/global/migrations.ts
+++ b/desktop/src/electron/database/global/migrations.ts
@@ -28,6 +28,7 @@ const createWorkspacesTable: Migration = {
.addColumn('account_id', 'integer', (col) => col.notNull())
.addColumn('role', 'integer', (col) => col.notNull())
.addColumn('user_node_id', 'text', (col) => col.notNull())
+ .addColumn('synced_at', 'text')
.execute();
},
down: async (db) => {
diff --git a/desktop/src/electron/database/global/schema.ts b/desktop/src/electron/database/global/schema.ts
index 5234821e..7a654fe6 100644
--- a/desktop/src/electron/database/global/schema.ts
+++ b/desktop/src/electron/database/global/schema.ts
@@ -15,6 +15,7 @@ export interface WorkspacesTableSchema {
account_id: string;
role: number;
user_node_id: string;
+ synced_at: string | null;
}
export interface TransactionsTableSchema {
diff --git a/desktop/src/electron/database/workspace/index.ts b/desktop/src/electron/database/workspace/index.ts
index ae63c3f3..d802838e 100644
--- a/desktop/src/electron/database/workspace/index.ts
+++ b/desktop/src/electron/database/workspace/index.ts
@@ -64,11 +64,15 @@ export class WorkspaceDatabase {
type: node.type,
parent_id: node.parentId,
workspace_id: this.workspaceId,
- created_at: node.createdAt.toISOString(),
+ created_at: new Date(node.createdAt).toISOString(),
created_by: node.createdBy,
version_id: node.versionId,
attrs: JSON.stringify(node.attrs),
content: JSON.stringify(node.content),
+ updated_at: node.updatedAt
+ ? new Date(node.updatedAt).toISOString()
+ : null,
+ updated_by: node.updatedBy,
})
.execute();
@@ -155,6 +159,56 @@ export class WorkspaceDatabase {
});
};
+ syncNodes = async (nodes: Node[]) => {
+ const nodeIds = nodes.map((node) => node.id);
+ const existingNodes = await this.database
+ .selectFrom('nodes')
+ .selectAll()
+ .where('id', 'in', nodeIds)
+ .execute();
+
+ for (const node of nodes) {
+ const existingNode = existingNodes.find((n) => n.id === node.id);
+ if (!existingNode) {
+ await this.database
+ .insertInto('nodes')
+ .values({
+ id: node.id,
+ type: node.type,
+ parent_id: node.parentId,
+ workspace_id: this.workspaceId,
+ created_at: new Date(node.createdAt).toISOString(),
+ created_by: node.createdBy,
+ version_id: node.versionId,
+ attrs: JSON.stringify(node.attrs),
+ content: JSON.stringify(node.content),
+ updated_at: node.updatedAt
+ ? new Date(node.updatedAt).toISOString()
+ : null,
+ updated_by: node.updatedBy,
+ })
+ .execute();
+ } else if (
+ existingNode.version_id != node.versionId &&
+ new Date(existingNode.updated_at) <= node.updatedAt
+ ) {
+ await this.database
+ .updateTable('nodes')
+ .set({
+ type: node.type,
+ parent_id: node.parentId,
+ updated_at: node.updatedAt.toISOString(),
+ updated_by: node.updatedBy,
+ version_id: node.versionId,
+ attrs: JSON.stringify(node.attrs),
+ content: JSON.stringify(node.content),
+ })
+ .where('id', '=', node.id)
+ .executeTakeFirst();
+ }
+ }
+ };
+
migrate = async () => {
const migrator = new Migrator({
db: this.database,
diff --git a/desktop/src/electron/event-loop.ts b/desktop/src/electron/event-loop.ts
index 49dc9edc..0d4847b1 100644
--- a/desktop/src/electron/event-loop.ts
+++ b/desktop/src/electron/event-loop.ts
@@ -1,14 +1,39 @@
import { globalDatabase } from '@/electron/database/global';
+import { Account } from '@/types/accounts';
+import { Transaction } from '@/types/transactions';
+import { Node } from '@/types/nodes';
export const initEventLoop = () => {
setInterval(async () => {
- await sendTransactions();
+ const accounts = await globalDatabase.getAccounts();
+ await sendTransactions(accounts);
+ await syncWorkspaces(accounts);
}, 10000);
};
-const sendTransactions = async () => {
- const groupedAccountTransactions =
- await globalDatabase.getGroupedAccountTransactions(20);
+const sendTransactions = async (accounts: Account[]) => {
+ const transactions = await globalDatabase.getTransactions(20);
+ const groupedAccountTransactions = transactions.reduce(
+ (acc, transaction) => {
+ const account = accounts.find((a) => a.id === transaction.accountId);
+ if (!account) {
+ return acc;
+ }
+
+ const accountTransactions = acc.find((a) => a.account === account);
+ if (accountTransactions) {
+ accountTransactions.transactions.push(transaction);
+ } else {
+ acc.push({
+ account,
+ transactions: [transaction],
+ });
+ }
+
+ return acc;
+ },
+ [] as { account: Account; transactions: Transaction[] }[],
+ );
for (const accountTransactions of groupedAccountTransactions) {
const transactionIds = accountTransactions.transactions.map((t) => t.id);
@@ -28,3 +53,45 @@ const sendTransactions = async () => {
}
}
};
+
+const syncWorkspaces = async (accounts: Account[]) => {
+ const workspaces = await globalDatabase.getWorkspaces();
+ for (const workspace of workspaces) {
+ const account = accounts.find((a) => a.id === workspace.accountId);
+ if (!account) {
+ continue;
+ }
+
+ const date = new Date();
+ let url = `http://localhost:3000/v1/workspaces/${workspace.id}/nodes`;
+ if (workspace.syncedAt) {
+ url += `?from=${workspace.syncedAt.toISOString()}`;
+ }
+ const response = await fetch(url, {
+ method: 'GET',
+ headers: {
+ 'Content-Type': 'application/json',
+ Authorization: `Bearer ${account.token}`,
+ },
+ });
+
+ if (response.ok) {
+ await globalDatabase.updateWorkspaceSyncedAt(workspace.id, date);
+ const { nodes }: { nodes: Node[] } = await response.json();
+ if (nodes.length == 0) {
+ return;
+ }
+
+ const workspaceDatabase = globalDatabase.getWorkspaceDatabase(
+ workspace.accountId,
+ workspace.id,
+ );
+
+ if (!workspaceDatabase) {
+ continue;
+ }
+
+ await workspaceDatabase.syncNodes(nodes);
+ }
+ }
+};
diff --git a/desktop/src/types/transactions.ts b/desktop/src/types/transactions.ts
index d4d0c648..12b68082 100644
--- a/desktop/src/types/transactions.ts
+++ b/desktop/src/types/transactions.ts
@@ -1,5 +1,3 @@
-import { Account } from '@/types/accounts';
-
export type Transaction = {
id: string;
workspaceId: string;
@@ -9,8 +7,3 @@ export type Transaction = {
input: string;
createdAt: Date;
};
-
-export type AccountTransactions = {
- account: Account;
- transactions: Transaction[];
-};
diff --git a/desktop/src/types/workspaces.ts b/desktop/src/types/workspaces.ts
index 9baaef01..3190864b 100644
--- a/desktop/src/types/workspaces.ts
+++ b/desktop/src/types/workspaces.ts
@@ -14,4 +14,5 @@ export type Workspace = {
accountId: string;
role: WorkspaceRole;
userNodeId: string;
+ syncedAt?: Date | null;
};
diff --git a/server/src/index.ts b/server/src/index.ts
index a90a236d..cf3671bb 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -26,6 +26,7 @@ app.post('/v1/workspaces', authMiddleware, workspaces.createWorkspace);
app.put('/v1/workspaces/:id', authMiddleware, workspaces.updateWorkspace);
app.delete('/v1/workspaces/:id', authMiddleware, workspaces.deleteWorkspace);
app.get('/v1/workspaces/:id', authMiddleware, workspaces.getWorkspace);
+app.get('/v1/workspaces/:id/nodes', authMiddleware, workspaces.getNodes);
//transactions
app.post('/v1/transactions', transactions.applyTransactions);
diff --git a/server/src/routes/accounts.ts b/server/src/routes/accounts.ts
index 3d0437cb..8fa02022 100644
--- a/server/src/routes/accounts.ts
+++ b/server/src/routes/accounts.ts
@@ -115,7 +115,13 @@ const registerWithEmail = async (req: Request, res: Response) => {
},
});
- return res.json(buildLoginOutput(account.id, account.name, account.email));
+ const output = await buildLoginOutput(
+ account.id,
+ account.name,
+ account.email,
+ );
+
+ return res.status(200).json(output);
};
const loginWithEmail = async (req: Request, res: Response) => {
diff --git a/server/src/routes/workspaces.ts b/server/src/routes/workspaces.ts
index 7a1f187e..df24ffc0 100644
--- a/server/src/routes/workspaces.ts
+++ b/server/src/routes/workspaces.ts
@@ -10,6 +10,8 @@ import {
import { ApiError, NeuronRequest, NeuronResponse } from '@/types/api';
import { generateId, IdType } from '@/lib/id';
import { prisma } from '@/data/db';
+import { Request, Response } from 'express';
+import { Node } from '@/types/nodes';
const createWorkspace = async (req: NeuronRequest, res: NeuronResponse) => {
const input: WorkspaceInput = req.body;
@@ -322,10 +324,68 @@ const getWorkspaces = async (req: NeuronRequest, res: NeuronResponse) => {
return res.status(200).json(outputs);
};
+const getNodes = async (req: Request, res: Response) => {
+ const workspaceId = req.params.id as string;
+ const from = req.query.from as string;
+ const nodes = await getNodesFromDatabase(workspaceId, from);
+ const outputs: Node[] = nodes.map((node) => {
+ return {
+ id: node.id,
+ parentId: node.parentId,
+ workspaceId: node.workspaceId,
+ type: node.type,
+ attrs: node.attrs,
+ createdAt: node.createdAt,
+ createdBy: node.createdBy,
+ versionId: node.versionId,
+ content: node.content ? JSON.parse(node.content as string) : null,
+ updatedAt: node.updatedAt,
+ updatedBy: node.updatedBy,
+ };
+ });
+
+ res.status(200).json({
+ nodes: outputs,
+ });
+};
+
+const getNodesFromDatabase = async (
+ workspaceId: string,
+ from?: string | null,
+) => {
+ if (from) {
+ const date = new Date(from);
+ return prisma.nodes.findMany({
+ where: {
+ workspaceId,
+ OR: [
+ {
+ createdAt: {
+ gte: date,
+ },
+ },
+ {
+ updatedAt: {
+ gte: date,
+ },
+ },
+ ],
+ },
+ });
+ } else {
+ return prisma.nodes.findMany({
+ where: {
+ workspaceId,
+ },
+ });
+ }
+};
+
export const workspaces = {
getWorkspaces,
createWorkspace,
updateWorkspace,
deleteWorkspace,
getWorkspace,
+ getNodes,
};
diff --git a/server/src/types/nodes.ts b/server/src/types/nodes.ts
index 1f2f63ef..b6284940 100644
--- a/server/src/types/nodes.ts
+++ b/server/src/types/nodes.ts
@@ -1,7 +1,7 @@
export type Node = {
id: string;
workspaceId: string;
- parentId?: string;
+ parentId?: string | null;
type: string;
attrs: any;
content?: NodeContent[] | null;