Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Initial node sync event loop
Hakan Shehuba4d8b315df3
diff --git a/desktop/src/electron/database/global/index.ts b/desktop/src/electron/database/global/index.tsindex f93cde17..93cba192 100644--- a/desktop/src/electron/database/global/index.ts+++ b/desktop/src/electron/database/global/index.ts@@ -4,7 +4,7 @@ import { Kysely, Migration, Migrator, SqliteDialect } from 'kysely';import { GlobalDatabaseSchema } from '@/electron/database/global/schema';import { Account } from '@/types/accounts';import { Workspace } from '@/types/workspaces';-import { AccountTransactions, Transaction } from '@/types/transactions';+import { Transaction } from '@/types/transactions';import { globalDatabaseMigrations } from '@/electron/database/global/migrations';import { GlobalDatabaseData, WorkspaceDatabaseData } from '@/types/global';import { WorkspaceDatabase } from '@/electron/database/workspace';@@ -123,6 +123,7 @@ export class GlobalDatabase {accountId: workspace.account_id,role: workspace.role,userNodeId: workspace.user_node_id,+ syncedAt: workspace.synced_at ? new Date(workspace.synced_at) : null,}));};@@ -138,6 +139,7 @@ export class GlobalDatabase {account_id: workspace.accountId,role: workspace.role,user_node_id: workspace.userNodeId,+ synced_at: workspace.syncedAt?.toISOString(),}).execute();@@ -159,56 +161,21 @@ export class GlobalDatabase {.execute();};- getGroupedAccountTransactions = async (- count: number,- ): Promise<AccountTransactions[]> => {+ getTransactions = async (count: number): Promise<Transaction[]> => {const transactions = await this.database.selectFrom('transactions').selectAll().limit(count).execute();- if (transactions.length === 0) {- return [];- }-- const accountIds = transactions.map(- (transaction) => transaction.account_id,- );- const accounts = await this.database- .selectFrom('accounts')- .selectAll()- .where('id', 'in', accountIds)- .execute();-- const transactionsMap = new Map<string, Transaction[]>();- transactions.forEach((transaction) => {- const transactionObj = {- id: transaction.id,- workspaceId: transaction.workspace_id,- accountId: transaction.account_id,- type: transaction.type,- nodeId: transaction.node_id,- input: transaction.input,- createdAt: new Date(transaction.created_at),- };-- if (transactionsMap.has(transaction.account_id)) {- transactionsMap.get(transaction.account_id)?.push(transactionObj);- } else {- transactionsMap.set(transaction.account_id, [transactionObj]);- }- });-- return accounts.map((account) => ({- account: {- id: account.id,- name: account.name,- email: account.email,- avatar: account.avatar,- token: account.token,- },- transactions: transactionsMap.get(account.id) || [],+ return transactions.map((transaction) => ({+ id: transaction.id,+ workspaceId: transaction.workspace_id,+ accountId: transaction.account_id,+ type: transaction.type,+ nodeId: transaction.node_id,+ input: transaction.input,+ createdAt: new Date(transaction.created_at),}));};@@ -218,6 +185,14 @@ export class GlobalDatabase {.where('id', 'in', ids).execute();};++ updateWorkspaceSyncedAt = async (workspaceId: string, date: Date) => {+ await this.database+ .updateTable('workspaces')+ .set('synced_at', date.toISOString())+ .where('id', '=', workspaceId)+ .execute();+ };}export const globalDatabase = new GlobalDatabase();diff --git a/desktop/src/electron/database/global/migrations.ts b/desktop/src/electron/database/global/migrations.tsindex 4bb2fcf4..a93664e0 100644--- a/desktop/src/electron/database/global/migrations.ts+++ b/desktop/src/electron/database/global/migrations.ts@@ -28,6 +28,7 @@ const createWorkspacesTable: Migration = {.addColumn('account_id', 'integer', (col) => col.notNull()).addColumn('role', 'integer', (col) => col.notNull()).addColumn('user_node_id', 'text', (col) => col.notNull())+ .addColumn('synced_at', 'text').execute();},down: async (db) => {diff --git a/desktop/src/electron/database/global/schema.ts b/desktop/src/electron/database/global/schema.tsindex 5234821e..7a654fe6 100644--- a/desktop/src/electron/database/global/schema.ts+++ b/desktop/src/electron/database/global/schema.ts@@ -15,6 +15,7 @@ export interface WorkspacesTableSchema {account_id: string;role: number;user_node_id: string;+ synced_at: string | null;}export interface TransactionsTableSchema {diff --git a/desktop/src/electron/database/workspace/index.ts b/desktop/src/electron/database/workspace/index.tsindex ae63c3f3..d802838e 100644--- a/desktop/src/electron/database/workspace/index.ts+++ b/desktop/src/electron/database/workspace/index.ts@@ -64,11 +64,15 @@ export class WorkspaceDatabase {type: node.type,parent_id: node.parentId,workspace_id: this.workspaceId,- created_at: node.createdAt.toISOString(),+ created_at: new Date(node.createdAt).toISOString(),created_by: node.createdBy,version_id: node.versionId,attrs: JSON.stringify(node.attrs),content: JSON.stringify(node.content),+ updated_at: node.updatedAt+ ? new Date(node.updatedAt).toISOString()+ : null,+ updated_by: node.updatedBy,}).execute();@@ -155,6 +159,56 @@ export class WorkspaceDatabase {});};+ syncNodes = async (nodes: Node[]) => {+ const nodeIds = nodes.map((node) => node.id);+ const existingNodes = await this.database+ .selectFrom('nodes')+ .selectAll()+ .where('id', 'in', nodeIds)+ .execute();++ for (const node of nodes) {+ const existingNode = existingNodes.find((n) => n.id === node.id);+ if (!existingNode) {+ await this.database+ .insertInto('nodes')+ .values({+ id: node.id,+ type: node.type,+ parent_id: node.parentId,+ workspace_id: this.workspaceId,+ created_at: new Date(node.createdAt).toISOString(),+ created_by: node.createdBy,+ version_id: node.versionId,+ attrs: JSON.stringify(node.attrs),+ content: JSON.stringify(node.content),+ updated_at: node.updatedAt+ ? new Date(node.updatedAt).toISOString()+ : null,+ updated_by: node.updatedBy,+ })+ .execute();+ } else if (+ existingNode.version_id != node.versionId &&+ new Date(existingNode.updated_at) <= node.updatedAt+ ) {+ await this.database+ .updateTable('nodes')+ .set({+ type: node.type,+ parent_id: node.parentId,+ updated_at: node.updatedAt.toISOString(),+ updated_by: node.updatedBy,+ version_id: node.versionId,+ attrs: JSON.stringify(node.attrs),+ content: JSON.stringify(node.content),+ })+ .where('id', '=', node.id)+ .executeTakeFirst();+ }+ }+ };+migrate = async () => {const migrator = new Migrator({db: this.database,diff --git a/desktop/src/electron/event-loop.ts b/desktop/src/electron/event-loop.tsindex 49dc9edc..0d4847b1 100644--- a/desktop/src/electron/event-loop.ts+++ b/desktop/src/electron/event-loop.ts@@ -1,14 +1,39 @@import { globalDatabase } from '@/electron/database/global';+import { Account } from '@/types/accounts';+import { Transaction } from '@/types/transactions';+import { Node } from '@/types/nodes';export const initEventLoop = () => {setInterval(async () => {- await sendTransactions();+ const accounts = await globalDatabase.getAccounts();+ await sendTransactions(accounts);+ await syncWorkspaces(accounts);}, 10000);};-const sendTransactions = async () => {- const groupedAccountTransactions =- await globalDatabase.getGroupedAccountTransactions(20);+const sendTransactions = async (accounts: Account[]) => {+ const transactions = await globalDatabase.getTransactions(20);+ const groupedAccountTransactions = transactions.reduce(+ (acc, transaction) => {+ const account = accounts.find((a) => a.id === transaction.accountId);+ if (!account) {+ return acc;+ }++ const accountTransactions = acc.find((a) => a.account === account);+ if (accountTransactions) {+ accountTransactions.transactions.push(transaction);+ } else {+ acc.push({+ account,+ transactions: [transaction],+ });+ }++ return acc;+ },+ [] as { account: Account; transactions: Transaction[] }[],+ );for (const accountTransactions of groupedAccountTransactions) {const transactionIds = accountTransactions.transactions.map((t) => t.id);@@ -28,3 +53,45 @@ const sendTransactions = async () => {}}};++const syncWorkspaces = async (accounts: Account[]) => {+ const workspaces = await globalDatabase.getWorkspaces();+ for (const workspace of workspaces) {+ const account = accounts.find((a) => a.id === workspace.accountId);+ if (!account) {+ continue;+ }++ const date = new Date();+ let url = `http://localhost:3000/v1/workspaces/${workspace.id}/nodes`;+ if (workspace.syncedAt) {+ url += `?from=${workspace.syncedAt.toISOString()}`;+ }+ const response = await fetch(url, {+ method: 'GET',+ headers: {+ 'Content-Type': 'application/json',+ Authorization: `Bearer ${account.token}`,+ },+ });++ if (response.ok) {+ await globalDatabase.updateWorkspaceSyncedAt(workspace.id, date);+ const { nodes }: { nodes: Node[] } = await response.json();+ if (nodes.length == 0) {+ return;+ }++ const workspaceDatabase = globalDatabase.getWorkspaceDatabase(+ workspace.accountId,+ workspace.id,+ );++ if (!workspaceDatabase) {+ continue;+ }++ await workspaceDatabase.syncNodes(nodes);+ }+ }+};diff --git a/desktop/src/types/transactions.ts b/desktop/src/types/transactions.tsindex d4d0c648..12b68082 100644--- a/desktop/src/types/transactions.ts+++ b/desktop/src/types/transactions.ts@@ -1,5 +1,3 @@-import { Account } from '@/types/accounts';-export type Transaction = {id: string;workspaceId: string;@@ -9,8 +7,3 @@ export type Transaction = {input: string;createdAt: Date;};--export type AccountTransactions = {- account: Account;- transactions: Transaction[];-};diff --git a/desktop/src/types/workspaces.ts b/desktop/src/types/workspaces.tsindex 9baaef01..3190864b 100644--- a/desktop/src/types/workspaces.ts+++ b/desktop/src/types/workspaces.ts@@ -14,4 +14,5 @@ export type Workspace = {accountId: string;role: WorkspaceRole;userNodeId: string;+ syncedAt?: Date | null;};diff --git a/server/src/index.ts b/server/src/index.tsindex a90a236d..cf3671bb 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -26,6 +26,7 @@ app.post('/v1/workspaces', authMiddleware, workspaces.createWorkspace);app.put('/v1/workspaces/:id', authMiddleware, workspaces.updateWorkspace);app.delete('/v1/workspaces/:id', authMiddleware, workspaces.deleteWorkspace);app.get('/v1/workspaces/:id', authMiddleware, workspaces.getWorkspace);+app.get('/v1/workspaces/:id/nodes', authMiddleware, workspaces.getNodes);//transactionsapp.post('/v1/transactions', transactions.applyTransactions);diff --git a/server/src/routes/accounts.ts b/server/src/routes/accounts.tsindex 3d0437cb..8fa02022 100644--- a/server/src/routes/accounts.ts+++ b/server/src/routes/accounts.ts@@ -115,7 +115,13 @@ const registerWithEmail = async (req: Request, res: Response) => {},});- return res.json(buildLoginOutput(account.id, account.name, account.email));+ const output = await buildLoginOutput(+ account.id,+ account.name,+ account.email,+ );++ return res.status(200).json(output);};const loginWithEmail = async (req: Request, res: Response) => {diff --git a/server/src/routes/workspaces.ts b/server/src/routes/workspaces.tsindex 7a1f187e..df24ffc0 100644--- a/server/src/routes/workspaces.ts+++ b/server/src/routes/workspaces.ts@@ -10,6 +10,8 @@ import {import { ApiError, NeuronRequest, NeuronResponse } from '@/types/api';import { generateId, IdType } from '@/lib/id';import { prisma } from '@/data/db';+import { Request, Response } from 'express';+import { Node } from '@/types/nodes';const createWorkspace = async (req: NeuronRequest, res: NeuronResponse) => {const input: WorkspaceInput = req.body;@@ -322,10 +324,68 @@ const getWorkspaces = async (req: NeuronRequest, res: NeuronResponse) => {return res.status(200).json(outputs);};+const getNodes = async (req: Request, res: Response) => {+ const workspaceId = req.params.id as string;+ const from = req.query.from as string;+ const nodes = await getNodesFromDatabase(workspaceId, from);+ const outputs: Node[] = nodes.map((node) => {+ return {+ id: node.id,+ parentId: node.parentId,+ workspaceId: node.workspaceId,+ type: node.type,+ attrs: node.attrs,+ createdAt: node.createdAt,+ createdBy: node.createdBy,+ versionId: node.versionId,+ content: node.content ? JSON.parse(node.content as string) : null,+ updatedAt: node.updatedAt,+ updatedBy: node.updatedBy,+ };+ });++ res.status(200).json({+ nodes: outputs,+ });+};++const getNodesFromDatabase = async (+ workspaceId: string,+ from?: string | null,+) => {+ if (from) {+ const date = new Date(from);+ return prisma.nodes.findMany({+ where: {+ workspaceId,+ OR: [+ {+ createdAt: {+ gte: date,+ },+ },+ {+ updatedAt: {+ gte: date,+ },+ },+ ],+ },+ });+ } else {+ return prisma.nodes.findMany({+ where: {+ workspaceId,+ },+ });+ }+};+export const workspaces = {getWorkspaces,createWorkspace,updateWorkspace,deleteWorkspace,getWorkspace,+ getNodes,};diff --git a/server/src/types/nodes.ts b/server/src/types/nodes.tsindex 1f2f63ef..b6284940 100644--- a/server/src/types/nodes.ts+++ b/server/src/types/nodes.ts@@ -1,7 +1,7 @@export type Node = {id: string;workspaceId: string;- parentId?: string;+ parentId?: string | null;type: string;attrs: any;content?: NodeContent[] | null;