Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Implement syncing and improvements in app queries
Hakan Shehu4491abec102f
diff --git a/desktop/src/components/accounts/account-logout.tsx b/desktop/src/components/accounts/account-logout.tsxindex 9d39b494..e9b87362 100644--- a/desktop/src/components/accounts/account-logout.tsx+++ b/desktop/src/components/accounts/account-logout.tsx@@ -17,7 +17,7 @@ interface AccountLogoutProps {}export const AccountLogout = ({ id, onCancel }: AccountLogoutProps) => {- const [isDeleting, setIsDeleting] = React.useState(false);+ const [isLoggingOut, setIsLoggingOut] = React.useState(false);return (<AlertDialogopen={true}@@ -38,15 +38,15 @@ export const AccountLogout = ({ id, onCancel }: AccountLogoutProps) => {<AlertDialogCancel>Cancel</AlertDialogCancel><Buttonvariant="destructive"- disabled={isDeleting}+ disabled={isLoggingOut}onClick={async () => {- setIsDeleting(true);+ setIsLoggingOut(true);await window.neuron.logout(id);window.location.href = '/';}}>- {isDeleting && <Spinner className="mr-1" />}- Delete+ {isLoggingOut && <Spinner className="mr-1" />}+ Logout</Button></AlertDialogFooter></AlertDialogContent>diff --git a/desktop/src/components/accounts/login.tsx b/desktop/src/components/accounts/login.tsxindex a3863cc0..66dd34a1 100644--- a/desktop/src/components/accounts/login.tsx+++ b/desktop/src/components/accounts/login.tsx@@ -24,7 +24,7 @@ export const Login = observer(() => {}).compile();- await appDatabase.executeQuery(insertAccountQuery);+ await appDatabase.mutate(insertAccountQuery);if (output.workspaces.length > 0) {const insertWorkspacesQuery = appDatabase.database@@ -37,14 +37,14 @@ export const Login = observer(() => {avatar: workspace.avatar,role: workspace.role,description: workspace.description,- synced_at: workspace.syncedAt,+ synced: 0,user_id: workspace.userId,version_id: workspace.versionId,})),).compile();- await appDatabase.executeQuery(insertWorkspacesQuery);+ await appDatabase.mutate(insertWorkspacesQuery);}window.location.href = '/';diff --git a/desktop/src/components/app.tsx b/desktop/src/components/app.tsxindex 37992e88..66bb9188 100644--- a/desktop/src/components/app.tsx+++ b/desktop/src/components/app.tsx@@ -36,7 +36,7 @@ export const App = observer(() => {queryKey: ['accounts'],queryFn: async () => {const query = appDatabase.selectFrom('accounts').selectAll().compile();- return await window.neuron.executeAppQuery(query);+ return await window.neuron.executeAppQueryAndSubscribe('accounts', query);},});@@ -44,7 +44,10 @@ export const App = observer(() => {queryKey: ['workspaces'],queryFn: async () => {const query = appDatabase.selectFrom('workspaces').selectAll().compile();- return await window.neuron.executeAppQuery(query);+ return await window.neuron.executeAppQueryAndSubscribe(+ 'workspaces',+ query,+ );},});@@ -58,7 +61,10 @@ export const App = observer(() => {<AppDatabaseContext.Providervalue={{database: appDatabase,- executeQuery: (query) => window.neuron.executeAppQuery(query),+ query: (query) => window.neuron.executeAppQuery(query),+ queryAndSubscribe: (queryId, query) =>+ window.neuron.executeAppQueryAndSubscribe(queryId, query),+ mutate: (mutation) => window.neuron.executeAppMutation(mutation),}}><Login />@@ -78,7 +84,10 @@ export const App = observer(() => {<AppDatabaseContext.Providervalue={{database: appDatabase,- executeQuery: (query) => window.neuron.executeAppQuery(query),+ query: (query) => window.neuron.executeAppQuery(query),+ queryAndSubscribe: (queryId, query) =>+ window.neuron.executeAppQueryAndSubscribe(queryId, query),+ mutate: (mutation) => window.neuron.executeAppMutation(mutation),}}><AccountContext.Provider@@ -99,7 +108,7 @@ export const App = observer(() => {accountId: row.account_id,role: row.role,userId: row.user_id,- syncedAt: row.synced_at,+ synced: row.synced,};}),logout: () => {diff --git a/desktop/src/components/messages/message.tsx b/desktop/src/components/messages/message.tsxindex 570213c3..f9f9690e 100644--- a/desktop/src/components/messages/message.tsx+++ b/desktop/src/components/messages/message.tsx@@ -22,8 +22,8 @@ interface MessageProps {export const Message = observer(({ message, previousMessage }: MessageProps) => {const author = message.author ?? {+ id: message.createdBy,attrs: {- id: message.createdBy,name: 'Deleted user',},};@@ -63,11 +63,7 @@ export const Message = observer(><div className="mr-2 w-10 pt-1">{shouldDisplayUserInfo() && (- <Avatar- id={author.attrs.id}- name={author.attrs.name}- size="medium"- />+ <Avatar id={author.id} name={author.attrs.name} size="medium" />)}</div>diff --git a/desktop/src/components/root.tsx b/desktop/src/components/root.tsxindex 7485ac68..cef66572 100644--- a/desktop/src/components/root.tsx+++ b/desktop/src/components/root.tsx@@ -9,6 +9,7 @@ import { WorkspaceRedirect } from '@/components/workspaces/workspace-redirect';import { createBrowserRouter, RouterProvider } from 'react-router-dom';import { Container } from '@/components/workspaces/container';import { QueryClient, QueryClientProvider } from '@tanstack/react-query';+import { eventBus } from '@/lib/event-bus';const router = createBrowserRouter([{@@ -37,15 +38,46 @@ const router = createBrowserRouter([},]);-const queryClient = new QueryClient({- defaultOptions: {- queries: {- retry: false,- },- },-});-export const Root = () => {+ const queryClient = React.useMemo(() => {+ return new QueryClient({+ defaultOptions: {+ queries: {+ retry: false,+ },+ },+ });+ }, []);++ React.useEffect(() => {+ const id = eventBus.subscribe((event) => {+ if (event.event === 'app_query_updated') {+ const result = event.payload.result;+ const queryKey = event.payload.queryId;++ if (result && queryKey) {+ queryClient.setQueryData([queryKey], result);+ }+ }+ });++ queryClient.getQueryCache().subscribe(async (event) => {+ if (+ event.type === 'removed' &&+ event.query &&+ event.query.queryKey &&+ event.query.queryKey.length > 0+ ) {+ const queryKey = event.query.queryKey[0];+ await window.neuron.unsubscribeAppQuery(queryKey);+ }+ });++ return () => {+ eventBus.unsubscribe(id);+ };+ }, []);+return (<QueryClientProvider client={queryClient}><TooltipProvider>diff --git a/desktop/src/components/ui/avatar.tsx b/desktop/src/components/ui/avatar.tsxindex 5f6c8f03..b7dd1b11 100644--- a/desktop/src/components/ui/avatar.tsx+++ b/desktop/src/components/ui/avatar.tsx@@ -16,7 +16,7 @@ const getAvatarSizeClasses = (size?: string) => {}return 'w-9 h-9';-}+};const colors = ['rgb(248 113 113)',@@ -34,7 +34,7 @@ const colors = [const getColorForId = (id: string) => {const index = Math.abs(hashCode(id)) % colors.length;return colors[index];-}+};const Avatar = ({id,diff --git a/desktop/src/components/workspaces/workspace-create.tsx b/desktop/src/components/workspaces/workspace-create.tsxindex 0cbda045..cbfed0bc 100644--- a/desktop/src/components/workspaces/workspace-create.tsx+++ b/desktop/src/components/workspaces/workspace-create.tsx@@ -59,13 +59,13 @@ export const WorkspaceCreate = observer(() => {description: data.description,avatar: data.avatar,role: data.role,- synced_at: new Date().toISOString(),+ synced: 0,user_id: data.userId,version_id: data.versionId,}).compile();- await appDatabase.executeQuery(insertWorkspaceQuery);+ await appDatabase.mutate(insertWorkspaceQuery);window.location.href = '/';},diff --git a/desktop/src/contexts/app-database.ts b/desktop/src/contexts/app-database.tsindex b2e8621e..4f1308c2 100644--- a/desktop/src/contexts/app-database.ts+++ b/desktop/src/contexts/app-database.ts@@ -4,7 +4,12 @@ import { AppDatabaseSchema } from '@/data/schemas/app';interface AppDatabaseContext {database: Kysely<AppDatabaseSchema>;- executeQuery: <R>(query: CompiledQuery<R>) => Promise<QueryResult<R>>;+ mutate: (mutation: CompiledQuery) => Promise<void>;+ query: <R>(query: CompiledQuery<R>) => Promise<QueryResult<R>>;+ queryAndSubscribe: <R>(+ queryId: string,+ query: CompiledQuery<R>,+ ) => Promise<QueryResult<R>>;}export const AppDatabaseContext = React.createContext<AppDatabaseContext>(diff --git a/desktop/src/data/account-manager.ts b/desktop/src/data/account-manager.tsindex a0a3a41f..6445630a 100644--- a/desktop/src/data/account-manager.ts+++ b/desktop/src/data/account-manager.ts@@ -4,17 +4,39 @@ import { Workspace } from '@/types/workspaces';import { WorkspaceManager } from '@/data/workspace-manager';import { SocketManager } from '@/data/socket-manager';import { ServerMutation } from '@/types/mutations';+import Axios, { AxiosInstance } from 'axios';+import { Kysely } from 'kysely';+import { AppDatabaseSchema } from '@/data/schemas/app';++const SERVER_URL = 'http://localhost:3000';export class AccountManager {+ private readonly account: Account;private readonly accountPath: string;private readonly socket: SocketManager;+ private readonly axios: AxiosInstance;private readonly workspaces: Map<string, WorkspaceManager>;+ private readonly database: Kysely<AppDatabaseSchema>;private interval: NodeJS.Timeout;- constructor(account: Account, appPath: string, workspaces: Workspace[]) {+ constructor(+ account: Account,+ appPath: string,+ workspaces: Workspace[],+ database: Kysely<AppDatabaseSchema>,+ ) {+ this.account = account;+ this.database = database;this.socket = new SocketManager(account);- this.workspaces = new Map<string, WorkspaceManager>();+ this.axios = Axios.create({+ baseURL: SERVER_URL,+ headers: {+ Authorization: `Bearer ${account.token}`,+ DeviceId: account.deviceId,+ },+ });+ this.workspaces = new Map<string, WorkspaceManager>();this.accountPath = `${appPath}/${account.id}`;if (!fs.existsSync(this.accountPath)) {fs.mkdirSync(this.accountPath);@@ -23,7 +45,7 @@ export class AccountManager {for (const workspace of workspaces) {this.workspaces.set(workspace.id,- new WorkspaceManager(account, workspace, this.accountPath),+ new WorkspaceManager(workspace, this.axios, this.accountPath),);}@@ -43,7 +65,7 @@ export class AccountManager {});}- public async init() {+ public async init(): Promise<void> {for (const workspace of this.workspaces.values()) {await workspace.init();}@@ -56,6 +78,17 @@ export class AccountManager {return this.workspaces.get(workspaceId);}+ public async addWorkspace(workspace: Workspace): Promise<void> {+ const workspaceManager = new WorkspaceManager(+ workspace,+ this.axios,+ this.accountPath,+ );++ await workspaceManager.init();+ this.workspaces.set(workspace.id, workspaceManager);+ }+public async logout(): Promise<void> {if (fs.existsSync(this.accountPath)) {fs.rm(this.accountPath, { recursive: true, force: true }, (err) => {@@ -74,7 +107,7 @@ export class AccountManager {}}- private startEventLoop() {+ private startEventLoop(): void {this.interval = setInterval(async () => {try {await this.executeEventLoop();@@ -84,10 +117,20 @@ export class AccountManager {}, 1000);}- private async executeEventLoop() {+ private async executeEventLoop(): Promise<void> {this.socket.checkConnection();for (const workspace of this.workspaces.values()) {+ if (!workspace.isSynced()) {+ const synced = await workspace.sync();+ if (synced) {+ await this.database+ .updateTable('workspaces')+ .set({ synced: 1 })+ .where('id', '=', workspace.getWorkspace().id)+ .execute();+ }+ }await workspace.sendMutations();}}diff --git a/desktop/src/data/app-manager.ts b/desktop/src/data/app-manager.tsindex 2fd5f199..100c31ae 100644--- a/desktop/src/data/app-manager.ts+++ b/desktop/src/data/app-manager.ts@@ -11,18 +11,26 @@ import { AccountManager } from '@/data/account-manager';import { AppDatabaseSchema } from '@/data/schemas/app';import { appDatabaseMigrations } from '@/data/migrations/app';import { Account } from '@/types/accounts';-import { buildSqlite } from '@/data/utils';+import {+ buildSqlite,+ extractTablesFromSql,+ resultHasChanged,+} from '@/data/utils';import { Workspace } from '@/types/workspaces';+import { SubscribedQueryData } from '@/types/databases';+import { eventBus } from '@/lib/event-bus';class AppManager {private readonly accounts: Map<string, AccountManager>;private readonly appPath: string;private readonly database: Kysely<AppDatabaseSchema>;+ private readonly subscribers: Map<string, SubscribedQueryData<unknown>>;private initPromise: Promise<void> | null = null;constructor() {this.accounts = new Map<string, AccountManager>();this.appPath = app.getPath('userData');+ this.subscribers = new Map();const dialect = new SqliteDialect({database: buildSqlite(`${this.appPath}/global.db`),@@ -40,7 +48,7 @@ class AppManager {return this.accounts.get(accountId);}- public async init() {+ public async init(): Promise<void> {if (!this.initPromise) {this.initPromise = this.startInit();}@@ -48,11 +56,113 @@ class AppManager {await this.initPromise;}- public async execute(- query: CompiledQuery<unknown>,- ): Promise<QueryResult<unknown>> {+ public async executeQuery<T>(+ query: CompiledQuery<T>,+ ): Promise<QueryResult<T>> {await this.waitForInit();- return await this.database.executeQuery(query);+ const result = await this.database.executeQuery(query);++ //only mutations should have side effects+ if (result.numAffectedRows > 0) {+ throw new Error('Query should not have any side effects');+ }++ return result;+ }++ public async executeQueryAndSubscribe<T>(+ queryId: string,+ query: CompiledQuery<T>,+ ): Promise<QueryResult<T>> {+ await this.waitForInit();+ const result = await this.database.executeQuery(query);++ // only mutations should have side effects+ if (result.numAffectedRows > 0) {+ throw new Error('Query should not have any side effects');+ }++ const selectedTables = extractTablesFromSql(query.sql);+ const subscriberData: SubscribedQueryData<T> = {+ query,+ tables: selectedTables,+ result: result,+ };+ this.subscribers.set(queryId, subscriberData);++ return result;+ }++ public async executeMutation(mutation: CompiledQuery): Promise<void> {+ const result = await this.database.executeQuery(mutation);+ if (result.numAffectedRows === 0n) {+ return;+ }++ const affectedTables = extractTablesFromSql(mutation.sql);+ if (affectedTables.length === 0) {+ return;+ }++ if (affectedTables.includes('accounts')) {+ const accounts = await this.getAccounts();+ for (const account of accounts) {+ if (this.accounts.has(account.id)) {+ continue;+ }++ const accountManager = new AccountManager(+ account,+ this.appPath,+ [],+ this.database,+ );++ await accountManager.init();+ this.accounts.set(account.id, accountManager);+ }+ }++ if (affectedTables.includes('workspaces')) {+ const workspaces = await this.getWorkspaces();+ for (const workspace of workspaces) {+ const accountManager = this.accounts.get(workspace.accountId);+ if (!accountManager) {+ continue;+ }++ const workspaceManager = accountManager.getWorkspace(workspace.id);+ if (!workspaceManager) {+ await accountManager.addWorkspace(workspace);+ }+ }+ }++ for (const [subscriberId, subscriberData] of this.subscribers) {+ const hasAffectedTables = subscriberData.tables.some((table) =>+ affectedTables.includes(table),+ );++ if (!hasAffectedTables) {+ continue;+ }++ const newResult = await this.database.executeQuery(subscriberData.query);++ if (resultHasChanged(subscriberData.result, newResult)) {+ eventBus.publish({+ event: 'app_query_updated',+ payload: {+ queryId: subscriberId,+ result: newResult,+ },+ });+ }+ }+ }++ public unsubscribeQuery(queryId: string): void {+ this.subscribers.delete(queryId);}public async logout(accountId: string): Promise<void> {@@ -64,9 +174,22 @@ class AppManager {await accountManager.logout();this.accounts.delete(accountId);++ const deleteAccountQuery = this.database+ .deleteFrom('accounts')+ .where('id', '=', accountId)+ .compile();++ const deleteWorkspacesQuery = this.database+ .deleteFrom('workspaces')+ .where('account_id', '=', accountId)+ .compile();++ await this.executeMutation(deleteAccountQuery);+ await this.executeMutation(deleteWorkspacesQuery);}- private async waitForInit() {+ private async waitForInit(): Promise<void> {if (!this.initPromise) {this.initPromise = this.startInit();}@@ -74,7 +197,7 @@ class AppManager {await this.initPromise;}- private async startInit() {+ private async startInit(): Promise<void> {await this.migrate();const accounts = await this.getAccounts();@@ -87,6 +210,7 @@ class AppManager {account,this.appPath,accountWorkspaces,+ this.database,);await accountManager.init();@@ -94,7 +218,7 @@ class AppManager {}}- private async migrate() {+ private async migrate(): Promise<void> {const migrator = new Migrator({db: this.database,provider: {@@ -129,17 +253,19 @@ class AppManager {.selectAll().execute();- return workspaces.map((workspace) => ({- id: workspace.id,- accountId: workspace.account_id,- name: workspace.name,- description: workspace.description,- avatar: workspace.avatar,- versionId: workspace.version_id,- role: workspace.role,- userId: workspace.user_id,- syncedAt: workspace.synced_at ? new Date(workspace.synced_at) : null,- }));+ return workspaces.map(+ (workspace): Workspace => ({+ id: workspace.id,+ accountId: workspace.account_id,+ name: workspace.name,+ description: workspace.description,+ avatar: workspace.avatar,+ versionId: workspace.version_id,+ role: workspace.role,+ userId: workspace.user_id,+ synced: workspace.synced === 1,+ }),+ );}}diff --git a/desktop/src/data/migrations/app.ts b/desktop/src/data/migrations/app.tsindex c9f6cf96..d4413f3f 100644--- a/desktop/src/data/migrations/app.ts+++ b/desktop/src/data/migrations/app.ts@@ -31,7 +31,7 @@ const createWorkspacesTable: Migration = {.addColumn('version_id', 'text', (col) => col.notNull()).addColumn('role', 'integer', (col) => col.notNull()).addColumn('user_id', 'text', (col) => col.notNull())- .addColumn('synced_at', 'text')+ .addColumn('synced', 'integer').execute();},down: async (db) => {diff --git a/desktop/src/data/mutation-manager.ts b/desktop/src/data/mutation-manager.tsindex 01561bfd..90e73d49 100644--- a/desktop/src/data/mutation-manager.ts+++ b/desktop/src/data/mutation-manager.ts@@ -16,15 +16,11 @@ import {ServerUpdateNodeMutation,} from '@/types/mutations';import { Workspace } from '@/types/workspaces';-import { Account } from '@/types/accounts';import { Node } from '@/types/nodes';-import Axios, { AxiosInstance } from 'axios';+import { AxiosInstance } from 'axios';import { NeuronId } from '@/lib/id';-const SERVER_URL = 'http://localhost:3000';-export class MutationManager {- private readonly account: Account;private readonly workspace: Workspace;private readonly database: Kysely<WorkspaceDatabaseSchema>;private readonly axios: AxiosInstance;@@ -34,37 +30,28 @@ export class MutationManager {>;constructor(- account: Account,+ axios: AxiosInstance,workspace: Workspace,database: Kysely<WorkspaceDatabaseSchema>,) {- this.account = account;this.workspace = workspace;+ this.axios = axios;this.database = database;- this.axios = Axios.create({- baseURL: SERVER_URL,- headers: {- Authorization: `Bearer ${account.token}`,- },- });this.mutationListeners = new Map();}- public onMutation(listener: (affectedTables: string[]) => void) {+ public onMutation(listener: (affectedTables: string[]) => void): void {const id = NeuronId.generate(NeuronId.Type.Subscriber);this.mutationListeners.set(id, listener);- return () => {- this.mutationListeners.delete(id);- };}- private notifyMutationListeners(affectedTables: string[]) {+ private notifyMutationListeners(affectedTables: string[]): void {for (const listener of this.mutationListeners.values()) {listener(affectedTables);}}- public async executeLocalMutation(mutation: LocalMutation) {+ public async executeLocalMutation(mutation: LocalMutation): Promise<void> {switch (mutation.type) {case 'create_node':return this.executeLocalCreateNodeMutation(mutation);@@ -81,7 +68,7 @@ export class MutationManager {private async executeLocalCreateNodeMutation(mutation: LocalCreateNodeMutation,- ) {+ ): Promise<void> {await this.database.transaction().execute(async (trx) => {const node = mutation.data.node;await trx@@ -115,7 +102,7 @@ export class MutationManager {private async executeLocalCreateNodesMutation(mutation: LocalCreateNodesMutation,- ) {+ ): Promise<void> {await this.database.transaction().execute(async (trx) => {const data = mutation.data;await trx@@ -151,7 +138,7 @@ export class MutationManager {private async executeLocalUpdateNodeMutation(mutation: LocalUpdateNodeMutation,- ) {+ ): Promise<void> {await this.database.transaction().execute(async (trx) => {const data = mutation.data;await trx@@ -184,7 +171,7 @@ export class MutationManager {private async executeLocalDeleteNodeMutation(mutation: LocalDeleteNodeMutation,- ) {+ ): Promise<void> {await this.database.transaction().execute(async (trx) => {await trx.deleteFrom('nodes')@@ -206,7 +193,7 @@ export class MutationManager {private async executeLocalDeleteNodesMutation(mutation: LocalDeleteNodesMutation,- ) {+ ): Promise<void> {await this.database.transaction().execute(async (trx) => {await trx.deleteFrom('nodes')@@ -226,7 +213,7 @@ export class MutationManager {this.notifyMutationListeners(['nodes']);}- public async executeServerMutation(mutation: ServerMutation) {+ public async executeServerMutation(mutation: ServerMutation): Promise<void> {switch (mutation.type) {case 'create_node':return this.executeServerCreateNodeMutation(mutation);@@ -243,14 +230,14 @@ export class MutationManager {public async executeServerCreateNodeMutation(mutation: ServerCreateNodeMutation,- ) {+ ): Promise<void> {const node = mutation.data.node;await this.syncNodeFromServer(node);}public async executeServerCreateNodesMutation(mutation: ServerCreateNodesMutation,- ) {+ ): Promise<void> {for (const node of mutation.data.nodes) {await this.syncNodeFromServer(node);}@@ -258,13 +245,13 @@ export class MutationManager {public async executeServerUpdateNodeMutation(mutation: ServerUpdateNodeMutation,- ) {+ ): Promise<void> {await this.syncNodeFromServer(mutation.data.node);}public async executeServerDeleteNodeMutation(mutation: LocalDeleteNodeMutation,- ) {+ ): Promise<void> {await this.database.deleteFrom('nodes').where('id', '=', mutation.data.id)@@ -273,7 +260,7 @@ export class MutationManager {public async executeServerDeleteNodesMutation(mutation: LocalDeleteNodesMutation,- ) {+ ): Promise<void> {await this.database.deleteFrom('nodes').where('id', 'in', mutation.data.ids)@@ -282,7 +269,7 @@ export class MutationManager {this.notifyMutationListeners(['nodes']);}- public async syncNodeFromServer(node: Node) {+ public async syncNodeFromServer(node: Node): Promise<void> {const existingNode = await this.database.selectFrom('nodes').selectAll()@@ -341,7 +328,7 @@ export class MutationManager {public shouldUpdateNodeFromServer(localNode: NodesTableSchema,serverNode: Node,- ) {+ ): boolean {if (localNode.server_version_id === serverNode.serverVersionId) {return false;}@@ -362,7 +349,7 @@ export class MutationManager {return true;}- public async sendMutations() {+ public async sendMutations(): Promise<void> {do {const nextMutation = await this.database.selectFrom('mutations')@@ -378,7 +365,6 @@ export class MutationManager {const { data, status } = await this.axios.post<ServerMutation>('v1/mutations',{- deviceId: this.account.deviceId,workspaceId: this.workspace.id,type: nextMutation.type,data: JSON.parse(nextMutation.data),diff --git a/desktop/src/data/schemas/app.ts b/desktop/src/data/schemas/app.tsindex 302c7fed..e5369d92 100644--- a/desktop/src/data/schemas/app.ts+++ b/desktop/src/data/schemas/app.ts@@ -16,7 +16,7 @@ export interface WorkspacesTableSchema {version_id: string;role: number;user_id: string;- synced_at: string | null;+ synced: number;}export interface AppDatabaseSchema {diff --git a/desktop/src/data/socket-manager.ts b/desktop/src/data/socket-manager.tsindex f3bf39bf..bd8a9049 100644--- a/desktop/src/data/socket-manager.ts+++ b/desktop/src/data/socket-manager.ts@@ -23,7 +23,7 @@ export class SocketManager {this.lastInitAt = 0;}- public init() {+ public init(): void {if (this.isConnected()) {return;}@@ -68,13 +68,13 @@ export class SocketManager {return this.socket !== null && this.socket.readyState === WebSocket.OPEN;}- public close() {+ public close(): void {if (this.socket) {this.socket.close();}}- public checkConnection() {+ public checkConnection(): void {if (this.isConnected()) {return;}@@ -94,13 +94,13 @@ export class SocketManager {}}- public send(message: SocketMessage) {+ public send(message: SocketMessage): void {if (this.isConnected()) {this.socket?.send(JSON.stringify(message));}}- public on(type: string, listener: (payload: any) => void) {+ public on(type: string, listener: (payload: any) => void): void {const listeners = this.listeners.get(type) || [];listeners.push(listener);this.listeners.set(type, listeners);diff --git a/desktop/src/data/utils.ts b/desktop/src/data/utils.tsindex 8da8457b..7d19f7bd 100644--- a/desktop/src/data/utils.ts+++ b/desktop/src/data/utils.ts@@ -5,7 +5,7 @@ import { QueryResult } from 'kysely';const parser = new Parser();const parseOptions = { database: 'Sqlite' };-export const buildSqlite = (filename: string) => {+export const buildSqlite = (filename: string): SQLite.Database => {const database = new SQLite(filename);database.pragma('journal_mode = WAL');return database;diff --git a/desktop/src/data/workspace-manager.ts b/desktop/src/data/workspace-manager.tsindex 126a174f..14012e6b 100644--- a/desktop/src/data/workspace-manager.ts+++ b/desktop/src/data/workspace-manager.ts@@ -7,10 +7,9 @@ import {QueryResult,SqliteDialect,} from 'kysely';-import { Workspace } from '@/types/workspaces';+import { Workspace, WorkspaceSyncData } from '@/types/workspaces';import { WorkspaceDatabaseSchema } from '@/data/schemas/workspace';import { workspaceDatabaseMigrations } from '@/data/migrations/workspace';-import { Account } from '@/types/accounts';import {buildSqlite,extractTablesFromSql,@@ -20,17 +19,18 @@ import { SubscribedQueryData } from '@/types/databases';import { LocalMutation, ServerMutation } from '@/types/mutations';import { eventBus } from '@/lib/event-bus';import { MutationManager } from '@/data/mutation-manager';+import { AxiosInstance } from 'axios';export class WorkspaceManager {- private readonly account: Account;private readonly workspace: Workspace;+ private readonly axios: AxiosInstance;private readonly database: Kysely<WorkspaceDatabaseSchema>;private readonly subscribers: Map<string, SubscribedQueryData<unknown>>;private readonly mutationManager: MutationManager;- constructor(account: Account, workspace: Workspace, accountPath: string) {- this.account = account;+ constructor(workspace: Workspace, axios: AxiosInstance, accountPath: string) {this.workspace = workspace;+ this.axios = axios;this.subscribers = new Map();const workspaceDir = `${accountPath}/${workspace.id}`;@@ -46,11 +46,7 @@ export class WorkspaceManager {dialect,});- this.mutationManager = new MutationManager(- account,- workspace,- this.database,- );+ this.mutationManager = new MutationManager(axios, workspace, this.database);this.mutationManager.onMutation(async (affectedTables) => {for (const [subscriberId, subscriberData] of this.subscribers) {@@ -79,7 +75,11 @@ export class WorkspaceManager {});}- public async init() {+ public getWorkspace(): Workspace {+ return this.workspace;+ }++ public async init(): Promise<void> {await this.migrate();}@@ -88,7 +88,7 @@ export class WorkspaceManager {): Promise<QueryResult<T>> {const result = await this.database.executeQuery(query);- //only transactions should have side effects+ //only mutations should have side effectsif (result.numAffectedRows > 0) {throw new Error('Query should not have any side effects');}@@ -102,7 +102,7 @@ export class WorkspaceManager {): Promise<QueryResult<T>> {const result = await this.database.executeQuery(query);- // only transactions should have side effects+ // only mutations should have side effectsif (result.numAffectedRows > 0) {throw new Error('Query should not have any side effects');}@@ -118,23 +118,48 @@ export class WorkspaceManager {return result;}- public unsubscribeQuery(queryId: string) {+ public unsubscribeQuery(queryId: string): void {this.subscribers.delete(queryId);}- public async executeLocalMutation(mutation: LocalMutation) {+ public async executeLocalMutation(mutation: LocalMutation): Promise<void> {await this.mutationManager.executeLocalMutation(mutation);}- public async executeServerMutation(mutation: ServerMutation) {+ public async executeServerMutation(mutation: ServerMutation): Promise<void> {await this.mutationManager.executeServerMutation(mutation);}- public async sendMutations() {+ public async sendMutations(): Promise<void> {await this.mutationManager.sendMutations();}- private async migrate() {+ public isSynced(): boolean {+ return this.workspace.synced;+ }++ public async sync(): Promise<boolean> {+ if (this.workspace.synced) {+ return true;+ }++ const { data, status } = await this.axios.get<WorkspaceSyncData>(+ `v1/${this.workspace.id}/sync`,+ );++ if (status !== 200) {+ return false;+ }++ for (const node of data.nodes) {+ await this.mutationManager.syncNodeFromServer(node);+ }++ this.workspace.synced = true;+ return true;+ }++ private async migrate(): Promise<void> {const migrator = new Migrator({db: this.database,provider: {diff --git a/desktop/src/main.ts b/desktop/src/main.tsindex d0fc73a0..ec0f9ad4 100644--- a/desktop/src/main.ts+++ b/desktop/src/main.ts@@ -2,7 +2,7 @@ import { app, ipcMain, BrowserWindow } from 'electron';import path from 'path';import { eventBus } from '@/lib/event-bus';import { appManager } from '@/data/app-manager';-import { CompiledQuery } from 'kysely';+import { CompiledQuery, QueryResult } from 'kysely';import { LocalMutation } from '@/types/mutations';let subscriptionId: string | null = null;@@ -70,8 +70,33 @@ app.on('before-quit', () => {ipcMain.handle('init', async () => appManager.init());ipcMain.handle('logout', async (_, accountId) => appManager.logout(accountId));-ipcMain.handle('execute-app-query', async (_, query: CompiledQuery) => {- return await appManager.execute(query);+ipcMain.handle(+ 'execute-app-mutation',+ async (_, mutation: CompiledQuery): Promise<void> => {+ await appManager.executeMutation(mutation);+ },+);++ipcMain.handle(+ 'execute-app-query',+ async (_, query: CompiledQuery<unknown>): Promise<QueryResult<unknown>> => {+ return await appManager.executeQuery(query);+ },+);++ipcMain.handle(+ 'execute-app-query-and-subscribe',+ async (+ _,+ queryId: string,+ query: CompiledQuery<unknown>,+ ): Promise<QueryResult<unknown>> => {+ return await appManager.executeQueryAndSubscribe(queryId, query);+ },+);++ipcMain.handle('unsubscribe-app-query', (_, queryId: string): void => {+ appManager.unsubscribeQuery(queryId);});ipcMain.handle(@@ -81,7 +106,7 @@ ipcMain.handle(accountId: string,workspaceId: string,mutation: LocalMutation,- ) => {+ ): Promise<void> => {const accountManager = await appManager.getAccount(accountId);if (!accountManager) {throw new Error(`Account not found: ${accountId}`);@@ -98,7 +123,12 @@ ipcMain.handle(ipcMain.handle('execute-workspace-query',- async (_, accountId: string, workspaceId: string, query: CompiledQuery) => {+ async (+ _,+ accountId: string,+ workspaceId: string,+ query: CompiledQuery<unknown>,+ ): Promise<QueryResult<unknown>> => {const accountManager = await appManager.getAccount(accountId);if (!accountManager) {throw new Error(`Account not found: ${accountId}`);@@ -120,8 +150,8 @@ ipcMain.handle(accountId: string,workspaceId: string,queryId: string,- query: CompiledQuery,- ) => {+ query: CompiledQuery<unknown>,+ ): Promise<QueryResult<unknown>> => {const accountManager = await appManager.getAccount(accountId);if (!accountManager) {throw new Error(`Account not found: ${accountId}`);@@ -138,7 +168,12 @@ ipcMain.handle(ipcMain.handle('unsubscribe-workspace-query',- async (_, accountId: string, workspaceId: string, queryId: string) => {+ async (+ _,+ accountId: string,+ workspaceId: string,+ queryId: string,+ ): Promise<void> => {const accountManager = await appManager.getAccount(accountId);if (!accountManager) {throw new Error(`Account not found: ${accountId}`);diff --git a/desktop/src/preload.ts b/desktop/src/preload.tsindex 3825b98b..2b9be6c2 100644--- a/desktop/src/preload.ts+++ b/desktop/src/preload.ts@@ -8,6 +8,17 @@ contextBridge.exposeInMainWorld('neuron', {logout: (accountId: string) => ipcRenderer.invoke('logout', accountId),executeAppQuery: <R>(query: CompiledQuery<R>): Promise<QueryResult<R>> =>ipcRenderer.invoke('execute-app-query', query),+ executeAppQueryAndSubscribe: <R>(+ queryId: string,+ query: CompiledQuery<R>,+ ): Promise<QueryResult<R>> =>+ ipcRenderer.invoke('execute-app-query-and-subscribe', queryId, query),++ executeAppMutation: (mutation: CompiledQuery): Promise<void> =>+ ipcRenderer.invoke('execute-app-mutation', mutation),++ unsubscribeAppQuery: (queryId: string): Promise<void> =>+ ipcRenderer.invoke('unsubscribe-app-query', queryId),executeWorkspaceMutation: (accountId: string,diff --git a/desktop/src/types/global.ts b/desktop/src/types/global.tsdeleted file mode 100644index 451b2368..00000000--- a/desktop/src/types/global.ts+++ /dev/null@@ -1,7 +0,0 @@-import { Account } from '@/types/accounts';-import { Workspace } from '@/types/workspaces';--export type GlobalDatabaseData = {- accounts: Account[];- workspaces: Workspace[];-};diff --git a/desktop/src/types/window.d.ts b/desktop/src/types/window.d.tsindex 2ceeb077..3535097c 100644--- a/desktop/src/types/window.d.ts+++ b/desktop/src/types/window.d.ts@@ -1,12 +1,17 @@-import { GlobalDatabaseData } from '@/types/global';import { EventBus } from '@/lib/event-bus';import { CompiledQuery, QueryResult } from 'kysely';import { LocalMutationInput } from '@/types/mutations';interface NeuronApi {- init: () => Promise<GlobalDatabaseData>;+ init: () => Promise<void>;logout: (accountId: string) => Promise<void>;+ executeAppMutation: (mutation: LocalMutationInput) => Promise<void>;executeAppQuery: (query: CompiledQuery<R>) => Promise<QueryResult<R>>;+ executeAppQueryAndSubscribe: (+ queryId: string,+ query: CompiledQuery<R>,+ ) => Promise<QueryResult<R>>;+ unsubscribeAppQuery: (queryId: string) => Promise<void>;executeWorkspaceMutation: (accountId: string,diff --git a/desktop/src/types/workspaces.ts b/desktop/src/types/workspaces.tsindex 2a11ab0f..8375c5f6 100644--- a/desktop/src/types/workspaces.ts+++ b/desktop/src/types/workspaces.ts@@ -1,3 +1,5 @@+import { Node } from '@/types/nodes';+export enum WorkspaceRole {Owner = 1,Admin = 2,@@ -14,5 +16,9 @@ export type Workspace = {accountId: string;role: WorkspaceRole;userId: string;- syncedAt?: Date | null;+ synced: boolean;+};++export type WorkspaceSyncData = {+ nodes: Node[];};diff --git a/server/src/api.ts b/server/src/api.tsindex 60e6ff69..c0a1be11 100644--- a/server/src/api.ts+++ b/server/src/api.ts@@ -10,6 +10,7 @@ import { authMiddleware } from '@/middlewares/auth';import { sockets } from '@/lib/sockets';import { SocketMessage } from '@/types/sockets';import { prisma } from '@/data/prisma';+import { syncRouter } from '@/routes/sync';export const initApi = () => {const app = express();@@ -25,6 +26,7 @@ export const initApi = () => {app.use('/v1/accounts', accountsRouter);app.use('/v1/workspaces', authMiddleware, workspacesRouter);app.use('/v1/mutations', authMiddleware, mutationsRouter);+ app.use('/v1/', authMiddleware, syncRouter);const server = http.createServer(app);diff --git a/server/src/routes/sync.ts b/server/src/routes/sync.tsnew file mode 100644index 00000000..d67b5a02--- /dev/null+++ b/server/src/routes/sync.ts@@ -0,0 +1,42 @@+import { NeuronRequest, NeuronResponse } from '@/types/api';+import { prisma } from '@/data/prisma';+import { Router } from 'express';+import { Node, NodeBlock } from '@/types/nodes';++export const syncRouter = Router();++syncRouter.get(+ '/:workspaceId/sync',+ async (req: NeuronRequest, res: NeuronResponse) => {+ const workspaceId = req.params.workspaceId as string;+ const nodes = await prisma.nodes.findMany({+ where: {+ workspaceId,+ },+ });++ const outputs: Node[] = nodes.map((node) => {+ return {+ id: node.id,+ parentId: node.parentId,+ workspaceId: node.workspaceId,+ type: node.type,+ index: node.index,+ attrs: node.attrs as Record<string, any>,+ createdAt: node.createdAt.toISOString(),+ createdBy: node.createdBy,+ versionId: node.versionId,+ content: node.content as NodeBlock[],+ updatedAt: node.updatedAt?.toISOString(),+ updatedBy: node.updatedBy,+ serverCreatedAt: node.serverCreatedAt.toISOString(),+ serverUpdatedAt: node.serverUpdatedAt?.toISOString(),+ serverVersionId: node.serverVersionId,+ };+ });++ res.status(200).json({+ nodes: outputs,+ });+ },+);diff --git a/server/src/routes/workspaces.ts b/server/src/routes/workspaces.tsindex 0c512850..399851d5 100644--- a/server/src/routes/workspaces.ts+++ b/server/src/routes/workspaces.ts@@ -11,7 +11,6 @@ import { ApiError, NeuronRequest, NeuronResponse } from '@/types/api';import { NeuronId } from '@/lib/id';import { prisma } from '@/data/prisma';import { Router } from 'express';-import { Node, NodeBlock } from '@/types/nodes';export const workspacesRouter = Router();@@ -334,67 +333,3 @@ workspacesRouter.get('/', async (req: NeuronRequest, res: NeuronResponse) => {return res.status(200).json(outputs);});--workspacesRouter.get(- '/:id/nodes',- async (req: NeuronRequest, res: NeuronResponse) => {- const workspaceId = req.params.id as string;- const from = req.query.from as string;- const nodes = await getNodesFromDatabase(workspaceId, from);- const outputs: Node[] = nodes.map((node) => {- return {- id: node.id,- parentId: node.parentId,- workspaceId: node.workspaceId,- type: node.type,- index: node.index,- attrs: node.attrs as Record<string, any>,- createdAt: node.createdAt.toISOString(),- createdBy: node.createdBy,- versionId: node.versionId,- content: node.content as NodeBlock[],- updatedAt: node.updatedAt?.toISOString(),- updatedBy: node.updatedBy,- serverCreatedAt: node.serverCreatedAt.toISOString(),- serverUpdatedAt: node.serverUpdatedAt?.toISOString(),- serverVersionId: node.serverVersionId,- };- });-- res.status(200).json({- nodes: outputs,- });- },-);--const getNodesFromDatabase = async (- workspaceId: string,- from?: string | null,-) => {- if (from) {- const date = new Date(from);- return prisma.nodes.findMany({- where: {- workspaceId,- OR: [- {- createdAt: {- gte: date,- },- },- {- updatedAt: {- gte: date,- },- },- ],- },- });- } else {- return prisma.nodes.findMany({- where: {- workspaceId,- },- });- }-};