A local-first personal finance app
:electron: Embed the sync-server (experimental) (#4526)
Michael Clarkbdf76f6c6319
diff --git a/bin/package-electron b/bin/package-electronindex dde72dc4d6..2ab7be3805 100755--- a/bin/package-electron+++ b/bin/package-electron@@ -34,21 +34,23 @@ if [ "$OSTYPE" == "msys" ]; thenfifi-yarn workspace loot-core build:node-# Get translationsecho "Updating translations..."if ! [ -d packages/desktop-client/locale ]; thengit clone https://github.com/actualbudget/translations packages/desktop-client/localefipushd packages/desktop-client/locale > /dev/null-git checkout .git pullpopd > /dev/nullpackages/desktop-client/bin/remove-untranslated-languages+yarn workspace loot-core build:nodeyarn workspace @actual-app/web build --mode=desktop # electron specific build+# required for running the sync-server server+yarn workspace loot-core build:browser+yarn workspace @actual-app/web build:browser+yarn workspace desktop-electron update-client(diff --git a/package.json b/package.jsonindex 54ee96a290..1e6fdf496c 100644--- a/package.json+++ b/package.json@@ -22,11 +22,12 @@"start:server": "yarn workspace @actual-app/sync-server start","start:server-monitor": "yarn workspace @actual-app/sync-server start-monitor","start:server-dev": "NODE_ENV=development BROWSER_OPEN=localhost:5006 yarn npm-run-all --parallel 'start:server-monitor' 'start'",- "start:desktop": "yarn rebuild-electron && npm-run-all --parallel 'start:desktop-*'",+ "start:desktop": "yarn desktop-dependencies && npm-run-all --parallel 'start:desktop-*'",+ "desktop-dependencies": "yarn rebuild-electron && yarn workspace loot-core build:browser","start:desktop-node": "yarn workspace loot-core watch:node","start:desktop-client": "yarn workspace @actual-app/web watch",+ "start:desktop-server-client": "yarn workspace @actual-app/web build:browser","start:desktop-electron": "yarn workspace desktop-electron watch",- "start:electron": "yarn start:desktop","start:browser": "npm-run-all --parallel 'start:browser-*'","start:browser-backend": "yarn workspace loot-core watch:browser","start:browser-frontend": "yarn workspace @actual-app/web start:browser",diff --git a/packages/desktop-electron/index.ts b/packages/desktop-electron/index.tsindex efd037f731..aedab61023 100644--- a/packages/desktop-electron/index.ts+++ b/packages/desktop-electron/index.ts@@ -2,6 +2,7 @@ import fs from 'fs';import { createServer, Server } from 'http';import path from 'path';+import ngrok from '@ngrok/ngrok';import {net,app,@@ -19,7 +20,7 @@ import {Env,ForkOptions,} from 'electron';-import { copy, exists, remove } from 'fs-extra';+import { copy, exists, mkdir, remove } from 'fs-extra';import promiseRetry from 'promise-retry';import type { GlobalPrefsJson } from '../loot-core/src/types/prefs';@@ -56,6 +57,7 @@ if (!isDev || !process.env.ACTUAL_DATA_DIR) {// be closed automatically when the JavaScript object is garbage collected.let clientWin: BrowserWindow | null;let serverProcess: UtilityProcess | null;+let actualServerProcess: UtilityProcess | null;let oAuthServer: ReturnType<typeof createServer> | null;@@ -63,25 +65,16 @@ let queuedClientWinLogs: string[] = []; // logs that are queued up until the cliconst logMessage = (loglevel: 'info' | 'error', message: string) => {// Electron main process logs- switch (loglevel) {- case 'info':- console.info(message);- break;- case 'error':- console.error(message);- break;- }+ const trimmedMessage = JSON.stringify(message.trim()); // ensure line endings are removed+ console[loglevel](trimmedMessage);if (!clientWin) {// queue up the logs until the client window is ready- queuedClientWinLogs.push(- // eslint-disable-next-line rulesdir/typography- `console.${loglevel}('Actual Sync Server Log:', ${JSON.stringify(message)})`,- );+ queuedClientWinLogs.push(`console.${loglevel}(${trimmedMessage})`);} else {// Send the queued up logs to the devtools consoleclientWin.webContents.executeJavaScript(- `console.${loglevel}('Actual Sync Server Log:', ${JSON.stringify(message)})`,+ `console.${loglevel}(${trimmedMessage})`,);}};@@ -130,7 +123,7 @@ if (isDev) {}async function loadGlobalPrefs() {- let state: GlobalPrefsJson | undefined = undefined;+ let state: GlobalPrefsJson = {};try {state = JSON.parse(fs.readFileSync(@@ -152,10 +145,10 @@ async function createBackgroundProcess() {...process.env, // required};- if (globalPrefs?.['server-self-signed-cert']) {+ if (globalPrefs['server-self-signed-cert']) {envVariables = {...envVariables,- NODE_EXTRA_CA_CERTS: globalPrefs?.['server-self-signed-cert'], // add self signed cert to env - fetch can pick it up+ NODE_EXTRA_CA_CERTS: globalPrefs['server-self-signed-cert'], // add self signed cert to env - fetch can pick it up};}@@ -176,14 +169,12 @@ async function createBackgroundProcess() {serverProcess.stdout?.on('data', (chunk: Buffer) => {// Send the Server log messages to the main browser window- clientWin?.webContents.executeJavaScript(`- console.info('Server Log:', ${JSON.stringify(chunk.toString('utf8'))})`);+ logMessage('info', `Server Log: ${chunk.toString('utf8')}`);});serverProcess.stderr?.on('data', (chunk: Buffer) => {// Send the Server log messages out to the main browser window- clientWin?.webContents.executeJavaScript(`- console.error('Server Log:', ${JSON.stringify(chunk.toString('utf8'))})`);+ logMessage('error', `Server Log: ${chunk.toString('utf8')}`);});serverProcess.on('message', msg => {@@ -204,6 +195,148 @@ async function createBackgroundProcess() {});}+async function startSyncServer() {+ try {+ const globalPrefs = await loadGlobalPrefs();++ const syncServerConfig = {+ port: globalPrefs.syncServerConfig?.port || 5007,+ ACTUAL_SERVER_DATA_DIR: path.resolve(+ process.env.ACTUAL_DATA_DIR!,+ 'actual-server',+ ),+ ACTUAL_SERVER_FILES: path.resolve(+ process.env.ACTUAL_DATA_DIR!,+ 'actual-server',+ 'server-files',+ ),+ ACTUAL_USER_FILES: path.resolve(+ process.env.ACTUAL_DATA_DIR!,+ 'actual-server',+ 'user-files',+ ),+ };++ const serverPath = path.join(+ // require.resolve will recursively search up the workspace for the module+ path.dirname(require.resolve('@actual-app/sync-server/package.json')),+ 'app.js',+ );++ const webRoot = path.join(+ // require.resolve will recursively search up the workspace for the module+ path.dirname(require.resolve('@actual-app/web/package.json')),+ 'build',+ );++ // Use env variables to configure the server+ const envVariables: Env = {+ ...process.env, // required+ ACTUAL_PORT: `${syncServerConfig.port}`,+ ACTUAL_SERVER_FILES: `${syncServerConfig.ACTUAL_SERVER_FILES}`,+ ACTUAL_USER_FILES: `${syncServerConfig.ACTUAL_USER_FILES}`,+ ACTUAL_DATA_DIR: `${syncServerConfig.ACTUAL_SERVER_DATA_DIR}`,+ ACTUAL_WEB_ROOT: webRoot,+ };++ // ACTUAL_SERVER_DATA_DIR is the root directory for the sync-server+ if (!fs.existsSync(syncServerConfig.ACTUAL_SERVER_DATA_DIR)) {+ mkdir(syncServerConfig.ACTUAL_SERVER_DATA_DIR, { recursive: true });+ }++ let forkOptions: ForkOptions = {+ stdio: 'pipe',+ env: envVariables,+ };++ if (isDev) {+ forkOptions = { ...forkOptions, execArgv: ['--inspect'] };+ }++ let syncServerStarted = false;++ const syncServerPromise = new Promise<void>(resolve => {+ actualServerProcess = utilityProcess.fork(serverPath, [], forkOptions);++ actualServerProcess.stdout?.on('data', (chunk: Buffer) => {+ // Send the Server console.log messages to the main browser window+ logMessage('info', `Sync-Server: ${chunk.toString('utf8')}`);+ });++ actualServerProcess.stderr?.on('data', (chunk: Buffer) => {+ // Send the Server console.error messages out to the main browser window+ logMessage('error', `Sync-Server: ${chunk.toString('utf8')}`);+ });++ actualServerProcess.on('message', msg => {+ switch (msg.type) {+ case 'server-started':+ logMessage('info', 'Sync-Server: Actual Sync Server has started!');+ syncServerStarted = true;+ resolve();+ break;+ default:+ logMessage(+ 'info',+ 'Sync-Server: Unknown server message: ' + msg.type,+ );+ }+ });+ });++ const SYNC_SERVER_WAIT_TIMEOUT = 20000; // wait 20 seconds for the server to start - if it doesn't, throw an error++ const syncServerTimeout = new Promise<void>((_, reject) => {+ setTimeout(() => {+ if (!syncServerStarted) {+ const errorMessage = `Sync-Server: Failed to start within ${SYNC_SERVER_WAIT_TIMEOUT / 1000} seconds. Something is wrong. Please raise a github issue.`;+ logMessage('error', errorMessage);+ reject(new Error(errorMessage));+ }+ }, SYNC_SERVER_WAIT_TIMEOUT);+ });++ return await Promise.race([syncServerPromise, syncServerTimeout]); // Either the server has started or the timeout is reached+ } catch (error) {+ logMessage('error', `Sync-Server: Error starting sync server: ${error}`);+ }+}++async function exposeSyncServer(+ syncServerConfig: GlobalPrefsJson['syncServerConfig'],+) {+ const hasRequiredConfig =+ syncServerConfig?.ngrokConfig?.authToken &&+ syncServerConfig?.ngrokConfig?.domain &&+ syncServerConfig?.port;++ if (!hasRequiredConfig) {+ logMessage(+ 'error',+ 'Sync-Server: Cannot expose sync server: missing ngrok settings',+ );+ return { error: 'Missing ngrok settings' };+ }++ try {+ const listener = await ngrok.forward({+ schemes: ['https'],+ addr: syncServerConfig.port,+ authtoken: syncServerConfig?.ngrokConfig?.authToken,+ domain: syncServerConfig?.ngrokConfig?.domain,+ });++ logMessage(+ 'info',+ `Sync-Server: Exposing actual server on url: ${listener.url()}`,+ );+ return { url: listener.url() };+ } catch (error) {+ logMessage('error', `Unable to run ngrok: ${error}`);+ return { error: `Unable to run ngrok. ${error}` };+ }+}+async function createWindow() {const windowState = await getWindowState();@@ -342,6 +475,17 @@ app.on('ready', async () => {// Install an `app://` protocol that always returns the base HTML// file no matter what URL it is. This allows us to use react-router// on the frontend++ const globalPrefs = await loadGlobalPrefs();++ if (globalPrefs.syncServerConfig?.autoStart) {+ // wait for both server and ngrok to start before starting the Actual client to ensure server is available+ await Promise.allSettled([+ startSyncServer(),+ exposeSyncServer(globalPrefs.syncServerConfig),+ ]);+ }+protocol.handle('app', request => {if (request.method !== 'GET') {return new Response(null, {diff --git a/packages/desktop-electron/package.json b/packages/desktop-electron/package.jsonindex fbefae9720..edebed32b4 100644--- a/packages/desktop-electron/package.json+++ b/packages/desktop-electron/package.json@@ -48,7 +48,7 @@"flatpak","AppImage"],- "artifactName": "${productName}-linux.${ext}"+ "artifactName": "${productName}-linux-${arch}.${ext}"},"flatpak": {"runtimeVersion": "23.08",@@ -85,6 +85,8 @@"npmRebuild": false},"dependencies": {+ "@actual-app/sync-server": "workspace:*",+ "@ngrok/ngrok": "^1.4.1","better-sqlite3": "^11.7.0","fs-extra": "^11.3.0","promise-retry": "^2.0.1"diff --git a/packages/loot-core/src/types/prefs.d.ts b/packages/loot-core/src/types/prefs.d.tsindex 8b6dacc0f1..68f848eb53 100644--- a/packages/loot-core/src/types/prefs.d.ts+++ b/packages/loot-core/src/types/prefs.d.ts@@ -89,6 +89,15 @@ export type GlobalPrefs = Partial<{preferredDarkTheme: DarkTheme;documentDir: string; // Electron onlyserverSelfSignedCert: string; // Electron only+ syncServerConfig?: {+ // Electron only+ autoStart?: boolean;+ port?: number;+ ngrokConfig?: {+ domain?: string;+ authToken?: string;+ };+ };}>;// GlobalPrefsJson represents what's saved in the global-store.json file@@ -109,6 +118,7 @@ export type GlobalPrefsJson = Partial<{theme?: GlobalPrefs['theme'];'preferred-dark-theme'?: GlobalPrefs['preferredDarkTheme'];'server-self-signed-cert'?: GlobalPrefs['serverSelfSignedCert'];+ syncServerConfig?: GlobalPrefs['syncServerConfig'];}>;export type AuthMethods = 'password' | 'openid';diff --git a/packages/sync-server/src/app.js b/packages/sync-server/src/app.jsindex 0dda48dd90..bb91e6ab16 100644--- a/packages/sync-server/src/app.js+++ b/packages/sync-server/src/app.js@@ -136,6 +136,9 @@ export async function run() {app.listen(config.get('port'), config.get('hostname'));}+ // Signify to any parent process that the server has started. Used in electron desktop app+ process.parentPort?.postMessage({ type: 'server-started' });+console.log('Listening on ' + config.get('hostname') + ':' + config.get('port') + '...',);diff --git a/upcoming-release-notes/4526.md b/upcoming-release-notes/4526.mdnew file mode 100644index 0000000000..32e06cf0bc--- /dev/null+++ b/upcoming-release-notes/4526.md@@ -0,0 +1,6 @@+---+category: Features+authors: [MikesGlitch]+---++Experimental: Embedding the sync server into the desktop appdiff --git a/yarn.lock b/yarn.lockindex eabc4e5aca..9543e00f18 100644--- a/yarn.lock+++ b/yarn.lock@@ -83,7 +83,7 @@ __metadata:languageName: unknownlinkType: soft-"@actual-app/sync-server@workspace:packages/sync-server":+"@actual-app/sync-server@workspace:*, @actual-app/sync-server@workspace:packages/sync-server":version: 0.0.0-use.localresolution: "@actual-app/sync-server@workspace:packages/sync-server"dependencies:@@ -2878,6 +2878,145 @@ __metadata:languageName: nodelinkType: hard+"@ngrok/ngrok-android-arm-eabi@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-android-arm-eabi@npm:1.4.1"+ conditions: os=android & cpu=arm+ languageName: node+ linkType: hard++"@ngrok/ngrok-android-arm64@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-android-arm64@npm:1.4.1"+ conditions: os=android & cpu=arm64+ languageName: node+ linkType: hard++"@ngrok/ngrok-darwin-arm64@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-darwin-arm64@npm:1.4.1"+ conditions: os=darwin & cpu=arm64+ languageName: node+ linkType: hard++"@ngrok/ngrok-darwin-universal@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-darwin-universal@npm:1.4.1"+ conditions: os=darwin+ languageName: node+ linkType: hard++"@ngrok/ngrok-darwin-x64@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-darwin-x64@npm:1.4.1"+ conditions: os=darwin & cpu=x64+ languageName: node+ linkType: hard++"@ngrok/ngrok-freebsd-x64@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-freebsd-x64@npm:1.4.1"+ conditions: os=freebsd & cpu=x64+ languageName: node+ linkType: hard++"@ngrok/ngrok-linux-arm-gnueabihf@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-linux-arm-gnueabihf@npm:1.4.1"+ conditions: os=linux & cpu=arm+ languageName: node+ linkType: hard++"@ngrok/ngrok-linux-arm64-gnu@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-linux-arm64-gnu@npm:1.4.1"+ conditions: os=linux & cpu=arm64 & libc=glibc+ languageName: node+ linkType: hard++"@ngrok/ngrok-linux-arm64-musl@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-linux-arm64-musl@npm:1.4.1"+ conditions: os=linux & cpu=arm64 & libc=musl+ languageName: node+ linkType: hard++"@ngrok/ngrok-linux-x64-gnu@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-linux-x64-gnu@npm:1.4.1"+ conditions: os=linux & cpu=x64 & libc=glibc+ languageName: node+ linkType: hard++"@ngrok/ngrok-linux-x64-musl@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-linux-x64-musl@npm:1.4.1"+ conditions: os=linux & cpu=x64 & libc=musl+ languageName: node+ linkType: hard++"@ngrok/ngrok-win32-ia32-msvc@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-win32-ia32-msvc@npm:1.4.1"+ conditions: os=win32 & cpu=ia32+ languageName: node+ linkType: hard++"@ngrok/ngrok-win32-x64-msvc@npm:1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok-win32-x64-msvc@npm:1.4.1"+ conditions: os=win32 & cpu=x64+ languageName: node+ linkType: hard++"@ngrok/ngrok@npm:^1.4.1":+ version: 1.4.1+ resolution: "@ngrok/ngrok@npm:1.4.1"+ dependencies:+ "@ngrok/ngrok-android-arm-eabi": "npm:1.4.1"+ "@ngrok/ngrok-android-arm64": "npm:1.4.1"+ "@ngrok/ngrok-darwin-arm64": "npm:1.4.1"+ "@ngrok/ngrok-darwin-universal": "npm:1.4.1"+ "@ngrok/ngrok-darwin-x64": "npm:1.4.1"+ "@ngrok/ngrok-freebsd-x64": "npm:1.4.1"+ "@ngrok/ngrok-linux-arm-gnueabihf": "npm:1.4.1"+ "@ngrok/ngrok-linux-arm64-gnu": "npm:1.4.1"+ "@ngrok/ngrok-linux-arm64-musl": "npm:1.4.1"+ "@ngrok/ngrok-linux-x64-gnu": "npm:1.4.1"+ "@ngrok/ngrok-linux-x64-musl": "npm:1.4.1"+ "@ngrok/ngrok-win32-ia32-msvc": "npm:1.4.1"+ "@ngrok/ngrok-win32-x64-msvc": "npm:1.4.1"+ dependenciesMeta:+ "@ngrok/ngrok-android-arm-eabi":+ optional: true+ "@ngrok/ngrok-android-arm64":+ optional: true+ "@ngrok/ngrok-darwin-arm64":+ optional: true+ "@ngrok/ngrok-darwin-universal":+ optional: true+ "@ngrok/ngrok-darwin-x64":+ optional: true+ "@ngrok/ngrok-freebsd-x64":+ optional: true+ "@ngrok/ngrok-linux-arm-gnueabihf":+ optional: true+ "@ngrok/ngrok-linux-arm64-gnu":+ optional: true+ "@ngrok/ngrok-linux-arm64-musl":+ optional: true+ "@ngrok/ngrok-linux-x64-gnu":+ optional: true+ "@ngrok/ngrok-linux-x64-musl":+ optional: true+ "@ngrok/ngrok-win32-ia32-msvc":+ optional: true+ "@ngrok/ngrok-win32-x64-msvc":+ optional: true+ checksum: 10/c86956756af6eb9f2cc47aba19ac14f59a0d031defc52ee6845b17363b051213abd29c984f11501d88a482f343fbadb0e7bf855803068bceb96bf5fb16dfb2cb+ languageName: node+ linkType: hard+"@nodelib/fs.scandir@npm:2.1.5":version: 2.1.5resolution: "@nodelib/fs.scandir@npm:2.1.5"@@ -9403,8 +9542,10 @@ __metadata:version: 0.0.0-use.localresolution: "desktop-electron@workspace:packages/desktop-electron"dependencies:+ "@actual-app/sync-server": "workspace:*""@electron/notarize": "npm:2.4.0""@electron/rebuild": "npm:3.6.0"+ "@ngrok/ngrok": "npm:^1.4.1""@types/copyfiles": "npm:^2""@types/fs-extra": "npm:^11"better-sqlite3: "npm:^11.7.0"