Skip to content

A local-first personal finance app

:electron: Embed the sync-server (experimental) (#4526)

Michael Clarkbdf76f6c6319
diff --git a/bin/package-electron b/bin/package-electron
index dde72dc4d6..2ab7be3805 100755
--- a/bin/package-electron
+++ b/bin/package-electron
@@ -34,21 +34,23 @@ if [ "$OSTYPE" == "msys" ]; then
fi
fi
-yarn workspace loot-core build:node
-
# Get translations
echo "Updating translations..."
if ! [ -d packages/desktop-client/locale ]; then
git clone https://github.com/actualbudget/translations packages/desktop-client/locale
fi
pushd packages/desktop-client/locale > /dev/null
-git checkout .
git pull
popd > /dev/null
packages/desktop-client/bin/remove-untranslated-languages
+yarn workspace loot-core build:node
yarn workspace @actual-app/web build --mode=desktop # electron specific build
+# required for running the sync-server server
+yarn workspace loot-core build:browser
+yarn workspace @actual-app/web build:browser
+
yarn workspace desktop-electron update-client
(
diff --git a/package.json b/package.json
index 54ee96a290..1e6fdf496c 100644
--- a/package.json
+++ b/package.json
@@ -22,11 +22,12 @@
"start:server": "yarn workspace @actual-app/sync-server start",
"start:server-monitor": "yarn workspace @actual-app/sync-server start-monitor",
"start:server-dev": "NODE_ENV=development BROWSER_OPEN=localhost:5006 yarn npm-run-all --parallel 'start:server-monitor' 'start'",
- "start:desktop": "yarn rebuild-electron && npm-run-all --parallel 'start:desktop-*'",
+ "start:desktop": "yarn desktop-dependencies && npm-run-all --parallel 'start:desktop-*'",
+ "desktop-dependencies": "yarn rebuild-electron && yarn workspace loot-core build:browser",
"start:desktop-node": "yarn workspace loot-core watch:node",
"start:desktop-client": "yarn workspace @actual-app/web watch",
+ "start:desktop-server-client": "yarn workspace @actual-app/web build:browser",
"start:desktop-electron": "yarn workspace desktop-electron watch",
- "start:electron": "yarn start:desktop",
"start:browser": "npm-run-all --parallel 'start:browser-*'",
"start:browser-backend": "yarn workspace loot-core watch:browser",
"start:browser-frontend": "yarn workspace @actual-app/web start:browser",
diff --git a/packages/desktop-electron/index.ts b/packages/desktop-electron/index.ts
index efd037f731..aedab61023 100644
--- a/packages/desktop-electron/index.ts
+++ b/packages/desktop-electron/index.ts
@@ -2,6 +2,7 @@ import fs from 'fs';
import { createServer, Server } from 'http';
import path from 'path';
+import ngrok from '@ngrok/ngrok';
import {
net,
app,
@@ -19,7 +20,7 @@ import {
Env,
ForkOptions,
} from 'electron';
-import { copy, exists, remove } from 'fs-extra';
+import { copy, exists, mkdir, remove } from 'fs-extra';
import promiseRetry from 'promise-retry';
import type { GlobalPrefsJson } from '../loot-core/src/types/prefs';
@@ -56,6 +57,7 @@ if (!isDev || !process.env.ACTUAL_DATA_DIR) {
// be closed automatically when the JavaScript object is garbage collected.
let clientWin: BrowserWindow | null;
let serverProcess: UtilityProcess | null;
+let actualServerProcess: UtilityProcess | null;
let oAuthServer: ReturnType<typeof createServer> | null;
@@ -63,25 +65,16 @@ let queuedClientWinLogs: string[] = []; // logs that are queued up until the cli
const logMessage = (loglevel: 'info' | 'error', message: string) => {
// Electron main process logs
- switch (loglevel) {
- case 'info':
- console.info(message);
- break;
- case 'error':
- console.error(message);
- break;
- }
+ const trimmedMessage = JSON.stringify(message.trim()); // ensure line endings are removed
+ console[loglevel](trimmedMessage);
if (!clientWin) {
// queue up the logs until the client window is ready
- queuedClientWinLogs.push(
- // eslint-disable-next-line rulesdir/typography
- `console.${loglevel}('Actual Sync Server Log:', ${JSON.stringify(message)})`,
- );
+ queuedClientWinLogs.push(`console.${loglevel}(${trimmedMessage})`);
} else {
// Send the queued up logs to the devtools console
clientWin.webContents.executeJavaScript(
- `console.${loglevel}('Actual Sync Server Log:', ${JSON.stringify(message)})`,
+ `console.${loglevel}(${trimmedMessage})`,
);
}
};
@@ -130,7 +123,7 @@ if (isDev) {
}
async function loadGlobalPrefs() {
- let state: GlobalPrefsJson | undefined = undefined;
+ let state: GlobalPrefsJson = {};
try {
state = JSON.parse(
fs.readFileSync(
@@ -152,10 +145,10 @@ async function createBackgroundProcess() {
...process.env, // required
};
- if (globalPrefs?.['server-self-signed-cert']) {
+ if (globalPrefs['server-self-signed-cert']) {
envVariables = {
...envVariables,
- NODE_EXTRA_CA_CERTS: globalPrefs?.['server-self-signed-cert'], // add self signed cert to env - fetch can pick it up
+ NODE_EXTRA_CA_CERTS: globalPrefs['server-self-signed-cert'], // add self signed cert to env - fetch can pick it up
};
}
@@ -176,14 +169,12 @@ async function createBackgroundProcess() {
serverProcess.stdout?.on('data', (chunk: Buffer) => {
// Send the Server log messages to the main browser window
- clientWin?.webContents.executeJavaScript(`
- console.info('Server Log:', ${JSON.stringify(chunk.toString('utf8'))})`);
+ logMessage('info', `Server Log: ${chunk.toString('utf8')}`);
});
serverProcess.stderr?.on('data', (chunk: Buffer) => {
// Send the Server log messages out to the main browser window
- clientWin?.webContents.executeJavaScript(`
- console.error('Server Log:', ${JSON.stringify(chunk.toString('utf8'))})`);
+ logMessage('error', `Server Log: ${chunk.toString('utf8')}`);
});
serverProcess.on('message', msg => {
@@ -204,6 +195,148 @@ async function createBackgroundProcess() {
});
}
+async function startSyncServer() {
+ try {
+ const globalPrefs = await loadGlobalPrefs();
+
+ const syncServerConfig = {
+ port: globalPrefs.syncServerConfig?.port || 5007,
+ ACTUAL_SERVER_DATA_DIR: path.resolve(
+ process.env.ACTUAL_DATA_DIR!,
+ 'actual-server',
+ ),
+ ACTUAL_SERVER_FILES: path.resolve(
+ process.env.ACTUAL_DATA_DIR!,
+ 'actual-server',
+ 'server-files',
+ ),
+ ACTUAL_USER_FILES: path.resolve(
+ process.env.ACTUAL_DATA_DIR!,
+ 'actual-server',
+ 'user-files',
+ ),
+ };
+
+ const serverPath = path.join(
+ // require.resolve will recursively search up the workspace for the module
+ path.dirname(require.resolve('@actual-app/sync-server/package.json')),
+ 'app.js',
+ );
+
+ const webRoot = path.join(
+ // require.resolve will recursively search up the workspace for the module
+ path.dirname(require.resolve('@actual-app/web/package.json')),
+ 'build',
+ );
+
+ // Use env variables to configure the server
+ const envVariables: Env = {
+ ...process.env, // required
+ ACTUAL_PORT: `${syncServerConfig.port}`,
+ ACTUAL_SERVER_FILES: `${syncServerConfig.ACTUAL_SERVER_FILES}`,
+ ACTUAL_USER_FILES: `${syncServerConfig.ACTUAL_USER_FILES}`,
+ ACTUAL_DATA_DIR: `${syncServerConfig.ACTUAL_SERVER_DATA_DIR}`,
+ ACTUAL_WEB_ROOT: webRoot,
+ };
+
+ // ACTUAL_SERVER_DATA_DIR is the root directory for the sync-server
+ if (!fs.existsSync(syncServerConfig.ACTUAL_SERVER_DATA_DIR)) {
+ mkdir(syncServerConfig.ACTUAL_SERVER_DATA_DIR, { recursive: true });
+ }
+
+ let forkOptions: ForkOptions = {
+ stdio: 'pipe',
+ env: envVariables,
+ };
+
+ if (isDev) {
+ forkOptions = { ...forkOptions, execArgv: ['--inspect'] };
+ }
+
+ let syncServerStarted = false;
+
+ const syncServerPromise = new Promise<void>(resolve => {
+ actualServerProcess = utilityProcess.fork(serverPath, [], forkOptions);
+
+ actualServerProcess.stdout?.on('data', (chunk: Buffer) => {
+ // Send the Server console.log messages to the main browser window
+ logMessage('info', `Sync-Server: ${chunk.toString('utf8')}`);
+ });
+
+ actualServerProcess.stderr?.on('data', (chunk: Buffer) => {
+ // Send the Server console.error messages out to the main browser window
+ logMessage('error', `Sync-Server: ${chunk.toString('utf8')}`);
+ });
+
+ actualServerProcess.on('message', msg => {
+ switch (msg.type) {
+ case 'server-started':
+ logMessage('info', 'Sync-Server: Actual Sync Server has started!');
+ syncServerStarted = true;
+ resolve();
+ break;
+ default:
+ logMessage(
+ 'info',
+ 'Sync-Server: Unknown server message: ' + msg.type,
+ );
+ }
+ });
+ });
+
+ const SYNC_SERVER_WAIT_TIMEOUT = 20000; // wait 20 seconds for the server to start - if it doesn't, throw an error
+
+ const syncServerTimeout = new Promise<void>((_, reject) => {
+ setTimeout(() => {
+ if (!syncServerStarted) {
+ const errorMessage = `Sync-Server: Failed to start within ${SYNC_SERVER_WAIT_TIMEOUT / 1000} seconds. Something is wrong. Please raise a github issue.`;
+ logMessage('error', errorMessage);
+ reject(new Error(errorMessage));
+ }
+ }, SYNC_SERVER_WAIT_TIMEOUT);
+ });
+
+ return await Promise.race([syncServerPromise, syncServerTimeout]); // Either the server has started or the timeout is reached
+ } catch (error) {
+ logMessage('error', `Sync-Server: Error starting sync server: ${error}`);
+ }
+}
+
+async function exposeSyncServer(
+ syncServerConfig: GlobalPrefsJson['syncServerConfig'],
+) {
+ const hasRequiredConfig =
+ syncServerConfig?.ngrokConfig?.authToken &&
+ syncServerConfig?.ngrokConfig?.domain &&
+ syncServerConfig?.port;
+
+ if (!hasRequiredConfig) {
+ logMessage(
+ 'error',
+ 'Sync-Server: Cannot expose sync server: missing ngrok settings',
+ );
+ return { error: 'Missing ngrok settings' };
+ }
+
+ try {
+ const listener = await ngrok.forward({
+ schemes: ['https'],
+ addr: syncServerConfig.port,
+ authtoken: syncServerConfig?.ngrokConfig?.authToken,
+ domain: syncServerConfig?.ngrokConfig?.domain,
+ });
+
+ logMessage(
+ 'info',
+ `Sync-Server: Exposing actual server on url: ${listener.url()}`,
+ );
+ return { url: listener.url() };
+ } catch (error) {
+ logMessage('error', `Unable to run ngrok: ${error}`);
+ return { error: `Unable to run ngrok. ${error}` };
+ }
+}
+
async function createWindow() {
const windowState = await getWindowState();
@@ -342,6 +475,17 @@ app.on('ready', async () => {
// Install an `app://` protocol that always returns the base HTML
// file no matter what URL it is. This allows us to use react-router
// on the frontend
+
+ const globalPrefs = await loadGlobalPrefs();
+
+ if (globalPrefs.syncServerConfig?.autoStart) {
+ // wait for both server and ngrok to start before starting the Actual client to ensure server is available
+ await Promise.allSettled([
+ startSyncServer(),
+ exposeSyncServer(globalPrefs.syncServerConfig),
+ ]);
+ }
+
protocol.handle('app', request => {
if (request.method !== 'GET') {
return new Response(null, {
diff --git a/packages/desktop-electron/package.json b/packages/desktop-electron/package.json
index fbefae9720..edebed32b4 100644
--- a/packages/desktop-electron/package.json
+++ b/packages/desktop-electron/package.json
@@ -48,7 +48,7 @@
"flatpak",
"AppImage"
],
- "artifactName": "${productName}-linux.${ext}"
+ "artifactName": "${productName}-linux-${arch}.${ext}"
},
"flatpak": {
"runtimeVersion": "23.08",
@@ -85,6 +85,8 @@
"npmRebuild": false
},
"dependencies": {
+ "@actual-app/sync-server": "workspace:*",
+ "@ngrok/ngrok": "^1.4.1",
"better-sqlite3": "^11.7.0",
"fs-extra": "^11.3.0",
"promise-retry": "^2.0.1"
diff --git a/packages/loot-core/src/types/prefs.d.ts b/packages/loot-core/src/types/prefs.d.ts
index 8b6dacc0f1..68f848eb53 100644
--- a/packages/loot-core/src/types/prefs.d.ts
+++ b/packages/loot-core/src/types/prefs.d.ts
@@ -89,6 +89,15 @@ export type GlobalPrefs = Partial<{
preferredDarkTheme: DarkTheme;
documentDir: string; // Electron only
serverSelfSignedCert: string; // Electron only
+ syncServerConfig?: {
+ // Electron only
+ autoStart?: boolean;
+ port?: number;
+ ngrokConfig?: {
+ domain?: string;
+ authToken?: string;
+ };
+ };
}>;
// GlobalPrefsJson represents what's saved in the global-store.json file
@@ -109,6 +118,7 @@ export type GlobalPrefsJson = Partial<{
theme?: GlobalPrefs['theme'];
'preferred-dark-theme'?: GlobalPrefs['preferredDarkTheme'];
'server-self-signed-cert'?: GlobalPrefs['serverSelfSignedCert'];
+ syncServerConfig?: GlobalPrefs['syncServerConfig'];
}>;
export type AuthMethods = 'password' | 'openid';
diff --git a/packages/sync-server/src/app.js b/packages/sync-server/src/app.js
index 0dda48dd90..bb91e6ab16 100644
--- a/packages/sync-server/src/app.js
+++ b/packages/sync-server/src/app.js
@@ -136,6 +136,9 @@ export async function run() {
app.listen(config.get('port'), config.get('hostname'));
}
+ // Signify to any parent process that the server has started. Used in electron desktop app
+ process.parentPort?.postMessage({ type: 'server-started' });
+
console.log(
'Listening on ' + config.get('hostname') + ':' + config.get('port') + '...',
);
diff --git a/upcoming-release-notes/4526.md b/upcoming-release-notes/4526.md
new file mode 100644
index 0000000000..32e06cf0bc
--- /dev/null
+++ b/upcoming-release-notes/4526.md
@@ -0,0 +1,6 @@
+---
+category: Features
+authors: [MikesGlitch]
+---
+
+Experimental: Embedding the sync server into the desktop app
diff --git a/yarn.lock b/yarn.lock
index eabc4e5aca..9543e00f18 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -83,7 +83,7 @@ __metadata:
languageName: unknown
linkType: soft
-"@actual-app/sync-server@workspace:packages/sync-server":
+"@actual-app/sync-server@workspace:*, @actual-app/sync-server@workspace:packages/sync-server":
version: 0.0.0-use.local
resolution: "@actual-app/sync-server@workspace:packages/sync-server"
dependencies:
@@ -2878,6 +2878,145 @@ __metadata:
languageName: node
linkType: hard
+"@ngrok/ngrok-android-arm-eabi@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-android-arm-eabi@npm:1.4.1"
+ conditions: os=android & cpu=arm
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-android-arm64@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-android-arm64@npm:1.4.1"
+ conditions: os=android & cpu=arm64
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-darwin-arm64@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-darwin-arm64@npm:1.4.1"
+ conditions: os=darwin & cpu=arm64
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-darwin-universal@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-darwin-universal@npm:1.4.1"
+ conditions: os=darwin
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-darwin-x64@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-darwin-x64@npm:1.4.1"
+ conditions: os=darwin & cpu=x64
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-freebsd-x64@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-freebsd-x64@npm:1.4.1"
+ conditions: os=freebsd & cpu=x64
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-linux-arm-gnueabihf@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-linux-arm-gnueabihf@npm:1.4.1"
+ conditions: os=linux & cpu=arm
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-linux-arm64-gnu@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-linux-arm64-gnu@npm:1.4.1"
+ conditions: os=linux & cpu=arm64 & libc=glibc
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-linux-arm64-musl@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-linux-arm64-musl@npm:1.4.1"
+ conditions: os=linux & cpu=arm64 & libc=musl
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-linux-x64-gnu@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-linux-x64-gnu@npm:1.4.1"
+ conditions: os=linux & cpu=x64 & libc=glibc
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-linux-x64-musl@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-linux-x64-musl@npm:1.4.1"
+ conditions: os=linux & cpu=x64 & libc=musl
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-win32-ia32-msvc@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-win32-ia32-msvc@npm:1.4.1"
+ conditions: os=win32 & cpu=ia32
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok-win32-x64-msvc@npm:1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok-win32-x64-msvc@npm:1.4.1"
+ conditions: os=win32 & cpu=x64
+ languageName: node
+ linkType: hard
+
+"@ngrok/ngrok@npm:^1.4.1":
+ version: 1.4.1
+ resolution: "@ngrok/ngrok@npm:1.4.1"
+ dependencies:
+ "@ngrok/ngrok-android-arm-eabi": "npm:1.4.1"
+ "@ngrok/ngrok-android-arm64": "npm:1.4.1"
+ "@ngrok/ngrok-darwin-arm64": "npm:1.4.1"
+ "@ngrok/ngrok-darwin-universal": "npm:1.4.1"
+ "@ngrok/ngrok-darwin-x64": "npm:1.4.1"
+ "@ngrok/ngrok-freebsd-x64": "npm:1.4.1"
+ "@ngrok/ngrok-linux-arm-gnueabihf": "npm:1.4.1"
+ "@ngrok/ngrok-linux-arm64-gnu": "npm:1.4.1"
+ "@ngrok/ngrok-linux-arm64-musl": "npm:1.4.1"
+ "@ngrok/ngrok-linux-x64-gnu": "npm:1.4.1"
+ "@ngrok/ngrok-linux-x64-musl": "npm:1.4.1"
+ "@ngrok/ngrok-win32-ia32-msvc": "npm:1.4.1"
+ "@ngrok/ngrok-win32-x64-msvc": "npm:1.4.1"
+ dependenciesMeta:
+ "@ngrok/ngrok-android-arm-eabi":
+ optional: true
+ "@ngrok/ngrok-android-arm64":
+ optional: true
+ "@ngrok/ngrok-darwin-arm64":
+ optional: true
+ "@ngrok/ngrok-darwin-universal":
+ optional: true
+ "@ngrok/ngrok-darwin-x64":
+ optional: true
+ "@ngrok/ngrok-freebsd-x64":
+ optional: true
+ "@ngrok/ngrok-linux-arm-gnueabihf":
+ optional: true
+ "@ngrok/ngrok-linux-arm64-gnu":
+ optional: true
+ "@ngrok/ngrok-linux-arm64-musl":
+ optional: true
+ "@ngrok/ngrok-linux-x64-gnu":
+ optional: true
+ "@ngrok/ngrok-linux-x64-musl":
+ optional: true
+ "@ngrok/ngrok-win32-ia32-msvc":
+ optional: true
+ "@ngrok/ngrok-win32-x64-msvc":
+ optional: true
+ checksum: 10/c86956756af6eb9f2cc47aba19ac14f59a0d031defc52ee6845b17363b051213abd29c984f11501d88a482f343fbadb0e7bf855803068bceb96bf5fb16dfb2cb
+ languageName: node
+ linkType: hard
+
"@nodelib/fs.scandir@npm:2.1.5":
version: 2.1.5
resolution: "@nodelib/fs.scandir@npm:2.1.5"
@@ -9403,8 +9542,10 @@ __metadata:
version: 0.0.0-use.local
resolution: "desktop-electron@workspace:packages/desktop-electron"
dependencies:
+ "@actual-app/sync-server": "workspace:*"
"@electron/notarize": "npm:2.4.0"
"@electron/rebuild": "npm:3.6.0"
+ "@ngrok/ngrok": "npm:^1.4.1"
"@types/copyfiles": "npm:^2"
"@types/fs-extra": "npm:^11"
better-sqlite3: "npm:^11.7.0"