Skip to content

A local-first personal finance app

:sparkles: ability to add migrations (#267)

Matiss Janis Aboltins1f79708ea3d2
diff --git a/.gitignore b/.gitignore
index e114b191b7..ad1e69fb0b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,8 @@ build/
*.pem
*.key
artifacts.json
+.migrate
+.migrate-test
# Yarn
.pnp.*
diff --git a/app.js b/app.js
index b76fc06eb0..4e254b5dab 100644
--- a/app.js
+++ b/app.js
@@ -1,6 +1,9 @@
import run from './src/app.js';
+import runMigrations from './src/migrations.js';
-run().catch((err) => {
- console.log('Error starting app:', err);
- process.exit(1);
-});
+runMigrations()
+ .then(run)
+ .catch((err) => {
+ console.log('Error starting app:', err);
+ process.exit(1);
+ });
diff --git a/jest.config.json b/jest.config.json
index 8785cee1ac..95fb27235b 100644
--- a/jest.config.json
+++ b/jest.config.json
@@ -1,5 +1,6 @@
{
- "setupFiles": ["./jest.setup.js"],
+ "globalSetup": "./jest.global-setup.js",
+ "globalTeardown": "./jest.global-teardown.js",
"testPathIgnorePatterns": ["dist", "/node_modules/", "/build/"],
"roots": ["<rootDir>"],
"moduleFileExtensions": ["ts", "js", "json"],
diff --git a/jest.global-setup.js b/jest.global-setup.js
new file mode 100644
index 0000000000..36f53cf1cc
--- /dev/null
+++ b/jest.global-setup.js
@@ -0,0 +1,10 @@
+import getAccountDb from './src/account-db.js';
+import runMigrations from './src/migrations.js';
+
+export default async function setup() {
+ await runMigrations();
+
+ // Insert a fake "valid-token" fixture that can be reused
+ const db = getAccountDb();
+ await db.mutate('INSERT INTO sessions (token) VALUES (?)', ['valid-token']);
+}
diff --git a/jest.global-teardown.js b/jest.global-teardown.js
new file mode 100644
index 0000000000..4e19fc3854
--- /dev/null
+++ b/jest.global-teardown.js
@@ -0,0 +1,5 @@
+import runMigrations from './src/migrations.js';
+
+export default async function teardown() {
+ await runMigrations('down');
+}
diff --git a/jest.setup.js b/jest.setup.js
deleted file mode 100644
index 18208ceb89..0000000000
--- a/jest.setup.js
+++ /dev/null
@@ -1,17 +0,0 @@
-import fs from 'node:fs';
-import path from 'node:path';
-import getAccountDb from './src/account-db.js';
-import config from './src/load-config.js';
-
-// Delete previous test database (force creation of a new one)
-const dbPath = path.join(config.serverFiles, 'account.sqlite');
-if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);
-
-// Create path for test user files and delete previous files there
-if (fs.existsSync(config.userFiles))
- fs.rmSync(config.userFiles, { recursive: true });
-fs.mkdirSync(config.userFiles);
-
-// Insert a fake "valid-token" fixture that can be reused
-const db = getAccountDb();
-db.mutate('INSERT INTO sessions (token) VALUES (?)', ['valid-token']);
diff --git a/migrations/1694360000000-create-folders.js b/migrations/1694360000000-create-folders.js
new file mode 100644
index 0000000000..2ba62b8df4
--- /dev/null
+++ b/migrations/1694360000000-create-folders.js
@@ -0,0 +1,24 @@
+import fs from 'node:fs/promises';
+import config from '../src/load-config.js';
+
+async function ensureExists(path) {
+ try {
+ await fs.mkdir(path);
+ } catch (err) {
+ if (err.code == 'EEXIST') {
+ return null;
+ }
+
+ throw err;
+ }
+}
+
+export const up = async function () {
+ await ensureExists(config.serverFiles);
+ await ensureExists(config.userFiles);
+};
+
+export const down = async function () {
+ await fs.rm(config.serverFiles, { recursive: true, force: true });
+ await fs.rm(config.userFiles, { recursive: true, force: true });
+};
diff --git a/migrations/1694360479680-create-account-db.js b/migrations/1694360479680-create-account-db.js
new file mode 100644
index 0000000000..fd57b79cc8
--- /dev/null
+++ b/migrations/1694360479680-create-account-db.js
@@ -0,0 +1,30 @@
+import getAccountDb from '../src/account-db.js';
+
+export const up = async function () {
+ await getAccountDb().exec(`
+ CREATE TABLE IF NOT EXISTS auth
+ (password TEXT PRIMARY KEY);
+
+ CREATE TABLE IF NOT EXISTS sessions
+ (token TEXT PRIMARY KEY);
+
+ CREATE TABLE IF NOT EXISTS files
+ (id TEXT PRIMARY KEY,
+ group_id TEXT,
+ sync_version SMALLINT,
+ encrypt_meta TEXT,
+ encrypt_keyid TEXT,
+ encrypt_salt TEXT,
+ encrypt_test TEXT,
+ deleted BOOLEAN DEFAULT FALSE,
+ name TEXT);
+ `);
+};
+
+export const down = async function () {
+ await getAccountDb().exec(`
+ DROP TABLE auth;
+ DROP TABLE sessions;
+ DROP TABLE files;
+ `);
+};
diff --git a/migrations/1694362247011-create-secret-table.js b/migrations/1694362247011-create-secret-table.js
new file mode 100644
index 0000000000..2f60a08151
--- /dev/null
+++ b/migrations/1694362247011-create-secret-table.js
@@ -0,0 +1,16 @@
+import getAccountDb from '../src/account-db.js';
+
+export const up = async function () {
+ await getAccountDb().exec(`
+ CREATE TABLE IF NOT EXISTS secrets (
+ name TEXT PRIMARY KEY,
+ value BLOB
+ );
+ `);
+};
+
+export const down = async function () {
+ await getAccountDb().exec(`
+ DROP TABLE secrets;
+ `);
+};
diff --git a/package.json b/package.json
index edd1868e72..71a378ba5a 100644
--- a/package.json
+++ b/package.json
@@ -28,6 +28,7 @@
"express-rate-limit": "^6.7.0",
"express-response-size": "^0.0.3",
"jws": "^4.0.0",
+ "migrate": "^2.0.0",
"nordigen-node": "^1.2.6",
"uuid": "^9.0.0"
},
diff --git a/src/account-db.js b/src/account-db.js
index 3c1e5845c3..b33fee8675 100644
--- a/src/account-db.js
+++ b/src/account-db.js
@@ -1,34 +1,15 @@
-import fs from 'node:fs';
import { join } from 'node:path';
import openDatabase from './db.js';
-import config, { sqlDir } from './load-config.js';
-import createDebug from 'debug';
+import config from './load-config.js';
import * as uuid from 'uuid';
import * as bcrypt from 'bcrypt';
-const debug = createDebug('actual:account-db');
-
let _accountDb = null;
export default function getAccountDb() {
if (_accountDb == null) {
- if (!fs.existsSync(config.serverFiles)) {
- debug(`creating server files directory: '${config.serverFiles}'`);
- fs.mkdirSync(config.serverFiles);
- }
-
- let dbPath = join(config.serverFiles, 'account.sqlite');
- let needsInit = !fs.existsSync(dbPath);
-
+ const dbPath = join(config.serverFiles, 'account.sqlite');
_accountDb = openDatabase(dbPath);
-
- if (needsInit) {
- debug(`initializing account database: '${dbPath}'`);
- let initSql = fs.readFileSync(join(sqlDir, 'account.sql'), 'utf8');
- _accountDb.exec(initSql);
- } else {
- debug(`opening account database: '${dbPath}'`);
- }
}
return _accountDb;
diff --git a/src/app-account.js b/src/app-account.js
index a5cd0b8a2e..f992a3dd6e 100644
--- a/src/app-account.js
+++ b/src/app-account.js
@@ -13,10 +13,6 @@ app.use(errorMiddleware);
export { app as handlers };
-export function init() {
- // eslint-disable-previous-line @typescript-eslint/no-empty-function
-}
-
// Non-authenticated endpoints:
//
// /needs-bootstrap
diff --git a/src/app-gocardless/services/gocardless-service.js b/src/app-gocardless/services/gocardless-service.js
index 66f4642709..7e67330cec 100644
--- a/src/app-gocardless/services/gocardless-service.js
+++ b/src/app-gocardless/services/gocardless-service.js
@@ -17,17 +17,12 @@ import jwt from 'jws';
import { SecretName, secretsService } from '../../services/secrets-service.js';
const GoCardlessClient = nordigenNode.default;
-const goCardlessClient = new GoCardlessClient({
- secretId: secretsService.get(SecretName.nordigen_secretId),
- secretKey: secretsService.get(SecretName.nordigen_secretKey),
-});
-
-secretsService.onUpdate(SecretName.nordigen_secretId, (newSecret) => {
- goCardlessClient.secretId = newSecret;
-});
-secretsService.onUpdate(SecretName.nordigen_secretKey, (newSecret) => {
- goCardlessClient.secretKey = newSecret;
-});
+
+const getGocardlessClient = () =>
+ new GoCardlessClient({
+ secretId: secretsService.get(SecretName.nordigen_secretId),
+ secretKey: secretsService.get(SecretName.nordigen_secretKey),
+ });
export const handleGoCardlessError = (response) => {
switch (response.status_code) {
@@ -58,7 +53,9 @@ export const goCardlessService = {
* @returns {boolean}
*/
isConfigured: () => {
- return !!(goCardlessClient.secretId && goCardlessClient.secretKey);
+ return !!(
+ getGocardlessClient().secretId && getGocardlessClient().secretKey
+ );
},
/**
@@ -76,7 +73,7 @@ export const goCardlessService = {
return clockTimestamp >= payload.exp;
};
- if (isExpiredJwtToken(goCardlessClient.token)) {
+ if (isExpiredJwtToken(getGocardlessClient().token)) {
// Generate new access token. Token is valid for 24 hours
// Note: access_token is automatically injected to other requests after you successfully obtain it
const tokenData = await client.generateToken();
@@ -479,25 +476,25 @@ export const goCardlessService = {
*/
export const client = {
getBalances: async (accountId) =>
- await goCardlessClient.account(accountId).getBalances(),
+ await getGocardlessClient().account(accountId).getBalances(),
getTransactions: async ({ accountId, dateFrom, dateTo }) =>
- await goCardlessClient.account(accountId).getTransactions({
+ await getGocardlessClient().account(accountId).getTransactions({
dateFrom,
dateTo,
country: undefined,
}),
getInstitutions: async (country) =>
- await goCardlessClient.institution.getInstitutions({ country }),
+ await getGocardlessClient().institution.getInstitutions({ country }),
getInstitutionById: async (institutionId) =>
- await goCardlessClient.institution.getInstitutionById(institutionId),
+ await getGocardlessClient().institution.getInstitutionById(institutionId),
getDetails: async (accountId) =>
- await goCardlessClient.account(accountId).getDetails(),
+ await getGocardlessClient().account(accountId).getDetails(),
getMetadata: async (accountId) =>
- await goCardlessClient.account(accountId).getMetadata(),
+ await getGocardlessClient().account(accountId).getMetadata(),
getRequisitionById: async (requisitionId) =>
- await goCardlessClient.requisition.getRequisitionById(requisitionId),
+ await getGocardlessClient().requisition.getRequisitionById(requisitionId),
deleteRequisition: async (requisitionId) =>
- await goCardlessClient.requisition.deleteRequisition(requisitionId),
+ await getGocardlessClient().requisition.deleteRequisition(requisitionId),
initSession: async ({
redirectUrl,
institutionId,
@@ -509,7 +506,7 @@ export const client = {
redirectImmediate,
accountSelection,
}) =>
- await goCardlessClient.initSession({
+ await getGocardlessClient().initSession({
redirectUrl,
institutionId,
referenceId,
@@ -520,7 +517,7 @@ export const client = {
redirectImmediate,
accountSelection,
}),
- generateToken: async () => await goCardlessClient.generateToken(),
+ generateToken: async () => await getGocardlessClient().generateToken(),
exchangeToken: async ({ refreshToken }) =>
- await goCardlessClient.exchangeToken({ refreshToken }),
+ await getGocardlessClient().exchangeToken({ refreshToken }),
};
diff --git a/src/app-sync.js b/src/app-sync.js
index b95e8262fc..5d036c3abe 100644
--- a/src/app-sync.js
+++ b/src/app-sync.js
@@ -15,9 +15,6 @@ const app = express();
app.use(errorMiddleware);
export { app as handlers };
-// eslint-disable-next-line
-export async function init() {}
-
// This is a version representing the internal format of sync
// messages. When this changes, all sync files need to be reset. We
// will check this version when syncing and notify the user if they
diff --git a/src/app.js b/src/app.js
index 6d588d2388..4c20da83db 100644
--- a/src/app.js
+++ b/src/app.js
@@ -71,17 +71,6 @@ function parseHTTPSConfig(value) {
}
export default async function run() {
- if (!fs.existsSync(config.serverFiles)) {
- fs.mkdirSync(config.serverFiles);
- }
-
- if (!fs.existsSync(config.userFiles)) {
- fs.mkdirSync(config.userFiles);
- }
-
- await accountApp.init();
- await syncApp.init();
-
if (config.https) {
const https = await import('node:https');
const httpsOptions = {
diff --git a/src/db.js b/src/db.js
index 80696fa00a..a4d57a6a9c 100644
--- a/src/db.js
+++ b/src/db.js
@@ -27,7 +27,7 @@ class WrappedDatabase {
* @param {string} sql
*/
exec(sql) {
- this.db.exec(sql);
+ return this.db.exec(sql);
}
/**
diff --git a/src/migrations.js b/src/migrations.js
new file mode 100644
index 0000000000..66f59d4a68
--- /dev/null
+++ b/src/migrations.js
@@ -0,0 +1,30 @@
+import migrate from 'migrate';
+import config from './load-config.js';
+
+export default function run(direction = 'up') {
+ console.log(
+ `Checking if there are any migrations to run for direction "${direction}"...`,
+ );
+
+ return new Promise((resolve) =>
+ migrate.load(
+ {
+ stateStore: `.migrate${config.mode === 'test' ? '-test' : ''}`,
+ },
+ (err, set) => {
+ if (err) {
+ throw err;
+ }
+
+ set[direction]((err) => {
+ if (err) {
+ throw err;
+ }
+
+ console.log('Migrations: DONE');
+ resolve();
+ });
+ },
+ ),
+ );
+}
diff --git a/src/services/secrets-service.js b/src/services/secrets-service.js
index 9e229588d0..ee64fc5c43 100644
--- a/src/services/secrets-service.js
+++ b/src/services/secrets-service.js
@@ -1,7 +1,4 @@
import createDebug from 'debug';
-import fs from 'node:fs';
-import { sqlDir } from '../load-config.js';
-import { join } from 'node:path';
import getAccountDb from '../account-db.js';
/**
@@ -18,18 +15,6 @@ class SecretsDb {
constructor() {
this.debug = createDebug('actual:secrets-db');
this.db = null;
- this.initialize();
- }
-
- initialize() {
- if (!this.db) {
- this.db = this.open();
- }
-
- this.debug(`initializing secrets table'`);
- //Create secret table if it doesn't exist
- const initSql = fs.readFileSync(join(sqlDir, 'secrets.sql'), 'utf8');
- this.db.exec(initSql);
}
open() {
@@ -64,7 +49,6 @@ class SecretsDb {
const secretsDb = new SecretsDb();
const _cachedSecrets = new Map();
-const _observers = new Map();
/**
* A service for managing secrets stored in `secretsDb`.
*/
@@ -78,25 +62,6 @@ export const secretsService = {
return _cachedSecrets.get(name) ?? secretsDb.get(name)?.value ?? null;
},
- /**
- * Callbacks new secret value when a secret changes.
- * @param {SecretName} name - The name of the secret to retrieve.
- * @param {function(string): void} callback - The new secret value callback.
- * @returns {void}
- */
- onUpdate: (name, callback) => {
- const observers = _observers.get(name) ?? [];
- observers.push(callback);
- _observers.set(name, observers);
- },
-
- _notifyObservers: (name, value) => {
- const observers = _observers.get(name) ?? [];
- for (const observer of observers) {
- observer(value);
- }
- },
-
/**
* Sets the value of a secret by name.
* @param {SecretName} name - The name of the secret to set.
@@ -108,7 +73,6 @@ export const secretsService = {
if (result.changes === 1) {
_cachedSecrets.set(name, value);
- secretsService._notifyObservers(name, value);
}
return result;
},
diff --git a/src/sql/account.sql b/src/sql/account.sql
deleted file mode 100644
index 8fe2662009..0000000000
--- a/src/sql/account.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-
-CREATE TABLE auth
- (password TEXT PRIMARY KEY);
-
-CREATE TABLE sessions
- (token TEXT PRIMARY KEY);
-
-CREATE TABLE files
- (id TEXT PRIMARY KEY,
- group_id TEXT,
- sync_version SMALLINT,
- encrypt_meta TEXT,
- encrypt_keyid TEXT,
- encrypt_salt TEXT,
- encrypt_test TEXT,
- deleted BOOLEAN DEFAULT FALSE,
- name TEXT);
diff --git a/src/sql/secrets.sql b/src/sql/secrets.sql
deleted file mode 100644
index 0cf078614e..0000000000
--- a/src/sql/secrets.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-CREATE TABLE IF NOT EXISTS secrets (
- name TEXT PRIMARY KEY,
- value BLOB
-);
diff --git a/upcoming-release-notes/267.md b/upcoming-release-notes/267.md
new file mode 100644
index 0000000000..24c96b5810
--- /dev/null
+++ b/upcoming-release-notes/267.md
@@ -0,0 +1,6 @@
+---
+category: Enhancements
+authors: [MatissJanis]
+---
+
+Ability to add and run database/fs migrations
diff --git a/yarn.lock b/yarn.lock
index c904942358..a9d0b629a7 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1638,6 +1638,7 @@ __metadata:
express-response-size: ^0.0.3
jest: ^29.3.1
jws: ^4.0.0
+ migrate: ^2.0.0
nordigen-node: ^1.2.6
prettier: ^2.8.3
supertest: ^6.3.1
@@ -2129,7 +2130,7 @@ __metadata:
languageName: node
linkType: hard
-"chalk@npm:^4.0.0":
+"chalk@npm:^4.0.0, chalk@npm:^4.1.2":
version: 4.1.2
resolution: "chalk@npm:4.1.2"
dependencies:
@@ -2256,6 +2257,13 @@ __metadata:
languageName: node
linkType: hard
+"commander@npm:^2.20.3":
+ version: 2.20.3
+ resolution: "commander@npm:2.20.3"
+ checksum: ab8c07884e42c3a8dbc5dd9592c606176c7eb5c1ca5ff274bcf907039b2c41de3626f684ea75ccf4d361ba004bbaff1f577d5384c155f3871e456bdf27becf9e
+ languageName: node
+ linkType: hard
+
"component-emitter@npm:^1.3.0":
version: 1.3.0
resolution: "component-emitter@npm:1.3.0"
@@ -2358,6 +2366,13 @@ __metadata:
languageName: node
linkType: hard
+"dateformat@npm:^4.6.3":
+ version: 4.6.3
+ resolution: "dateformat@npm:4.6.3"
+ checksum: c3aa0617c0a5b30595122bc8d1bee6276a9221e4d392087b41cbbdf175d9662ae0e50d0d6dcdf45caeac5153c4b5b0844265f8cd2b2245451e3da19e39e3b65d
+ languageName: node
+ linkType: hard
+
"dayjs@npm:^1.11.3":
version: 1.11.7
resolution: "dayjs@npm:1.11.7"
@@ -2526,6 +2541,13 @@ __metadata:
languageName: node
linkType: hard
+"dotenv@npm:^16.0.0":
+ version: 16.3.1
+ resolution: "dotenv@npm:16.3.1"
+ checksum: 15d75e7279018f4bafd0ee9706593dd14455ddb71b3bcba9c52574460b7ccaf67d5cf8b2c08a5af1a9da6db36c956a04a1192b101ee102a3e0cf8817bbcf3dfd
+ languageName: node
+ linkType: hard
+
"ecdsa-sig-formatter@npm:1.0.11":
version: 1.0.11
resolution: "ecdsa-sig-formatter@npm:1.0.11"
@@ -4447,6 +4469,29 @@ __metadata:
languageName: node
linkType: hard
+"migrate@npm:^2.0.0":
+ version: 2.0.0
+ resolution: "migrate@npm:2.0.0"
+ dependencies:
+ chalk: ^4.1.2
+ commander: ^2.20.3
+ dateformat: ^4.6.3
+ dotenv: ^16.0.0
+ inherits: ^2.0.3
+ minimatch: ^9.0.1
+ mkdirp: ^3.0.1
+ slug: ^8.2.2
+ bin:
+ migrate: bin/migrate
+ migrate-create: bin/migrate-create
+ migrate-down: bin/migrate-down
+ migrate-init: bin/migrate-init
+ migrate-list: bin/migrate-list
+ migrate-up: bin/migrate-up
+ checksum: d7e5f476d32c638e7c6ee15e36e0a049f69ad3cb011011623658defa656cba2c75c0128d005ed0a06ea6d6426200297d809d1338db63cd580c03e8d42001a7bd
+ languageName: node
+ linkType: hard
+
"mime-db@npm:1.52.0":
version: 1.52.0
resolution: "mime-db@npm:1.52.0"
@@ -4513,6 +4558,15 @@ __metadata:
languageName: node
linkType: hard
+"minimatch@npm:^9.0.1":
+ version: 9.0.3
+ resolution: "minimatch@npm:9.0.3"
+ dependencies:
+ brace-expansion: ^2.0.1
+ checksum: 253487976bf485b612f16bf57463520a14f512662e592e95c571afdab1442a6a6864b6c88f248ce6fc4ff0b6de04ac7aa6c8bb51e868e99d1d65eb0658a708b5
+ languageName: node
+ linkType: hard
+
"minimist@npm:^1.2.0, minimist@npm:^1.2.3":
version: 1.2.6
resolution: "minimist@npm:1.2.6"
@@ -4624,6 +4678,15 @@ __metadata:
languageName: node
linkType: hard
+"mkdirp@npm:^3.0.1":
+ version: 3.0.1
+ resolution: "mkdirp@npm:3.0.1"
+ bin:
+ mkdirp: dist/cjs/src/bin.js
+ checksum: 972deb188e8fb55547f1e58d66bd6b4a3623bf0c7137802582602d73e6480c1c2268dcbafbfb1be466e00cc7e56ac514d7fd9334b7cf33e3e2ab547c16f83a8d
+ languageName: node
+ linkType: hard
+
"ms@npm:2.0.0":
version: 2.0.0
resolution: "ms@npm:2.0.0"
@@ -5503,6 +5566,15 @@ __metadata:
languageName: node
linkType: hard
+"slug@npm:^8.2.2":
+ version: 8.2.3
+ resolution: "slug@npm:8.2.3"
+ bin:
+ slug: cli.js
+ checksum: eb2fbf8d13df0a94f09ffd7c20e02d5e88c1fdd51e178fe8e670937747dec5a97efd172956b914efd5eb3fadd65a306a68f4eed0327a8c5c9a8af30f2c95a46b
+ languageName: node
+ linkType: hard
+
"smart-buffer@npm:^4.2.0":
version: 4.2.0
resolution: "smart-buffer@npm:4.2.0"