A local-first personal finance app
:sparkles: ability to add migrations (#267)
Matiss Janis Aboltins1f79708ea3d2
diff --git a/.gitignore b/.gitignoreindex e114b191b7..ad1e69fb0b 100644--- a/.gitignore+++ b/.gitignore@@ -15,6 +15,8 @@ build/*.pem*.keyartifacts.json+.migrate+.migrate-test# Yarn.pnp.*diff --git a/app.js b/app.jsindex b76fc06eb0..4e254b5dab 100644--- a/app.js+++ b/app.js@@ -1,6 +1,9 @@import run from './src/app.js';+import runMigrations from './src/migrations.js';-run().catch((err) => {- console.log('Error starting app:', err);- process.exit(1);-});+runMigrations()+ .then(run)+ .catch((err) => {+ console.log('Error starting app:', err);+ process.exit(1);+ });diff --git a/jest.config.json b/jest.config.jsonindex 8785cee1ac..95fb27235b 100644--- a/jest.config.json+++ b/jest.config.json@@ -1,5 +1,6 @@{- "setupFiles": ["./jest.setup.js"],+ "globalSetup": "./jest.global-setup.js",+ "globalTeardown": "./jest.global-teardown.js","testPathIgnorePatterns": ["dist", "/node_modules/", "/build/"],"roots": ["<rootDir>"],"moduleFileExtensions": ["ts", "js", "json"],diff --git a/jest.global-setup.js b/jest.global-setup.jsnew file mode 100644index 0000000000..36f53cf1cc--- /dev/null+++ b/jest.global-setup.js@@ -0,0 +1,10 @@+import getAccountDb from './src/account-db.js';+import runMigrations from './src/migrations.js';++export default async function setup() {+ await runMigrations();++ // Insert a fake "valid-token" fixture that can be reused+ const db = getAccountDb();+ await db.mutate('INSERT INTO sessions (token) VALUES (?)', ['valid-token']);+}diff --git a/jest.global-teardown.js b/jest.global-teardown.jsnew file mode 100644index 0000000000..4e19fc3854--- /dev/null+++ b/jest.global-teardown.js@@ -0,0 +1,5 @@+import runMigrations from './src/migrations.js';++export default async function teardown() {+ await runMigrations('down');+}diff --git a/jest.setup.js b/jest.setup.jsdeleted file mode 100644index 18208ceb89..0000000000--- a/jest.setup.js+++ /dev/null@@ -1,17 +0,0 @@-import fs from 'node:fs';-import path from 'node:path';-import getAccountDb from './src/account-db.js';-import config from './src/load-config.js';--// Delete previous test database (force creation of a new one)-const dbPath = path.join(config.serverFiles, 'account.sqlite');-if (fs.existsSync(dbPath)) fs.unlinkSync(dbPath);--// Create path for test user files and delete previous files there-if (fs.existsSync(config.userFiles))- fs.rmSync(config.userFiles, { recursive: true });-fs.mkdirSync(config.userFiles);--// Insert a fake "valid-token" fixture that can be reused-const db = getAccountDb();-db.mutate('INSERT INTO sessions (token) VALUES (?)', ['valid-token']);diff --git a/migrations/1694360000000-create-folders.js b/migrations/1694360000000-create-folders.jsnew file mode 100644index 0000000000..2ba62b8df4--- /dev/null+++ b/migrations/1694360000000-create-folders.js@@ -0,0 +1,24 @@+import fs from 'node:fs/promises';+import config from '../src/load-config.js';++async function ensureExists(path) {+ try {+ await fs.mkdir(path);+ } catch (err) {+ if (err.code == 'EEXIST') {+ return null;+ }++ throw err;+ }+}++export const up = async function () {+ await ensureExists(config.serverFiles);+ await ensureExists(config.userFiles);+};++export const down = async function () {+ await fs.rm(config.serverFiles, { recursive: true, force: true });+ await fs.rm(config.userFiles, { recursive: true, force: true });+};diff --git a/migrations/1694360479680-create-account-db.js b/migrations/1694360479680-create-account-db.jsnew file mode 100644index 0000000000..fd57b79cc8--- /dev/null+++ b/migrations/1694360479680-create-account-db.js@@ -0,0 +1,30 @@+import getAccountDb from '../src/account-db.js';++export const up = async function () {+ await getAccountDb().exec(`+ CREATE TABLE IF NOT EXISTS auth+ (password TEXT PRIMARY KEY);++ CREATE TABLE IF NOT EXISTS sessions+ (token TEXT PRIMARY KEY);++ CREATE TABLE IF NOT EXISTS files+ (id TEXT PRIMARY KEY,+ group_id TEXT,+ sync_version SMALLINT,+ encrypt_meta TEXT,+ encrypt_keyid TEXT,+ encrypt_salt TEXT,+ encrypt_test TEXT,+ deleted BOOLEAN DEFAULT FALSE,+ name TEXT);+ `);+};++export const down = async function () {+ await getAccountDb().exec(`+ DROP TABLE auth;+ DROP TABLE sessions;+ DROP TABLE files;+ `);+};diff --git a/migrations/1694362247011-create-secret-table.js b/migrations/1694362247011-create-secret-table.jsnew file mode 100644index 0000000000..2f60a08151--- /dev/null+++ b/migrations/1694362247011-create-secret-table.js@@ -0,0 +1,16 @@+import getAccountDb from '../src/account-db.js';++export const up = async function () {+ await getAccountDb().exec(`+ CREATE TABLE IF NOT EXISTS secrets (+ name TEXT PRIMARY KEY,+ value BLOB+ );+ `);+};++export const down = async function () {+ await getAccountDb().exec(`+ DROP TABLE secrets;+ `);+};diff --git a/package.json b/package.jsonindex edd1868e72..71a378ba5a 100644--- a/package.json+++ b/package.json@@ -28,6 +28,7 @@"express-rate-limit": "^6.7.0","express-response-size": "^0.0.3","jws": "^4.0.0",+ "migrate": "^2.0.0","nordigen-node": "^1.2.6","uuid": "^9.0.0"},diff --git a/src/account-db.js b/src/account-db.jsindex 3c1e5845c3..b33fee8675 100644--- a/src/account-db.js+++ b/src/account-db.js@@ -1,34 +1,15 @@-import fs from 'node:fs';import { join } from 'node:path';import openDatabase from './db.js';-import config, { sqlDir } from './load-config.js';-import createDebug from 'debug';+import config from './load-config.js';import * as uuid from 'uuid';import * as bcrypt from 'bcrypt';-const debug = createDebug('actual:account-db');-let _accountDb = null;export default function getAccountDb() {if (_accountDb == null) {- if (!fs.existsSync(config.serverFiles)) {- debug(`creating server files directory: '${config.serverFiles}'`);- fs.mkdirSync(config.serverFiles);- }-- let dbPath = join(config.serverFiles, 'account.sqlite');- let needsInit = !fs.existsSync(dbPath);-+ const dbPath = join(config.serverFiles, 'account.sqlite');_accountDb = openDatabase(dbPath);-- if (needsInit) {- debug(`initializing account database: '${dbPath}'`);- let initSql = fs.readFileSync(join(sqlDir, 'account.sql'), 'utf8');- _accountDb.exec(initSql);- } else {- debug(`opening account database: '${dbPath}'`);- }}return _accountDb;diff --git a/src/app-account.js b/src/app-account.jsindex a5cd0b8a2e..f992a3dd6e 100644--- a/src/app-account.js+++ b/src/app-account.js@@ -13,10 +13,6 @@ app.use(errorMiddleware);export { app as handlers };-export function init() {- // eslint-disable-previous-line @typescript-eslint/no-empty-function-}-// Non-authenticated endpoints://// /needs-bootstrapdiff --git a/src/app-gocardless/services/gocardless-service.js b/src/app-gocardless/services/gocardless-service.jsindex 66f4642709..7e67330cec 100644--- a/src/app-gocardless/services/gocardless-service.js+++ b/src/app-gocardless/services/gocardless-service.js@@ -17,17 +17,12 @@ import jwt from 'jws';import { SecretName, secretsService } from '../../services/secrets-service.js';const GoCardlessClient = nordigenNode.default;-const goCardlessClient = new GoCardlessClient({- secretId: secretsService.get(SecretName.nordigen_secretId),- secretKey: secretsService.get(SecretName.nordigen_secretKey),-});--secretsService.onUpdate(SecretName.nordigen_secretId, (newSecret) => {- goCardlessClient.secretId = newSecret;-});-secretsService.onUpdate(SecretName.nordigen_secretKey, (newSecret) => {- goCardlessClient.secretKey = newSecret;-});++const getGocardlessClient = () =>+ new GoCardlessClient({+ secretId: secretsService.get(SecretName.nordigen_secretId),+ secretKey: secretsService.get(SecretName.nordigen_secretKey),+ });export const handleGoCardlessError = (response) => {switch (response.status_code) {@@ -58,7 +53,9 @@ export const goCardlessService = {* @returns {boolean}*/isConfigured: () => {- return !!(goCardlessClient.secretId && goCardlessClient.secretKey);+ return !!(+ getGocardlessClient().secretId && getGocardlessClient().secretKey+ );},/**@@ -76,7 +73,7 @@ export const goCardlessService = {return clockTimestamp >= payload.exp;};- if (isExpiredJwtToken(goCardlessClient.token)) {+ if (isExpiredJwtToken(getGocardlessClient().token)) {// Generate new access token. Token is valid for 24 hours// Note: access_token is automatically injected to other requests after you successfully obtain itconst tokenData = await client.generateToken();@@ -479,25 +476,25 @@ export const goCardlessService = {*/export const client = {getBalances: async (accountId) =>- await goCardlessClient.account(accountId).getBalances(),+ await getGocardlessClient().account(accountId).getBalances(),getTransactions: async ({ accountId, dateFrom, dateTo }) =>- await goCardlessClient.account(accountId).getTransactions({+ await getGocardlessClient().account(accountId).getTransactions({dateFrom,dateTo,country: undefined,}),getInstitutions: async (country) =>- await goCardlessClient.institution.getInstitutions({ country }),+ await getGocardlessClient().institution.getInstitutions({ country }),getInstitutionById: async (institutionId) =>- await goCardlessClient.institution.getInstitutionById(institutionId),+ await getGocardlessClient().institution.getInstitutionById(institutionId),getDetails: async (accountId) =>- await goCardlessClient.account(accountId).getDetails(),+ await getGocardlessClient().account(accountId).getDetails(),getMetadata: async (accountId) =>- await goCardlessClient.account(accountId).getMetadata(),+ await getGocardlessClient().account(accountId).getMetadata(),getRequisitionById: async (requisitionId) =>- await goCardlessClient.requisition.getRequisitionById(requisitionId),+ await getGocardlessClient().requisition.getRequisitionById(requisitionId),deleteRequisition: async (requisitionId) =>- await goCardlessClient.requisition.deleteRequisition(requisitionId),+ await getGocardlessClient().requisition.deleteRequisition(requisitionId),initSession: async ({redirectUrl,institutionId,@@ -509,7 +506,7 @@ export const client = {redirectImmediate,accountSelection,}) =>- await goCardlessClient.initSession({+ await getGocardlessClient().initSession({redirectUrl,institutionId,referenceId,@@ -520,7 +517,7 @@ export const client = {redirectImmediate,accountSelection,}),- generateToken: async () => await goCardlessClient.generateToken(),+ generateToken: async () => await getGocardlessClient().generateToken(),exchangeToken: async ({ refreshToken }) =>- await goCardlessClient.exchangeToken({ refreshToken }),+ await getGocardlessClient().exchangeToken({ refreshToken }),};diff --git a/src/app-sync.js b/src/app-sync.jsindex b95e8262fc..5d036c3abe 100644--- a/src/app-sync.js+++ b/src/app-sync.js@@ -15,9 +15,6 @@ const app = express();app.use(errorMiddleware);export { app as handlers };-// eslint-disable-next-line-export async function init() {}-// This is a version representing the internal format of sync// messages. When this changes, all sync files need to be reset. We// will check this version when syncing and notify the user if theydiff --git a/src/app.js b/src/app.jsindex 6d588d2388..4c20da83db 100644--- a/src/app.js+++ b/src/app.js@@ -71,17 +71,6 @@ function parseHTTPSConfig(value) {}export default async function run() {- if (!fs.existsSync(config.serverFiles)) {- fs.mkdirSync(config.serverFiles);- }-- if (!fs.existsSync(config.userFiles)) {- fs.mkdirSync(config.userFiles);- }-- await accountApp.init();- await syncApp.init();-if (config.https) {const https = await import('node:https');const httpsOptions = {diff --git a/src/db.js b/src/db.jsindex 80696fa00a..a4d57a6a9c 100644--- a/src/db.js+++ b/src/db.js@@ -27,7 +27,7 @@ class WrappedDatabase {* @param {string} sql*/exec(sql) {- this.db.exec(sql);+ return this.db.exec(sql);}/**diff --git a/src/migrations.js b/src/migrations.jsnew file mode 100644index 0000000000..66f59d4a68--- /dev/null+++ b/src/migrations.js@@ -0,0 +1,30 @@+import migrate from 'migrate';+import config from './load-config.js';++export default function run(direction = 'up') {+ console.log(+ `Checking if there are any migrations to run for direction "${direction}"...`,+ );++ return new Promise((resolve) =>+ migrate.load(+ {+ stateStore: `.migrate${config.mode === 'test' ? '-test' : ''}`,+ },+ (err, set) => {+ if (err) {+ throw err;+ }++ set[direction]((err) => {+ if (err) {+ throw err;+ }++ console.log('Migrations: DONE');+ resolve();+ });+ },+ ),+ );+}diff --git a/src/services/secrets-service.js b/src/services/secrets-service.jsindex 9e229588d0..ee64fc5c43 100644--- a/src/services/secrets-service.js+++ b/src/services/secrets-service.js@@ -1,7 +1,4 @@import createDebug from 'debug';-import fs from 'node:fs';-import { sqlDir } from '../load-config.js';-import { join } from 'node:path';import getAccountDb from '../account-db.js';/**@@ -18,18 +15,6 @@ class SecretsDb {constructor() {this.debug = createDebug('actual:secrets-db');this.db = null;- this.initialize();- }-- initialize() {- if (!this.db) {- this.db = this.open();- }-- this.debug(`initializing secrets table'`);- //Create secret table if it doesn't exist- const initSql = fs.readFileSync(join(sqlDir, 'secrets.sql'), 'utf8');- this.db.exec(initSql);}open() {@@ -64,7 +49,6 @@ class SecretsDb {const secretsDb = new SecretsDb();const _cachedSecrets = new Map();-const _observers = new Map();/*** A service for managing secrets stored in `secretsDb`.*/@@ -78,25 +62,6 @@ export const secretsService = {return _cachedSecrets.get(name) ?? secretsDb.get(name)?.value ?? null;},- /**- * Callbacks new secret value when a secret changes.- * @param {SecretName} name - The name of the secret to retrieve.- * @param {function(string): void} callback - The new secret value callback.- * @returns {void}- */- onUpdate: (name, callback) => {- const observers = _observers.get(name) ?? [];- observers.push(callback);- _observers.set(name, observers);- },-- _notifyObservers: (name, value) => {- const observers = _observers.get(name) ?? [];- for (const observer of observers) {- observer(value);- }- },-/*** Sets the value of a secret by name.* @param {SecretName} name - The name of the secret to set.@@ -108,7 +73,6 @@ export const secretsService = {if (result.changes === 1) {_cachedSecrets.set(name, value);- secretsService._notifyObservers(name, value);}return result;},diff --git a/src/sql/account.sql b/src/sql/account.sqldeleted file mode 100644index 8fe2662009..0000000000--- a/src/sql/account.sql+++ /dev/null@@ -1,17 +0,0 @@--CREATE TABLE auth- (password TEXT PRIMARY KEY);--CREATE TABLE sessions- (token TEXT PRIMARY KEY);--CREATE TABLE files- (id TEXT PRIMARY KEY,- group_id TEXT,- sync_version SMALLINT,- encrypt_meta TEXT,- encrypt_keyid TEXT,- encrypt_salt TEXT,- encrypt_test TEXT,- deleted BOOLEAN DEFAULT FALSE,- name TEXT);diff --git a/src/sql/secrets.sql b/src/sql/secrets.sqldeleted file mode 100644index 0cf078614e..0000000000--- a/src/sql/secrets.sql+++ /dev/null@@ -1,4 +0,0 @@-CREATE TABLE IF NOT EXISTS secrets (- name TEXT PRIMARY KEY,- value BLOB-);diff --git a/upcoming-release-notes/267.md b/upcoming-release-notes/267.mdnew file mode 100644index 0000000000..24c96b5810--- /dev/null+++ b/upcoming-release-notes/267.md@@ -0,0 +1,6 @@+---+category: Enhancements+authors: [MatissJanis]+---++Ability to add and run database/fs migrationsdiff --git a/yarn.lock b/yarn.lockindex c904942358..a9d0b629a7 100644--- a/yarn.lock+++ b/yarn.lock@@ -1638,6 +1638,7 @@ __metadata:express-response-size: ^0.0.3jest: ^29.3.1jws: ^4.0.0+ migrate: ^2.0.0nordigen-node: ^1.2.6prettier: ^2.8.3supertest: ^6.3.1@@ -2129,7 +2130,7 @@ __metadata:languageName: nodelinkType: hard-"chalk@npm:^4.0.0":+"chalk@npm:^4.0.0, chalk@npm:^4.1.2":version: 4.1.2resolution: "chalk@npm:4.1.2"dependencies:@@ -2256,6 +2257,13 @@ __metadata:languageName: nodelinkType: hard+"commander@npm:^2.20.3":+ version: 2.20.3+ resolution: "commander@npm:2.20.3"+ checksum: ab8c07884e42c3a8dbc5dd9592c606176c7eb5c1ca5ff274bcf907039b2c41de3626f684ea75ccf4d361ba004bbaff1f577d5384c155f3871e456bdf27becf9e+ languageName: node+ linkType: hard+"component-emitter@npm:^1.3.0":version: 1.3.0resolution: "component-emitter@npm:1.3.0"@@ -2358,6 +2366,13 @@ __metadata:languageName: nodelinkType: hard+"dateformat@npm:^4.6.3":+ version: 4.6.3+ resolution: "dateformat@npm:4.6.3"+ checksum: c3aa0617c0a5b30595122bc8d1bee6276a9221e4d392087b41cbbdf175d9662ae0e50d0d6dcdf45caeac5153c4b5b0844265f8cd2b2245451e3da19e39e3b65d+ languageName: node+ linkType: hard+"dayjs@npm:^1.11.3":version: 1.11.7resolution: "dayjs@npm:1.11.7"@@ -2526,6 +2541,13 @@ __metadata:languageName: nodelinkType: hard+"dotenv@npm:^16.0.0":+ version: 16.3.1+ resolution: "dotenv@npm:16.3.1"+ checksum: 15d75e7279018f4bafd0ee9706593dd14455ddb71b3bcba9c52574460b7ccaf67d5cf8b2c08a5af1a9da6db36c956a04a1192b101ee102a3e0cf8817bbcf3dfd+ languageName: node+ linkType: hard+"ecdsa-sig-formatter@npm:1.0.11":version: 1.0.11resolution: "ecdsa-sig-formatter@npm:1.0.11"@@ -4447,6 +4469,29 @@ __metadata:languageName: nodelinkType: hard+"migrate@npm:^2.0.0":+ version: 2.0.0+ resolution: "migrate@npm:2.0.0"+ dependencies:+ chalk: ^4.1.2+ commander: ^2.20.3+ dateformat: ^4.6.3+ dotenv: ^16.0.0+ inherits: ^2.0.3+ minimatch: ^9.0.1+ mkdirp: ^3.0.1+ slug: ^8.2.2+ bin:+ migrate: bin/migrate+ migrate-create: bin/migrate-create+ migrate-down: bin/migrate-down+ migrate-init: bin/migrate-init+ migrate-list: bin/migrate-list+ migrate-up: bin/migrate-up+ checksum: d7e5f476d32c638e7c6ee15e36e0a049f69ad3cb011011623658defa656cba2c75c0128d005ed0a06ea6d6426200297d809d1338db63cd580c03e8d42001a7bd+ languageName: node+ linkType: hard+"mime-db@npm:1.52.0":version: 1.52.0resolution: "mime-db@npm:1.52.0"@@ -4513,6 +4558,15 @@ __metadata:languageName: nodelinkType: hard+"minimatch@npm:^9.0.1":+ version: 9.0.3+ resolution: "minimatch@npm:9.0.3"+ dependencies:+ brace-expansion: ^2.0.1+ checksum: 253487976bf485b612f16bf57463520a14f512662e592e95c571afdab1442a6a6864b6c88f248ce6fc4ff0b6de04ac7aa6c8bb51e868e99d1d65eb0658a708b5+ languageName: node+ linkType: hard+"minimist@npm:^1.2.0, minimist@npm:^1.2.3":version: 1.2.6resolution: "minimist@npm:1.2.6"@@ -4624,6 +4678,15 @@ __metadata:languageName: nodelinkType: hard+"mkdirp@npm:^3.0.1":+ version: 3.0.1+ resolution: "mkdirp@npm:3.0.1"+ bin:+ mkdirp: dist/cjs/src/bin.js+ checksum: 972deb188e8fb55547f1e58d66bd6b4a3623bf0c7137802582602d73e6480c1c2268dcbafbfb1be466e00cc7e56ac514d7fd9334b7cf33e3e2ab547c16f83a8d+ languageName: node+ linkType: hard+"ms@npm:2.0.0":version: 2.0.0resolution: "ms@npm:2.0.0"@@ -5503,6 +5566,15 @@ __metadata:languageName: nodelinkType: hard+"slug@npm:^8.2.2":+ version: 8.2.3+ resolution: "slug@npm:8.2.3"+ bin:+ slug: cli.js+ checksum: eb2fbf8d13df0a94f09ffd7c20e02d5e88c1fdd51e178fe8e670937747dec5a97efd172956b914efd5eb3fadd65a306a68f4eed0327a8c5c9a8af30f2c95a46b+ languageName: node+ linkType: hard+"smart-buffer@npm:^4.2.0":version: 4.2.0resolution: "smart-buffer@npm:4.2.0"