A local-first personal finance app
integrate FileService for app-sync.js (#432)
Tom Crassetcc347aef089e
diff --git a/src/app-simplefin/app-simplefin.js b/src/app-simplefin/app-simplefin.jsindex 7d27c1bd7d..7951526e3f 100644--- a/src/app-simplefin/app-simplefin.js+++ b/src/app-simplefin/app-simplefin.js@@ -42,7 +42,7 @@ app.post(}}}- } catch (error) {+ } catch {invalidToken(res);return;}diff --git a/src/app-sync.js b/src/app-sync.jsindex 447f9d9661..7930321660 100644--- a/src/app-sync.js+++ b/src/app-sync.js@@ -13,6 +13,16 @@ import { getPathForUserFile, getPathForGroupFile } from './util/paths.js';import * as simpleSync from './sync-simple.js';import { SyncProtoBuf } from '@actual-app/crdt';+import {+ File,+ FilesService,+ FileUpdate,+} from './app-sync/services/files-service.js';+import { FileNotFound } from './app-sync/errors.js';+import {+ validateSyncedFile,+ validateUploadedFile,+} from './app-sync/validation.js';const app = express();app.use(errorMiddleware);@@ -26,11 +36,24 @@ export { app as handlers };const OK_RESPONSE = { status: 'ok' };-// This is a version representing the internal format of sync-// messages. When this changes, all sync files need to be reset. We-// will check this version when syncing and notify the user if they-// need to reset.-const SYNC_FORMAT_VERSION = 2;+function boolToInt(deleted) {+ return deleted ? 1 : 0;+}++const verifyFileExists = (fileId, filesService, res, errorObject) => {+ try {+ return filesService.get(fileId);+ } catch (e) {+ if (e instanceof FileNotFound) {+ //FIXME: error code should be 404. Need to make sure frontend is ok with it.+ //TODO: put this into a middleware that checks if FileNotFound is thrown and returns 404 and same error message+ // for every FileNotFound error+ res.status(400).send(errorObject);+ return;+ }+ throw e;+ }+};app.post('/sync', async (req, res) => {let requestPb;@@ -43,10 +66,9 @@ app.post('/sync', async (req, res) => {return;}- let accountDb = getAccountDb();- let file_id = requestPb.getFileid() || null;- let group_id = requestPb.getGroupid() || null;- let key_id = requestPb.getKeyid() || null;+ let fileId = requestPb.getFileid() || null;+ let groupId = requestPb.getGroupid() || null;+ let keyId = requestPb.getKeyid() || null;let since = requestPb.getSince() || null;let messages = requestPb.getMessagesList();@@ -58,68 +80,27 @@ app.post('/sync', async (req, res) => {});}- let currentFiles = accountDb.all(- 'SELECT group_id, encrypt_keyid, encrypt_meta, sync_version FROM files WHERE id = ?',- [file_id],- );+ const filesService = new FilesService(getAccountDb());- if (currentFiles.length === 0) {- res.status(400);- res.send('file-not-found');- return;- }-- let currentFile = currentFiles[0];-- if (- currentFile.sync_version == null ||- currentFile.sync_version < SYNC_FORMAT_VERSION- ) {- res.status(400);- res.send('file-old-version');- return;- }-- // When resetting sync state, something went wrong. There is no- // group id and it's awaiting a file to be uploaded.- if (currentFile.group_id == null) {- res.status(400);- res.send('file-needs-upload');- return;- }+ const currentFile = verifyFileExists(+ fileId,+ filesService,+ res,+ 'file-not-found',+ );- // Check to make sure the uploaded file is valid and has been- // encrypted with the same key it is registered with (this might- // be wrong if there was an error during the key creation- // process)- let uploadedKeyId = currentFile.encrypt_meta- ? JSON.parse(currentFile.encrypt_meta).keyId- : null;- if (uploadedKeyId !== currentFile.encrypt_keyid) {- res.status(400);- res.send('file-key-mismatch');+ if (!currentFile) {return;}- // The changes being synced are part of an old group, which- // means the file has been reset. User needs to re-download.- if (group_id !== currentFile.group_id) {+ const errorMessage = validateSyncedFile(groupId, keyId, currentFile);+ if (errorMessage) {res.status(400);- res.send('file-has-reset');+ res.send(errorMessage);return;}- // The data is encrypted with a different key which is- // unacceptable. We can't accept these changes. Reject them and- // tell the user that they need to generate the correct key- // (which necessitates a sync reset so they need to re-download).- if (key_id !== currentFile.encrypt_keyid) {- res.status(400);- res.send('file-has-new-key');- return false;- }-- let { trie, newMessages } = simpleSync.sync(messages, since, group_id);+ let { trie, newMessages } = simpleSync.sync(messages, since, groupId);// encode it back...let responsePb = new SyncProtoBuf.SyncResponse();@@ -132,57 +113,70 @@ app.post('/sync', async (req, res) => {});app.post('/user-get-key', (req, res) => {- let accountDb = getAccountDb();let { fileId } = req.body;- let rows = accountDb.all(- 'SELECT encrypt_salt, encrypt_keyid, encrypt_test FROM files WHERE id = ?',- [fileId],- );- if (rows.length === 0) {- res.status(400).send('file-not-found');+ const filesService = new FilesService(getAccountDb());+ const file = verifyFileExists(fileId, filesService, res, 'file-not-found');++ if (!file) {return;}- let { encrypt_salt, encrypt_keyid, encrypt_test } = rows[0];res.send({status: 'ok',- data: { id: encrypt_keyid, salt: encrypt_salt, test: encrypt_test },+ data: {+ id: file.encryptKeyId,+ salt: file.encryptSalt,+ test: file.encryptTest,+ },});});app.post('/user-create-key', (req, res) => {- let accountDb = getAccountDb();let { fileId, keyId, keySalt, testContent } = req.body;- accountDb.mutate(- 'UPDATE files SET encrypt_salt = ?, encrypt_keyid = ?, encrypt_test = ? WHERE id = ?',- [keySalt, keyId, testContent, fileId],+ const filesService = new FilesService(getAccountDb());++ if (!verifyFileExists(fileId, filesService, res, 'file not found')) {+ return;+ }++ filesService.update(+ fileId,+ new FileUpdate({+ encryptSalt: keySalt,+ encryptKeyId: keyId,+ encryptTest: testContent,+ }),);res.send(OK_RESPONSE);});app.post('/reset-user-file', async (req, res) => {- let accountDb = getAccountDb();let { fileId } = req.body;- let files = accountDb.all('SELECT group_id FROM files WHERE id = ?', [+ const filesService = new FilesService(getAccountDb());+ const file = verifyFileExists(fileId,- ]);- if (files.length === 0) {- res.status(400).send('User or file not found');+ filesService,+ res,+ 'User or file not found',+ );++ if (!file) {return;}- let { group_id } = files[0];- accountDb.mutate('UPDATE files SET group_id = NULL WHERE id = ?', [fileId]);+ const groupId = file.groupId;++ filesService.update(fileId, new FileUpdate({ groupId: null }));- if (group_id) {+ if (groupId) {try {- await fs.unlink(getPathForGroupFile(group_id));- } catch (e) {- console.log(`Unable to delete sync data for group "${group_id}"`);+ await fs.unlink(getPathForGroupFile(groupId));+ } catch {+ console.log(`Unable to delete sync data for group "${groupId}"`);}}@@ -190,7 +184,6 @@ app.post('/reset-user-file', async (req, res) => {});app.post('/upload-user-file', async (req, res) => {- let accountDb = getAccountDb();if (typeof req.headers['x-actual-name'] !== 'string') {// FIXME: Not sure how this cannot be a string when the header is// set.@@ -215,38 +208,25 @@ app.post('/upload-user-file', async (req, res) => {? JSON.parse(encryptMeta).keyId: null;- let currentFiles = accountDb.all(- 'SELECT group_id, encrypt_keyid, encrypt_meta FROM files WHERE id = ?',- [fileId],- );- if (currentFiles.length > 0) {- let currentFile = currentFiles[0];-- // The uploading file is part of an old group, so reject- // it. All of its internal sync state is invalid because its- // old. The sync state has been reset, so user needs to- // either reset again or download from the current group.- if (groupId !== currentFile.group_id) {- res.status(400);- res.send('file-has-reset');- return;- }+ const filesService = new FilesService(getAccountDb());+ let currentFile;- // The key that the file is encrypted with is different than- // the current registered key. All data must always be- // encrypted with the registered key for consistency. Key- // changes always necessitate a sync reset, which means this- // upload is trying to overwrite another reset. That might- // be be fine, but since we definitely cannot accept a file- // encrypted with the wrong key, we bail and suggest the- // user download the latest file.- if (keyId !== currentFile.encrypt_keyid) {- res.status(400);- res.send('file-has-new-key');- return;+ try {+ currentFile = filesService.get(fileId);+ } catch (e) {+ if (e instanceof FileNotFound) {+ currentFile = null;+ } else {+ throw e;}}+ const errorMessage = validateUploadedFile(groupId, keyId, currentFile);+ if (errorMessage) {+ res.status(400).send(errorMessage);+ return;+ }+try {await fs.writeFile(getPathForUserFile(fileId), req.body);} catch (err) {@@ -255,37 +235,44 @@ app.post('/upload-user-file', async (req, res) => {return;}- let rows = accountDb.all('SELECT id FROM files WHERE id = ?', [fileId]);- if (rows.length === 0) {+ if (!currentFile) {// it's newgroupId = uuid.v4();- accountDb.mutate(- 'INSERT INTO files (id, group_id, sync_version, name, encrypt_meta) VALUES (?, ?, ?, ?, ?)',- [fileId, groupId, syncFormatVersion, name, encryptMeta],- );- res.send({ status: 'ok', groupId });- } else {- if (!groupId) {- // sync state was reset, create new group- groupId = uuid.v4();- accountDb.mutate('UPDATE files SET group_id = ? WHERE id = ?', [- groupId,- fileId,- ]);- }- // Regardless, update some properties- accountDb.mutate(- 'UPDATE files SET sync_version = ?, encrypt_meta = ?, name = ? WHERE id = ?',- [syncFormatVersion, encryptMeta, name, fileId],+ filesService.set(+ new File({+ id: fileId,+ groupId: groupId,+ syncVersion: syncFormatVersion,+ name: name,+ encryptMeta: encryptMeta,+ }),);res.send({ status: 'ok', groupId });+ return;}++ if (!groupId) {+ // sync state was reset, create new group+ groupId = uuid.v4();+ filesService.update(fileId, new FileUpdate({ groupId: groupId }));+ }++ // Regardless, update some properties+ filesService.update(+ fileId,+ new FileUpdate({+ syncVersion: syncFormatVersion,+ encryptMeta: encryptMeta,+ name: name,+ }),+ );++ res.send({ status: 'ok', groupId });});app.get('/download-user-file', async (req, res) => {- let accountDb = getAccountDb();let fileId = req.headers['x-actual-file-id'];if (typeof fileId !== 'string') {// FIXME: Not sure how this cannot be a string when the header is@@ -294,13 +281,8 @@ app.get('/download-user-file', async (req, res) => {return;}- // Do some authentication- let rows = accountDb.all(- 'SELECT id FROM files WHERE id = ? AND deleted = FALSE',- [fileId],- );- if (rows.length === 0) {- res.status(400).send('User or file not found');+ const filesService = new FilesService(getAccountDb());+ if (!verifyFileExists(fileId, filesService, res, 'User or file not found')) {return;}@@ -309,70 +291,69 @@ app.get('/download-user-file', async (req, res) => {});app.post('/update-user-filename', (req, res) => {- let accountDb = getAccountDb();let { fileId, name } = req.body;- // Do some authentication- let rows = accountDb.all(- 'SELECT id FROM files WHERE id = ? AND deleted = FALSE',- [fileId],- );- if (rows.length === 0) {- res.status(400).send('file not found');+ const filesService = new FilesService(getAccountDb());++ if (!verifyFileExists(fileId, filesService, res, 'file not found')) {return;}- accountDb.mutate('UPDATE files SET name = ? WHERE id = ?', [name, fileId]);-+ filesService.update(fileId, new FileUpdate({ name: name }));res.send(OK_RESPONSE);});app.get('/list-user-files', (req, res) => {- let accountDb = getAccountDb();- let rows = accountDb.all('SELECT * FROM files');-+ const fileService = new FilesService(getAccountDb());+ const rows = fileService.find();res.send({status: 'ok',data: rows.map((row) => ({- deleted: row.deleted,+ deleted: boolToInt(row.deleted),fileId: row.id,- groupId: row.group_id,+ groupId: row.groupId,name: row.name,- encryptKeyId: row.encrypt_keyid,+ encryptKeyId: row.encryptKeyId,})),});});app.get('/get-user-file-info', (req, res) => {- let accountDb = getAccountDb();let fileId = req.headers['x-actual-file-id'];- let rows = accountDb.all(- 'SELECT * FROM files WHERE id = ? AND deleted = FALSE',- [fileId],- );+ // TODO: Return 422 if fileId is not provided. Need to make sure frontend can handle it+ // if (!fileId) {+ // return res.status(422).send({+ // details: 'fileId-required',+ // reason: 'unprocessable-entity',+ // status: 'error',+ // });+ // }- if (rows.length === 0) {- res.status(400).send({ status: 'error', reason: 'file-not-found' });+ const fileService = new FilesService(getAccountDb());++ const file = verifyFileExists(fileId, fileService, res, {+ status: 'error',+ reason: 'file-not-found',+ });++ if (!file) {return;}- let row = rows[0];-res.send({status: 'ok',data: {- deleted: row.deleted,- fileId: row.id,- groupId: row.group_id,- name: row.name,- encryptMeta: row.encrypt_meta ? JSON.parse(row.encrypt_meta) : null,+ deleted: boolToInt(file.deleted), // FIXME: convert to boolean, make sure it works in the frontend+ fileId: file.id,+ groupId: file.groupId,+ name: file.name,+ encryptMeta: file.encryptMeta ? JSON.parse(file.encryptMeta) : null,},});});app.post('/delete-user-file', (req, res) => {- let accountDb = getAccountDb();let { fileId } = req.body;if (!fileId) {@@ -383,12 +364,12 @@ app.post('/delete-user-file', (req, res) => {});}- let rows = accountDb.all('SELECT * FROM files WHERE id = ?', [fileId]);-- if (rows.length === 0) {- return res.status(400).send('file-not-found');+ const filesService = new FilesService(getAccountDb());+ if (!verifyFileExists(fileId, filesService, res, 'file-not-found')) {+ return;}- accountDb.mutate('UPDATE files SET deleted = TRUE WHERE id = ?', [fileId]);+ filesService.update(fileId, new FileUpdate({ deleted: true }));+res.send(OK_RESPONSE);});diff --git a/src/app-sync.test.js b/src/app-sync.test.jsindex 4c3328abd5..b81d5f1446 100644--- a/src/app-sync.test.js+++ b/src/app-sync.test.js@@ -67,6 +67,54 @@ describe('/user-create-key', () => {status: 'error',});});++ it('returns 400 if the file is not found', async () => {+ const res = await request(app)+ .post('/user-create-key')+ .set('x-actual-token', 'valid-token')+ .send({ fileId: 'non-existent-file-id' });++ expect(res.statusCode).toEqual(400);+ expect(res.text).toBe('file not found');+ });++ it('creates a new encryption key for the file', async () => {+ const fileId = crypto.randomBytes(16).toString('hex');++ const old_encrypt_salt = 'old-salt';+ const old_encrypt_keyid = 'old-key';+ const old_encrypt_test = 'old-encrypt-test';+ const encrypt_salt = 'test-salt';+ const encrypt_keyid = 'test-key-id';+ const encrypt_test = 'test-encrypt-test';++ getAccountDb().mutate(+ 'INSERT INTO files (id, encrypt_salt, encrypt_keyid, encrypt_test) VALUES (?, ?, ?, ?)',+ [fileId, old_encrypt_salt, old_encrypt_keyid, old_encrypt_test],+ );++ const res = await request(app)+ .post('/user-create-key')+ .set('x-actual-token', 'valid-token')+ .send({+ fileId,+ keyId: encrypt_keyid,+ keySalt: encrypt_salt,+ testContent: encrypt_test,+ });++ expect(res.statusCode).toEqual(200);+ expect(res.body).toEqual({ status: 'ok' });++ const rows = getAccountDb().all(+ 'SELECT encrypt_salt, encrypt_keyid, encrypt_test FROM files WHERE id = ?',+ [fileId],+ );++ expect(rows[0].encrypt_salt).toEqual(encrypt_salt);+ expect(rows[0].encrypt_keyid).toEqual(encrypt_keyid);+ expect(rows[0].encrypt_test).toEqual(encrypt_test);+ });});describe('/reset-user-file', () => {@@ -104,7 +152,7 @@ describe('/reset-user-file', () => {fileId,]);- expect(rows[0].group_id).toBeNull;+ expect(rows[0].group_id).toBeNull();});it('returns 400 if the file is not found', async () => {diff --git a/src/app-sync/errors.js b/src/app-sync/errors.jsnew file mode 100644index 0000000000..b57867cdda--- /dev/null+++ b/src/app-sync/errors.js@@ -0,0 +1,13 @@+export class FileNotFound extends Error {+ constructor(params = {}) {+ super("File does not exist or you don't have access to it");+ this.details = params;+ }+}++export class GenericFileError extends Error {+ constructor(message, params = {}) {+ super(message);+ this.details = params;+ }+}diff --git a/src/app-sync/services/files-service.js b/src/app-sync/services/files-service.jsnew file mode 100644index 0000000000..63fe7f373b--- /dev/null+++ b/src/app-sync/services/files-service.js@@ -0,0 +1,197 @@+import getAccountDb from '../../account-db.js';+import { FileNotFound, GenericFileError } from '../errors.js';++class FileBase {+ constructor(+ name,+ groupId,+ encryptSalt,+ encryptTest,+ encryptKeyId,+ encryptMeta,+ syncVersion,+ deleted,+ ) {+ this.name = name;+ this.groupId = groupId;+ this.encryptSalt = encryptSalt;+ this.encryptTest = encryptTest;+ this.encryptKeyId = encryptKeyId;+ this.encryptMeta = encryptMeta;+ this.syncVersion = syncVersion;+ this.deleted = typeof deleted === 'boolean' ? deleted : Boolean(deleted);+ }+}++class File extends FileBase {+ constructor({+ id,+ name = null,+ groupId = null,+ encryptSalt = null,+ encryptTest = null,+ encryptKeyId = null,+ encryptMeta = null,+ syncVersion = null,+ deleted = false,+ }) {+ super(+ name,+ groupId,+ encryptSalt,+ encryptTest,+ encryptKeyId,+ encryptMeta,+ syncVersion,+ deleted,+ );+ this.id = id;+ }+}++/**+ * Represents a file update. Will only update the fields that are defined.+ * @class+ * @extends FileBase+ */+class FileUpdate extends FileBase {+ constructor({+ name = undefined,+ groupId = undefined,+ encryptSalt = undefined,+ encryptTest = undefined,+ encryptKeyId = undefined,+ encryptMeta = undefined,+ syncVersion = undefined,+ deleted = undefined,+ }) {+ super(+ name,+ groupId,+ encryptSalt,+ encryptTest,+ encryptKeyId,+ encryptMeta,+ syncVersion,+ deleted,+ );+ }+}++const boolToInt = (bool) => {+ return bool ? 1 : 0;+};++class FilesService {+ constructor(accountDb) {+ this.accountDb = accountDb;+ }++ get(fileId) {+ const rawFile = this.getRaw(fileId);+ if (!rawFile || (rawFile && rawFile.deleted)) {+ throw new FileNotFound();+ }++ return this.validate(rawFile);+ }++ set(file) {+ const deletedInt = boolToInt(file.deleted);+ this.accountDb.mutate(+ 'INSERT INTO files (id, group_id, sync_version, name, encrypt_meta, encrypt_salt, encrypt_test, encrypt_keyid, deleted) VALUES (?, ?, ?, ?, ?, ?, ?,? ,?)',+ [+ file.id,+ file.groupId,+ file.syncVersion.toString(),+ file.name,+ file.encryptMeta,+ file.encryptSalt,+ file.encrypt_test,+ file.encrypt_keyid,+ deletedInt,+ ],+ );+ }++ find(limit = 1000) {+ return this.accountDb+ .all('SELECT * FROM files WHERE deleted = 0 LIMIT ?', [limit])+ .map(this.validate);+ }++ update(id, fileUpdate) {+ let query = 'UPDATE files SET';+ const params = [];+ const updates = [];++ if (fileUpdate.name !== undefined) {+ updates.push('name = ?');+ params.push(fileUpdate.name);+ }+ if (fileUpdate.groupId !== undefined) {+ updates.push('group_id = ?');+ params.push(fileUpdate.groupId);+ }+ if (fileUpdate.encryptSalt !== undefined) {+ updates.push('encrypt_salt = ?');+ params.push(fileUpdate.encryptSalt);+ }+ if (fileUpdate.encryptTest !== undefined) {+ updates.push('encrypt_test = ?');+ params.push(fileUpdate.encryptTest);+ }+ if (fileUpdate.encryptKeyId !== undefined) {+ updates.push('encrypt_keyid = ?');+ params.push(fileUpdate.encryptKeyId);+ }+ if (fileUpdate.encryptMeta !== undefined) {+ updates.push('encrypt_meta = ?');+ params.push(fileUpdate.encryptMeta);+ }+ if (fileUpdate.syncVersion !== undefined) {+ updates.push('sync_version = ?');+ params.push(fileUpdate.syncVersion);+ }+ if (fileUpdate.deleted !== undefined) {+ updates.push('deleted = ?');+ params.push(boolToInt(fileUpdate.deleted));+ }++ if (updates.length > 0) {+ query += ' ' + updates.join(', ') + ' WHERE id = ?';+ params.push(id);++ const res = this.accountDb.mutate(query, params);++ if (res.changes != 1) {+ throw new GenericFileError('Could not update File', { id });+ }+ }++ // Return the modified object+ return this.validate(this.getRaw(id));+ }++ getRaw(fileId) {+ return this.accountDb.first(`SELECT * FROM files WHERE id = ?`, [fileId]);+ }++ validate(rawFile) {+ return new File({+ id: rawFile.id,+ name: rawFile.name,+ groupId: rawFile.group_id,+ encryptSalt: rawFile.encrypt_salt,+ encryptTest: rawFile.encrypt_test,+ encryptKeyId: rawFile.encrypt_keyid,+ encryptMeta: rawFile.encrypt_meta,+ syncVersion: rawFile.sync_version,+ deleted: Boolean(rawFile.deleted),+ });+ }+}++const filesService = new FilesService(getAccountDb());++export { filesService, FilesService, File, FileUpdate };diff --git a/src/app-sync/tests/services/files-service.test.js b/src/app-sync/tests/services/files-service.test.jsnew file mode 100644index 0000000000..7d3dadd6d1--- /dev/null+++ b/src/app-sync/tests/services/files-service.test.js@@ -0,0 +1,214 @@+import getAccountDb from '../../../account-db.js';+import { FileNotFound } from '../../errors.js';+import {+ FilesService,+ File,+ FileUpdate,+} from '../../services/files-service.js'; // Adjust the path as necessary+import crypto from 'node:crypto';+describe('FilesService', () => {+ let filesService;+ let accountDb;++ const insertToyExampleData = () => {+ accountDb.mutate(+ 'INSERT INTO files (id, group_id, sync_version, name, encrypt_meta, encrypt_salt, encrypt_test, encrypt_keyid, deleted) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',+ [+ '1',+ 'group1',+ 1,+ 'file1',+ '{"key":"value"}',+ 'salt',+ 'test',+ 'keyid',+ 0,+ ],+ );+ };++ const clearDatabase = () => {+ accountDb.mutate('DELETE FROM files');+ };++ beforeAll((done) => {+ accountDb = getAccountDb();+ filesService = new FilesService(accountDb);+ done();+ });++ beforeEach((done) => {+ insertToyExampleData();+ done();+ });++ afterEach((done) => {+ clearDatabase();+ done();+ });++ test('get should return a file', () => {+ const file = filesService.get('1');+ const expectedFile = new File({+ id: '1',+ groupId: 'group1',+ syncVersion: 1,+ name: 'file1',+ encryptMeta: '{"key":"value"}',+ encryptSalt: 'salt',+ encryptTest: 'test',+ encryptKeyId: 'keyid',+ deleted: false,+ });++ expect(file).toEqual(expectedFile);+ });++ test('get should throw FileNotFound if file is deleted or does not exist', () => {+ const fileId = crypto.randomBytes(16).toString('hex');+ accountDb.mutate(+ 'INSERT INTO files (id, group_id, sync_version, name, encrypt_meta, encrypt_salt, encrypt_test, encrypt_keyid, deleted) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',+ [+ fileId,+ 'group1',+ 1,+ 'file1',+ '{"key":"value"}',+ 'salt',+ 'test',+ 'keyid',+ 1,+ ],+ );++ expect(() => {+ filesService.get(fileId);+ }).toThrow(FileNotFound);++ expect(() => {+ filesService.get(crypto.randomBytes(16).toString('hex'));+ }).toThrow(FileNotFound);+ });++ test.each([true, false])(+ 'set should insert a new file with deleted: %p',+ (deleted) => {+ const fileId = crypto.randomBytes(16).toString('hex');+ const newFile = new File({+ id: fileId,+ groupId: 'group2',+ syncVersion: 1,+ name: 'file2',+ encryptMeta: '{"key":"value2"}',+ deleted: deleted,+ });++ filesService.set(newFile);++ const file = filesService.validate(filesService.getRaw(fileId));+ const expectedFile = new File({+ id: fileId,+ groupId: 'group2',+ syncVersion: 1,+ name: 'file2',+ encryptMeta: '{"key":"value2"}',+ encryptSalt: null, // default value+ encryptTest: null, // default value+ encryptKeyId: null, // default value+ deleted: deleted,+ });++ expect(file).toEqual(expectedFile);+ },+ );++ test('find should return a list of files', () => {+ const files = filesService.find();+ expect(files.length).toBe(1);+ expect(files[0]).toEqual(+ new File({+ id: '1',+ groupId: 'group1',+ syncVersion: 1,+ name: 'file1',+ encryptMeta: '{"key":"value"}',+ encryptSalt: 'salt',+ encryptTest: 'test',+ encryptKeyId: 'keyid',+ deleted: false,+ }),+ );+ });++ test('find should respect the limit parameter', () => {+ filesService.set(+ new File({+ id: crypto.randomBytes(16).toString('hex'),+ groupId: 'group2',+ syncVersion: 1,+ name: 'file2',+ encryptMeta: '{"key":"value2"}',+ deleted: false,+ }),+ );+ // Make sure that the file was inserted+ const allFiles = filesService.find();+ expect(allFiles.length).toBe(2);++ // Limit the number of files returned+ const limitedFiles = filesService.find(1);+ expect(limitedFiles.length).toBe(1);+ });++ test('update should modify all attributes of an existing file', () => {+ const fileUpdate = new FileUpdate({+ name: 'updatedFile1',+ groupId: 'updatedGroup1',+ encryptSalt: 'updatedSalt',+ encryptTest: 'updatedTest',+ encryptKeyId: 'updatedKeyId',+ encryptMeta: '{"key":"updatedValue"}',+ syncVersion: 2,+ deleted: true,+ });+ const updatedFile = filesService.update('1', fileUpdate);++ expect(updatedFile).toEqual(+ new File({+ id: '1',+ name: 'updatedFile1',+ groupId: 'updatedGroup1',+ encryptSalt: 'updatedSalt',+ encryptTest: 'updatedTest',+ encryptMeta: '{"key":"updatedValue"}',+ encryptKeyId: 'updatedKeyId',+ syncVersion: 2,+ deleted: true,+ }),+ );+ });++ test.each([['update-group', null]])(+ 'update should modify a single attribute with groupId = $groupId',+ (newGroupId) => {+ const fileUpdate = new FileUpdate({+ groupId: newGroupId,+ });+ const updatedFile = filesService.update('1', fileUpdate);++ expect(updatedFile).toEqual(+ new File({+ id: '1',+ name: 'file1',+ groupId: newGroupId,+ syncVersion: 1,+ encryptMeta: '{"key":"value"}',+ encryptSalt: 'salt',+ encryptTest: 'test',+ encryptKeyId: 'keyid',+ deleted: false,+ }),+ );+ },+ );+});diff --git a/src/app-sync/validation.js b/src/app-sync/validation.jsnew file mode 100644index 0000000000..4fe3fff2c0--- /dev/null+++ b/src/app-sync/validation.js@@ -0,0 +1,77 @@+// This is a version representing the internal format of sync+// messages. When this changes, all sync files need to be reset. We+// will check this version when syncing and notify the user if they+// need to reset.+const SYNC_FORMAT_VERSION = 2;++const validateSyncedFile = (groupId, keyId, currentFile) => {+ if (+ currentFile.syncVersion == null ||+ currentFile.syncVersion < SYNC_FORMAT_VERSION+ ) {+ return 'file-old-version';+ }++ // When resetting sync state, something went wrong. There is no+ // group id and it's awaiting a file to be uploaded.+ if (currentFile.groupId == null) {+ return 'file-needs-upload';+ }++ // Check to make sure the uploaded file is valid and has been+ // encrypted with the same key it is registered with (this might+ // be wrong if there was an error during the key creation+ // process)+ let uploadedKeyId = currentFile.encryptMeta+ ? JSON.parse(currentFile.encryptMeta).keyId+ : null;+ if (uploadedKeyId !== currentFile.encryptKeyId) {+ return 'file-key-mismatch';+ }++ // The changes being synced are part of an old group, which+ // means the file has been reset. User needs to re-download.+ if (groupId !== currentFile.groupId) {+ return 'file-has-reset';+ }++ // The data is encrypted with a different key which is+ // unacceptable. We can't accept these changes. Reject them and+ // tell the user that they need to generate the correct key+ // (which necessitates a sync reset so they need to re-download).+ if (keyId !== currentFile.encryptKeyId) {+ return 'file-has-new-key';+ }++ return null;+};++const validateUploadedFile = (groupId, keyId, currentFile) => {+ if (!currentFile) {+ // File is new, so no need to validate+ return null;+ }+ // The uploading file is part of an old group, so reject+ // it. All of its internal sync state is invalid because its+ // old. The sync state has been reset, so user needs to+ // either reset again or download from the current group.+ if (groupId !== currentFile.groupId) {+ return 'file-has-reset';+ }++ // The key that the file is encrypted with is different than+ // the current registered key. All data must always be+ // encrypted with the registered key for consistency. Key+ // changes always necessitate a sync reset, which means this+ // upload is trying to overwrite another reset. That might+ // be be fine, but since we definitely cannot accept a file+ // encrypted with the wrong key, we bail and suggest the+ // user download the latest file.+ if (keyId !== currentFile.encryptKeyId) {+ return 'file-has-new-key';+ }++ return null;+};++export { validateSyncedFile, validateUploadedFile };diff --git a/src/util/middlewares.js b/src/util/middlewares.jsindex 14e6e4dc14..5bbec6946a 100644--- a/src/util/middlewares.js+++ b/src/util/middlewares.js@@ -7,10 +7,22 @@ import * as expressWinston from 'express-winston';* @param {Error} err* @param {import('express').Request} req* @param {import('express').Response} res- * @param {import('express').NextFunction} _next+ * @param {import('express').NextFunction} next*/-async function errorMiddleware(err, req, res, _next) {- console.log('ERROR', err);+async function errorMiddleware(err, req, res, next) {+ if (res.headersSent) {+ // If you call next() with an error after you have started writing the response+ // (for example, if you encounter an error while streaming the response+ // to the client), the Express default error handler closes+ // the connection and fails the request.++ // So when you add a custom error handler, you must delegate+ // to the default Express error handler, when the headers+ // have already been sent to the client+ // Source: https://expressjs.com/en/guide/error-handling.html+ return next(err);+ }+ console.log(`Error on endpoint ${req.url}`, err.message, err.stack);res.status(500).send({ status: 'error', reason: 'internal-error' });}diff --git a/upcoming-release-notes/432.md b/upcoming-release-notes/432.mdnew file mode 100644index 0000000000..4eb7e7d639--- /dev/null+++ b/upcoming-release-notes/432.md@@ -0,0 +1,6 @@+---+category: Maintenance+authors: [tcrasset]+---++Integrate FileService for app-sync.js\ No newline at end of file