A local-first personal finance app
// @ts-strict-ignore
import { getClock, Timestamp } from '@actual-app/crdt';
import * as db from '#server/db';
import * as prefs from '#server/prefs';
import * as sheet from '#server/sheet';
import * as mockSyncServer from '#server/tests/mockSyncServer';
import { q } from '#shared/query';
import * as encoder from './encoder';
import { isError } from './utils';
import {
addSyncListener,
applyMessages,
fullSync,
receiveMessages,
sendMessages,
setSyncingMode,
} from './index';
beforeEach(() => {
mockSyncServer.reset();
setSyncingMode('enabled');
return global.emptyDatabase()();
});
afterEach(() => {
global.resetTime();
setSyncingMode('disabled');
});
describe('Sync', () => {
it('should send messages to the server', async () => {
void prefs.loadPrefs();
void prefs.savePrefs({ groupId: 'group' });
let timestamp = Timestamp.send();
await sendMessages([
{
dataset: 'transactions',
row: 'foo',
column: 'amount',
value: 3200,
timestamp,
},
]);
global.stepForwardInTime();
timestamp = Timestamp.send();
await sendMessages([
{
dataset: 'transactions',
row: 'foo',
column: 'amount',
value: 4200,
timestamp,
},
]);
expect(getClock().timestamp.toString()).toEqual(timestamp.toString());
expect(mockSyncServer.getClock().merkle).toEqual(getClock().merkle);
expect(
await db.all<db.DbCrdtMessage>('SELECT * FROM messages_crdt'),
).toMatchSnapshot();
expect(
await db.all<db.DbClockMessage>('SELECT * FROM messages_clock'),
).toMatchSnapshot();
});
it('should resend old messages to the server', async () => {
void prefs.loadPrefs();
void prefs.savePrefs({ groupId: 'group' });
global.stepForwardInTime(Date.parse('2018-11-13T13:20:00.000Z'));
await applyMessages([
global.stepForwardInTime() || {
dataset: 'transactions',
row: 'foo',
column: 'amount',
value: 3200,
timestamp: Timestamp.send(),
},
global.stepForwardInTime() || {
dataset: 'transactions',
row: 'foo',
column: 'amount',
value: 3200,
timestamp: Timestamp.send(),
},
]);
// Move the clock forward so that the above 2 messages are not
// automatically sent out, but will need to be re-sent by way of
// the merkle tree
void prefs.savePrefs({
lastSyncedTimestamp: getClock().timestamp.toString(),
});
expect(mockSyncServer.getMessages().length).toBe(0);
const result = await fullSync();
if (isError(result)) throw result.error;
expect(result.messages.length).toBe(0);
expect(mockSyncServer.getMessages().length).toBe(2);
});
it('should sync multiple clients', async () => {
void prefs.loadPrefs();
void prefs.savePrefs({
groupId: 'group',
lastSyncedTimestamp: Timestamp.zero.toString(),
});
await mockSyncServer.handlers['/sync/sync'](
await encoder.encode(
'group',
'client',
Timestamp.parse('1970-01-01T01:17:37.000Z-0000-0000testinguuid2'),
[
{
dataset: 'transactions',
row: 'foo',
column: 'amount',
value: 'N:3200',
timestamp: Timestamp.parse(
'1970-01-02T05:17:36.789Z-0000-0000testinguuid2',
),
},
{
dataset: 'transactions',
row: 'foo',
column: 'amount',
value: 'N:4200',
timestamp: Timestamp.parse(
'1970-01-02T10:17:36.999Z-0000-0000testinguuid2',
),
},
],
),
);
await applyMessages([
global.stepForwardInTime(Date.parse('1970-01-03T10:17:37.000Z')) || {
dataset: 'transactions',
row: 'foo',
column: 'amount',
value: 5000,
timestamp: Timestamp.send(),
},
]);
const result = await fullSync();
if (isError(result)) throw result.error;
expect(result.messages.length).toBe(2);
expect(mockSyncServer.getMessages().length).toBe(3);
});
it('should advance the clock once per received batch', async () => {
// `Date.now()` is frozen here, standing in for the 100ms rounding Firefox
// applies under resistFingerprinting. The counter only resets when
// `Date.now()` advances, so one tick per message overflowed its 16 bits
// once a sync carried more than 65535 of them.
const node = '0000testinguuid2';
const base = Date.parse('1970-01-02T00:00:00.000Z');
const messages = [];
for (let i = 0; i < 100; i++) {
messages.push({
dataset: 'transactions',
row: 'foo',
column: 'amount',
value: 'N:1',
timestamp: Timestamp.parse(
`${new Date(base + i).toISOString()}-0000-${node}`,
),
});
}
await receiveMessages(messages);
expect(getClock().timestamp.counter()).toBe(0);
// The clock must still sort above every timestamp it received.
const latest = messages[messages.length - 1].timestamp.toString();
expect(getClock().timestamp.toString() > latest).toBe(true);
});
it('should reject a batch holding a far-future timestamp', async () => {
// `toISOString` writes years above 9999 as `+010000-…`, which sorts below
// every normal timestamp. The batch maximum must not rank it last, which
// would let it skip the clock drift check.
const message = (millis: number) => ({
dataset: 'transactions',
row: 'foo',
column: 'amount',
value: 'N:1',
timestamp: Timestamp.parse(
`${new Date(millis).toISOString()}-0000-0000testinguuid2`,
),
});
await expect(
receiveMessages([
message(Date.parse('1970-01-02T00:00:00.000Z')),
message(253402300800000),
]),
).rejects.toMatchObject({ reason: 'clock-drift' });
});
});
describe('receiveMessages', () => {
function conflictingMessages() {
const timestamp = new Timestamp(Date.now() + 5000, 0, '0000000000000001');
// Duplicate timestamps make the batch fail inside the database transaction.
return ['foo', 'bar'].map(row => ({
dataset: 'transactions',
row,
column: 'amount',
value: 3200,
timestamp,
}));
}
it('discards failed clock advances before receiving the next queued batch', async () => {
const messages = conflictingMessages();
const failedBatch = receiveMessages(messages);
const nextBatch = receiveMessages([
{
...messages[0],
timestamp: new Timestamp(Date.now() + 1000, 0, '0000000000000002'),
},
]);
await expect(failedBatch).rejects.toThrow(/UNIQUE constraint failed/);
await expect(nextBatch).resolves.toHaveLength(1);
expect(getClock().timestamp.millis()).toBe(Date.now() + 1000);
expect(getClock().timestamp.counter()).toBe(1);
});
it('does not reuse timestamps issued while a failed receive is pending', async () => {
const failedBatch = receiveMessages(conflictingMessages());
const issued = Timestamp.send();
await expect(failedBatch).rejects.toThrow(/UNIQUE constraint failed/);
expect(Timestamp.send().toString() > issued.toString()).toBe(true);
});
it('keeps the received clock when a step after the commit fails', async () => {
const [message] = conflictingMessages();
const removeListener = addSyncListener(() => {
throw new Error('listener failed');
});
try {
await expect(receiveMessages([message])).rejects.toThrow(
'listener failed',
);
} finally {
removeListener();
}
expect(getClock().timestamp.millis()).toBe(message.timestamp.millis());
});
});
function registerBudgetMonths(months) {
const createdMonths = new Set();
for (const month of months) {
createdMonths.add(month);
}
sheet.get().meta().createdMonths = months;
}
async function asSecondClient(func) {
void prefs.loadPrefs();
void prefs.savePrefs({
groupId: 'group',
lastSyncedTimestamp: Timestamp.zero.toString(),
});
await func();
await global.emptyDatabase()();
void prefs.savePrefs({
groupId: 'group',
lastSyncedTimestamp: Timestamp.zero.toString(),
});
}
function expectCellToExist(sheetName, name) {
const value = sheet.get().getCellValueLoose(sheetName, name);
expect(value).not.toBe(null);
}
function expectCellNotToExist(sheetName, name, voided?: boolean) {
const value = sheet.get().getCellValueLoose(sheetName, name);
expect(value).toBe(voided ? 0 : null);
}
describe('Sync projections', () => {
test('synced categories should have budgets created', async () => {
let groupId, fooId, barId;
await asSecondClient(async () => {
await sheet.loadSpreadsheet(db);
groupId = await db.insertCategoryGroup({ id: 'group1', name: 'group1' });
fooId = await db.insertCategory({ name: 'foo', cat_group: 'group1' });
barId = await db.insertCategory({ name: 'bar', cat_group: 'group1' });
});
await sheet.loadSpreadsheet(db);
registerBudgetMonths(['2017-01', '2017-02']);
expectCellNotToExist('budget201701', 'sum-amount-' + fooId);
expectCellNotToExist('budget201701', 'sum-amount-' + barId);
expectCellNotToExist('budget201701', 'group-sum-amount-' + barId);
await fullSync();
// Make sure the budget cells have been created
expectCellToExist('budget201701', 'sum-amount-' + fooId);
expectCellToExist('budget201702', 'sum-amount-' + fooId);
expectCellToExist('budget201701', 'sum-amount-' + barId);
expectCellToExist('budget201702', 'sum-amount-' + barId);
expectCellToExist('budget201701', 'group-sum-amount-' + groupId);
expectCellToExist('budget201702', 'group-sum-amount-' + groupId);
});
test('creating and deleting categories in same sync', async () => {
// It should work when the client creates a category and deletes
// it in the same sync (should do nothing)
let fooId;
await asSecondClient(async () => {
await sheet.loadSpreadsheet(db);
await db.insertCategoryGroup({ id: 'group1', name: 'group1' });
fooId = await db.insertCategory({ name: 'foo', cat_group: 'group1' });
await db.deleteCategory({ id: fooId });
});
await sheet.loadSpreadsheet(db);
registerBudgetMonths(['2017-01', '2017-02']);
expectCellNotToExist('budget201701', 'sum-amount-' + fooId);
await fullSync();
expectCellNotToExist('budget201701', 'sum-amount-' + fooId);
});
test('synced categories should have budgets deleted', async () => {
let fooId;
await asSecondClient(async () => {
await sheet.loadSpreadsheet(db);
await db.insertCategoryGroup({
id: 'group1',
name: 'group1',
});
fooId = await db.insertCategory({ name: 'foo', cat_group: 'group1' });
await db.deleteCategory({ id: fooId });
});
await sheet.loadSpreadsheet(db);
registerBudgetMonths(['2017-01', '2017-02']);
// Get all the messages. We'll apply them in two passes
const messages = mockSyncServer.getMessages();
// Apply all but the last message (which deletes the category)
await applyMessages(messages.slice(0, -1));
expect((await db.getCategories()).length).toBe(1);
expectCellToExist('budget201701', 'sum-amount-' + fooId);
// Apply the last message and make sure it deleted the appropriate
// budget cells
await applyMessages([messages[messages.length - 1]]);
expect((await db.getCategories()).length).toBe(0);
expectCellNotToExist('budget201701', 'sum-amount-' + fooId, true);
});
test('creating and deleting groups in same sync', async () => {
// It should work when the client creates a category and deletes
// it in the same sync (should do nothing)
let groupId;
await asSecondClient(async () => {
await sheet.loadSpreadsheet(db);
groupId = await db.insertCategoryGroup({ id: 'group1', name: 'group1' });
await db.deleteCategoryGroup({ id: groupId });
});
await sheet.loadSpreadsheet(db);
registerBudgetMonths(['2017-01', '2017-02']);
expectCellNotToExist('budget201701', 'group-sum-amount-' + groupId);
await fullSync();
expectCellNotToExist('budget201701', 'group-sum-amount-' + groupId);
});
test('synced groups should have budgets deleted', async () => {
// Create the message list which creates categories and groups and
// then deletes them. Go ahead and include a category deletion in
// the same pass to make sure that works.
let groupId, fooId;
await asSecondClient(async () => {
await sheet.loadSpreadsheet(db);
groupId = await db.insertCategoryGroup({
id: 'group1',
name: 'group1',
});
fooId = await db.insertCategory({ name: 'foo', cat_group: 'group1' });
await db.deleteCategory({ id: fooId });
await db.deleteCategoryGroup({ id: groupId });
});
await sheet.loadSpreadsheet(db);
registerBudgetMonths(['2017-01', '2017-02']);
// Get all the messages. We'll apply them in two passes
const messages = mockSyncServer.getMessages();
const firstMessages = messages.filter(m => m.column !== 'tombstone');
const secondMessages = messages.filter(m => m.column === 'tombstone');
// Apply all the good messages
await applyMessages(firstMessages);
expect((await db.getCategories()).length).toBe(1);
expect((await db.getCategoriesGrouped()).length).toBe(1);
expectCellToExist('budget201701', 'sum-amount-' + fooId);
expectCellToExist('budget201701', 'group-sum-amount-' + groupId);
// Apply the messages that deletes it
await applyMessages(secondMessages);
expect((await db.getCategories()).length).toBe(0);
expect((await db.getCategoriesGrouped()).length).toBe(0);
expectCellNotToExist('budget201701', 'sum-amount-' + fooId, true);
expectCellNotToExist('budget201701', 'group-sum-amount-' + groupId, true);
});
test('categories should update the budget when moved', async () => {
let groupId, fooId;
await asSecondClient(async () => {
await sheet.loadSpreadsheet(db);
groupId = await db.insertCategoryGroup({ id: 'group1', name: 'group1' });
await db.insertCategoryGroup({ id: 'group2', name: 'group2' });
fooId = await db.insertCategory({ name: 'foo', cat_group: 'group1' });
await db.moveCategory(fooId, 'group2', null);
});
await sheet.loadSpreadsheet(db);
registerBudgetMonths(['2017-01', '2017-02']);
// Get all the messages. We'll apply them in two passes
const messages = mockSyncServer.getMessages();
const firstMessages = messages.slice(0, -2);
const secondMessages = messages.slice(-2);
// Apply all the good messages
await applyMessages(firstMessages);
const [cat] = await db.getCategories();
expect(cat.cat_group).toBe('group1');
expectCellToExist('budget201701', 'group-sum-amount-' + groupId);
// Apply the messages that deletes it
await applyMessages(secondMessages);
});
});
describe('Sync account balance cells', () => {
test('recomputes account totals and group subtotals when transactions sync', async () => {
void prefs.loadPrefs();
void prefs.savePrefs({ groupId: 'group' });
await sheet.loadSpreadsheet(db);
const spreadsheet = sheet.get();
function sumOf(filter: Record<string, unknown>) {
return q('transactions')
.filter(filter)
.calculate({ $sum: '$amount' })
.serialize();
}
spreadsheet.createQuery(
'__global',
'onbudget-accounts-balance',
sumOf({ 'account.offbudget': false }),
);
spreadsheet.createQuery(
'__global',
'account-group-balance-g1-on',
sumOf({ 'account.account_group_id': 'g1', 'account.offbudget': false }),
);
spreadsheet.createQuery(
'__global',
'balance-acct1',
sumOf({ account: 'acct1' }),
);
await sheet.waitOnSpreadsheet();
const recompute = vi.spyOn(spreadsheet, 'recompute');
await applyMessages([
global.stepForwardInTime() || {
dataset: 'transactions',
row: 'foo',
column: 'amount',
value: 3200,
timestamp: Timestamp.send(),
},
]);
const recomputed = recompute.mock.calls.map(([name]) => name);
expect(recomputed).toContain('__global!onbudget-accounts-balance');
expect(recomputed).toContain('__global!account-group-balance-g1-on');
expect(recomputed).not.toContain('__global!balance-acct1');
});
});