A local-first personal finance app
chore: add types to `crdt` package (#1076)
Tom Frenche4ec5b3eb179
diff --git a/packages/crdt/src/crdt/index.ts b/packages/crdt/src/crdt/index.tsindex 18ad17bf14..a42b13ead3 100644--- a/packages/crdt/src/crdt/index.ts+++ b/packages/crdt/src/crdt/index.ts@@ -8,5 +8,6 @@ export {makeClientId,serializeClock,deserializeClock,+ Clock,Timestamp,} from './timestamp';diff --git a/packages/crdt/src/crdt/merkle.test.ts b/packages/crdt/src/crdt/merkle.test.tsindex cf7828a1b7..a0b7d98a68 100644--- a/packages/crdt/src/crdt/merkle.test.ts+++ b/packages/crdt/src/crdt/merkle.test.ts@@ -17,7 +17,7 @@ function insertMessages(trie, messages) {describe('merkle trie', () => {test('adding an item works', () => {let trie = merkle.insert(- {},+ merkle.emptyTrie(),Timestamp.parse('2018-11-12T13:21:40.122Z-0000-0123456789ABCDEF'),);trie = merkle.insert(@@ -28,8 +28,8 @@ describe('merkle trie', () => {});test('diff returns the correct time difference', () => {- let trie1: { hash?: unknown } = {};- let trie2: { hash?: unknown } = {};+ let trie1 = merkle.emptyTrie();+ let trie2 = merkle.emptyTrie();const messages = [// First client messages@@ -65,9 +65,9 @@ describe('merkle trie', () => {});test('diffing works with empty tries', () => {- let trie1 = {};+ let trie1 = merkle.emptyTrie();let trie2 = merkle.insert(- {},+ merkle.emptyTrie(),Timestamp.parse('2009-01-02T10:17:37.789Z-0000-0000testinguuid1'),);@@ -90,7 +90,7 @@ describe('merkle trie', () => {message('2018-11-01T02:37:00.000Z-0000-0123456789ABCDEF', 2100),];- let trie: { hash?: unknown } = {};+ let trie = merkle.emptyTrie();messages.forEach(msg => {trie = merkle.insert(trie, msg.timestamp);});@@ -122,10 +122,10 @@ describe('merkle trie', () => {// Case 0: It always returns a base time when comparing with an// empty trie- expect(new Date(merkle.diff({}, trie)).toISOString()).toBe(+ expect(new Date(merkle.diff(merkle.emptyTrie(), trie)).toISOString()).toBe('1970-01-01T00:00:00.000Z',);- expect(new Date(merkle.diff(trie, {})).toISOString()).toBe(+ expect(new Date(merkle.diff(trie, merkle.emptyTrie())).toISOString()).toBe('1970-01-01T00:00:00.000Z',);diff --git a/packages/crdt/src/crdt/merkle.ts b/packages/crdt/src/crdt/merkle.tsindex 48df9b0efe..d77b4f8077 100644--- a/packages/crdt/src/crdt/merkle.ts+++ b/packages/crdt/src/crdt/merkle.ts@@ -7,11 +7,27 @@// * Need to check to make sure if account exists when handling// * transaction changes in syncing-export function getKeys(trie) {- return Object.keys(trie).filter(x => x !== 'hash');+import { Timestamp } from './timestamp';++/**+ * Represents a node within a trinary radix trie.+ */+export type TrieNode = {+ '0'?: TrieNode;+ '1'?: TrieNode;+ '2'?: TrieNode;+ hash: number;+};++export function emptyTrie(): TrieNode {+ return { hash: 0 };}-export function keyToTimestamp(key) {+export function getKeys(trie: TrieNode): ('0' | '1' | '2')[] {+ return Object.keys(trie).filter(x => x !== 'hash') as ('0' | '1' | '2')[];+}++export function keyToTimestamp(key: string): number {// 16 is the length of the base 3 value of the current time in// minutes. Ensure it's padded to create the full valuelet fullkey = key + '0'.repeat(16 - key.length);@@ -20,7 +36,10 @@ export function keyToTimestamp(key) {return parseInt(fullkey, 3) * 1000 * 60;}-export function insert(trie, timestamp) {+/**+ * Mutates `trie` to insert a node at `timestamp`+ */+export function insert(trie: TrieNode, timestamp: Timestamp) {let hash = timestamp.hash();let key = Number(Math.floor(timestamp.millis() / 1000 / 60)).toString(3);@@ -28,7 +47,7 @@ export function insert(trie, timestamp) {return insertKey(trie, key, hash);}-function insertKey(trie, key, hash) {+function insertKey(trie: TrieNode, key: string, hash: number) {if (key.length === 0) {return trie;}@@ -41,15 +60,15 @@ function insertKey(trie, key, hash) {});}-export function build(timestamps) {- let trie = {};+export function build(timestamps: Timestamp[]) {+ let trie = emptyTrie();for (let timestamp of timestamps) {insert(trie, timestamp);}return trie;}-export function diff(trie1, trie2) {+export function diff(trie1: TrieNode, trie2: TrieNode): number {if (trie1.hash === trie2.hash) {return null;}@@ -103,12 +122,12 @@ export function diff(trie1, trie2) {}k += diffkey;- node1 = node1[diffkey] || {};- node2 = node2[diffkey] || {};+ node1 = node1[diffkey] || emptyTrie();+ node2 = node2[diffkey] || emptyTrie();}}-export function prune(trie, n = 2) {+export function prune(trie: TrieNode, n = 2): TrieNode {// Do nothing if emptyif (!trie.hash) {return trie;@@ -118,12 +137,16 @@ export function prune(trie, n = 2) {keys.sort();let next = { hash: trie.hash };- keys = keys.slice(-n).map(k => (next[k] = prune(trie[k], n)));++ // Prune child nodes.+ for (let k of keys.slice(-n)) {+ next[k] = prune(trie[k], n);+ }return next;}-export function debug(trie, k = '', indent = 0) {+export function debug(trie: TrieNode, k = '', indent = 0) {const str =' '.repeat(indent) +(k !== '' ? `k: ${k} ` : '') +diff --git a/packages/crdt/src/crdt/timestamp.test.ts b/packages/crdt/src/crdt/timestamp.test.tsindex 5dde3525ef..c1fb77f217 100644--- a/packages/crdt/src/crdt/timestamp.test.ts+++ b/packages/crdt/src/crdt/timestamp.test.ts@@ -16,10 +16,10 @@ describe('Timestamp', function () {describe('comparison', function () {it('should be in order', function () {- expect(Timestamp.zero()).toBe(Timestamp.zero());- expect(Timestamp.max() > Timestamp.zero()).toBeTruthy();- expect(Timestamp.send() > Timestamp.zero()).toBeTruthy();- expect(Timestamp.send() < Timestamp.max()).toBeTruthy();+ expect(Timestamp.zero).toBe(Timestamp.zero);+ expect(Timestamp.max > Timestamp.zero).toBeTruthy();+ expect(Timestamp.send() > Timestamp.zero).toBeTruthy();+ expect(Timestamp.send() < Timestamp.max).toBeTruthy();});});@@ -42,7 +42,7 @@ describe('Timestamp', function () {'9999-12-31T23:59:59.999Z-FFFF-10000000000000000',];for (let invalidInput of invalidInputs) {- expect(Timestamp.parse(invalidInput)).toBe(null);+ expect(Timestamp.parse(invalidInput as string)).toBe(null);}});diff --git a/packages/crdt/src/crdt/timestamp.ts b/packages/crdt/src/crdt/timestamp.tsindex 16a3d4a0cb..190924c7ea 100644--- a/packages/crdt/src/crdt/timestamp.ts+++ b/packages/crdt/src/crdt/timestamp.ts@@ -1,6 +1,8 @@import murmurhash from 'murmurhash';import { v4 as uuidv4 } from 'uuid';+import { TrieNode } from './merkle';+/*** Hybrid Unique Logical Clock (HULC) timestamp generator*@@ -24,29 +26,37 @@ import { v4 as uuidv4 } from 'uuid';* http://www.cse.buffalo.edu/tech-reports/2014-04.pdf*/+export type Clock = {+ timestamp: MutableTimestamp;+ merkle: TrieNode;+};+// A mutable global clock-let clock = null;+let clock: Clock = null;-export function setClock(clock_) {+export function setClock(clock_: Clock): void {clock = clock_;}-export function getClock() {+export function getClock(): Clock {return clock;}-export function makeClock(timestamp, merkle = {}) {+export function makeClock(+ timestamp: Timestamp,+ merkle: TrieNode = { hash: 0 },+) {return { timestamp: MutableTimestamp.from(timestamp), merkle };}-export function serializeClock(clock) {+export function serializeClock(clock: Clock): string {return JSON.stringify({timestamp: clock.timestamp.toString(),merkle: clock.merkle,});}-export function deserializeClock(clock) {+export function deserializeClock(clock: string): Clock {let data;try {data = JSON.parse(clock);@@ -79,18 +89,7 @@ const MAX_NODE_LENGTH = 16;* timestamp instance class*/export class Timestamp {- static init;- static max;- static parse;- static recv;- static send;- static since;- static zero;- static ClockDriftError;- static DuplicateNodeError;- static OverflowError;-- _state;+ _state: { millis: number; counter: number; node: string };constructor(millis: number, counter: number, node: string) {this._state = {@@ -127,222 +126,221 @@ export class Timestamp {hash() {return murmurhash.v3(this.toString());}-}--class MutableTimestamp extends Timestamp {- static from;-- setMillis(n) {- this._state.millis = n;- }- setCounter(n) {- this._state.counter = n;- }+ // Timestamp generator initialization+ // * sets the node ID to an arbitrary value+ // * useful for mocking/unit testing+ static init(options: { maxDrift?: number; node?: string } = {}) {+ if (options.maxDrift) {+ config.maxDrift = options.maxDrift;+ }- setNode(n) {- this._state.node = n;+ setClock(+ makeClock(+ new Timestamp(+ 0,+ 0,+ options.node+ ? ('0000000000000000' + options.node).toString().slice(-16)+ : '',+ ),+ ),+ );}-}-MutableTimestamp.from = timestamp => {- return new MutableTimestamp(- timestamp.millis(),- timestamp.counter(),- timestamp.node(),+ /**+ * maximum timestamp+ */+ static max = Timestamp.parse(+ '9999-12-31T23:59:59.999Z-FFFF-FFFFFFFFFFFFFFFF',);-};-// Timestamp generator initialization-// * sets the node ID to an arbitrary value-// * useful for mocking/unit testing-Timestamp.init = function (options: { maxDrift?: number; node?: string } = {}) {- if (options.maxDrift) {- config.maxDrift = options.maxDrift;+ /**+ * timestamp parsing+ * converts a fixed-length string timestamp to the structured value+ */+ static parse(timestamp: string | Timestamp): Timestamp | null {+ if (timestamp instanceof Timestamp) {+ return timestamp;+ }+ if (typeof timestamp === 'string') {+ let parts = timestamp.split('-');+ if (parts && parts.length === 5) {+ let millis = Date.parse(parts.slice(0, 3).join('-')).valueOf();+ let counter = parseInt(parts[3], 16);+ let node = parts[4];+ if (+ !isNaN(millis) &&+ millis >= 0 &&+ !isNaN(counter) &&+ counter <= MAX_COUNTER &&+ typeof node === 'string' &&+ node.length <= MAX_NODE_LENGTH+ ) {+ return new Timestamp(millis, counter, node);+ }+ }+ }+ return null;}- setClock(- makeClock(- new Timestamp(- 0,- 0,- options.node- ? ('0000000000000000' + options.node).toString().slice(-16)- : '',- ),- ),- );-};+ /**+ * Timestamp send. Generates a unique, monotonic timestamp suitable+ * for transmission to another system in string format+ */+ static send(): Timestamp | null {+ if (!clock) {+ return null;+ }-/**- * Timestamp send. Generates a unique, monotonic timestamp suitable- * for transmission to another system in string format- */-Timestamp.send = function () {- if (!clock) {- return null;- }+ // retrieve the local wall time+ let phys = Date.now();- // retrieve the local wall time- let phys = Date.now();+ // unpack the clock.timestamp logical time and counter+ let lOld = clock.timestamp.millis();+ let cOld = clock.timestamp.counter();- // unpack the clock.timestamp logical time and counter- let lOld = clock.timestamp.millis();- let cOld = clock.timestamp.counter();+ // calculate the next logical time and counter+ // * ensure that the logical time never goes backward+ // * increment the counter if phys time does not advance+ let lNew = Math.max(lOld, phys);+ let cNew = lOld === lNew ? cOld + 1 : 0;- // calculate the next logical time and counter- // * ensure that the logical time never goes backward- // * increment the counter if phys time does not advance- let lNew = Math.max(lOld, phys);- let cNew = lOld === lNew ? cOld + 1 : 0;+ // check the result for drift and counter overflow+ if (lNew - phys > config.maxDrift) {+ throw new Timestamp.ClockDriftError(lNew, phys, config.maxDrift);+ }+ if (cNew > MAX_COUNTER) {+ throw new Timestamp.OverflowError();+ }- // check the result for drift and counter overflow- if (lNew - phys > config.maxDrift) {- throw new Timestamp.ClockDriftError(lNew, phys, config.maxDrift);- }- if (cNew > MAX_COUNTER) {- throw new Timestamp.OverflowError();+ // repack the logical time/counter+ clock.timestamp.setMillis(lNew);+ clock.timestamp.setCounter(cNew);++ return new Timestamp(+ clock.timestamp.millis(),+ clock.timestamp.counter(),+ clock.timestamp.node(),+ );}- // repack the logical time/counter- clock.timestamp.setMillis(lNew);- clock.timestamp.setCounter(cNew);+ // Timestamp receive. Parses and merges a timestamp from a remote+ // system with the local timeglobal uniqueness and monotonicity are+ // preserved+ static recv(msg: Timestamp): Timestamp | null {+ if (!clock) {+ return null;+ }- return new Timestamp(- clock.timestamp.millis(),- clock.timestamp.counter(),- clock.timestamp.node(),- );-};+ // retrieve the local wall time+ let phys = Date.now();-// Timestamp receive. Parses and merges a timestamp from a remote-// system with the local timeglobal uniqueness and monotonicity are-// preserved-Timestamp.recv = function (msg) {- if (!clock) {- return null;- }+ // unpack the message wall time/counter+ let lMsg = msg.millis();+ let cMsg = msg.counter();- // retrieve the local wall time- let phys = Date.now();+ // assert the node id and remote clock drift+ // if (msg.node() === clock.timestamp.node()) {+ // throw new Timestamp.DuplicateNodeError(clock.timestamp.node());+ // }+ if (lMsg - phys > config.maxDrift) {+ throw new Timestamp.ClockDriftError();+ }- // unpack the message wall time/counter- let lMsg = msg.millis();- let cMsg = msg.counter();+ // unpack the clock.timestamp logical time and counter+ let lOld = clock.timestamp.millis();+ let cOld = clock.timestamp.counter();++ // calculate the next logical time and counter+ // . ensure that the logical time never goes backward+ // . if all logical clocks are equal, increment the max counter+ // . if max = old > message, increment local counter+ // . if max = messsage > old, increment message counter+ // . otherwise, clocks are monotonic, reset counter+ let lNew = Math.max(Math.max(lOld, phys), lMsg);+ let cNew =+ lNew === lOld && lNew === lMsg+ ? Math.max(cOld, cMsg) + 1+ : lNew === lOld+ ? cOld + 1+ : lNew === lMsg+ ? cMsg + 1+ : 0;++ // check the result for drift and counter overflow+ if (lNew - phys > config.maxDrift) {+ throw new Timestamp.ClockDriftError();+ }+ if (cNew > MAX_COUNTER) {+ throw new Timestamp.OverflowError();+ }- // assert the node id and remote clock drift- // if (msg.node() === clock.timestamp.node()) {- // throw new Timestamp.DuplicateNodeError(clock.timestamp.node());- // }- if (lMsg - phys > config.maxDrift) {- throw new Timestamp.ClockDriftError();- }+ // repack the logical time/counter+ clock.timestamp.setMillis(lNew);+ clock.timestamp.setCounter(cNew);- // unpack the clock.timestamp logical time and counter- let lOld = clock.timestamp.millis();- let cOld = clock.timestamp.counter();-- // calculate the next logical time and counter- // . ensure that the logical time never goes backward- // . if all logical clocks are equal, increment the max counter- // . if max = old > message, increment local counter- // . if max = messsage > old, increment message counter- // . otherwise, clocks are monotonic, reset counter- let lNew = Math.max(Math.max(lOld, phys), lMsg);- let cNew =- lNew === lOld && lNew === lMsg- ? Math.max(cOld, cMsg) + 1- : lNew === lOld- ? cOld + 1- : lNew === lMsg- ? cMsg + 1- : 0;-- // check the result for drift and counter overflow- if (lNew - phys > config.maxDrift) {- throw new Timestamp.ClockDriftError();- }- if (cNew > MAX_COUNTER) {- throw new Timestamp.OverflowError();+ return new Timestamp(+ clock.timestamp.millis(),+ clock.timestamp.counter(),+ clock.timestamp.node(),+ );}- // repack the logical time/counter- clock.timestamp.setMillis(lNew);- clock.timestamp.setCounter(cNew);-- return new Timestamp(- clock.timestamp.millis(),- clock.timestamp.counter(),- clock.timestamp.node(),+ /**+ * zero/minimum timestamp+ */+ static zero = Timestamp.parse(+ '1970-01-01T00:00:00.000Z-0000-0000000000000000',);-};-/**- * timestamp parsing- * converts a fixed-length string timestamp to the structured value- */-Timestamp.parse = function (timestamp: string): Timestamp | null {- if (typeof timestamp === 'string') {- let parts = timestamp.split('-');- if (parts && parts.length === 5) {- let millis = Date.parse(parts.slice(0, 3).join('-')).valueOf();- let counter = parseInt(parts[3], 16);- let node = parts[4];- if (- !isNaN(millis) &&- millis >= 0 &&- !isNaN(counter) &&- counter <= MAX_COUNTER &&- typeof node === 'string' &&- node.length <= MAX_NODE_LENGTH- ) {- return new Timestamp(millis, counter, node);- }+ static since = isoString => isoString + '-0000-0000000000000000';++ /**+ * error classes+ */+ static DuplicateNodeError = class DuplicateNodeError extends Error {+ constructor(node: string) {+ super('duplicate node identifier ' + node);+ this.name = 'DuplicateNodeError';}- }- return null;-};+ };-/**- * zero/minimum timestamp- */-let zero = Timestamp.parse('1970-01-01T00:00:00.000Z-0000-0000000000000000');-Timestamp.zero = function () {- return zero;-};+ static ClockDriftError = class ClockDriftError extends Error {+ constructor(...args: unknown[]) {+ super(+ ['maximum clock drift exceeded'].concat(args as string[]).join(' '),+ );+ this.name = 'ClockDriftError';+ }+ };-/**- * maximum timestamp- */-let max = Timestamp.parse('9999-12-31T23:59:59.999Z-FFFF-FFFFFFFFFFFFFFFF');-Timestamp.max = function () {- return max;-};+ static OverflowError = class OverflowError extends Error {+ constructor() {+ super('timestamp counter overflow');+ this.name = 'OverflowError';+ }+ };+}-Timestamp.since = isoString => {- return isoString + '-0000-0000000000000000';-};+class MutableTimestamp extends Timestamp {+ static from(timestamp) {+ return new MutableTimestamp(+ timestamp.millis(),+ timestamp.counter(),+ timestamp.node(),+ );+ }-/**- * error classes- */-Timestamp.DuplicateNodeError = class extends Error {- constructor(node) {- super('duplicate node identifier ' + node);- this.name = 'DuplicateNodeError';+ setMillis(n) {+ this._state.millis = n;}-};-Timestamp.ClockDriftError = class extends Error {- constructor(...args) {- super(['maximum clock drift exceeded'].concat(args).join(' '));- this.name = 'ClockDriftError';+ setCounter(n) {+ this._state.counter = n;}-};-Timestamp.OverflowError = class extends Error {- constructor() {- super('timestamp counter overflow');- this.name = 'OverflowError';+ setNode(n) {+ this._state.node = n;}-};+}diff --git a/packages/crdt/src/main.ts b/packages/crdt/src/main.tsindex 76c4159a53..ff5d5dacb3 100644--- a/packages/crdt/src/main.ts+++ b/packages/crdt/src/main.ts@@ -7,6 +7,7 @@ export {makeClientId,serializeClock,deserializeClock,+ Clock,Timestamp,} from './crdt';diff --git a/packages/loot-core/src/server/main.ts b/packages/loot-core/src/server/main.tsindex a9eacee60d..b4fcf52eac 100644--- a/packages/loot-core/src/server/main.ts+++ b/packages/loot-core/src/server/main.ts@@ -61,7 +61,6 @@ import {setSyncingMode,makeTestMessage,clearFullSyncTimeout,- syncAndReceiveMessages,resetSync,repairSync,} from './sync';@@ -2462,7 +2461,6 @@ export const lib = {return res;},on: (name, func) => app.events.on(name, func),- syncAndReceiveMessages,q,db,diff --git a/packages/loot-core/src/server/migrate/migrations.ts b/packages/loot-core/src/server/migrate/migrations.tsindex ca229ca682..f4b6b3993d 100644--- a/packages/loot-core/src/server/migrate/migrations.ts+++ b/packages/loot-core/src/server/migrate/migrations.ts@@ -1,6 +1,7 @@// We have to bundle in JS migrations manually to avoid having to `eval`// them which doesn't play well with CSP. There isn't great, and eventually// we can remove this migration.+import { Database } from 'better-sqlite3';import { v4 as uuidv4 } from 'uuid';import m1632571489012 from '../../../migrations/1632571489012_remove_cache';@@ -13,18 +14,21 @@ let javascriptMigrations = {1632571489012: m1632571489012,};-export async function withMigrationsDir(dir, func) {+export async function withMigrationsDir(+ dir: string,+ func: () => Promise<void>,+): Promise<void> {let oldDir = MIGRATIONS_DIR;MIGRATIONS_DIR = dir;await func();MIGRATIONS_DIR = oldDir;}-export function getMigrationsDir() {+export function getMigrationsDir(): string {return MIGRATIONS_DIR;}-function getMigrationId(name) {+function getMigrationId(name: string): number {return parseInt(name.match(/^(\d)+/)[0]);}@@ -36,7 +40,7 @@ export function getUpMigration(id, names) {}}-export async function getAppliedMigrations(db) {+export async function getAppliedMigrations(db: Database): Promise<number[]> {const rows = await sqlite.runQuery<{ id: number }>(db,'SELECT * FROM __migrations__ ORDER BY id ASC',@@ -46,7 +50,9 @@ export async function getAppliedMigrations(db) {return rows.map(row => row.id);}-export async function getMigrationList(migrationsDir) {+export async function getMigrationList(+ migrationsDir: string,+): Promise<string[]> {const files = await fs.listDir(migrationsDir);return files.filter(name => name.match(/(\.sql|\.js)$/))@@ -62,7 +68,7 @@ export async function getMigrationList(migrationsDir) {});}-export function getPending(appliedIds, all) {+export function getPending(appliedIds: number[], all: string[]): string[] {return all.filter(name => {const id = getMigrationId(name);return appliedIds.indexOf(id) === -1;@@ -94,7 +100,11 @@ async function applySql(db, sql) {}}-export async function applyMigration(db, name, migrationsDir) {+export async function applyMigration(+ db: Database,+ name: string,+ migrationsDir: string,+): Promise<void> {const code = await fs.readFile(fs.join(migrationsDir, name));if (name.match(/\.js$/)) {await applyJavaScript(db, getMigrationId(name));@@ -106,7 +116,10 @@ export async function applyMigration(db, name, migrationsDir) {]);}-function checkDatabaseValidity(appliedIds, available) {+function checkDatabaseValidity(+ appliedIds: number[],+ available: string[],+): void {for (let i = 0; i < appliedIds.length; i++) {if (i >= available.length ||@@ -121,7 +134,7 @@ function checkDatabaseValidity(appliedIds, available) {}}-export async function migrate(db) {+export async function migrate(db: Database): Promise<string[]> {let appliedIds = await getAppliedMigrations(db);let available = await getMigrationList(MIGRATIONS_DIR);diff --git a/packages/loot-core/src/server/sync/encoder.ts b/packages/loot-core/src/server/sync/encoder.tsindex aa45156404..fc6df6c329 100644--- a/packages/loot-core/src/server/sync/encoder.ts+++ b/packages/loot-core/src/server/sync/encoder.ts@@ -1,4 +1,4 @@-import { SyncProtoBuf } from '@actual-app/crdt';+import { Timestamp, SyncProtoBuf } from '@actual-app/crdt';import * as encryption from '../encryption';import { SyncError } from '../errors';@@ -20,7 +20,7 @@ function coerceBuffer(value) {export async function encode(groupId: string,fileId: string,- since: string,+ since: Timestamp,messages: Message[],): Promise<Uint8Array> {let { encryptKeyId } = prefs.getPrefs();@@ -29,7 +29,7 @@ export async function encode(for (let i = 0; i < messages.length; i++) {let msg = messages[i];let envelopePb = new SyncProtoBuf.MessageEnvelope();- envelopePb.setTimestamp(msg.timestamp);+ envelopePb.setTimestamp(msg.timestamp.toString());let messagePb = new SyncProtoBuf.Message();messagePb.setDataset(msg.dataset);@@ -66,7 +66,7 @@ export async function encode(requestPb.setGroupid(groupId);requestPb.setFileid(fileId);requestPb.setKeyid(encryptKeyId);- requestPb.setSince(since);+ requestPb.setSince(since.toString());return requestPb.serializeBinary();}@@ -83,7 +83,7 @@ export async function decode(for (let i = 0; i < list.length; i++) {let envelopePb = list[i];- let timestamp = envelopePb.getTimestamp();+ let timestamp = Timestamp.parse(envelopePb.getTimestamp());let encrypted = envelopePb.getIsencrypted();let msg;diff --git a/packages/loot-core/src/server/sync/index.ts b/packages/loot-core/src/server/sync/index.tsindex d2ee55476d..6b0a1b9dc6 100644--- a/packages/loot-core/src/server/sync/index.ts+++ b/packages/loot-core/src/server/sync/index.ts@@ -249,7 +249,7 @@ export type Message = {dataset: string;old?: unknown;row: string;- timestamp: string;+ timestamp: Timestamp;value: string | number | null;};@@ -333,9 +333,8 @@ export const applyMessages = sequential(async (messages: Message[]) => {db.transaction(() => {let added = new Set();- for (let i = 0; i < messages.length; i++) {- let msg = messages[i];- let { dataset, row, column, timestamp, value } = msg;+ for (const msg of messages) {+ const { dataset, row, column, timestamp, value } = msg;if (!msg.old) {apply(msg, getIn(oldData, [dataset, row]) || added.has(dataset + row));@@ -357,7 +356,7 @@ export const applyMessages = sequential(async (messages: Message[]) => {[timestamp.toString(), dataset, row, column, serializeValue(value)],);- currentMerkle = merkle.insert(currentMerkle, msg.timestamp);+ currentMerkle = merkle.insert(currentMerkle, timestamp);}}@@ -487,7 +486,7 @@ export async function sendMessages(messages: Message[]) {}}-export function getMessagesSince(since: string) {+export function getMessagesSince(since: string): Message[] {return db.runQuery('SELECT timestamp, dataset, row, column, value FROM messages_crdt WHERE timestamp > ?',[since],@@ -495,21 +494,6 @@ export function getMessagesSince(since: string) {);}-export async function syncAndReceiveMessages(- messages: Message[],- since: string,-): Promise<Message[]> {- let localMessages = await getMessagesSince(since);- await receiveMessages(- messages.map(msg => ({- ...msg,- value: deserializeValue(msg.value as string),- timestamp: Timestamp.parse(msg.timestamp),- })),- );- return localMessages;-}-export function clearFullSyncTimeout(): void {if (syncTimeout) {clearTimeout(syncTimeout);@@ -691,7 +675,6 @@ async function _fullSync(res.messages.map(msg => ({...msg,value: deserializeValue(msg.value as string),- timestamp: Timestamp.parse(msg.timestamp),})),);}diff --git a/packages/loot-core/src/server/sync/migrate.test.ts b/packages/loot-core/src/server/sync/migrate.test.tsindex b2fd82fedf..459f8da0c9 100644--- a/packages/loot-core/src/server/sync/migrate.test.ts+++ b/packages/loot-core/src/server/sync/migrate.test.ts@@ -1,3 +1,4 @@+import { Timestamp } from '@actual-app/crdt';import fc from 'fast-check';import * as arbs from '../../mocks/arbitrary-schema';@@ -39,7 +40,8 @@ let messageArb: fc.Arbitrary<Message> = fc}).noBias().noShrink()- .map(date => date.toISOString() + '-0000-0123456789ABCDEF');+ .map(date => date.toISOString() + '-0000-0123456789ABCDEF')+ .map(Timestamp.parse);return fc.record<Message>({timestamp: timestamp,diff --git a/packages/loot-core/src/server/sync/migrate.ts b/packages/loot-core/src/server/sync/migrate.tsindex 6238aa41ae..dbc0fd951e 100644--- a/packages/loot-core/src/server/sync/migrate.ts+++ b/packages/loot-core/src/server/sync/migrate.ts@@ -1,11 +1,11 @@import { Timestamp } from '@actual-app/crdt';-import { addSyncListener, applyMessages } from './index';+import { Message, addSyncListener, applyMessages } from './index';function migrateParentIds(_oldValues, newValues) {newValues.forEach((items, table) => {if (table === 'transactions') {- let toApply = [];+ let toApply: Message[] = [];items.forEach(newValue => {if (diff --git a/packages/loot-core/src/server/sync/repair.ts b/packages/loot-core/src/server/sync/repair.tsindex e8736eb692..ba55acd13e 100644--- a/packages/loot-core/src/server/sync/repair.ts+++ b/packages/loot-core/src/server/sync/repair.ts@@ -2,9 +2,16 @@ import { serializeClock, getClock, Timestamp, merkle } from '@actual-app/crdt';import * as db from '../db';-export function rebuildMerkleHash() {- let rows = db.runQuery('SELECT timestamp FROM messages_crdt', [], true);- let trie: Record<string, unknown> = {};+export function rebuildMerkleHash(): {+ numMessages: number;+ trie: merkle.TrieNode;+} {+ let rows: { timestamp: string }[] = db.runQuery(+ 'SELECT timestamp FROM messages_crdt',+ [],+ true,+ );+ let trie = merkle.emptyTrie();for (let i = 0; i < rows.length; i++) {trie = merkle.insert(trie, Timestamp.parse(rows[i].timestamp));@@ -16,7 +23,7 @@ export function rebuildMerkleHash() {};}-export default async function repairSync() {+export default async function repairSync(): Promise<void> {let rebuilt = rebuildMerkleHash();let clock = getClock();diff --git a/packages/loot-core/src/server/sync/sync.property.test.ts b/packages/loot-core/src/server/sync/sync.property.test.tsindex f43f63b28e..94ad71c2ac 100644--- a/packages/loot-core/src/server/sync/sync.property.test.ts+++ b/packages/loot-core/src/server/sync/sync.property.test.ts@@ -251,7 +251,7 @@ async function run(msgs) {await encoder.encode('group',clientId2,- Timestamp.zero(),+ Timestamp.zero,res.secondMessages.map(x => ({...x,value: sync.serializeValue(x.value),@@ -270,7 +270,7 @@ async function run(msgs) {await encoder.encode('group',clientId2,- Timestamp.zero(),+ Timestamp.zero,res.secondMessages.map(x => ({...x,value: sync.serializeValue(x.value),diff --git a/packages/loot-core/src/server/sync/sync.test.ts b/packages/loot-core/src/server/sync/sync.test.tsindex 91cc537d9e..a0a8673c77 100644--- a/packages/loot-core/src/server/sync/sync.test.ts+++ b/packages/loot-core/src/server/sync/sync.test.ts@@ -97,28 +97,32 @@ describe('Sync', () => {prefs.loadPrefs();prefs.savePrefs({groupId: 'group',- lastSyncedTimestamp: Timestamp.zero().toString(),+ lastSyncedTimestamp: Timestamp.zero.toString(),});await mockSyncServer.handlers['/sync/sync'](await encoder.encode('group','client',- '1970-01-01T01:17:37.000Z-0000-0000testinguuid2',+ Timestamp.parse('1970-01-01T01:17:37.000Z-0000-0000testinguuid2'),[{dataset: 'transactions',row: 'foo',column: 'amount',value: 'N:3200',- timestamp: '1970-01-02T05:17:36.789Z-0000-0000testinguuid2',+ timestamp: Timestamp.parse(+ '1970-01-02T05:17:36.789Z-0000-0000testinguuid2',+ ),},{dataset: 'transactions',row: 'foo',column: 'amount',value: 'N:4200',- timestamp: '1970-01-02T10:17:36.999Z-0000-0000testinguuid2',+ timestamp: Timestamp.parse(+ '1970-01-02T10:17:36.999Z-0000-0000testinguuid2',+ ),},],),@@ -153,7 +157,7 @@ async function asSecondClient(func) {prefs.loadPrefs();prefs.savePrefs({groupId: 'group',- lastSyncedTimestamp: Timestamp.zero().toString(),+ lastSyncedTimestamp: Timestamp.zero.toString(),});await func();@@ -161,7 +165,7 @@ async function asSecondClient(func) {await global.emptyDatabase()();prefs.savePrefs({groupId: 'group',- lastSyncedTimestamp: Timestamp.zero().toString(),+ lastSyncedTimestamp: Timestamp.zero.toString(),});}@@ -236,10 +240,7 @@ describe('Sync projections', () => {registerBudgetMonths(['2017-01', '2017-02']);// Get all the messages. We'll apply them in two passes- let messages = mockSyncServer.getMessages().map(msg => ({- ...msg,- timestamp: Timestamp.parse(msg.timestamp),- }));+ let messages = mockSyncServer.getMessages();// Apply all but the last message (which deletes the category)await applyMessages(messages.slice(0, -1));@@ -290,10 +291,7 @@ describe('Sync projections', () => {registerBudgetMonths(['2017-01', '2017-02']);// Get all the messages. We'll apply them in two passes- let messages = mockSyncServer.getMessages().map(msg => ({- ...msg,- timestamp: Timestamp.parse(msg.timestamp),- }));+ let messages = mockSyncServer.getMessages();let firstMessages = messages.filter(m => m.column !== 'tombstone');let secondMessages = messages.filter(m => m.column === 'tombstone');@@ -327,10 +325,7 @@ describe('Sync projections', () => {registerBudgetMonths(['2017-01', '2017-02']);// Get all the messages. We'll apply them in two passes- let messages = mockSyncServer.getMessages().map(msg => ({- ...msg,- timestamp: Timestamp.parse(msg.timestamp),- }));+ let messages = mockSyncServer.getMessages();let firstMessages = messages.slice(0, -2);let secondMessages = messages.slice(-2);diff --git a/packages/loot-core/src/server/tests/mockSyncServer.ts b/packages/loot-core/src/server/tests/mockSyncServer.tsindex 1e64534eaf..09db3b7632 100644--- a/packages/loot-core/src/server/tests/mockSyncServer.ts+++ b/packages/loot-core/src/server/tests/mockSyncServer.ts@@ -1,11 +1,23 @@-import { makeClock, Timestamp, merkle, SyncProtoBuf } from '@actual-app/crdt';+import {+ Clock,+ makeClock,+ Timestamp,+ merkle,+ SyncProtoBuf,+} from '@actual-app/crdt';++import { Message } from '../sync';import { basic as defaultMockData } from './mockData.json';const handlers = {};let currentMockData = defaultMockData;let currentClock = makeClock(new Timestamp(0, 0, '0000000000000000'));-let currentMessages = [];+let currentMessages: {+ timestamp: string;+ is_encrypted: boolean;+ content: Uint8Array;+}[] = [];// Ugh, this is duplicated...function deserializeValue(value) {@@ -27,7 +39,7 @@ handlers['/'] = () => {return 'development';};-handlers['/sync/sync'] = async data => {+handlers['/sync/sync'] = async (data: Uint8Array): Promise<Uint8Array> => {let requestPb = SyncProtoBuf.SyncRequest.deserializeBinary(data);let since = requestPb.getSince();let messages = requestPb.getMessagesList();@@ -39,7 +51,7 @@ handlers['/sync/sync'] = async data => {currentMessages.push({timestamp: msg.getTimestamp(),is_encrypted: msg.getIsencrypted(),- content: msg.getContent(),+ content: msg.getContent_asU8(),});currentClock.merkle = merkle.insert(@@ -104,17 +116,17 @@ export const reset = () => {currentMessages = [];};-export const getClock = () => {+export const getClock = (): Clock => {return currentClock;};-export const getMessages = () => {+export const getMessages = (): Message[] => {return currentMessages.map(msg => {let { timestamp, content } = msg;let fields = SyncProtoBuf.Message.deserializeBinary(content);return {- timestamp: timestamp,+ timestamp: Timestamp.parse(timestamp),dataset: fields.getDataset(),row: fields.getRow(),column: fields.getColumn(),diff --git a/packages/loot-core/src/server/undo.ts b/packages/loot-core/src/server/undo.tsindex f21766309b..4fffa90490 100644--- a/packages/loot-core/src/server/undo.ts+++ b/packages/loot-core/src/server/undo.ts@@ -4,7 +4,7 @@ import * as connection from '../platform/server/connection';import { getIn } from '../shared/util';import { withMutatorContext, getMutatorContext } from './mutators';-import { sendMessages } from './sync';+import { Message, sendMessages } from './sync';// A marker always sits as the first entry to simplify logictype MarkerMessage = { type: 'marker'; meta?: unknown };@@ -228,7 +228,7 @@ export async function redo() {}}-function redoResurrections(messages, oldData) {+function redoResurrections(messages, oldData): Message[] {let resurrect = new Set<string>();messages.forEach(message => {diff --git a/upcoming-release-notes/1076.md b/upcoming-release-notes/1076.mdnew file mode 100644index 0000000000..d5e0350fcf--- /dev/null+++ b/upcoming-release-notes/1076.md@@ -0,0 +1,6 @@+---+category: Maintenance+authors: [TomAFrench]+---++Add types to `crdt` directory\ No newline at end of file