A local-first personal finance app
Move the rest of the syncing protobuf code to the CRDT package (#1217)
Jed Fox982d57c9ae06
diff --git a/.prettierignore b/.prettierignorenew file mode 100644index 0000000000..2faa2ef87d--- /dev/null+++ b/.prettierignore@@ -0,0 +1 @@+sync_pb.*diff --git a/packages/crdt/.gitignore b/packages/crdt/.gitignorenew file mode 100644index 0000000000..f1772b9098--- /dev/null+++ b/packages/crdt/.gitignore@@ -0,0 +1,2 @@+bin/protoc-gen-js+bin/protocdiff --git a/packages/loot-core/src/server/sync/README.md b/packages/crdt/README.mdsimilarity index 57%rename from packages/loot-core/src/server/sync/README.mdrename to packages/crdt/README.mdindex d9f4c17be7..bb046aa2f8 100644--- a/packages/loot-core/src/server/sync/README.md+++ b/packages/crdt/README.md@@ -1,25 +1,18 @@-# protobuf+# `@actual-app/crdt`-We use [protobuf](https://developers.google.com/protocol-buffers/) to encode messages as binary data to send across the network.+This package contains the core CRDT logic that enables Actual’s syncing. It is shared between the client and server. We may or may not follow semver when updating this package; any usage of it outside Actual is undocumented and at your own risk.-## Generating protobuf+## protobuf-The protobuf is generated by using the [protoc](https://github.com/protocolbuffers/protobuf) compiler.+We use [protobuf](https://developers.google.com/protocol-buffers/) to encode messages as binary data to send across the network.-This can be installed by downloading one of the [pre-built binaries](https://github.com/protocolbuffers/protobuf/releases/) and placing it in your `$PATH`. The version used to build the current protobuf is [v3.20.1](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.1).+### Generating protobuf-Once installed, the protobuf can be generated by running the below commands.+The protobuf is generated by using the [protoc](https://github.com/protocolbuffers/protobuf) compiler.-```bash-cd packages/loot-core+This can be installed by downloading one of the [pre-built binaries](https://github.com/protocolbuffers/protobuf/releases/) and placing it in your `$PATH`. The version used to build the current protobuf is [v3.20.1](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.1). You’ll also need to [download the latest version of `protoc-gen-js`](https://github.com/protocolbuffers/protobuf-javascript/releases/latest). For convenience, you can put both of these binaries in `./bin`.-protoc --plugin="protoc-gen-ts=../../node_modules/.bin/protoc-gen-ts" \- --ts_opt=esModuleInterop=true \- --ts_out="src/server/sync/proto" \- --js_out=import_style=commonjs,binary:src/server/sync/proto \- --proto_path=src/server/sync \- sync.proto-```+Once installed, the protobuf can be generated by running `./bin/generate-proto`.However there is one very important thing to remember! The default output includes this near the top:diff --git a/packages/crdt/bin/generate-proto b/packages/crdt/bin/generate-protonew file mode 100755index 0000000000..e539c56334--- /dev/null+++ b/packages/crdt/bin/generate-proto@@ -0,0 +1,21 @@+#!/bin/bash++cd "$(dirname "$(dirname "$0")")"++if ! [ -x "$(command -v protoc)" ]; then+ echo 'Error: protoc is not installed. See the readme for installation instructions.' >&2+ exit 1+fi++export PATH="$PWD/bin:$PATH"++protoc --plugin="protoc-gen-ts=../../node_modules/.bin/protoc-gen-ts" \+ --ts_opt=esModuleInterop=true \+ --ts_out="src/proto" \+ --js_out=import_style=commonjs,binary:src/proto \+ --proto_path=src/proto \+ sync.proto++../../node_modules/.bin/prettier --write src/proto/*.d.ts++echo 'One more step! Find the `var global = ...` declaration in src/proto/sync_pb.js and change it to `var global = globalThis;`'diff --git a/packages/crdt/package.json b/packages/crdt/package.jsonindex dbff3f1358..d6dd400c6e 100644--- a/packages/crdt/package.json+++ b/packages/crdt/package.json@@ -10,6 +10,7 @@],"scripts": {"build:node": "tsc --p tsconfig.dist.json",+ "proto:generate": "./bin/generate-proto","build": "rm -rf dist && yarn run build:node && cp src/proto/sync_pb.d.ts dist/src/proto/","test": "jest -c jest.config.js"},@@ -22,6 +23,7 @@"@types/jest": "^27.5.0","jest": "^27.0.0","ts-jest": "^27.0.0",+ "ts-protoc-gen": "^0.15.0","typescript": "^5.0.2"}}diff --git a/packages/loot-core/src/server/sync/sync.proto b/packages/crdt/src/proto/sync.protosimilarity index 100%rename from packages/loot-core/src/server/sync/sync.protorename to packages/crdt/src/proto/sync.protodiff --git a/packages/crdt/src/proto/sync_pb.d.ts b/packages/crdt/src/proto/sync_pb.d.tsindex 5a31387bf1..966b8a3385 100644--- a/packages/crdt/src/proto/sync_pb.d.ts+++ b/packages/crdt/src/proto/sync_pb.d.ts@@ -20,11 +20,11 @@ export class EncryptedData extends jspb.Message {setData(value: Uint8Array | string): void;serializeBinary(): Uint8Array;- toObject(includeInstance?: boolean): EncryptedDataAsObject;+ toObject(includeInstance?: boolean): EncryptedData.AsObject;static toObject(includeInstance: boolean,msg: EncryptedData,- ): EncryptedDataAsObject;+ ): EncryptedData.AsObject;static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;@@ -40,11 +40,13 @@ export class EncryptedData extends jspb.Message {): EncryptedData;}-export type EncryptedDataAsObject = {- iv: Uint8Array | string;- authtag: Uint8Array | string;- data: Uint8Array | string;-};+export namespace EncryptedData {+ export type AsObject = {+ iv: Uint8Array | string;+ authtag: Uint8Array | string;+ data: Uint8Array | string;+ };+}export class Message extends jspb.Message {getDataset(): string;@@ -60,8 +62,8 @@ export class Message extends jspb.Message {setValue(value: string): void;serializeBinary(): Uint8Array;- toObject(includeInstance?: boolean): MessageAsObject;- static toObject(includeInstance: boolean, msg: Message): MessageAsObject;+ toObject(includeInstance?: boolean): Message.AsObject;+ static toObject(includeInstance: boolean, msg: Message): Message.AsObject;static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;@@ -77,12 +79,14 @@ export class Message extends jspb.Message {): Message;}-export type MessageAsObject = {- dataset: string;- row: string;- column: string;- value: string;-};+export namespace Message {+ export type AsObject = {+ dataset: string;+ row: string;+ column: string;+ value: string;+ };+}export class MessageEnvelope extends jspb.Message {getTimestamp(): string;@@ -97,11 +101,11 @@ export class MessageEnvelope extends jspb.Message {setContent(value: Uint8Array | string): void;serializeBinary(): Uint8Array;- toObject(includeInstance?: boolean): MessageEnvelopeAsObject;+ toObject(includeInstance?: boolean): MessageEnvelope.AsObject;static toObject(includeInstance: boolean,msg: MessageEnvelope,- ): MessageEnvelopeAsObject;+ ): MessageEnvelope.AsObject;static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;@@ -117,11 +121,13 @@ export class MessageEnvelope extends jspb.Message {): MessageEnvelope;}-export type MessageEnvelopeAsObject = {- timestamp: string;- isencrypted: boolean;- content: Uint8Array | string;-};+export namespace MessageEnvelope {+ export type AsObject = {+ timestamp: string;+ isencrypted: boolean;+ content: Uint8Array | string;+ };+}export class SyncRequest extends jspb.Message {clearMessagesList(): void;@@ -142,11 +148,11 @@ export class SyncRequest extends jspb.Message {setSince(value: string): void;serializeBinary(): Uint8Array;- toObject(includeInstance?: boolean): SyncRequestAsObject;+ toObject(includeInstance?: boolean): SyncRequest.AsObject;static toObject(includeInstance: boolean,msg: SyncRequest,- ): SyncRequestAsObject;+ ): SyncRequest.AsObject;static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;@@ -162,13 +168,15 @@ export class SyncRequest extends jspb.Message {): SyncRequest;}-export type SyncRequestAsObject = {- messagesList: Array<MessageEnvelopeAsObject>;- fileid: string;- groupid: string;- keyid: string;- since: string;-};+export namespace SyncRequest {+ export type AsObject = {+ messagesList: Array<MessageEnvelope.AsObject>;+ fileid: string;+ groupid: string;+ keyid: string;+ since: string;+ };+}export class SyncResponse extends jspb.Message {clearMessagesList(): void;@@ -180,11 +188,11 @@ export class SyncResponse extends jspb.Message {setMerkle(value: string): void;serializeBinary(): Uint8Array;- toObject(includeInstance?: boolean): SyncResponseAsObject;+ toObject(includeInstance?: boolean): SyncResponse.AsObject;static toObject(includeInstance: boolean,msg: SyncResponse,- ): SyncResponseAsObject;+ ): SyncResponse.AsObject;static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;@@ -200,7 +208,9 @@ export class SyncResponse extends jspb.Message {): SyncResponse;}-export type SyncResponseAsObject = {- messagesList: Array<MessageEnvelopeAsObject>;- merkle: string;-};+export namespace SyncResponse {+ export type AsObject = {+ messagesList: Array<MessageEnvelope.AsObject>;+ merkle: string;+ };+}diff --git a/packages/loot-core/package.json b/packages/loot-core/package.jsonindex 14a1a16161..d32b9df5c9 100644--- a/packages/loot-core/package.json+++ b/packages/loot-core/package.json@@ -72,7 +72,6 @@"throttleit": "^1.0.0","ts-jest": "^27.0.0","ts-node": "^10.7.0",- "ts-protoc-gen": "^0.15.0","typescript": "^4.6.4","uuid": "3.3.2","webpack": "^5.86.0",diff --git a/upcoming-release-notes/1217.md b/upcoming-release-notes/1217.mdnew file mode 100644index 0000000000..137c23d64d--- /dev/null+++ b/upcoming-release-notes/1217.md@@ -0,0 +1,6 @@+---+category: Maintenance+authors: [j-f1]+---++Move the rest of the syncing protobuf code to the CRDT packagediff --git a/yarn.lock b/yarn.lockindex 588e148eba..5759cfa18b 100644--- a/yarn.lock+++ b/yarn.lock@@ -32,6 +32,7 @@ __metadata:jest: ^27.0.0murmurhash: ^0.0.2ts-jest: ^27.0.0+ ts-protoc-gen: ^0.15.0typescript: ^5.0.2uuid: 3.3.2languageName: unknown@@ -12380,7 +12381,6 @@ __metadata:throttleit: ^1.0.0ts-jest: ^27.0.0ts-node: ^10.7.0- ts-protoc-gen: ^0.15.0typescript: ^4.6.4uuid: 3.3.2webpack: ^5.86.0