Skip to content

A local-first personal finance app

Move the rest of the syncing protobuf code to the CRDT package (#1217)

Jed Fox982d57c9ae06
diff --git a/.prettierignore b/.prettierignore
new file mode 100644
index 0000000000..2faa2ef87d
--- /dev/null
+++ b/.prettierignore
@@ -0,0 +1 @@
+sync_pb.*
diff --git a/packages/crdt/.gitignore b/packages/crdt/.gitignore
new file mode 100644
index 0000000000..f1772b9098
--- /dev/null
+++ b/packages/crdt/.gitignore
@@ -0,0 +1,2 @@
+bin/protoc-gen-js
+bin/protoc
diff --git a/packages/loot-core/src/server/sync/README.md b/packages/crdt/README.md
similarity index 57%
rename from packages/loot-core/src/server/sync/README.md
rename to packages/crdt/README.md
index d9f4c17be7..bb046aa2f8 100644
--- a/packages/loot-core/src/server/sync/README.md
+++ b/packages/crdt/README.md
@@ -1,25 +1,18 @@
-# protobuf
+# `@actual-app/crdt`
-We use [protobuf](https://developers.google.com/protocol-buffers/) to encode messages as binary data to send across the network.
+This package contains the core CRDT logic that enables Actual’s syncing. It is shared between the client and server. We may or may not follow semver when updating this package; any usage of it outside Actual is undocumented and at your own risk.
-## Generating protobuf
+## protobuf
-The protobuf is generated by using the [protoc](https://github.com/protocolbuffers/protobuf) compiler.
+We use [protobuf](https://developers.google.com/protocol-buffers/) to encode messages as binary data to send across the network.
-This can be installed by downloading one of the [pre-built binaries](https://github.com/protocolbuffers/protobuf/releases/) and placing it in your `$PATH`. The version used to build the current protobuf is [v3.20.1](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.1).
+### Generating protobuf
-Once installed, the protobuf can be generated by running the below commands.
+The protobuf is generated by using the [protoc](https://github.com/protocolbuffers/protobuf) compiler.
-```bash
-cd packages/loot-core
+This can be installed by downloading one of the [pre-built binaries](https://github.com/protocolbuffers/protobuf/releases/) and placing it in your `$PATH`. The version used to build the current protobuf is [v3.20.1](https://github.com/protocolbuffers/protobuf/releases/tag/v3.20.1). You’ll also need to [download the latest version of `protoc-gen-js`](https://github.com/protocolbuffers/protobuf-javascript/releases/latest). For convenience, you can put both of these binaries in `./bin`.
-protoc --plugin="protoc-gen-ts=../../node_modules/.bin/protoc-gen-ts" \
- --ts_opt=esModuleInterop=true \
- --ts_out="src/server/sync/proto" \
- --js_out=import_style=commonjs,binary:src/server/sync/proto \
- --proto_path=src/server/sync \
- sync.proto
-```
+Once installed, the protobuf can be generated by running `./bin/generate-proto`.
However there is one very important thing to remember! The default output includes this near the top:
diff --git a/packages/crdt/bin/generate-proto b/packages/crdt/bin/generate-proto
new file mode 100755
index 0000000000..e539c56334
--- /dev/null
+++ b/packages/crdt/bin/generate-proto
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+cd "$(dirname "$(dirname "$0")")"
+
+if ! [ -x "$(command -v protoc)" ]; then
+ echo 'Error: protoc is not installed. See the readme for installation instructions.' >&2
+ exit 1
+fi
+
+export PATH="$PWD/bin:$PATH"
+
+protoc --plugin="protoc-gen-ts=../../node_modules/.bin/protoc-gen-ts" \
+ --ts_opt=esModuleInterop=true \
+ --ts_out="src/proto" \
+ --js_out=import_style=commonjs,binary:src/proto \
+ --proto_path=src/proto \
+ sync.proto
+
+../../node_modules/.bin/prettier --write src/proto/*.d.ts
+
+echo 'One more step! Find the `var global = ...` declaration in src/proto/sync_pb.js and change it to `var global = globalThis;`'
diff --git a/packages/crdt/package.json b/packages/crdt/package.json
index dbff3f1358..d6dd400c6e 100644
--- a/packages/crdt/package.json
+++ b/packages/crdt/package.json
@@ -10,6 +10,7 @@
],
"scripts": {
"build:node": "tsc --p tsconfig.dist.json",
+ "proto:generate": "./bin/generate-proto",
"build": "rm -rf dist && yarn run build:node && cp src/proto/sync_pb.d.ts dist/src/proto/",
"test": "jest -c jest.config.js"
},
@@ -22,6 +23,7 @@
"@types/jest": "^27.5.0",
"jest": "^27.0.0",
"ts-jest": "^27.0.0",
+ "ts-protoc-gen": "^0.15.0",
"typescript": "^5.0.2"
}
}
diff --git a/packages/loot-core/src/server/sync/sync.proto b/packages/crdt/src/proto/sync.proto
similarity index 100%
rename from packages/loot-core/src/server/sync/sync.proto
rename to packages/crdt/src/proto/sync.proto
diff --git a/packages/crdt/src/proto/sync_pb.d.ts b/packages/crdt/src/proto/sync_pb.d.ts
index 5a31387bf1..966b8a3385 100644
--- a/packages/crdt/src/proto/sync_pb.d.ts
+++ b/packages/crdt/src/proto/sync_pb.d.ts
@@ -20,11 +20,11 @@ export class EncryptedData extends jspb.Message {
setData(value: Uint8Array | string): void;
serializeBinary(): Uint8Array;
- toObject(includeInstance?: boolean): EncryptedDataAsObject;
+ toObject(includeInstance?: boolean): EncryptedData.AsObject;
static toObject(
includeInstance: boolean,
msg: EncryptedData,
- ): EncryptedDataAsObject;
+ ): EncryptedData.AsObject;
static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
static extensionsBinary: {
[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -40,11 +40,13 @@ export class EncryptedData extends jspb.Message {
): EncryptedData;
}
-export type EncryptedDataAsObject = {
- iv: Uint8Array | string;
- authtag: Uint8Array | string;
- data: Uint8Array | string;
-};
+export namespace EncryptedData {
+ export type AsObject = {
+ iv: Uint8Array | string;
+ authtag: Uint8Array | string;
+ data: Uint8Array | string;
+ };
+}
export class Message extends jspb.Message {
getDataset(): string;
@@ -60,8 +62,8 @@ export class Message extends jspb.Message {
setValue(value: string): void;
serializeBinary(): Uint8Array;
- toObject(includeInstance?: boolean): MessageAsObject;
- static toObject(includeInstance: boolean, msg: Message): MessageAsObject;
+ toObject(includeInstance?: boolean): Message.AsObject;
+ static toObject(includeInstance: boolean, msg: Message): Message.AsObject;
static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
static extensionsBinary: {
[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -77,12 +79,14 @@ export class Message extends jspb.Message {
): Message;
}
-export type MessageAsObject = {
- dataset: string;
- row: string;
- column: string;
- value: string;
-};
+export namespace Message {
+ export type AsObject = {
+ dataset: string;
+ row: string;
+ column: string;
+ value: string;
+ };
+}
export class MessageEnvelope extends jspb.Message {
getTimestamp(): string;
@@ -97,11 +101,11 @@ export class MessageEnvelope extends jspb.Message {
setContent(value: Uint8Array | string): void;
serializeBinary(): Uint8Array;
- toObject(includeInstance?: boolean): MessageEnvelopeAsObject;
+ toObject(includeInstance?: boolean): MessageEnvelope.AsObject;
static toObject(
includeInstance: boolean,
msg: MessageEnvelope,
- ): MessageEnvelopeAsObject;
+ ): MessageEnvelope.AsObject;
static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
static extensionsBinary: {
[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -117,11 +121,13 @@ export class MessageEnvelope extends jspb.Message {
): MessageEnvelope;
}
-export type MessageEnvelopeAsObject = {
- timestamp: string;
- isencrypted: boolean;
- content: Uint8Array | string;
-};
+export namespace MessageEnvelope {
+ export type AsObject = {
+ timestamp: string;
+ isencrypted: boolean;
+ content: Uint8Array | string;
+ };
+}
export class SyncRequest extends jspb.Message {
clearMessagesList(): void;
@@ -142,11 +148,11 @@ export class SyncRequest extends jspb.Message {
setSince(value: string): void;
serializeBinary(): Uint8Array;
- toObject(includeInstance?: boolean): SyncRequestAsObject;
+ toObject(includeInstance?: boolean): SyncRequest.AsObject;
static toObject(
includeInstance: boolean,
msg: SyncRequest,
- ): SyncRequestAsObject;
+ ): SyncRequest.AsObject;
static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
static extensionsBinary: {
[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -162,13 +168,15 @@ export class SyncRequest extends jspb.Message {
): SyncRequest;
}
-export type SyncRequestAsObject = {
- messagesList: Array<MessageEnvelopeAsObject>;
- fileid: string;
- groupid: string;
- keyid: string;
- since: string;
-};
+export namespace SyncRequest {
+ export type AsObject = {
+ messagesList: Array<MessageEnvelope.AsObject>;
+ fileid: string;
+ groupid: string;
+ keyid: string;
+ since: string;
+ };
+}
export class SyncResponse extends jspb.Message {
clearMessagesList(): void;
@@ -180,11 +188,11 @@ export class SyncResponse extends jspb.Message {
setMerkle(value: string): void;
serializeBinary(): Uint8Array;
- toObject(includeInstance?: boolean): SyncResponseAsObject;
+ toObject(includeInstance?: boolean): SyncResponse.AsObject;
static toObject(
includeInstance: boolean,
msg: SyncResponse,
- ): SyncResponseAsObject;
+ ): SyncResponse.AsObject;
static extensions: { [key: number]: jspb.ExtensionFieldInfo<jspb.Message> };
static extensionsBinary: {
[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>;
@@ -200,7 +208,9 @@ export class SyncResponse extends jspb.Message {
): SyncResponse;
}
-export type SyncResponseAsObject = {
- messagesList: Array<MessageEnvelopeAsObject>;
- merkle: string;
-};
+export namespace SyncResponse {
+ export type AsObject = {
+ messagesList: Array<MessageEnvelope.AsObject>;
+ merkle: string;
+ };
+}
diff --git a/packages/loot-core/package.json b/packages/loot-core/package.json
index 14a1a16161..d32b9df5c9 100644
--- a/packages/loot-core/package.json
+++ b/packages/loot-core/package.json
@@ -72,7 +72,6 @@
"throttleit": "^1.0.0",
"ts-jest": "^27.0.0",
"ts-node": "^10.7.0",
- "ts-protoc-gen": "^0.15.0",
"typescript": "^4.6.4",
"uuid": "3.3.2",
"webpack": "^5.86.0",
diff --git a/upcoming-release-notes/1217.md b/upcoming-release-notes/1217.md
new file mode 100644
index 0000000000..137c23d64d
--- /dev/null
+++ b/upcoming-release-notes/1217.md
@@ -0,0 +1,6 @@
+---
+category: Maintenance
+authors: [j-f1]
+---
+
+Move the rest of the syncing protobuf code to the CRDT package
diff --git a/yarn.lock b/yarn.lock
index 588e148eba..5759cfa18b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -32,6 +32,7 @@ __metadata:
jest: ^27.0.0
murmurhash: ^0.0.2
ts-jest: ^27.0.0
+ ts-protoc-gen: ^0.15.0
typescript: ^5.0.2
uuid: 3.3.2
languageName: unknown
@@ -12380,7 +12381,6 @@ __metadata:
throttleit: ^1.0.0
ts-jest: ^27.0.0
ts-node: ^10.7.0
- ts-protoc-gen: ^0.15.0
typescript: ^4.6.4
uuid: 3.3.2
webpack: ^5.86.0