A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: fix types & syncing issues
Abdullah Atta5b537ab4707b
diff --git a/packages/core/__e2e__/sync.test.js b/packages/core/__e2e__/sync.test.jsindex b2b7fc9d2..55172e815 100644--- a/packages/core/__e2e__/sync.test.js+++ b/packages/core/__e2e__/sync.test.js@@ -27,7 +27,7 @@ import { delay } from "../__tests__/utils";import { test, expect, vitest } from "vitest";import { login } from "./utils";-const TEST_TIMEOUT = 30 * 1000;+const TEST_TIMEOUT = 60 * 1000;test("case 1: device A & B should only download the changes from device C (no uploading)",@@ -311,12 +311,16 @@ test(expect(deviceA.notebooks.topics(id).has("Topic 1")).toBeTruthy();expect(deviceB.notebooks.topics(id).has("Topic 2")).toBeTruthy();+ expect(+ deviceB.notebooks.topics(id).topic("Topic 2").dateModified >+ deviceA.notebooks.topics(id).topic("Topic 1").dateModified+ ).toBeTruthy();+ expect(+ deviceB.notebooks.notebook(id).dateModified >+ deviceA.notebooks.notebook(id).dateModified+ ).toBeTruthy();- await syncAndWait(deviceA, deviceB, false);-- // await delay(1000);-- // await syncAndWait(deviceB, deviceB, false);+ await syncAndWait(deviceB, deviceA, false);expect(deviceA.notebooks.topics(id).has("Topic 1")).toBeTruthy();expect(deviceB.notebooks.topics(id).has("Topic 1")).toBeTruthy();@@ -360,15 +364,27 @@ test(expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(1);+ ctx.onTestFailed(() => {+ console.log(deviceA.notes.topicReferences.get(topic.id), noteA);+ console.log(deviceB.notes.topicReferences.get(topic.id), noteB);++ deviceB.notes.topicReferences.rebuild();+ deviceA.notes.topicReferences.rebuild();++ console.log(deviceA.notes.topicReferences.get(topic.id), noteA);+ console.log(deviceB.notes.topicReferences.get(topic.id), noteB);+ });await syncAndWait(deviceB, deviceA, false);- await syncAndWait(deviceA, deviceB, false);- expect(deviceA.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);- expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);+ expect(deviceA.notes.note(noteB)).toBeDefined();+ expect(deviceB.notes.note(noteA)).toBeDefined();expect(deviceA.notes.note(noteA).data.notebooks).toHaveLength(1);expect(deviceA.notes.note(noteB).data.notebooks).toHaveLength(1);+ expect(deviceA.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);+ expect(deviceB.notebooks.topics(id).topic(topic.id).totalNotes).toBe(2);+await cleanup(deviceA, deviceB);},TEST_TIMEOUT@@ -456,6 +472,7 @@ function syncAndWait(deviceA, deviceB, force = false) {console.log("sync completed.");resolve();});+console.log("waiting for sync...","Device A:",diff --git a/packages/core/__tests__/trash.test.js b/packages/core/__tests__/trash.test.jsindex 01a3c785a..10b9a0706 100644--- a/packages/core/__tests__/trash.test.js+++ b/packages/core/__tests__/trash.test.js@@ -45,7 +45,7 @@ test("permanently delete a note", () =>expect(await note.content()).toBeDefined();await db.trash.delete(db.trash.all[0].id);expect(db.trash.all).toHaveLength(0);- const content = await db.content.raw(note.data.contentId);+ const content = await db.content.get(note.data.contentId);expect(content).toBeUndefined();sessions = await db.noteHistory.get(noteId);@@ -229,7 +229,7 @@ test("clear trash should delete note content", () =>expect(db.trash.all).toHaveLength(0);- const content = await db.content.raw(note.contentId);+ const content = await db.content.get(note.contentId);expect(content).toBeUndefined();sessions = await db.noteHistory.get(note.id);diff --git a/packages/core/src/api/sync/collector.ts b/packages/core/src/api/sync/collector.tsindex 8123500aa..e2e2bd5c3 100644--- a/packages/core/src/api/sync/collector.ts+++ b/packages/core/src/api/sync/collector.ts@@ -19,103 +19,131 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { Cipher, SerializedKey } from "@notesnook/crypto";import Database from "..";-import { CURRENT_DATABASE_VERSION } from "../../common";+import { CURRENT_DATABASE_VERSION, EV, EVENTS } from "../../common";+import { logger } from "../../logger";+import { SYNC_COLLECTIONS_MAP, SyncItem, SyncTransferItem } from "./types";import { Item, MaybeDeletedItem } from "../../types";-export type SyncableItemType =- | "note"- | "shortcut"- | "notebook"- | "content"- | "attachment"- | "reminder"- | "relation"- | "color"- | "tag"- | "settings";-export type CollectedResult = {- items: (MaybeDeletedItem<Item> | Cipher)[];- types: (SyncableItemType | "vaultKey")[];-};--export type SyncItem = {- id: string;- v: number;-} & Cipher;-+const ASYNC_COLLECTIONS_MAP = {+ content: "content"+} as const;class Collector {- private lastSyncedTimestamp = 0;- private key?: SerializedKey;+ logger = logger.scope("SyncCollector");constructor(private readonly db: Database) {}- async collect(lastSyncedTimestamp: number, isForceSync?: boolean) {- await this.db.notes.init();-- this.lastSyncedTimestamp = lastSyncedTimestamp;- this.key = await this.db.user.getEncryptionKey();- const vaultKey = await this.db.vault.getKey();-- const collections = {- note: this.db.notes.raw,- shortcut: this.db.shortcuts.raw,- notebook: this.db.notebooks.raw,- content: await this.db.content.all(),- attachment: this.db.attachments.syncable,- reminder: this.db.reminders.raw,- relation: this.db.relations.raw,- color: this.db.colors.raw,- tag: this.db.tags.raw,- settings: [this.db.settings.raw]- };-- const result: CollectedResult = {- items: [],- types: []- };- for (const type in collections) {- this.collectInternal(- type as SyncableItemType,- collections[type as SyncableItemType],- result,- isForceSync- );+ async *collect(+ chunkSize: number,+ lastSyncedTimestamp: number,+ isForceSync = false+ ): AsyncGenerator<SyncTransferItem, void, unknown> {+ const key = await this.db.user.getEncryptionKey();+ if (!key || !key.key || !key.salt) {+ EV.publish(EVENTS.userSessionExpired);+ throw new Error("User encryption key not generated. Please relogin.");}- if (vaultKey) {- result.items.push(vaultKey);- result.types.push("vaultKey");+ const settings = await this.prepareChunk(+ [this.db.settings.raw],+ lastSyncedTimestamp,+ isForceSync,+ key+ );+ if (settings) yield { items: settings, type: "settings" };++ const attachments = await this.prepareChunk(+ this.db.attachments.syncable,+ lastSyncedTimestamp,+ isForceSync,+ key+ );+ if (attachments) yield { items: attachments, type: "attachment" };++ for (const itemType in ASYNC_COLLECTIONS_MAP) {+ const collectionKey =+ ASYNC_COLLECTIONS_MAP[itemType as keyof typeof ASYNC_COLLECTIONS_MAP];+ const collection = this.db[collectionKey].collection;+ for await (const chunk of collection.iterate(chunkSize)) {+ const items = await this.prepareChunk(+ chunk.map((item) => item[1]),+ lastSyncedTimestamp,+ isForceSync,+ key+ );+ if (!items) continue;+ yield { items, type: itemType as keyof typeof ASYNC_COLLECTIONS_MAP };+ }}- return result;+ for (const itemType in SYNC_COLLECTIONS_MAP) {+ const collectionKey =+ SYNC_COLLECTIONS_MAP[itemType as keyof typeof SYNC_COLLECTIONS_MAP];+ const collection = this.db[collectionKey].collection;+ for (const chunk of collection.iterateSync(chunkSize)) {+ const items = await this.prepareChunk(+ chunk,+ lastSyncedTimestamp,+ isForceSync,+ key+ );+ if (!items) continue;+ yield { items, type: itemType as keyof typeof SYNC_COLLECTIONS_MAP };+ }+ }}- private serialize(item: MaybeDeletedItem<Item>) {- if (!this.key) throw new Error("No encryption key found.");- return this.db.storage().encrypt(this.key, JSON.stringify(item));+ async prepareChunk(+ chunk: MaybeDeletedItem<Item>[],+ lastSyncedTimestamp: number,+ isForceSync: boolean,+ key: SerializedKey+ ) {+ const { ids, items } = filterSyncableItems(+ chunk,+ lastSyncedTimestamp,+ isForceSync+ );+ if (!ids.length) return;+ const ciphers = await this.db.storage().encryptMulti(key, items);+ return toPushItem(ids, ciphers);}+}+export default Collector;- encrypt(array: MaybeDeletedItem<Item>[]) {- if (!array.length) return [];- return Promise.all(array.map(this.map, this));+function toPushItem(ids: string[], ciphers: Cipher<"base64">[]) {+ if (ids.length !== ciphers.length)+ throw new Error("ids.length must be equal to ciphers.length");++ const items: SyncItem[] = [];+ for (let i = 0; i < ids.length; ++i) {+ const id = ids[i];+ const cipher = ciphers[i];+ items.push({ ...cipher, v: CURRENT_DATABASE_VERSION, id });}+ return items;+}- private collectInternal(- itemType: SyncableItemType,- items: MaybeDeletedItem<Item>[],- result: CollectedResult,- isForceSync?: boolean- ) {- if (!items || !items.length) return;+function filterSyncableItems(+ items: MaybeDeletedItem<Item>[],+ lastSyncedTimestamp: number,+ isForceSync = false+): { items: string[]; ids: string[] } {+ if (!items || !items.length) return { items: [], ids: [] };- for (const item of items) {- if (!item) continue;+ const ids = [];+ const syncableItems = [];+ for (const item of items) {+ if (!item) continue;- const isSyncable = !item.synced || isForceSync;- const isUnsynced =- item.dateModified > this.lastSyncedTimestamp || isForceSync;+ const isSyncable = !item.synced || isForceSync;+ const isUnsynced = item.dateModified > lastSyncedTimestamp || isForceSync;- if (isUnsynced && isSyncable) {- result.items.push(+ // synced is a local only property+ delete item.synced;++ if (isUnsynced && isSyncable) {+ ids.push(item.id);+ syncableItems.push(+ JSON.stringify("localOnly" in item && item.localOnly? {id: item.id,@@ -124,21 +152,9 @@ class Collector {deleteReason: "localOnly"}: item- );- result.types.push(itemType);- }+ )+ );}}-- private async map(item: MaybeDeletedItem<Item>) {- // synced is a local only property- delete item.synced;-- return {- id: item.id,- v: CURRENT_DATABASE_VERSION,- ...(await this.serialize(item))- };- }+ return { items: syncableItems, ids };}-export default Collector;diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.tsindex 73c1d9fc6..ab74fb685 100644--- a/packages/core/src/api/sync/index.ts+++ b/packages/core/src/api/sync/index.ts@@ -26,11 +26,7 @@ import {} from "../../common";import Constants from "../../utils/constants";import TokenManager from "../token-manager";-import Collector, {- CollectedResult,- SyncableItemType,- SyncItem-} from "./collector";+import Collector from "./collector";import * as signalr from "@microsoft/signalr";import Merger from "./merger";import Conflicts from "./conflicts";@@ -40,6 +36,14 @@ import { Mutex } from "async-mutex";import Database from "..";import { migrateItem } from "../../migrations";import { SerializedKey } from "@notesnook/crypto";+import {+ ItemMap,+ MaybeDeletedItem,+ Note,+ Notebook,+ TrashOrItem+} from "../../types";+import { SyncableItemType, SyncTransferItem } from "./types";const ITEM_TYPE_TO_COLLECTION_TYPE = {note: "notes",@@ -48,7 +52,9 @@ const ITEM_TYPE_TO_COLLECTION_TYPE = {attachment: "attachments",relation: "relations",reminder: "reminders",- shortcut: "shortcuts"+ shortcut: "shortcuts",+ tag: "tags",+ color: "colors"};export type SyncOptions = {@@ -57,11 +63,6 @@ export type SyncOptions = {serverLastSynced?: number;};-type SyncTransferItem = {- items: SyncItem[];- type: SyncableItemType;-};-export default class SyncManager {sync = new Sync(this.db);constructor(private readonly db: Database) {}@@ -169,6 +170,11 @@ class Sync {}, 15000) as unknown as number;const key = await this.db.user.getEncryptionKey();+ if (!key || !key.key || !key.salt) {+ EV.publish(EVENTS.userSessionExpired);+ throw new Error("User encryption key not generated. Please relogin.");+ }+const dbLastSynced = await this.db.lastSynced();await this.processChunk(chunk, key, dbLastSynced, true);});@@ -208,7 +214,7 @@ class Sync {if ((options.type === "send" || options.type === "full") &&- (await this.send(lastSynced, options.force, newLastSynced))+ (await this.send(lastSynced, newLastSynced, options.force))) {this.logger.info("New data sent");await this.stop(newLastSynced);@@ -278,7 +284,7 @@ class Sync {serverResponse.vaultKey.salt !== null &&serverResponse.vaultKey.length > 0) {- await this.db.vault._setKey(serverResponse.vaultKey);+ await this.db.vault.setKey(serverResponse.vaultKey);}this.connection.off("SendItems");@@ -328,7 +334,7 @@ class Sync {// return true;}- async stop(lastSynced) {+ async stop(lastSynced: number) {// refresh topic referencesthis.db.notes.topicReferences.rebuild();// refresh monographs on sync completed@@ -366,7 +372,7 @@ class Sync {/*** @private*/- async onPushCompleted(lastSynced) {+ async onPushCompleted(lastSynced: number) {// refresh topic referencesthis.db.notes.topicReferences.rebuild();@@ -395,12 +401,14 @@ class Sync {));- let items = [];- if (this.merger.isSyncCollection(chunk.type)) {- items = deserialized.map((item) =>- this.merger.mergeItemSync(item, chunk.type, dbLastSynced)- );- } else if (chunk.type === "content") {+ const itemType = chunk.type;+ let items: (+ | MaybeDeletedItem<+ ItemMap[SyncableItemType] | TrashOrItem<Note> | TrashOrItem<Notebook>+ >+ | undefined+ )[] = [];+ if (itemType === "content") {const localItems = await this.db.content.multi(chunk.items.map((i) => i.id));@@ -409,28 +417,33 @@ class Sync {this.merger.mergeContent(item, localItems[item.id], dbLastSynced)));+ } else if (itemType === "settings") {+ await this.merger.mergeItem(deserialized[0], itemType, dbLastSynced);+ return;} else {- items = await Promise.all(- deserialized.map((item) =>- this.merger.mergeItem(item, chunk.type, dbLastSynced)- )- );+ items = this.merger.isSyncCollection(itemType)+ ? deserialized.map((item) =>+ this.merger.mergeItemSync(item, itemType, dbLastSynced)+ )+ : await Promise.all(+ deserialized.map((item) =>+ this.merger.mergeItem(item, itemType, dbLastSynced)+ )+ );}+ const collectionType = ITEM_TYPE_TO_COLLECTION_TYPE[itemType];+ await this.db[collectionType].collection.setItems(items as any);+if (notify &&- (chunk.type === "content" || chunk.type === "note") &&+ (itemType === "note" || itemType === "content") &&items.length > 0) {items.forEach((item) =>this.db.eventManager.publish(EVENTS.syncItemMerged, item));}-- const collectionType = this.itemTypeToCollection[chunk.type];- if (collectionType && this.db[collectionType]) {- await this.db[collectionType]._collection.setItems(items);- }}private async pushItem(item: SyncTransferItem, newLastSynced: number) {@@ -477,7 +490,11 @@ function promiseTimeout(ms: number, promise: Promise<unknown>) {return Promise.race([promise, timeout]);}-async function deserializeItem(decryptedItem, version, database) {+async function deserializeItem(+ decryptedItem: string,+ version: number,+ database: Database+) {const deserialized = JSON.parse(decryptedItem);deserialized.remote = true;deserialized.synced = true;diff --git a/packages/core/src/api/sync/merger.ts b/packages/core/src/api/sync/merger.tsindex 8e44b8bb2..511be0bdb 100644--- a/packages/core/src/api/sync/merger.ts+++ b/packages/core/src/api/sync/merger.ts@@ -17,292 +17,215 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { migrateItem } from "../../migrations";import { set } from "../../utils/set";import { logger } from "../../logger";import { isHTMLEqual } from "../../utils/html-diff";-import { EV, EVENTS } from "../../common";import Database from "..";-import { SyncItem, SyncableItemType } from "./collector";-import { Item, ItemMap, MaybeDeletedItem, isDeleted } from "../../types";-import { SerializedKey } from "@notesnook/crypto";-import { isCipher } from "../../database/crypto";--type Conflict<P extends SyncableItemType> = (- local: MaybeDeletedItem<ItemMap[P]>,- remote: MaybeDeletedItem<ItemMap[P]>-) => Promise<void>;--type Set<P extends SyncableItemType> = (- item: MaybeDeletedItem<ItemMap[P]>-) => Promise<void>;--type Get<P extends SyncableItemType> = (- id: string-) =>- | MaybeDeletedItem<ItemMap[P]>- | undefined- | Promise<MaybeDeletedItem<ItemMap[P]> | undefined>;--type MergeDefinition = {- [P in SyncableItemType]: {- threshold?: number;- get?: Get<P>;- set: Set<P>;- conflict?: Conflict<P>;- };-};+import { SYNC_COLLECTIONS_MAP } from "./types";+import {+ Attachment,+ ContentItem,+ Item,+ ItemMap,+ MaybeDeletedItem,+ Note,+ Notebook,+ SettingsItem,+ TrashOrItem,+ isDeleted+} from "../../types";class Merger {- private mergeDefinition: MergeDefinition;- private logger = logger.scope("Merger");- private lastSynced = 0;- private key?: SerializedKey;- constructor(private readonly db: Database) {- this.mergeDefinition = {- settings: {- threshold: 1000,- get: () => this.db.settings.raw,- set: (item) => this.db.settings.merge(item),- conflict: (_local, remote) => this.db.settings.merge(remote)- },- note: {- get: (id) => this.db.notes.note(id)?.data,- set: (item) => this.db.notes.merge(item)- },- shortcut: {- get: (id) => this.db.shortcuts.shortcut(id),- set: (item) => this.db.shortcuts.merge(item)- },- reminder: {- get: (id) => this.db.reminders.reminder(id),- set: (item) => this.db.reminders.merge(item)- },- relation: {- get: (id) => this.db.relations.relation(id),- set: (item) => this.db.relations.merge(item)- },- tag: {- get: (id) => this.db.tags.tag(id),- set: (item) => this.db.tags.merge(item)- },- color: {- get: (id) => this.db.colors.color(id),- set: (item) => this.db.colors.merge(item)- },- notebook: {- threshold: 1000,- get: (id) => this.db.notebooks.notebook(id)?.data,- set: (item) => this.db.notebooks.merge(item),- conflict: (_local, remote) => this.db.notebooks.merge(remote)- },- content: {- threshold: process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000,- get: (id) => this.db.content.raw(id),- set: async (item) => {- await this.db.content.merge(item);- },- conflict: async (local, remote) => {- if (isDeleted(local) || isDeleted(remote)) {- if (remote.dateModified > local.dateModified)- await db.content.merge(remote);- return;- }-- const note = this.db.notes.note(local.noteId);- if (!note || !note.data) return;+ logger = logger.scope("Merger");+ constructor(private readonly db: Database) {}- // if hashes are equal do nothing- if (- !note.locked &&- (!remote ||- !local ||- !local.data ||- !remote.data ||- isHTMLEqual(local.data, remote.data))- )- return;-- if (note.locked) {- // if note is locked or content is deleted we keep the most recent version.- if (remote.dateModified > local.dateModified)- await this.db.content.merge({ ...remote, id: local.id });- } else {- // otherwise we trigger the conflicts- await this.db.content.merge({ ...local, conflicted: remote });- await this.db.notes.add({ id: local.noteId, conflicted: true });- await this.db.storage().write("hasConflicts", true);- }- }- },- attachment: {- set: async (remoteAttachment) => {- if (isDeleted(remoteAttachment)) {- await this.db.attachments.merge(remoteAttachment);- return;- }+ isSyncCollection(type: string): type is keyof typeof SYNC_COLLECTIONS_MAP {+ return type in SYNC_COLLECTIONS_MAP;+ }- const localAttachment = this.db.attachments.attachment(- remoteAttachment.metadata.hash- );- if (- localAttachment &&- localAttachment.dateUploaded !== remoteAttachment.dateUploaded- ) {- const noteIds = localAttachment.noteIds.slice();- const isRemoved = await this.db.attachments.remove(- localAttachment.metadata.hash,- true- );- if (!isRemoved)- throw new Error(- "Conflict could not be resolved in one of the attachments."- );- remoteAttachment.noteIds = set.union(- remoteAttachment.noteIds,- noteIds- );- }- await this.db.attachments.merge(remoteAttachment);+ isConflicted(+ localItem: MaybeDeletedItem<Item>,+ remoteItem: MaybeDeletedItem<Item>,+ lastSynced: number,+ conflictThreshold: number+ ) {+ const isResolved =+ "dateResolved" in localItem &&+ localItem.dateResolved === remoteItem.dateModified;+ const isModified =+ // the local item is modified if it was changed/modified after the last+ // sync i.e. it wasn't synced yet.+ // However, in case a sync is interrupted the local item's date modified+ // will be ahead of last sync. In that case, we also have to check if the+ // synced flag is false (it is only false if a user makes edits on the+ // local device).+ localItem.dateModified > lastSynced && !localItem.synced;+ if (isModified && !isResolved) {+ // If time difference between local item's edits & remote item's edits+ // is less than threshold, we shouldn't trigger a merge conflict; instead+ // we will keep the most recently changed item.+ const timeDiff =+ Math.max(remoteItem.dateModified, localItem.dateModified) -+ Math.min(remoteItem.dateModified, localItem.dateModified);++ if (timeDiff < conflictThreshold) {+ if (remoteItem.dateModified > localItem.dateModified) {+ return "merge";}+ return;}- };- }-- async _migrate(deserialized: Item, version: number) {- // it is a locked note, bail out.- if (isCipher(deserialized) && deserialized.alg && deserialized.cipher)- return deserialized;-- return migrateItem(deserialized, version, deserialized.type, this.db);- }-- async _deserialize(item: SyncItem, migrate = true) {- if (!this.key) throw new Error("User encryption key not found.");- const decrypted = await this.db.storage().decrypt(this.key, item);- if (!decrypted) {- throw new Error("Decrypted item cannot be undefined or empty.");+ return "conflict";+ } else if (!isResolved) {+ return "merge";}-- const deserialized = JSON.parse(decrypted);- deserialized.remote = true;- deserialized.synced = true;- if (!migrate) return deserialized;- await this._migrate(deserialized, item.v);- return deserialized;}- async _mergeItem<TItemType extends SyncableItemType>(- syncItem: SyncItem,- get: Get<TItemType>,- add: Set<TItemType>+ mergeItemSync<TType extends keyof typeof SYNC_COLLECTIONS_MAP>(+ remoteItem: MaybeDeletedItem<+ ItemMap[TType] | TrashOrItem<Note> | TrashOrItem<Notebook>+ >,+ type: TType,+ lastSynced: number) {- const remoteItem = (await this._deserialize(syncItem)) as MaybeDeletedItem<- ItemMap[TItemType]- >;- const localItem = await get(remoteItem.id);- if (!localItem || remoteItem.dateModified > localItem.dateModified) {- await add(remoteItem);- return remoteItem;+ switch (type) {+ case "shortcut":+ case "reminder":+ case "tag":+ case "color":+ case "note":+ case "relation": {+ const localItem = this.db[SYNC_COLLECTIONS_MAP[type]].collection.getRaw(+ remoteItem.id+ );+ if (!localItem || remoteItem.dateModified > localItem.dateModified) {+ return remoteItem;+ }+ break;+ }+ // case "note": {+ // const localItem = this.db.notes.collection.getRaw(remoteItem.id);+ // if (!localItem || remoteItem.dateModified > localItem.dateModified) {+ // return this.db.notes.merge(+ // localItem,+ // remoteItem as MaybeDeletedItem<TrashOrItem<Note>>+ // );+ // }+ // break;+ // }+ case "notebook": {+ const THRESHOLD = 1000;+ const localItem = this.db.notebooks.collection.getRaw(remoteItem.id);+ if (+ !localItem ||+ this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD)+ ) {+ return this.db.notebooks.merge(+ localItem,+ remoteItem as MaybeDeletedItem<TrashOrItem<Notebook>>,+ lastSynced+ );+ }+ break;+ }}}- async _mergeItemWithConflicts<TItemType extends SyncableItemType>(- syncItem: SyncItem,- get: Get<TItemType>,- add: Set<TItemType>,- markAsConflicted: Conflict<TItemType>,- threshold: number+ async mergeContent(+ remoteItem: MaybeDeletedItem<ContentItem>,+ localItem: MaybeDeletedItem<ContentItem>,+ lastSynced: number) {- const remoteItem = (await this._deserialize(syncItem)) as MaybeDeletedItem<- ItemMap[TItemType]- >;- const localItem = await get(remoteItem.id);+ if (localItem && "localOnly" in localItem && localItem.localOnly) return;- if (!localItem || isDeleted(localItem)) {- await add(remoteItem);+ const THRESHOLD = process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000;+ const conflicted =+ localItem &&+ this.isConflicted(localItem, remoteItem, lastSynced, THRESHOLD);+ if (!localItem || conflicted === "merge") {return remoteItem;- } else {- const isResolved =- "dateResolved" in localItem &&- localItem.dateResolved === remoteItem.dateModified;- const isModified =- // the local item is modified if it was changed/modified after the last sync- // i.e. it wasn't synced yet.- // However, in case a sync is interrupted the local item's date modified will- // be ahead of last sync. In that case, we also have to check if the synced flag- // is false (it is only false if a user makes edits on the local device).- localItem.dateModified > this.lastSynced && !localItem.synced;- if (isModified && !isResolved) {- // If time difference between local item's edits & remote item's edits- // is less than threshold, we shouldn't trigger a merge conflict; instead- // we will keep the most recently changed item.- const timeDiff =- Math.max(remoteItem.dateModified, localItem.dateModified) -- Math.min(remoteItem.dateModified, localItem.dateModified);-- if (timeDiff < threshold) {- if (remoteItem.dateModified > localItem.dateModified) {- await add(remoteItem);- return remoteItem;- }- return;- }+ } else if (conflicted === "conflict") {+ if (isDeleted(localItem) || isDeleted(remoteItem)) {+ if (remoteItem.dateModified > localItem.dateModified) return remoteItem;+ return;+ }- this.logger.info("Conflict detected", {- itemId: remoteItem.id,- isResolved,- isModified,- timeDiff,- remote: remoteItem.dateModified,- local: localItem.dateModified,- lastSynced: this.lastSynced+ const note = this.db.notes.collection.get(localItem.noteId);+ if (!note) return;++ // if hashes are equal do nothing+ if (+ !note.locked &&+ (!remoteItem ||+ !remoteItem ||+ !localItem.data ||+ !remoteItem.data ||+ isHTMLEqual(localItem.data, remoteItem.data))+ )+ return;++ if (note.locked) {+ // if note is locked or content is deleted we keep the most recent version.+ if (remoteItem.dateModified > localItem.dateModified) return remoteItem;+ } else {+ // otherwise we trigger the conflicts+ await this.db.notes.add({+ id: localItem.noteId,+ conflicted: true});-- await markAsConflicted(localItem, remoteItem);- } else if (!isResolved) {- await add(remoteItem);- return remoteItem;+ await this.db.storage().write("hasConflicts", true);+ return {+ ...localItem,+ conflicted: remoteItem+ };}}}- async mergeItem<TItemType extends SyncableItemType>(- type: TItemType | "vaultKey",- item: SyncItem+ async mergeItem(+ remoteItem: SettingsItem | MaybeDeletedItem<Attachment>,+ type: "settings" | "attachment",+ lastSynced: number) {- this.lastSynced = await this.db.lastSynced();-- if (!this.key) this.key = await this.db.user.getEncryptionKey();- if (!this.key || !this.key.key || !this.key.salt) {- EV.publish(EVENTS.userSessionExpired);- throw new Error("User encryption key not generated. Please relogin.");- }-- if (type === "vaultKey") {- await this.db.vault.setKey(await this._deserialize(item, false));- return;- }-- const definition = this.mergeDefinition[type];- if (definition.conflict && definition.get && definition.threshold) {- return await this._mergeItemWithConflicts<TItemType>(- item,- definition.get,- definition.set,- definition.conflict,- definition.threshold- );- } else if (definition.get && definition.set) {- return await this._mergeItem<TItemType>(- item,- definition.get,- definition.set- );- } else if (!definition.get && !!definition.set) {- const remote = await this._deserialize(item);- await definition.set(remote);+ switch (type) {+ case "settings": {+ if (isDeleted(remoteItem) || remoteItem.type !== "settings") return;++ const localItem = this.db.settings.raw;+ if (+ !localItem ||+ this.isConflicted(localItem, remoteItem, lastSynced, 1000)+ ) {+ await this.db.settings.merge(remoteItem, lastSynced);+ }+ break;+ }+ case "attachment": {+ if (isDeleted(remoteItem)) {+ return this.db.attachments.merge(undefined, remoteItem);+ }+ if (remoteItem.type !== "attachment") return;++ const localAttachment = this.db.attachments.attachment(+ remoteItem.metadata.hash+ );+ if (+ localAttachment &&+ localAttachment.dateUploaded !== remoteItem.dateUploaded+ ) {+ const noteIds = localAttachment.noteIds.slice();+ const isRemoved = await this.db.attachments.remove(+ localAttachment.metadata.hash,+ true+ );+ if (!isRemoved)+ throw new Error(+ "Conflict could not be resolved in one of the attachments."+ );+ remoteItem.noteIds = set.union(remoteItem.noteIds, noteIds);+ }+ return this.db.attachments.merge(undefined, remoteItem);+ }}}}diff --git a/packages/core/src/api/sync/types.ts b/packages/core/src/api/sync/types.tsnew file mode 100644index 000000000..4d53748e5--- /dev/null+++ b/packages/core/src/api/sync/types.ts@@ -0,0 +1,52 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { Cipher } from "@notesnook/crypto";++export type SyncableItemType =+ | "note"+ | "shortcut"+ | "notebook"+ | "content"+ | "attachment"+ | "reminder"+ | "relation"+ | "color"+ | "tag"+ | "settings";++export type SyncItem = {+ id: string;+ v: number;+} & Cipher<"base64">;++export const SYNC_COLLECTIONS_MAP = {+ note: "notes",+ notebook: "notebooks",+ shortcut: "shortcuts",+ reminder: "reminders",+ relation: "relations",+ tag: "tags",+ color: "colors"+} as const;++export type SyncTransferItem = {+ items: SyncItem[];+ type: SyncableItemType;+};diff --git a/packages/core/src/api/vault.ts b/packages/core/src/api/vault.tsindex 55a7a4067..5abc4bd50 100644--- a/packages/core/src/api/vault.ts+++ b/packages/core/src/api/vault.ts@@ -209,7 +209,7 @@ export default class Vault {return await this.lockNote(note, await this.getVaultPassword());}- async exists(vaultKey?: Cipher) {+ async exists(vaultKey?: Cipher<"base64">) {if (!vaultKey) vaultKey = await this.getKey();return vaultKey && isCipher(vaultKey);}@@ -370,10 +370,10 @@ export default class Vault {}async getKey() {- return await this.db.storage().read<Cipher>("vaultKey");+ return await this.db.storage().read<Cipher<"base64">>("vaultKey");}- async setKey(vaultKey: Cipher) {+ async setKey(vaultKey: Cipher<"base64">) {await this.db.storage().write("vaultKey", vaultKey);}}diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.tsindex 0d48284be..1dcfda2b9 100644--- a/packages/core/src/collections/attachments.ts+++ b/packages/core/src/collections/attachments.ts@@ -43,7 +43,7 @@ import Database from "../api";export class Attachments implements ICollection {name = "attachments";key: Cipher<"base64"> | null = null;- private readonly collection: CachedCollection<"attachments", Attachment>;+ readonly collection: CachedCollection<"attachments", Attachment>;constructor(private readonly db: Database) {this.collection = new CachedCollection(db.storage,@@ -109,7 +109,7 @@ export class Attachments implements ICollection {}merge(- localAttachment: MaybeDeletedItem<Attachment>,+ localAttachment: MaybeDeletedItem<Attachment> | undefined,remoteAttachment: MaybeDeletedItem<Attachment>) {if (isDeleted(remoteAttachment)) return remoteAttachment;@@ -323,7 +323,10 @@ export class Attachments implements ICollection {return (outputType === "base64"- ? dataurl.fromObject({ type: attachment.metadata.type, data })+ ? dataurl.fromObject({+ type: attachment.metadata.type,+ data+ }): data) as Output<TOutputFormat>;}diff --git a/packages/core/src/collections/colors.ts b/packages/core/src/collections/colors.tsindex a1710a86b..ded823f59 100644--- a/packages/core/src/collections/colors.ts+++ b/packages/core/src/collections/colors.ts@@ -36,7 +36,7 @@ export const DefaultColors: Record<string, string> = {export class Colors implements ICollection {name = "colors";- private readonly collection: CachedCollection<"colors", Color>;+ readonly collection: CachedCollection<"colors", Color>;constructor(private readonly db: Database) {this.collection = new CachedCollection(db.storage,diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.tsindex ef119c9d8..efd437b6f 100644--- a/packages/core/src/collections/content.ts+++ b/packages/core/src/collections/content.ts@@ -49,7 +49,7 @@ export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({export class Content implements ICollection {name = "content";- private readonly collection: IndexedCollection<"content", ContentItem>;+ readonly collection: IndexedCollection<"content", ContentItem>;constructor(private readonly db: Database) {this.collection = new IndexedCollection(db.storage,@@ -130,7 +130,7 @@ export class Content implements ICollection {async get(id: string) {const content = await this.raw(id);if (!content || isDeleted(content)) return;- return content.data;+ return content;}async raw(id: string) {diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.tsindex 1fd5d5f1d..2142ca14d 100644--- a/packages/core/src/collections/notebooks.ts+++ b/packages/core/src/collections/notebooks.ts@@ -27,7 +27,6 @@ import {MaybeDeletedItem,Notebook,Topic,- TrashItem,TrashOrItem,isDeleted,isTrashItem@@ -52,15 +51,23 @@ export class Notebooks implements ICollection {return this.collection.init();}- async merge(remoteNotebook: MaybeDeletedItem<TrashOrItem<Notebook>>) {+ merge(+ localNotebook: MaybeDeletedItem<TrashOrItem<Notebook>> | undefined,+ remoteNotebook: MaybeDeletedItem<TrashOrItem<Notebook>>,+ lastSyncedTimestamp: number+ ) {if (isDeleted(remoteNotebook) || isTrashItem(remoteNotebook))- return await this.collection.add(remoteNotebook);+ return remoteNotebook;- const id = remoteNotebook.id;- const localNotebook = this.collection.get(id);+ if (+ localNotebook &&+ (isTrashItem(localNotebook) || isDeleted(localNotebook))+ ) {+ if (localNotebook.dateModified > remoteNotebook.dateModified) return;+ return remoteNotebook;+ }if (localNotebook && localNotebook.topics?.length) {- const lastSyncedTimestamp = await this.db.lastSynced();let isChanged = false;// merge new and old topicsfor (const oldTopic of localNotebook.topics) {@@ -93,7 +100,7 @@ export class Notebooks implements ICollection {}remoteNotebook.remote = !isChanged;}- return await this.collection.add(remoteNotebook);+ return remoteNotebook;}async add(diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.tsindex 8eaff927e..dcbb2ff57 100644--- a/packages/core/src/collections/notes.ts+++ b/packages/core/src/collections/notes.ts@@ -27,13 +27,7 @@ import { Tiptap } from "../content-types/tiptap";import { EMPTY_CONTENT, isUnencryptedContent } from "./content";import { CHECK_IDS, checkIsUserPremium } from "../common";import { buildFromTemplate } from "../utils/templates";-import {- Note,- TrashOrItem,- isTrashItem,- MaybeDeletedItem,- isDeleted-} from "../types";+import { Note, TrashOrItem, isTrashItem, isDeleted } from "../types";import Database from "../api";import { CachedCollection } from "../database/cached-collection";import { ICollection } from "./collection";@@ -67,17 +61,6 @@ export class Notes implements ICollection {this.topicReferences.rebuild();}- async merge(remoteNote: MaybeDeletedItem<TrashOrItem<Note>>) {- if (!remoteNote) return;-- const id = remoteNote.id;- const localNote = this.collection.get(id);-- if (localNote && localNote.localOnly) return;-- return await this.collection.add(remoteNote);- }-async add(item: Partial<Note & { content: NoteContent<false>; sessionId: string }>): Promise<string | undefined> {@@ -430,6 +413,9 @@ export class Notes implements ICollection {this.topicReferences.rebuild();}+ /**+ * @internal+ */async _clearAllNotebookReferences(notebookId: string) {const notes = this.db.notes.all;@@ -482,7 +468,7 @@ class NoteIdCache {for (const note of notes) {const { notebooks } = note;- if (!notebooks) continue;+ if (!notebooks) return;for (const notebook of notebooks) {for (const topic of notebook.topics) {diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.tsindex b5050ed56..9ba70a8c2 100644--- a/packages/core/src/collections/relations.ts+++ b/packages/core/src/collections/relations.ts@@ -29,7 +29,7 @@ type RelationsArray<TType extends keyof ItemMap> = Relation[] & {export class Relations implements ICollection {name = "relations";- private readonly collection: CachedCollection<"relations", Relation>;+ readonly collection: CachedCollection<"relations", Relation>;constructor(private readonly db: Database) {this.collection = new CachedCollection(db.storage,diff --git a/packages/core/src/collections/reminders.ts b/packages/core/src/collections/reminders.tsindex d0178c231..ff3e2e41b 100644--- a/packages/core/src/collections/reminders.ts+++ b/packages/core/src/collections/reminders.ts@@ -36,7 +36,7 @@ dayjs.extend(isToday);export class Reminders implements ICollection {name = "reminders";- private readonly collection: CachedCollection<"reminders", Reminder>;+ readonly collection: CachedCollection<"reminders", Reminder>;constructor(private readonly db: Database) {this.collection = new CachedCollection(db.storage,diff --git a/packages/core/src/collections/session-content.ts b/packages/core/src/collections/session-content.tsindex 16d5942b7..0936ab6f7 100644--- a/packages/core/src/collections/session-content.ts+++ b/packages/core/src/collections/session-content.ts@@ -27,7 +27,7 @@ import Database from "../api";import { ContentType, SessionContentItem, isDeleted } from "../types";export type NoteContent<TLocked extends boolean> = {- data: TLocked extends true ? Cipher : string;+ data: TLocked extends true ? Cipher<"base64"> : string;type: ContentType;};diff --git a/packages/core/src/collections/shortcuts.ts b/packages/core/src/collections/shortcuts.tsindex ecd4a8ea7..e032e71aa 100644--- a/packages/core/src/collections/shortcuts.ts+++ b/packages/core/src/collections/shortcuts.ts@@ -25,7 +25,7 @@ import { ICollection } from "./collection";const ALLOWED_SHORTCUT_TYPES = ["notebook", "topic", "tag"];export class Shortcuts implements ICollection {name = "shortcuts";- private readonly collection: CachedCollection<"shortcuts", Shortcut>;+ readonly collection: CachedCollection<"shortcuts", Shortcut>;constructor(private readonly db: Database) {this.collection = new CachedCollection(db.storage,diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.tsindex 730c4fb07..92b90ba56 100644--- a/packages/core/src/collections/tags.ts+++ b/packages/core/src/collections/tags.ts@@ -25,7 +25,7 @@ import { ICollection } from "./collection";export class Tags implements ICollection {name = "tags";- private readonly collection: CachedCollection<"tags", Tag>;+ readonly collection: CachedCollection<"tags", Tag>;constructor(private readonly db: Database) {this.collection = new CachedCollection(db.storage, "tags", db.eventManager);}diff --git a/packages/core/src/database/cached-collection.ts b/packages/core/src/database/cached-collection.tsindex 12a85df87..1c6a13726 100644--- a/packages/core/src/database/cached-collection.ts+++ b/packages/core/src/database/cached-collection.ts@@ -28,7 +28,7 @@ import {} from "../types";import { StorageAccessor } from "../interfaces";import EventManager from "../utils/event-manager";-import { toChunks } from "../utils/array";+import { chunkedIterate } from "../utils/array";export class CachedCollection<TCollectionType extends CollectionType,@@ -109,6 +109,11 @@ export class CachedCollection<return item;}+ getRaw(id: string) {+ const item = this.cache.get(id);+ return item;+ }+raw() {return Array.from(this.cache.values());}@@ -140,10 +145,7 @@ export class CachedCollection<}*iterateSync(chunkSize: number) {- const chunks = toChunks(Array.from(this.cache.values()), chunkSize);- for (const chunk of chunks) {- yield chunk;- }+ yield* chunkedIterate(Array.from(this.cache.values()), chunkSize);}invalidateCache() {diff --git a/packages/core/src/database/crypto.ts b/packages/core/src/database/crypto.tsindex 1128e0b91..4673c2d51 100644--- a/packages/core/src/database/crypto.ts+++ b/packages/core/src/database/crypto.ts@@ -31,7 +31,7 @@ export class Crypto {}}-export function isCipher(item: any): item is Cipher {+export function isCipher(item: any): item is Cipher<"base64"> {return (typeof item === "object" &&"cipher" in item &&diff --git a/packages/core/src/database/indexed-collection.ts b/packages/core/src/database/indexed-collection.tsindex ac9b1c151..99168e291 100644--- a/packages/core/src/database/indexed-collection.ts+++ b/packages/core/src/database/indexed-collection.ts@@ -108,7 +108,7 @@ export class IndexedCollection<return Object.fromEntries(data);}- setItems(items) {+ setItems(items: (MaybeDeletedItem<T> | undefined)[]) {const entries = items.reduce((array, item) => {if (!item) return array;@@ -120,7 +120,7 @@ export class IndexedCollection<array.push([item.id, item]);return array;- }, []);+ }, [] as [string, MaybeDeletedItem<T>][]);return this.indexer.writeMulti(entries);}diff --git a/packages/core/src/database/indexer.ts b/packages/core/src/database/indexer.tsindex 8625b083b..8d8813ad5 100644--- a/packages/core/src/database/indexer.ts+++ b/packages/core/src/database/indexer.ts@@ -86,13 +86,15 @@ export default class Indexer<T> {* @param {any[]} items* @returns*/- async writeMulti(items) {- const entries = items.map(([id, item]) => {- if (!this.indices.includes(id)) this.indices.push(id);- return [this.makeId(id), item];- });+ async writeMulti(items: [string, MaybeDeletedItem<T>][]) {+ const entries: [string, MaybeDeletedItem<T> | string[]][] = items.map(+ ([id, item]) => {+ if (!this.indices.includes(id)) this.indices.push(id);+ return [this.makeId(id), item];+ }+ );entries.push([this.type, this.indices]);- await super.writeMulti(entries);+ await this.storage().writeMulti(entries);}async migrateIndices() {diff --git a/packages/core/src/utils/array.js b/packages/core/src/utils/array.tssimilarity index 64%rename from packages/core/src/utils/array.jsrename to packages/core/src/utils/array.tsindex 4c92b0cdb..9f20fd5b0 100644--- a/packages/core/src/utils/array.js+++ b/packages/core/src/utils/array.ts@@ -17,48 +17,58 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-export function findItemAndDelete(array, predicate) {+export function findItemAndDelete<T>(+ array: T[],+ predicate: (item: T) => boolean+) {return deleteAtIndex(array, array.findIndex(predicate));}-export function addItem(array, item) {+export function addItem<T>(array: T[], item: T) {const index = array.indexOf(item);if (index > -1) return false;array.push(item);return true;}-export function deleteItem(array, item) {+export function deleteItem<T>(array: T[], item: T) {return deleteAtIndex(array, array.indexOf(item));}-export function deleteItems(array, ...items) {- for (let item of items) {+export function deleteItems<T>(array: T[], ...items: T[]) {+ for (const item of items) {deleteItem(array, item);}}-export function findById(array, id) {+export function findById<T extends { id: string }>(array: T[], id: string) {if (!array) return false;return array.find((item) => item.id === id);}-export function hasItem(array, item) {+export function hasItem<T>(array: T[], item: T) {if (!array) return false;return array.indexOf(item) > -1;}-function deleteAtIndex(array, index) {+function deleteAtIndex<T>(array: T[], index: number) {if (index === -1) return false;array.splice(index, 1);return true;}-export function toChunks(array, chunkSize) {- let chunks = [];+export function toChunks<T>(array: T[], chunkSize: number) {+ const chunks: T[][] = [];for (let i = 0; i < array.length; i += chunkSize) {const chunk = array.slice(i, i + chunkSize);chunks.push(chunk);}return chunks;}++export function* chunkedIterate<T>(array: T[], chunkSize: number) {+ for (let i = 0; i < array.length; i += chunkSize) {+ const chunk = array.slice(i, i + chunkSize);+ yield chunk;+ }+}