A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: add tests for sync merger
Abdullah Attaf127203a5e3c
diff --git a/packages/core/src/api/sync/__tests__/merger.test.js b/packages/core/src/api/sync/__tests__/merger.test.jsnew file mode 100644index 000000000..c70d5f629--- /dev/null+++ b/packages/core/src/api/sync/__tests__/merger.test.js@@ -0,0 +1,409 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import {+ TEST_NOTE,+ databaseTest,+ loginFakeUser+} from "../../../../__tests__/utils";+import { expect, describe, vi } from "vitest";+import Merger from "../merger";++describe.concurrent("merge item synchronously", (test) => {+ test("accept remote item if no local item is found", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const merged = merger.mergeItem(+ {+ type: "color"+ },+ undefined+ );++ expect(merged).toBeDefined();+ expect(merged.type).toBe("color");+ }));++ test("accept remote item if it is newer than local item", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const merged = merger.mergeItem(+ {+ type: "color",+ dateModified: Date.now()+ },+ {+ type: "color",+ dateModified: Date.now() - 1000+ }+ );++ expect(merged).toBeDefined();+ expect(merged.type).toBe("color");+ }));++ test("accept local item if it is newer than remote item", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const merged = merger.mergeItem(+ {+ type: "color",+ dateModified: Date.now() - 1000+ },+ {+ type: "color",+ dateModified: Date.now()+ }+ );++ expect(merged).toBeUndefined();+ }));+});++describe.concurrent("merge content", (test) => {+ test("do nothing if local item is localOnly", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const merged = await merger.mergeContent(+ {+ type: "tiptap",+ data: "Hello"+ },+ {+ type: "tiptap",+ localOnly: true+ }+ );++ expect(merged).toBeUndefined();+ }));++ test("accept remote item if local item is not defined", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const merged = await merger.mergeContent(+ {+ type: "tiptap"+ },+ undefined+ );++ expect(merged).toBeDefined();+ expect(merged.type).toBe("tiptap");+ }));++ test("accept remote item if it is newer than local item", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const merged = await merger.mergeContent(+ {+ type: "tiptap",+ data: "Remote",+ dateEdited: Date.now()+ },+ {+ type: "tiptap",+ data: "Local",+ dateEdited: Date.now() - 1000+ }+ );++ expect(merged).toBeDefined();+ expect(merged.data).toBe("Remote");+ }));++ test("trigger conflict if local item dateEdited is newer", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const noteId = await db.notes.add(TEST_NOTE);+ const merged = await merger.mergeContent(+ {+ type: "tiptap",+ data: "Remote",+ noteId,+ dateEdited: Date.now() - 60000+ },+ {+ type: "tiptap",+ data: "Local",+ noteId,++ dateEdited: Date.now()+ }+ );++ expect(merged).toBeDefined();+ expect(merged.data).toBe("Local");+ expect(merged.conflicted).toBeDefined();+ expect(merged.conflicted.data).toBe("Remote");+ expect(await db.notes.conflicted.has(noteId)).toBe(true);+ }));++ test("merge conflicts if local item is already conflicted", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const noteId = await db.notes.add({ ...TEST_NOTE, conflicted: true });+ const merged = await merger.mergeContent(+ {+ type: "tiptap",+ data: "Conflicted remote 2",+ noteId+ },+ {+ type: "tiptap",+ data: "Local",+ noteId,+ conflicted: { type: "tiptap", data: "Conflicted remote" }+ }+ );++ expect(merged).toBeDefined();+ expect(merged.data).toBe("Local");+ expect(merged.conflicted).toBeDefined();+ expect(merged.conflicted.data).toBe("Conflicted remote 2");+ expect(await db.notes.conflicted.has(noteId)).toBe(true);+ }));++ describe("auto resolve conflict", () => {+ describe("edits under the conflict threshold", (test) => {+ test("keep remote if it is newer", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const noteId = await db.notes.add(TEST_NOTE);+ const merged = await merger.mergeContent(+ {+ type: "tiptap",+ data: "Remote",+ noteId,+ dateEdited: Date.now() - 3000,+ dateModified: Date.now()+ },+ {+ type: "tiptap",+ data: "Local",+ noteId,+ dateEdited: Date.now(),+ dateModified: Date.now() - 6000+ }+ );++ expect(merged).toBeDefined();+ expect(merged.data).toBe("Remote");+ }));++ test("keep local if it is newer", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const noteId = await db.notes.add(TEST_NOTE);+ const merged = await merger.mergeContent(+ {+ type: "tiptap",+ data: "Remote",+ noteId,+ dateEdited: Date.now() - 3000,+ dateModified: Date.now() - 6000+ },+ {+ type: "tiptap",+ data: "Local",+ noteId,+ dateEdited: Date.now(),+ dateModified: Date.now()+ }+ );++ expect(merged).toBeUndefined();+ }));+ });++ describe("edits are equal", (test) => {+ test("keep remote if it is newer", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const noteId = await db.notes.add(TEST_NOTE);+ const merged = await merger.mergeContent(+ {+ type: "tiptap",+ data: "Remote",+ noteId,+ dateEdited: Date.now() - 60000,+ dateModified: Date.now()+ },+ {+ type: "tiptap",+ data: "Remote",+ noteId,+ dateEdited: Date.now(),+ dateModified: Date.now() - 6000+ }+ );++ expect(merged).toBeDefined();+ expect(merged.data).toBe("Remote");+ }));++ test("keep local if it is newer", () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const noteId = await db.notes.add(TEST_NOTE);+ const merged = await merger.mergeContent(+ {+ type: "tiptap",+ data: "Remote",+ noteId,+ dateEdited: Date.now() - 60000,+ dateModified: Date.now() - 6000+ },+ {+ type: "tiptap",+ data: "Remote",+ noteId,+ dateEdited: Date.now(),+ dateModified: Date.now()+ }+ );++ expect(merged).toBeUndefined();+ }));+ });+ });+});++describe.concurrent("merge attachment", () => {+ describe("accept remote item", (test) => {+ const cases = [+ {+ name: "local item is undefined",+ remote: { type: "attachment" },+ local: undefined+ },+ {+ name: "local item is deleted (remote is newer)",+ remote: { type: "attachment", dateModified: Date.now() },+ local: {+ type: "attachment",+ deleted: true,+ dateModified: Date.now() - 1000+ }+ },+ {+ name: "remote item is deleted (remote is newer)",+ remote: { type: "attachment", deleted: true, dateModified: Date.now() },+ local: { type: "attachment", dateModified: Date.now() - 1000 }+ },+ {+ name: "remote item's dateUploaded is more recent",+ remote: {+ type: "attachment",+ dateUploaded: Date.now()+ },+ local: { type: "attachment", dateUploaded: Date.now() - 1000 }+ }+ ];++ for (const testCase of cases) {+ test(testCase.name, () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);+ db.attachments.remove = vi.fn(() => true);++ const merged = await merger.mergeAttachment(+ testCase.remote,+ testCase.local+ );++ expect(merged).toBeDefined();+ expect(merged).toStrictEqual(testCase.remote);+ })+ );+ }+ });++ describe("accept local item", (test) => {+ const cases = [+ {+ name: "local item is deleted (local is newer)",+ remote: { type: "attachment", dateModified: Date.now() - 1000 },+ local: {+ type: "attachment",+ deleted: true,+ dateModified: Date.now()+ }+ },+ {+ name: "remote item is deleted (local is newer)",+ remote: {+ type: "attachment",+ deleted: true,+ dateModified: Date.now() - 1000+ },+ local: { type: "attachment", dateModified: Date.now() }+ },+ {+ name: "local item's dateUploaded is more recent",+ remote: {+ type: "attachment",+ dateUploaded: Date.now() - 1000+ },+ local: { type: "attachment", dateUploaded: Date.now() }+ }+ ];++ for (const testCase of cases) {+ test(testCase.name, () =>+ databaseTest().then(async (db) => {+ await loginFakeUser(db);+ const merger = new Merger(db);++ const merged = await merger.mergeAttachment(+ testCase.remote,+ testCase.local+ );++ expect(merged).toBeUndefined();+ })+ );+ }+ });+});diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.tsindex 0101824ce..ae6342efb 100644--- a/packages/core/src/api/sync/index.ts+++ b/packages/core/src/api/sync/index.ts@@ -286,8 +286,6 @@ class Sync {async processChunk(chunk: SyncTransferItem, key: SerializedKey) {const itemType = chunk.type;- if (itemType === "settings") return;-const decrypted = await this.db.storage().decryptMulti(key, chunk.items);const deserialized: MaybeDeletedItem<Item>[] = [];@@ -313,11 +311,11 @@ class Sync {itemType === "attachment"? await Promise.all(deserialized.map((item) =>- this.merger.mergeItemAsync(item, localItems[item.id], itemType)+ this.merger.mergeAttachment(item, localItems[item.id]))): deserialized.map((item) =>- this.merger.mergeItemSync(item, localItems[item.id], itemType)+ this.merger.mergeItem(item, localItems[item.id]));}diff --git a/packages/core/src/api/sync/merger.ts b/packages/core/src/api/sync/merger.tsindex 2f2990195..2e2262421 100644--- a/packages/core/src/api/sync/merger.ts+++ b/packages/core/src/api/sync/merger.ts@@ -20,7 +20,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { logger } from "../../logger";import { isHTMLEqual } from "../../utils/html-diff";import Database from "..";-import { ContentItem, Item, MaybeDeletedItem, isDeleted } from "../../types";+import {+ Attachment,+ ContentItem,+ Item,+ MaybeDeletedItem,+ isDeleted+} from "../../types";const THRESHOLD = process.env.NODE_ENV === "test" ? 6 * 1000 : 60 * 1000;class Merger {@@ -31,33 +37,12 @@ class Merger {// return type in SYNC_COLLECTIONS_MAP;// }- mergeItemSync(+ mergeItem(remoteItem: MaybeDeletedItem<Item>,- localItem: MaybeDeletedItem<Item> | undefined,- type:- | "shortcut"- | "reminder"- | "tag"- | "color"- | "note"- | "relation"- | "notebook"- | "settingitem"+ localItem: MaybeDeletedItem<Item> | undefined) {- switch (type) {- case "shortcut":- case "reminder":- case "tag":- case "color":- case "note":- case "relation":- case "notebook":- case "settingitem": {- if (!localItem || remoteItem.dateModified > localItem.dateModified) {- return remoteItem;- }- break;- }+ if (!localItem || remoteItem.dateModified > localItem.dateModified) {+ return remoteItem;}}@@ -78,9 +63,7 @@ class Merger {!localItem.data ||!remoteItem.data) {- if (!localItem || remoteItem.dateModified > localItem.dateModified)- return remoteItem;- return;+ return this.mergeItem(remoteItem, localItem);} else {// it's possible that the local item already has a conflict so// we can just replace the conflicted content@@ -101,48 +84,41 @@ class Merger {}}- async mergeItemAsync(- remoteItem: MaybeDeletedItem<Item>,- localItem: MaybeDeletedItem<Item> | undefined,- type: "attachment"+ async mergeAttachment(+ remoteItem: MaybeDeletedItem<Attachment>,+ localItem: MaybeDeletedItem<Attachment> | undefined) {- switch (type) {- case "attachment": {- if (!localItem) return remoteItem;- if (- isDeleted(localItem) ||- isDeleted(remoteItem) ||- remoteItem.type !== "attachment" ||- localItem.type !== "attachment"- ) {- if (remoteItem.dateModified > localItem.dateModified)- return remoteItem;- return;- }-- if (localItem.dateUploaded !== remoteItem.dateUploaded) {- const isRemoved = await this.db.attachments.remove(- localItem.hash,- true- );- if (!isRemoved)- throw new Error(- "Conflict could not be resolved in one of the attachments."- );- }- return remoteItem;- }+ if (+ !localItem ||+ isDeleted(localItem) ||+ isDeleted(remoteItem) ||+ !localItem.dateUploaded ||+ !remoteItem.dateUploaded+ ) {+ return this.mergeItem(remoteItem, localItem);}++ if (localItem.dateUploaded > remoteItem.dateUploaded) return;++ const isRemoved = await this.db.attachments.remove(localItem.hash, true);+ if (!isRemoved)+ throw new Error(+ "Conflict could not be resolved in one of the attachments."+ );+ return remoteItem;}}export default Merger;-function isContentConflicted(+export function isContentConflicted(localItem: ContentItem,remoteItem: ContentItem,conflictThreshold: number) {- const isResolved = localItem.dateResolved === remoteItem.dateModified;+ const isResolved =+ localItem.dateResolved &&+ remoteItem.dateModified &&+ localItem.dateResolved === remoteItem.dateModified;const isEdited =// the local item is edited if it was changed/edited after the remote// note and it also wasn't synced yet.diff --git a/packages/core/src/api/sync/types.ts b/packages/core/src/api/sync/types.tsindex 50ec5f2b6..574b9c179 100644--- a/packages/core/src/api/sync/types.ts+++ b/packages/core/src/api/sync/types.ts@@ -19,38 +19,12 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { Cipher } from "@notesnook/crypto";-export type SyncableItemType =- | "note"- | "shortcut"- | "notebook"- | "content"- | "attachment"- | "reminder"- | "relation"- | "color"- | "tag"- | "settings"- | "settingitem";-export type SyncItem = {id: string;v: number;} & Cipher<"base64">;-// export const SYNC_COLLECTIONS_MAP = {-// note: "notes",-// notebook: "notebooks",-// shortcut: "shortcuts",-// reminder: "reminders",-// relation: "relations",-// tag: "tags",-// color: "colors",-// settingitem: "settings"-// } as const;--// export const ASYNC_COLLECTIONS_MAP = {-// content: "content"-// } as const;+export type SyncableItemType = keyof typeof SYNC_COLLECTIONS_MAP;export const SYNC_COLLECTIONS_MAP = {settingitem: "settings",@@ -62,21 +36,13 @@ export const SYNC_COLLECTIONS_MAP = {relation: "relations",tag: "tags",color: "colors",- note: "notes"+ note: "notes",+ vault: "vaults"} as const;-export const SYNC_ITEM_TYPES = [- "settingitem",- "attachment",- "content",- "notebook",- "shortcut",- "reminder",- "relation",- "tag",- "color",- "note"-] as const;+export const SYNC_ITEM_TYPES = Object.keys(+ SYNC_COLLECTIONS_MAP+) as SyncableItemType[];export type SyncTransferItem = {items: SyncItem[];diff --git a/packages/core/src/models/note.ts b/packages/core/src/models/note.tsdeleted file mode 100644index 68914a28f..000000000--- a/packages/core/src/models/note.ts+++ /dev/null@@ -1,52 +0,0 @@-/*-This file is part of the Notesnook project (https://notesnook.com/)--Copyright (C) 2023 Streetwriters (Private) Limited--This program is free software: you can redistribute it and/or modify-it under the terms of the GNU General Public License as published by-the Free Software Foundation, either version 3 of the License, or-(at your option) any later version.--This program is distributed in the hope that it will be useful,-but WITHOUT ANY WARRANTY; without even the implied warranty of-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-GNU General Public License for more details.--You should have received a copy of the GNU General Public License-along with this program. If not, see <http://www.gnu.org/licenses/>.-*/--import Database from "../api";-import { isDeleted, type Note } from "../types";--export function createNoteModel(note: Note, db: Database) {- return {- ...note,- data: note,- async content() {- if (!note.contentId) return null;- const content = await db.content.get(note.contentId);- return content && !isDeleted(content) ? content.data : null;- },- synced() {- return !note.contentId || db.content.exists(note.contentId);- },- localOnly() {- return toggleProperty(db, note, "localOnly");- },- favorite() {- return toggleProperty(db, note, "favorite");- },- pin() {- return toggleProperty(db, note, "pinned");- },- readonly() {- return toggleProperty(db, note, "readonly");- }- };-}--function toggleProperty(db: Database, note: Note, property: keyof Note) {- return db.notes.add({ id: note.id, [property]: !note[property] });-}diff --git a/packages/core/src/models/notebook.ts b/packages/core/src/models/notebook.tsdeleted file mode 100644index e26b64305..000000000--- a/packages/core/src/models/notebook.ts+++ /dev/null@@ -1,46 +0,0 @@-/*-This file is part of the Notesnook project (https://notesnook.com/)--Copyright (C) 2023 Streetwriters (Private) Limited--This program is free software: you can redistribute it and/or modify-it under the terms of the GNU General Public License as published by-the Free Software Foundation, either version 3 of the License, or-(at your option) any later version.--This program is distributed in the hope that it will be useful,-but WITHOUT ANY WARRANTY; without even the implied warranty of-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-GNU General Public License for more details.--You should have received a copy of the GNU General Public License-along with this program. If not, see <http://www.gnu.org/licenses/>.-*/--import Database from "../api";-import { Notebook } from "../types";--export function createNotebookModel(notebook: Notebook, db: Database) {- return {- ...notebook,- /**- * @deprecated please use `notebook` directly instead- */- data: notebook,- /**- * @deprecated please use `db.notebooks.totalNotes()` instead- */- totalNotes: (function () {- return db.notebooks.totalNotes(notebook.id);- })(),- /**- * @deprecated please use `db.notebooks.pin()` & `db.notebooks.unpin()` instead.- */- pin() {- return db.notebooks?.add({- id: notebook.id,- pinned: !notebook.pinned- });- }- };-}diff --git a/packages/core/src/utils/buffer-pool.ts b/packages/core/src/utils/buffer-pool.tsdeleted file mode 100644index dfae03a3e..000000000--- a/packages/core/src/utils/buffer-pool.ts+++ /dev/null@@ -1,36 +0,0 @@-/*-This file is part of the Notesnook project (https://notesnook.com/)--Copyright (C) 2023 Streetwriters (Private) Limited--This program is free software: you can redistribute it and/or modify-it under the terms of the GNU General Public License as published by-the Free Software Foundation, either version 3 of the License, or-(at your option) any later version.--This program is distributed in the hope that it will be useful,-but WITHOUT ANY WARRANTY; without even the implied warranty of-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-GNU General Public License for more details.--You should have received a copy of the GNU General Public License-along with this program. If not, see <http://www.gnu.org/licenses/>.-*/--export class BufferPool {- private freeBuffers: Buffer[] = [];- constructor(private readonly size: number) {}-- alloc() {- return this.freeBuffers.pop() || this.allocNew();- }-- private allocNew() {- return Buffer.alloc(this.size);- }-- free(buf: Buffer) {- this.freeBuffers.push(buf);- return true;- }-}