A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: migrate everything to sqlite
Abdullah Atta21b3866df431
diff --git a/packages/core/__tests__/backup.test.js b/packages/core/__tests__/backup.test.jsindex f88751013..ab4bff8b8 100644--- a/packages/core/__tests__/backup.test.js+++ b/packages/core/__tests__/backup.test.js@@ -17,14 +17,8 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { TEST_NOTE, loginFakeUser, notebookTest } from "./utils";-// import v52Backup from "./__fixtures__/backup.v5.2.json";-// import v52BackupCopy from "./__fixtures__/backup.v5.2.copy.json";-// import v56BackupCopy from "./__fixtures__/backup.v5.6.json";-// import v58BackupCopy from "./__fixtures__/backup.v5.8.json";-// import qclone from "qclone";+import { TEST_NOTE, databaseTest, loginFakeUser, notebookTest } from "./utils";import { test, expect } from "vitest";-// import { getId, makeId } from "../src/utils/id";test("export backup", () =>notebookTest().then(async ({ db }) => {@@ -70,34 +64,32 @@ test("export encrypted backup", () =>expect(backup.encrypted).toBe(true);}));-test("import backup", () =>- notebookTest().then(async ({ db, id }) => {- await db.notes.add(TEST_NOTE);-- const exp = [];- for await (const file of db.backup.export("node", false)) {- exp.push(file);- }-- await db.storage().clear();- await db.backup.import(JSON.parse(exp[1].data));- expect(db.notebooks.notebook(id).data.id).toBe(id);- }));--test("import encrypted backup", () =>- notebookTest().then(async ({ db, id }) => {- await loginFakeUser(db);- await db.notes.add(TEST_NOTE);-- const exp = [];- for await (const file of db.backup.export("node", true)) {- exp.push(file);- }-- await db.storage().clear();- await db.backup.import(JSON.parse(exp[1].data), "password");- expect(db.notebooks.notebook(id).data.id).toBe(id);- }));+test("import backup", async () => {+ const { db, id } = await notebookTest();+ const exp = [];+ for await (const file of db.backup.export("node", false)) {+ exp.push(file);+ }++ const db2 = await databaseTest();+ await db2.backup.import(JSON.parse(exp[1].data));+ expect((await db2.notebooks.notebook(id)).id).toBe(id);+});++test("import encrypted backup", async () => {+ const { db, id } = await notebookTest();+ await loginFakeUser(db);+ await db.notes.add(TEST_NOTE);++ const exp = [];+ for await (const file of db.backup.export("node", true)) {+ exp.push(file);+ }++ const db2 = await databaseTest();+ await db2.backup.import(JSON.parse(exp[1].data), "password");+ expect((await db2.notebooks.notebook(id)).id).toBe(id);+});test("import tempered backup", () =>notebookTest().then(async ({ db }) => {@@ -113,104 +105,3 @@ test("import tempered backup", () =>backup.data += "hello";await expect(db.backup.import(backup)).rejects.toThrow(/tempered/);}));--// describe.each([-// ["v5.2", v52Backup],-// ["v5.2 copy", v52BackupCopy],-// ["v5.6", v56BackupCopy],-// ["v5.8", v58BackupCopy]-// ])("testing backup version: %s", (version, data) => {-// test(`import ${version} backup`, () => {-// return databaseTest().then(async (db) => {-// await db.backup.import(qclone(data));--// expect(db.settings.raw.id).toBeDefined();-// expect(db.settings.raw.dateModified).toBeDefined();-// expect(db.settings.raw.dateEdited).toBeUndefined();-// expect(db.settings.raw.pins).toBeUndefined();--// expect(-// db.notes.all.every((v) => {-// const doesNotHaveContent = !v.content;-// const doesNotHaveColors = !v.colors; // && (!v.color || v.color.length);-// const hasTopicsInAllNotebooks =-// !v.notebooks ||-// v.notebooks.every((nb) => !!nb.id && !!nb.topics && !nb.topic);-// const hasDateModified = v.dateModified > 0;-// const doesNotHaveTags = !v.tags;-// const doesNotHaveColor = !v.color;-// if (!doesNotHaveTags) console.log(v);-// return (-// doesNotHaveTags &&-// doesNotHaveColor &&-// doesNotHaveContent &&-// !v.notebook &&-// hasTopicsInAllNotebooks &&-// doesNotHaveColors &&-// hasDateModified-// );-// })-// ).toBeTruthy();--// expect(-// db.tags.all.every((t) => makeId(t.title) !== t.id && !t.noteIds)-// ).toBeTruthy();--// expect(-// db.colors.all.every(-// (t) => makeId(t.title) !== t.id && !t.noteIds && !!t.colorCode-// )-// ).toBeTruthy();--// expect(-// db.notebooks.all.every((v) => v.title != null && v.dateModified > 0)-// ).toBeTruthy();--// expect(db.notebooks.all.every((v) => !v.topics)).toBeTruthy();--// expect(-// db.attachments.all.every((v) => v.dateModified > 0 && !v.dateEdited)-// ).toBeTruthy();--// expect(db.attachments.all.every((a) => !a.noteIds)).toBeTruthy();--// if (data.data.settings.pins)-// expect(db.shortcuts.all).toHaveLength(data.data.settings.pins.length);--// const allContent = await db.content.all();-// expect(-// allContent.every((v) => v.type === "tiptap" || v.deleted)-// ).toBeTruthy();-// expect(allContent.every((v) => !v.persistDateEdited)).toBeTruthy();-// expect(allContent.every((v) => v.dateModified > 0)).toBeTruthy();--// expect(-// allContent.every(-// (v) =>-// !v.data.includes("tox-checklist") &&-// !v.data.includes("tox-checklist--checked")-// )-// ).toBeTruthy();-// });-// });--// test(`verify indices of ${version} backup`, () => {-// return databaseTest().then(async (db) => {-// await db.backup.import(qclone(data));--// const keys = await db.storage().getAllKeys();-// for (let key in data.data) {-// const item = data.data[key];-// if (item && !item.type && item.deleted) continue;-// if (-// key.startsWith("_uk_") ||-// key === "hasConflicts" ||-// key === "monographs" ||-// key === "token"-// )-// continue;-// expect(keys.some((k) => k.startsWith(key))).toBeTruthy();-// }-// });-// });-// });diff --git a/packages/core/__tests__/lookup.test.js b/packages/core/__tests__/lookup.test.jsindex 324a6101a..b556e35b2 100644--- a/packages/core/__tests__/lookup.test.js+++ b/packages/core/__tests__/lookup.test.js@@ -31,7 +31,7 @@ test("search notes", () =>content: content}).then(async ({ db }) => {await db.notes.add(TEST_NOTE);- let filtered = await db.lookup.notes(db.notes.all, "note of the world");+ let filtered = await db.lookup.notes("note the world");expect(filtered).toHaveLength(1);}));@@ -42,8 +42,8 @@ test("search notes with a locked note", () =>const noteId = await db.notes.add(TEST_NOTE);await db.vault.create("password");await db.vault.add(noteId);- let filtered = await db.lookup.notes(db.notes.all, "note of the world");- expect(filtered).toHaveLength(1);+ expect(await db.lookup.notes("note the world")).toHaveLength(1);+ expect(await db.lookup.notes("format")).toHaveLength(0);}));test("search notes with an empty note", () =>@@ -54,13 +54,16 @@ test("search notes with an empty note", () =>title: "world is a heavy tune",content: { type: "tiptap", data: "<p><br></p>" }});- let filtered = await db.lookup.notes(db.notes.all, "heavy tune");+ let filtered = await db.lookup.notes("heavy tune");expect(filtered).toHaveLength(1);}));test("search notebooks", () =>notebookTest().then(async ({ db }) => {await db.notebooks.add(TEST_NOTEBOOK2);- let filtered = db.lookup.notebooks(db.notebooks.all, "Description");+ let filtered = db.lookup.notebooks(+ await db.notebooks.all.items(),+ "Description"+ );expect(filtered.length).toBeGreaterThan(0);}));diff --git a/packages/core/__tests__/migrations.test.ts b/packages/core/__tests__/migrations.test.tsindex 7fbea81b8..1c7fd4e5f 100644--- a/packages/core/__tests__/migrations.test.ts+++ b/packages/core/__tests__/migrations.test.ts@@ -171,22 +171,16 @@ test("[5.6] move pins to shortcuts", () =>expect(await migrateItem(item, 5.6, 5.7, "settings", db, "local")).toBe(true);+ const shortcuts = await db.shortcuts.all.items();expect(item.pins).toBeUndefined();expect(- db.shortcuts.all.find(- (s) =>- s.item.type === "topic" &&- s.item.id === "hello" &&- s.item.notebookId === "world"- )+ shortcuts.find((s) => s.itemType === "notebook" && s.itemId === "hello")).toBeDefined();expect(- db.shortcuts.all.find(- (s) => s.item.type === "notebook" && s.item.id === "world"- )+ shortcuts.find((s) => s.itemType === "notebook" && s.itemId === "world")).toBeDefined();expect(- db.shortcuts.all.find((s) => s.item.type === "tag" && s.item.id === "tag")+ shortcuts.find((s) => s.itemType === "tag" && s.itemId === "tag")).toBeDefined();}));@@ -279,7 +273,7 @@ describe("[5.9] make tags syncable", () => {databaseTest().then(async (db) => {const noteId = getId();const tags = ["hello", "world", "i am here"];- await db.notes.collection.add({+ await db.legacyNotes.add({type: "note",title: "I am a note",tags,@@ -287,7 +281,7 @@ describe("[5.9] make tags syncable", () => {});for (const tag of tags) {- await db.tags.collection.add({+ await db.legacyTags.add({id: makeId(tag),noteIds: [noteId],type: "tag",@@ -302,27 +296,24 @@ describe("[5.9] make tags syncable", () => {});await db.legacySettings.init();- const note = db.notes.note(noteId);+ const note = db.legacyNotes.get(noteId);if (!note) throw new Error("Failed to find note.");- expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(+ expect(await migrateItem(note, 5.9, 6.0, "note", db, "backup")).toBe(true);- const resolvedTags = db.relations- .to({ type: "note", id: noteId }, "tag")- .resolved()- .sort((a, b) => a.title.localeCompare(b.title));+ const resolvedTags = (+ await db.relations.to({ type: "note", id: noteId }, "tag").resolve()+ ).sort((a, b) => a.title.localeCompare(b.title));- expect(note.data.tags).toBeUndefined();- expect(db.tags.all).toHaveLength(3);+ expect(note.tags).toBeUndefined();+ expect(await db.tags.all.count()).toBe(3);expect(resolvedTags).toHaveLength(3);expect(resolvedTags[0].title).toBe("hello");expect(resolvedTags[1].title).toBe("I AM GOOD!");expect(resolvedTags[2].title).toBe("i am here");- expect(- tags.every((t) => !db.tags.collection.exists(makeId(t)))- ).toBeTruthy();+ expect(db.legacyTags.items()).toHaveLength(0);}));test("migrate old tag item to new one", () =>@@ -367,7 +358,7 @@ describe("[5.9] make tags syncable", () => {databaseTest().then(async (db) => {const noteId = getId();const tags = ["hello", "world", "i am here"];- await db.notes.collection.add({+ await db.legacyNotes.add({type: "note",title: "I am a note",tags,@@ -388,30 +379,27 @@ describe("[5.9] make tags syncable", () => {title: tag};await migrateItem(item, 5.9, 6.0, "tag", db, "backup");- await db.tags.collection.add(item);+ await db.tags.add(item);}- const note = db.notes.note(noteId);+ const note = db.legacyNotes.get(noteId);if (!note) throw new Error("Failed to find note.");- expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(+ expect(await migrateItem(note, 5.9, 6.0, "note", db, "backup")).toBe(true);- const resolvedTags = db.relations- .to({ type: "note", id: noteId }, "tag")- .resolved()- .sort((a, b) => a.title.localeCompare(b.title));+ const resolvedTags = (+ await db.relations.to({ type: "note", id: noteId }, "tag").resolve()+ ).sort((a, b) => a.title.localeCompare(b.title));- expect(note.data.tags).toBeUndefined();- expect(db.tags.all).toHaveLength(3);+ expect(note.tags).toBeUndefined();+ expect(await db.tags.all.count()).toBe(3);expect(resolvedTags).toHaveLength(3);expect(resolvedTags[0].title).toBe("hello");expect(resolvedTags[1].title).toBe("I AM GOOD!");expect(resolvedTags[2].title).toBe("i am here");- expect(- tags.every((t) => !db.tags.collection.exists(makeId(t)))- ).toBeTruthy();+ expect(db.legacyTags.items()).toHaveLength(0);}));});@@ -419,37 +407,37 @@ describe("[5.9] make colors syncable", () => {test("create colors from notes & link to them using relations", () =>databaseTest().then(async (db) => {const noteId = getId();- await db.notes.collection.add({+ await db.legacyNotes.add({type: "note",title: "I am a note",color: "blue",id: noteId});- await db.colors.collection.add({+ await db.legacyColors.add({id: makeId("blue"),noteIds: [noteId],type: "tag",title: "blue"});- const note = db.notes.note(noteId);+ const note = db.legacyNotes.get(noteId);if (!note) throw new Error("Failed to find note.");- expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(+ expect(await migrateItem(note, 5.9, 6.0, "note", db, "backup")).toBe(true);- const resolvedColors = db.relations+ const resolvedColors = await db.relations.to({ type: "note", id: noteId }, "color")- .resolved();+ .resolve();- expect(note.data.color).toBeUndefined();- expect(db.colors.all).toHaveLength(1);+ expect(note.color).toBeUndefined();+ expect(await db.colors.all.count()).toBe(1);expect(resolvedColors).toHaveLength(1);expect(resolvedColors[0].title).toBe("blue");expect(resolvedColors[0].colorCode).toBe("#2196F3");- expect(db.colors.collection.exists(makeId("blue"))).toBeFalsy();+ expect(db.legacyColors.exists(makeId("blue"))).toBeFalsy();}));test("migrate old color item to new one", () =>@@ -501,7 +489,7 @@ describe("[5.9] make colors syncable", () => {test("migrate color before notes", () =>databaseTest().then(async (db) => {const noteId = getId();- await db.notes.collection.add({+ await db.legacyNotes.add({type: "note",title: "I am a note",color: "blue",@@ -522,25 +510,25 @@ describe("[5.9] make colors syncable", () => {title: "blue"};await migrateItem(color, 5.9, 6.0, "tag", db, "backup");- await db.colors.collection.add(color);+ await db.colors.add(color);- const note = db.notes.note(noteId);+ const note = db.legacyNotes.get(noteId);if (!note) throw new Error("Failed to find note.");- expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(+ expect(await migrateItem(note, 5.9, 6.0, "note", db, "backup")).toBe(true);- const resolvedColors = db.relations+ const resolvedColors = await db.relations.to({ type: "note", id: noteId }, "color")- .resolved();+ .resolve();- expect(note.data.color).toBeUndefined();- expect(db.colors.all).toHaveLength(1);+ expect(note.color).toBeUndefined();+ expect(await db.colors.all.count()).toBe(1);expect(resolvedColors).toHaveLength(1);expect(resolvedColors[0].title).toBe("I AM GOOD!");expect(resolvedColors[0].colorCode).toBe("#2196F3");- expect(db.colors.collection.exists(makeId("blue"))).toBeFalsy();+ expect(db.legacyColors.exists(makeId("blue"))).toBeFalsy();}));});@@ -553,15 +541,31 @@ test("[5.9] move attachments.noteIds to relations", () =>};await migrateItem(attachment, 5.9, 6.0, "attachment", db, "backup");- const linkedNotes = db.relations.from(- { type: "attachment", id: "ATTACHMENT_ID" },- "note"- );+ const linkedNotes = await db.relations+ .from({ type: "attachment", id: "ATTACHMENT_ID" }, "note")+ .get();expect(attachment.noteIds).toBeUndefined();expect(linkedNotes).toHaveLength(1);- expect(linkedNotes[0].to.id).toBe("HELLO_NOTE_ID");+ expect(linkedNotes[0]).toBe("HELLO_NOTE_ID");}));+test.todo("[5.9] flatten attachment object", () =>+ databaseTest().then(async (db) => {+ // const attachment = {+ // id: "ATTACHMENT_ID",+ // type: "attachment",+ // noteIds: ["HELLO_NOTE_ID"]+ // };+ // await migrateItem(attachment, 5.9, 6.0, "attachment", db, "backup");+ // const linkedNotes = await db.relations+ // .from({ type: "attachment", id: "ATTACHMENT_ID" }, "note")+ // .get();+ // expect(attachment.noteIds).toBeUndefined();+ // expect(linkedNotes).toHaveLength(1);+ // expect(linkedNotes[0]).toBe("HELLO_NOTE_ID");+ })+);+describe("[5.9] move topics out of notebooks & use relations", () => {test("convert topics to subnotebooks", () =>databaseTest().then(async (db) => {@@ -575,17 +579,16 @@ describe("[5.9] move topics out of notebooks & use relations", () => {};await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup");- const linkedNotebooks = db.relations.from(- { type: "notebook", id: "parent_notebook" },- "notebook"- );+ const linkedNotebooks = await db.relations+ .from({ type: "notebook", id: "parent_notebook" }, "notebook")+ .get();expect(notebook.topics).toBeUndefined();expect(linkedNotebooks).toHaveLength(2);- expect(linkedNotebooks.some((a) => a.to.id === "topics1")).toBeTruthy();- expect(linkedNotebooks.some((a) => a.to.id === "topics2")).toBeTruthy();- expect(db.notebooks.all).toHaveLength(2);- expect(db.notebooks.notebook("topics1")).toBeDefined();- expect(db.notebooks.notebook("topics2")).toBeDefined();+ expect(linkedNotebooks.some((a) => a === "topics1")).toBeTruthy();+ expect(linkedNotebooks.some((a) => a === "topics2")).toBeTruthy();+ expect(await db.notebooks.all.count()).toBe(2);+ expect(await db.notebooks.notebook("topics1")).toBeDefined();+ expect(await db.notebooks.notebook("topics2")).toBeDefined();}));test("convert topic shortcuts to notebook shortcuts", () =>@@ -600,8 +603,8 @@ describe("[5.9] move topics out of notebooks & use relations", () => {};await migrateItem(shortcut, 5.9, 6.0, "shortcut", db, "backup");- expect(shortcut.item.type).toBe("notebook");- expect(shortcut.item.id).toBe("topics1");+ expect(shortcut.itemType).toBe("notebook");+ expect(shortcut.itemId).toBe("topics1");}));test("convert topic links in note to relations", () =>@@ -613,13 +616,13 @@ describe("[5.9] move topics out of notebooks & use relations", () => {};await migrateItem(note, 5.9, 6.0, "note", db, "backup");- const linkedNotebooks = db.relations+ const linkedNotebooks = await db.relations.to({ type: "note", id: "note1" }, "notebook")- .sort((a, b) => a.to.id.localeCompare(b.to.id));+ .get();expect(note.notebooks).toBeUndefined();expect(linkedNotebooks).toHaveLength(2);- expect(linkedNotebooks.some((a) => a.from.id === "topic1")).toBeTruthy();- expect(linkedNotebooks.some((a) => a.from.id === "topic2")).toBeTruthy();+ expect(linkedNotebooks.some((a) => a === "topic1")).toBeTruthy();+ expect(linkedNotebooks.some((a) => a === "topic2")).toBeTruthy();}));});diff --git a/packages/core/__tests__/note-history.test.js b/packages/core/__tests__/note-history.test.jsindex 487ce2cfb..e8793fa8b 100644--- a/packages/core/__tests__/note-history.test.js+++ b/packages/core/__tests__/note-history.test.js@@ -80,9 +80,8 @@ test("restoring an old session should replace note's content", () =>const [, firstVersion] = await db.noteHistory.get(id);await db.noteHistory.restore(firstVersion.id);- await expect(db.notes.note(id).content()).resolves.toBe(- TEST_NOTE.content.data- );+ const contentId = (await db.notes.note(id)).contentId;+ expect((await db.content.get(contentId)).data).toBe(TEST_NOTE.content.data);}));test("date created of session should not change on edit", () =>@@ -152,7 +151,7 @@ test("auto clear sessions if they exceed the limit", () =>await db.noteHistory.cleanup(id, 1);sessions = await db.noteHistory.get(id);- expect(await db.noteHistory.get(id)).toHaveLength(1);+ expect(sessions).toHaveLength(1);const content = await db.noteHistory.content(sessions[0].id);expect(content.data).toBe(editedContent.data);@@ -163,7 +162,7 @@ test("save a locked note should add a locked session to note history", () =>await db.vault.create("password");await db.vault.add(id);- const note = db.notes.note(id).data;+ const note = await db.notes.note(id);const editedContent = { type: "tiptap", data: "<p>hello world</p>" };await db.vault.save({...note,diff --git a/packages/core/__tests__/notebooks.test.js b/packages/core/__tests__/notebooks.test.jsindex 37379b72f..0fd9e4ad1 100644--- a/packages/core/__tests__/notebooks.test.js+++ b/packages/core/__tests__/notebooks.test.js@@ -21,35 +21,30 @@ import { notebookTest, TEST_NOTEBOOK } from "./utils";import { test, expect } from "vitest";test("add a notebook", () =>- notebookTest().then(({ db, id }) => {+ notebookTest().then(async ({ db, id }) => {expect(id).toBeDefined();- let notebook = db.notebooks.notebook(id);+ const notebook = await db.notebooks.notebook(id);expect(notebook).toBeDefined();expect(notebook.title).toBe(TEST_NOTEBOOK.title);}));test("get all notebooks", () =>- notebookTest().then(({ db }) => {- expect(db.notebooks.all.length).toBeGreaterThan(0);+ notebookTest().then(async ({ db }) => {+ expect(await db.notebooks.all.count()).toBeGreaterThan(0);}));test("pin a notebook", () =>notebookTest().then(async ({ db, id }) => {- let notebook = db.notebooks.notebook(id);- await notebook.pin();- notebook = db.notebooks.notebook(id);- expect(notebook.data.pinned).toBe(true);+ await db.notebooks.pin(true, id);+ const notebook = await db.notebooks.notebook(id);+ expect(notebook.pinned).toBe(true);}));test("unpin a notebook", () =>notebookTest().then(async ({ db, id }) => {- let notebook = db.notebooks.notebook(id);- await notebook.pin();- notebook = db.notebooks.notebook(id);- expect(notebook.data.pinned).toBe(true);- await notebook.pin();- notebook = db.notebooks.notebook(id);- expect(notebook.data.pinned).toBe(false);+ await db.notebooks.pin(false, id);+ const notebook = await db.notebooks.notebook(id);+ expect(notebook.pinned).toBe(false);}));test("updating notebook with empty title should throw", () =>diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.tsindex 31f31c7ed..66d8fb27c 100644--- a/packages/core/__tests__/notes.test.ts+++ b/packages/core/__tests__/notes.test.ts@@ -68,18 +68,16 @@ test("add invalid note", () =>expect(db.notes.add({ hello: "world" })).rejects.toThrow();}));-test.only("add note", () =>+test("add note", () =>noteTest().then(async ({ db, id }) => {- const note = await db.notes.note$(id);- expect(note).toBeDefined();- const content = await db.content.get(note!.contentId!);- expect(content!.data).toStrictEqual(TEST_NOTE.content.data);+ expect(await db.notes.exists(id)).toBe(true);}));test("get note content", () =>noteTest().then(async ({ db, id }) => {- const content = await db.notes.note(id)?.content();- expect(content).toStrictEqual(TEST_NOTE.content.data);+ const note = await db.notes.note(id);+ const content = await db.content.get(note.contentId);+ expect(content?.data).toStrictEqual(TEST_NOTE.content.data);}));test("delete note", () =>@@ -97,21 +95,21 @@ test("delete note", () =>await db.notes.addToNotebook(subNotebookId, id);- await db.notes.delete(id);+ await db.notes.moveToTrash(id);- expect(db.notes.note(id)).toBeUndefined();- expect(db.notebooks.totalNotes(notebookId)).toBe(0);- expect(db.notebooks.totalNotes(subNotebookId)).toBe(0);+ expect(await db.notes.note(id)).toBeUndefined();+ expect(await db.notebooks.totalNotes(notebookId)).toBe(0);+ expect(await db.notebooks.totalNotes(subNotebookId)).toBe(0);}));test("get all notes", () =>noteTest().then(async ({ db }) => {- expect(db.notes.all.length).toBeGreaterThan(0);+ expect(await db.notes.all.count()).toBeGreaterThan(0);}));test("note without a title should get a premade title", () =>noteTest().then(async ({ db, id }) => {- const note = db.notes.note(id);+ const note = await db.notes.note(id);expect(note?.title.startsWith("Note ")).toBe(true);}));@@ -123,7 +121,7 @@ test("note should get headline from content", () =>data: "<p>This is a very colorful existence.</p>"}}).then(async ({ db, id }) => {- const note = db.notes.note(id);+ const note = await db.notes.note(id);expect(note?.headline).toBe("This is a very colorful existence.");}));@@ -135,14 +133,14 @@ test("note should not get headline if there is no p tag", () =>data: `<ol style="list-style-type: decimal;" data-mce-style="list-style-type: decimal;"><li>Hello I won't be a headline :(</li><li>Me too.</li><li>Gold.</li></ol>`}}).then(async ({ db, id }) => {- const note = db.notes.note(id);+ const note = await db.notes.note(id);expect(note?.headline).toBe("");}));test("note title should allow trailing space", () =>noteTest({ title: "Hello ", content: TEST_NOTE.content }).then(async ({ db, id }) => {- const note = db.notes.note(id);+ const note = await db.notes.note(id);expect(note?.title).toBe("Hello ");}));@@ -150,7 +148,7 @@ test("note title should allow trailing space", () =>test("note title should not allow newlines", () =>noteTest({ title: "Hello\nhello", content: TEST_NOTE.content }).then(async ({ db, id }) => {- const note = db.notes.note(id);+ const note = await db.notes.note(id);expect(note?.title).toBe("Hello hello");}));@@ -169,53 +167,53 @@ test("update note", () =>// colors: ["red", "blue"]};await db.notes.add(noteData);- const note = db.notes.note(id);+ const note = await db.notes.note(id);+ const content = await db.content.get(note.contentId);+expect(note?.title).toBe(noteData.title);- expect(await note?.content()).toStrictEqual(noteData.content.data);- expect(note?.data.pinned).toBe(true);- expect(note?.data.favorite).toBe(true);+ expect(content.data).toStrictEqual(noteData.content.data);+ expect(note?.pinned).toBe(true);+ expect(note?.favorite).toBe(true);}));test("get favorite notes", () =>noteTest({...TEST_NOTE,favorite: true- }).then(({ db }) => {- expect(db.notes.favorites.length).toBeGreaterThan(0);+ }).then(async ({ db }) => {+ expect(await db.notes.favorites.count()).toBeGreaterThan(0);}));test("get pinned notes", () =>noteTest({...TEST_NOTE,pinned: true- }).then(({ db }) => {- expect(db.notes.pinned.length).toBeGreaterThan(0);+ }).then(async ({ db }) => {+ expect(await db.notes.pinned.count()).toBeGreaterThan(0);}));-test("get grouped notes by abc", () => groupedTest("abc"));+test.todo("get grouped notes by abc", () => groupedTest("abc"));-test("get grouped notes by month", () => groupedTest("month"));+test.todo("get grouped notes by month", () => groupedTest("month"));-test("get grouped notes by year", () => groupedTest("year"));+test.todo("get grouped notes by year", () => groupedTest("year"));-test("get grouped notes by weak", () => groupedTest("week"));+test.todo("get grouped notes by weak", () => groupedTest("week"));-test("get grouped notes default", () => groupedTest("default"));+test.todo("get grouped notes default", () => groupedTest("default"));test("pin note", () =>noteTest().then(async ({ db, id }) => {- let note = db.notes.note(id);- await note?.pin();- note = db.notes.note(id);- expect(note?.data.pinned).toBe(true);+ await db.notes.pin(true, id);+ const note = await db.notes.note(id);+ expect(note.pinned).toBe(true);}));test("favorite note", () =>noteTest().then(async ({ db, id }) => {- let note = db.notes.note(id);- await note?.favorite();- note = db.notes.note(id);- expect(note?.data.favorite).toBe(true);+ await db.notes.favorite(true, id);+ const note = await db.notes.note(id);+ expect(note.favorite).toBe(true);}));test("add note to subnotebook", () =>@@ -230,10 +228,12 @@ test("add note to subnotebook", () =>);expect(- db.relations.from({ type: "notebook", id: notebookId }, "notebook")- ).toHaveLength(1);- expect(db.notebooks.totalNotes(subNotebookId)).toBe(1);- expect(db.notebooks.totalNotes(notebookId)).toBe(1);+ await db.relations+ .from({ type: "notebook", id: notebookId }, "notebook")+ .count()+ ).toBe(1);+ expect(await db.notebooks.totalNotes(subNotebookId)).toBe(1);+ expect(await db.notebooks.totalNotes(notebookId)).toBe(1);}));test("duplicate note to topic should not be added", () =>@@ -242,7 +242,7 @@ test("duplicate note to topic should not be added", () =>notebookTitle: "Hello",subNotebookTitle: "Home"});- expect(db.notebooks.totalNotes(subNotebookId)).toBe(1);+ expect(await db.notebooks.totalNotes(subNotebookId)).toBe(1);}));test("add the same note to 2 notebooks", () =>@@ -257,16 +257,18 @@ test("add the same note to 2 notebooks", () =>});expect(- db.relations+ await db.relations.from({ type: "notebook", id: nb1.subNotebookId }, "note").has(id)).toBe(true);expect(- db.relations+ await db.relations.from({ type: "notebook", id: nb2.subNotebookId }, "note").has(id)).toBe(true);- expect(db.relations.to({ type: "note", id }, "notebook")).toHaveLength(2);+ expect(+ await db.relations.to({ type: "note", id }, "notebook").count()+ ).toBe(2);}));test("moving note to same notebook and topic should do nothing", () =>@@ -278,7 +280,9 @@ test("moving note to same notebook and topic should do nothing", () =>await db.notes.addToNotebook(subNotebookId, id);- expect(db.relations.to({ type: "note", id }, "notebook")).toHaveLength(1);+ expect(+ await db.relations.to({ type: "note", id }, "notebook").count()+ ).toBe(1);}));test("export note to html", () =>@@ -319,31 +323,29 @@ test("deleting a colored note should remove it from that color", () =>);expect(- db.relations.from({ id: colorId, type: "color" }, "note").has(id)+ await db.relations.from({ id: colorId, type: "color" }, "note").has(id)).toBe(true);- await db.notes.delete(id);+ await db.notes.moveToTrash(id);expect(- db.relations.from({ id: colorId, type: "color" }, "note").has(id)+ await db.relations.from({ id: colorId, type: "color" }, "note").has(id)).toBe(false);}));test("note's content should follow note's localOnly property", () =>noteTest().then(async ({ db, id }) => {- await db.notes.note(id)?.localOnly();- let note = db.notes.note(id);- if (!note?.contentId) throw new Error("No content in note.");+ await db.notes.localOnly(true, id);+ let note = await db.notes.note(id);- expect(note?.data.localOnly).toBe(true);+ expect(note.localOnly).toBe(true);let content = await db.content.get(note.contentId);expect(content?.localOnly).toBe(true);- await db.notes.note(id)?.localOnly();- note = db.notes.note(id);- if (!note?.contentId) throw new Error("No content in note.");+ await db.notes.localOnly(false, id);+ note = await db.notes.note(id);- expect(note?.data.localOnly).toBe(false);+ expect(note.localOnly).toBe(false);content = await db.content.get(note.contentId);expect(content?.localOnly).toBe(false);}));@@ -365,11 +367,11 @@ test("note content should not contain image base64 data after save", () =>await loginFakeUser(db);await db.notes.add({ id, content: { type: "tiptap", data: IMG_CONTENT } });- const note = db.notes.note(id);- const content = await note?.content();+ const note = await db.notes.note(id);+ const content = await db.content.get(note.contentId);- expect(content).not.toContain(`src="data:image/png;`);- expect(content).not.toContain(`src=`);+ expect(content.data).not.toContain(`src="data:image/png;`);+ expect(content.data).not.toContain(`src=`);}));test("adding a note with an invalid tag should clean the tag array", () =>@@ -383,6 +385,6 @@ test("adding a note with an invalid tag should clean the tag array", () =>).resolves.toBe("helloworld");expect(- db.relations.to({ id: "helloworld", type: "note" }, "tag")- ).toHaveLength(0);+ await db.relations.to({ id: "helloworld", type: "note" }, "tag").count()+ ).toBe(0);}));diff --git a/packages/core/__tests__/shortcuts.test.js b/packages/core/__tests__/shortcuts.test.jsindex 8f06aaf61..294288a2e 100644--- a/packages/core/__tests__/shortcuts.test.js+++ b/packages/core/__tests__/shortcuts.test.js@@ -23,44 +23,45 @@ import { test, expect } from "vitest";test("create a shortcut of an invalid item should throw", () =>databaseTest().then(async (db) => {await expect(() =>- db.shortcuts.add({ item: { type: "HELLO!" } })+ db.shortcuts.add({ itemType: "HELLO!" })).rejects.toThrow(/cannot create a shortcut/i);}));test("create a shortcut of notebook", () =>notebookTest().then(async ({ db, id }) => {- await db.shortcuts.add({ item: { type: "notebook", id } });- expect(db.shortcuts.exists(id)).toBe(true);- expect(db.shortcuts.all[0].item.id).toBe(id);+ await db.shortcuts.add({ itemType: "notebook", itemId: id });+ expect(await db.shortcuts.exists(id)).toBe(true);+ expect(await db.shortcuts.all.has(id)).toBe(true);}));test("create a duplicate shortcut of notebook", () =>notebookTest().then(async ({ db, id }) => {- await db.shortcuts.add({ item: { type: "notebook", id } });- await db.shortcuts.add({ item: { type: "notebook", id } });+ await db.shortcuts.add({ itemType: "notebook", itemId: id });+ await db.shortcuts.add({ itemType: "notebook", itemId: id });- expect(db.shortcuts.all).toHaveLength(1);- expect(db.shortcuts.all[0].item.id).toBe(id);+ expect(await db.shortcuts.all.count()).toBe(1);+ expect(await db.shortcuts.all.has(id)).toBe(true);}));test("pin a tag", () =>databaseTest().then(async (db) => {const tagId = await db.tags.add({ title: "HELLO!" });- await db.shortcuts.add({ item: { type: "tag", id: tagId } });+ await db.shortcuts.add({ itemType: "tag", itemId: tagId });- expect(db.shortcuts.all).toHaveLength(1);- expect(db.shortcuts.all[0].item.id).toBe(tagId);+ expect(await db.shortcuts.all.count()).toBe(1);+ expect(await db.shortcuts.all.has(tagId)).toBe(true);}));test("remove shortcut", () =>databaseTest().then(async (db) => {const tagId = await db.tags.add({ title: "HELLO!" });const shortcutId = await db.shortcuts.add({- item: { type: "tag", id: tagId }+ itemType: "tag",+ itemId: tagId});- expect(db.shortcuts.all).toHaveLength(1);+ expect(await db.shortcuts.all.count()).toBe(1);await db.shortcuts.remove(shortcutId);- expect(db.shortcuts.all).toHaveLength(0);+ expect(await db.shortcuts.all.count()).toBe(0);}));diff --git a/packages/core/__tests__/tags.test.ts b/packages/core/__tests__/tags.test.tsindex 3dff2774b..5724813f3 100644--- a/packages/core/__tests__/tags.test.ts+++ b/packages/core/__tests__/tags.test.ts@@ -37,8 +37,10 @@ for (const type of ["tag", "color"] as const) {const tagId = await db[collection].add(item("hello"));await db.relations.add({ id: tagId, type }, { id, type: "note" });- expect(db[collection].all[0].title).toBe("hello");- expect(db.relations.from({ id: tagId, type }, "note")).toHaveLength(1);+ expect((await db[collection][type](tagId)).title).toBe("hello");+ expect(await db.relations.from({ id: tagId, type }, "note").count()).toBe(+ 1+ );}));test(`${type} 2 notes`, () =>@@ -50,15 +52,17 @@ for (const type of ["tag", "color"] as const) {await db.relations.add({ id: tagId, type }, { id, type: "note" });await db.relations.add({ id: tagId, type }, { id: id2, type: "note" });- expect(db[collection].all[0].title).toBe("hello");- expect(db.relations.from({ id: tagId, type }, "note")).toHaveLength(2);+ expect((await db[collection][type](tagId)).title).toBe("hello");+ expect(await db.relations.from({ id: tagId, type }, "note").count()).toBe(+ 2+ );}));test(`rename a ${type}`, () =>databaseTest().then(async (db) => {const tagId = await db[collection].add(item("hello"));await db[collection].add({ id: tagId, title: `hello (new)` });- expect(db[collection].all[0].title).toBe("hello (new)");+ expect((await db[collection][type](tagId)).title).toBe("hello (new)");}));test(`remove a ${type}`, () =>@@ -67,16 +71,20 @@ for (const type of ["tag", "color"] as const) {await db.relations.add({ id: tagId, type }, { id, type: "note" });await db[collection].remove(tagId);- expect(db[collection].all).toHaveLength(0);- expect(db.relations.from({ id: tagId, type }, "note")).toHaveLength(0);+ expect(await db[collection].collection.count()).toBe(0);+ expect(await db.relations.from({ id: tagId, type }, "note").count()).toBe(+ 0+ );}));test(`invalid characters from ${type} title are removed`, () =>databaseTest().then(async (db) => {- await db[collection].add(+ const tagId = await db[collection].add(item(" \n\n\n\t\t\thello l\n\n\n\t\t "));- expect(db[collection].all[0].title).toBe("hello l");+ expect((await db[collection][type](tagId)).title).toBe(+ "hello l"+ );}));test(`remove a note from ${type}`, () =>@@ -85,6 +93,8 @@ for (const type of ["tag", "color"] as const) {await db.relations.add({ id: tagId, type }, { id, type: "note" });await db.relations.unlink({ id: tagId, type }, { id, type: "note" });- expect(db.relations.from({ id: tagId, type }, "note")).toHaveLength(0);+ expect(await db.relations.from({ id: tagId, type }, "note").count()).toBe(+ 0+ );}));}diff --git a/packages/core/__tests__/trash.test.ts b/packages/core/__tests__/trash.test.tsindex d55543f78..ebba17b45 100644--- a/packages/core/__tests__/trash.test.ts+++ b/packages/core/__tests__/trash.test.ts@@ -28,8 +28,8 @@ import {import { test, expect } from "vitest";test("trash should be empty", () =>- databaseTest().then((db) => {- expect(db.trash.all).toHaveLength(0);+ databaseTest().then(async (db) => {+ expect(await db.trash.all()).toHaveLength(0);}));test("permanently delete a note", () =>@@ -38,19 +38,19 @@ test("permanently delete a note", () =>...TEST_NOTE,sessionId: Date.now().toString()});- const note = db.notes.note(noteId);+ const note = await db.notes.note(noteId);if (!note) throw new Error("Could not find note.");let sessions = await db.noteHistory.get(noteId);expect(sessions).toHaveLength(1);- await db.notes.delete(noteId);- expect(db.trash.all).toHaveLength(1);- expect(await note.content()).toBeDefined();- await db.trash.delete(db.trash.all[0].id);- expect(db.trash.all).toHaveLength(0);- const content = note.contentId && (await db.content.get(note.contentId));- expect(content).toBeUndefined();+ await db.notes.moveToTrash(noteId);++ expect(await db.trash.all()).toHaveLength(1);+ expect(await db.content.get(note.contentId)).toBeDefined();+ await db.trash.delete({ id: noteId, type: "note" });+ expect(await db.trash.all()).toHaveLength(0);+ expect(await db.content.get(note.contentId)).toBeUndefined();sessions = await db.noteHistory.get(noteId);expect(sessions).toHaveLength(0);@@ -67,45 +67,45 @@ test("restore a deleted note that was in a notebook", () =>);await db.notes.addToNotebook(subNotebookId, id);- await db.notes.delete(id);- await db.trash.restore(db.trash.all[0].id);- expect(db.trash.all).toHaveLength(0);+ await db.notes.moveToTrash(id);+ await db.trash.restore({ type: "note", id });+ expect(await db.trash.all()).toHaveLength(0);- const note = db.notes.note(id);+ const note = await db.notes.note(id);+ const content = await db.content.get(note.contentId);expect(note).toBeDefined();- expect(await note?.content()).toBe(TEST_NOTE.content.data);+ expect(content.data).toBe(TEST_NOTE.content.data);expect(- db.relations.from({ type: "notebook", id: subNotebookId }, "note").has(id)+ await db.relations+ .from({ type: "notebook", id: subNotebookId }, "note")+ .has(id)).toBe(true);}));test("delete a locked note", () =>noteTest().then(async ({ db, id }) => {- const note = db.notes.note(id);+ const note = await db.notes.note(id);await db.vault.create("password");await db.vault.add(id);- await db.notes.delete(id);- expect(db.trash.all).toHaveLength(1);- expect(- note && note.contentId && (await db.content.get(note.contentId))- ).toBeDefined();+ await db.notes.moveToTrash(id);+ expect(await db.trash.all()).toHaveLength(1);+ expect(await db.content.get(note.contentId)).toBeDefined();}));test("restore a deleted locked note", () =>noteTest().then(async ({ db, id }) => {- let note = db.notes.note(id);+ let note = await db.notes.note(id);await db.vault.create("password");await db.vault.add(id);- await db.notes.delete(id);- expect(db.trash.all).toHaveLength(1);- expect(- note && note.contentId && (await db.content.get(note.contentId))- ).toBeDefined();- await db.trash.restore(db.trash.all[0].id);- expect(db.trash.all).toHaveLength(0);- note = db.notes.note(id);+ await db.notes.moveToTrash(id);+ expect(await db.trash.all()).toHaveLength(1);+ expect(await db.content.get(note.contentId)).toBeDefined();+ await db.trash.restore({ type: "note", id });++ note = await db.notes.note(id);+ expect(await db.trash.all()).toHaveLength(0);expect(note).toBeDefined();}));@@ -114,13 +114,15 @@ test("restore a deleted note that's in a deleted notebook", () =>const notebookId = await db.notebooks.add(TEST_NOTEBOOK);await db.notes.addToNotebook(notebookId, id);- await db.notes.delete(id);- await db.notebooks.delete(notebookId);+ await db.notes.moveToTrash(id);+ await db.notebooks.moveToTrash(notebookId);- await db.trash.restore(id);- const note = db.notes.note(id);+ await db.trash.restore({ type: "note", id });+ const note = await db.notes.note(id);expect(note).toBeDefined();- expect(db.relations.to({ type: "note", id }, "notebook")).toHaveLength(0);+ expect(+ await db.relations.to({ type: "note", id }, "notebook").count()+ ).toBe(0);}));test("delete a notebook", () =>@@ -129,11 +131,11 @@ test("delete a notebook", () =>await db.notes.addToNotebook(id, noteId);- await db.notebooks.delete(id);- expect(db.notebooks.notebook(id)).toBeUndefined();+ await db.notebooks.moveToTrash(id);+ expect(await db.notebooks.notebook(id)).toBeUndefined();expect(- db.relations.to({ type: "note", id: noteId }, "notebook")- ).toHaveLength(0);+ await db.relations.to({ type: "note", id: noteId }, "notebook").count()+ ).toBe(0);}));test("restore a deleted notebook", () =>@@ -141,17 +143,17 @@ test("restore a deleted notebook", () =>const noteId = await db.notes.add(TEST_NOTE);await db.notes.addToNotebook(id, noteId);- await db.notebooks.delete(id);- await db.trash.restore(id);+ await db.notebooks.moveToTrash(id);+ await db.trash.restore({ type: "notebook", id });const notebook = db.notebooks.notebook(id);expect(notebook).toBeDefined();expect(- db.relations.to({ type: "note", id: noteId }, "notebook")- ).toHaveLength(1);+ await db.relations.to({ type: "note", id: noteId }, "notebook").count()+ ).toBe(1);expect(- db.relations.to({ type: "note", id: noteId }, "notebook").has(id)+ await db.relations.to({ type: "note", id: noteId }, "notebook").has(id)).toBe(true);}));@@ -160,14 +162,14 @@ test("restore a notebook that has deleted notes", () =>const noteId = await db.notes.add(TEST_NOTE);await db.notes.addToNotebook(id, noteId);- await db.notebooks.delete(id);- await db.notes.delete(noteId);- await db.trash.restore(id);+ await db.notebooks.moveToTrash(id);+ await db.notes.moveToTrash(noteId);+ await db.trash.restore({ type: "notebook", id });const notebook = db.notebooks.notebook(id);expect(notebook).toBeDefined();expect(- db.relations.from({ type: "notebook", id: id }, "note").has(noteId)+ await db.relations.from({ type: "notebook", id: id }, "note").has(noteId)).toBe(false);}));@@ -177,37 +179,28 @@ test("permanently delete items older than 7 days", () =>const noteId = await db.notes.add(TEST_NOTE);const notebookId = await db.notebooks.add(TEST_NOTEBOOK);- await db.notebooks.delete(notebookId);- await db.notes.delete(noteId);+ await db.notebooks.moveToTrash(notebookId);+ await db.notes.moveToTrash(noteId);- const note = db.trash.all.find((t) => t.id === noteId);- if (!note || note.itemType !== "note")- throw new Error("Could not find note in trash.");-- await db.notes.collection.update({- ...note,+ await db.notes.collection.update([noteId], {type: "trash",+ itemType: "note",id: noteId,dateDeleted: sevenDaysEarlier});- const notebook = db.trash.all.find((t) => t.id === notebookId);- if (!notebook || notebook.itemType !== "notebook")- throw new Error("Could not find notebook in trash.");-- await db.notebooks.collection.update({- ...notebook,+ await db.notebooks.collection.update([notebookId], {type: "trash",id: notebookId,dateDeleted: sevenDaysEarlier,itemType: "notebook"});- expect(db.trash.all).toHaveLength(2);+ expect(await db.trash.all()).toHaveLength(2);await db.trash.cleanup();- expect(db.trash.all).toHaveLength(0);+ expect(await db.trash.all()).toHaveLength(0);}));test("trash cleanup should not delete items newer than 7 days", () =>@@ -215,14 +208,14 @@ test("trash cleanup should not delete items newer than 7 days", () =>const noteId = await db.notes.add(TEST_NOTE);const notebookId = await db.notebooks.add(TEST_NOTEBOOK);- await db.notebooks.delete(notebookId);- await db.notes.delete(noteId);+ await db.notebooks.moveToTrash(notebookId);+ await db.notes.moveToTrash(noteId);- expect(db.trash.all).toHaveLength(2);+ expect(await db.trash.all()).toHaveLength(2);await db.trash.cleanup();- expect(db.trash.all).toHaveLength(2);+ expect(await db.trash.all()).toHaveLength(2);}));test("clear trash should delete note content", () =>@@ -237,16 +230,16 @@ test("clear trash should delete note content", () =>let sessions = await db.noteHistory.get(noteId);expect(sessions).toHaveLength(1);- const note = { ...db.notes.note(noteId)?.data };+ const note = { ...(await db.notes.note(noteId)) };- await db.notebooks.delete(notebookId);- await db.notes.delete(noteId);+ await db.notebooks.moveToTrash(notebookId);+ await db.notes.moveToTrash(noteId);- expect(db.trash.all).toHaveLength(2);+ expect(await db.trash.all()).toHaveLength(2);await db.trash.clear();- expect(db.trash.all).toHaveLength(0);+ expect(await db.trash.all()).toHaveLength(0);const content = note.contentId && (await db.content.get(note.contentId));expect(content).toBeUndefined();diff --git a/packages/core/__tests__/utils/index.ts b/packages/core/__tests__/utils/index.tsindex f1c317164..cece9f50e 100644--- a/packages/core/__tests__/utils/index.ts+++ b/packages/core/__tests__/utils/index.ts@@ -28,6 +28,8 @@ import { EventSourcePolyfill as EventSource } from "event-source-polyfill";import { randomBytes } from "../../src/utils/random";import { GroupOptions, Note, Notebook } from "../../src/types";import { NoteContent } from "../../src/collections/session-content";+import { SqliteDriver } from "kysely";+import BetterSQLite3 from "better-sqlite3";const TEST_NOTEBOOK: Partial<Notebook> = {title: "Test Notebook",@@ -45,7 +47,8 @@ function databaseTest() {storage: new NodeStorageInterface(),eventsource: EventSource,fs: FS,- compressor: Compressor+ compressor: Compressor,+ sqlite: new SqliteDriver({ database: BetterSQLite3(":memory:") })});return db.init().then(() => db);}diff --git a/packages/core/__tests__/vault.test.js b/packages/core/__tests__/vault.test.jsindex b7c01af86..956ba86d6 100644--- a/packages/core/__tests__/vault.test.js+++ b/packages/core/__tests__/vault.test.js@@ -58,11 +58,11 @@ test("lock a note", () =>noteTest().then(async ({ db, id }) => {await db.vault.create("password");await db.vault.add(id);- const note = db.notes.note(id);+ const note = await db.notes.note(id);expect(note.headline).toBe("");- const content = await db.content.raw(note.data.contentId, false);+ const content = await db.content.get(note.contentId);expect(content.noteId).toBeDefined();expect(content.data.iv).toBeDefined();expect(content.data.cipher).toBeDefined();@@ -72,9 +72,9 @@ test("locked note is not favorited", () =>noteTest().then(async ({ db, id }) => {await db.vault.create("password");await db.vault.add(id);- const note = db.notes.note(id);+ const note = await db.notes.note(id);- expect(note.data.favorite).toBeFalsy();+ expect(note.favorite).toBeFalsy();}));test("unlock a note", () =>@@ -92,12 +92,12 @@ test("unlock a note permanently", () =>await db.vault.create("password");await db.vault.add(id);await db.vault.remove(id, "password");- const note = db.notes.note(id);+ const note = await db.notes.note(id);expect(note.id).toBe(id);expect(note.headline).not.toBe("");- const content = await db.content.raw(note.data.contentId);+ const content = await db.content.get(note.contentId);expect(content.data).toBeDefined();- expect(typeof content.data).toBe("string");+ expect(typeof content.data).toBe("object");}));test("save a locked note", () =>@@ -105,13 +105,12 @@ test("save a locked note", () =>await db.vault.create("password");await db.vault.add(id);- const note = db.notes.note(id).data;+ const note = await db.notes.note(id);await db.vault.save(note);- const content = await db.content.raw(note.contentId);+ const content = await db.content.get(note.contentId);expect(content.data.cipher).toBeTypeOf("string");- expect(() => JSON.parse()).toThrow();}));test("save an edited locked note", () =>@@ -119,13 +118,13 @@ test("save an edited locked note", () =>await db.vault.create("password");await db.vault.add(id);- const note = db.notes.note(id).data;+ const note = await db.notes.note(id);await db.vault.save({...note,content: { type: "tiptap", data: "<p>hello world</p>" }});- const content = await db.content.raw(note.contentId);+ const content = await db.content.get(note.contentId);expect(content.data.cipher).toBeTypeOf("string");expect(() => JSON.parse(content.data.cipher)).toThrow();diff --git a/packages/core/package-lock.json b/packages/core/package-lock.jsonindex 2258b8bed..f47c28244 100644--- a/packages/core/package-lock.json+++ b/packages/core/package-lock.json@@ -10,6 +10,7 @@"hasInstallScript": true,"license": "GPL-3.0-or-later","dependencies": {+ "@leeoniya/ufuzzy": "^1.0.10","@microsoft/signalr": "^8.0.0","@notesnook/logger": "file:../logger","@readme/data-urls": "^3.0.0",@@ -1884,6 +1885,11 @@"@jridgewell/sourcemap-codec": "^1.4.14"}},+ "node_modules/@leeoniya/ufuzzy": {+ "version": "1.0.14",+ "resolved": "https://registry.npmjs.org/@leeoniya/ufuzzy/-/ufuzzy-1.0.14.tgz",+ "integrity": "sha512-/xF4baYuCQMo+L/fMSUrZnibcu0BquEGnbxfVPiZhs/NbJeKj4c/UmFpQzW9Us0w45ui/yYW3vyaqawhNYsTzA=="+ },"node_modules/@microsoft/signalr": {"version": "8.0.0","license": "MIT",diff --git a/packages/core/package.json b/packages/core/package.jsonindex c8137e13a..6abf6ee0f 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -47,6 +47,7 @@},"dependencies": {"@microsoft/signalr": "^8.0.0",+ "@leeoniya/ufuzzy": "^1.0.10","@notesnook/logger": "file:../logger","@readme/data-urls": "^3.0.0","@streetwriters/showdown": "^3.0.5-alpha",diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.tsindex ad92931de..d3f261953 100644--- a/packages/core/src/api/index.ts+++ b/packages/core/src/api/index.ts@@ -22,9 +22,9 @@ import { Crypto, CryptoAccessor } from "../database/crypto";import { FileStorage, FileStorageAccessor } from "../database/fs";import { Notebooks } from "../collections/notebooks";import Trash from "../collections/trash";-import { Tags } from "../collections/tags";-import { Colors } from "../collections/colors";import Sync, { SyncOptions } from "./sync";+import { Tags } from "../collections/tags";+import { Colors, } from "../collections/colors";import Vault from "./vault";import Lookup from "./lookup";import { Content } from "../collections/content";@@ -62,13 +62,14 @@ import { Attachment } from "../types";import { Settings } from "../collections/settings";import { DatabaseAccessor, DatabaseSchema, createDatabase } from "../database";import { Kysely, SqliteDriver, Transaction } from "kysely";-import BetterSQLite3 from "better-sqlite3";+import { CachedCollection } from "../database/cached-collection";type EventSourceConstructor = new (uri: string,init: EventSourceInit & { headers?: Record<string, string> }) => EventSource;type Options = {+ sqlite: SqliteDriver;storage: IStorage;eventsource?: EventSourceConstructor;fs: IFileStorage;@@ -126,18 +127,20 @@ class Database {};private _transaction?: Transaction<DatabaseSchema>;+ private transactionMutex = new Mutex();transaction = (executor: (tr: Transaction<DatabaseSchema>) => void | Promise<void>) => {- if (this._transaction) return executor(this._transaction);- return this.sql()- .transaction()- .execute(async (tr) => {- this._transaction = tr;- await executor(tr);- this._transaction = undefined;- })- .finally(() => (this._transaction = undefined));+ return this.transactionMutex.runExclusive(() =>+ this.sql()+ .transaction()+ .execute(async (tr) => {+ this._transaction = tr;+ await executor(tr);+ this._transaction = undefined;+ })+ .finally(() => (this._transaction = undefined))+ );};private options?: Options;@@ -174,6 +177,23 @@ class Database {relations = new Relations(this);notes = new Notes(this);+ /**+ * @deprecated only kept here for migration purposes+ */+ legacyTags = new CachedCollection(this.storage, "tags", this.eventManager);+ /**+ * @deprecated only kept here for migration purposes+ */+ legacyColors = new CachedCollection(+ this.storage,+ "colors",+ this.eventManager+ );+ /**+ * @deprecated only kept here for migration purposes+ */+ legacyNotes = new CachedCollection(this.storage, "notes", this.eventManager);+// constructor() {// this.sseMutex = new Mutex();// // this.lastHeartbeat = undefined; // { local: 0, server: 0 };@@ -209,9 +229,7 @@ class Database {this.disconnectSSE();});- this._sql = await createDatabase(- new SqliteDriver({ database: BetterSQLite3("nn.db") })- );+ if (this.options) this._sql = await createDatabase(this.options.sqlite);await this._validate();@@ -242,6 +260,11 @@ class Database {await this.trash.init();+ // legacy collections+ await this.legacyTags.init();+ await this.legacyColors.init();+ await this.legacyNotes.init();+// we must not wait on network requests that's why// no awaitthis.monographs.refresh();diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.tsindex 695f12955..723ed53c6 100644--- a/packages/core/src/api/lookup.ts+++ b/packages/core/src/api/lookup.ts@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { filter, parse } from "liqe";+import uFuzzy from "@leeoniya/ufuzzy";import Database from ".";import {Attachment,@@ -26,31 +26,41 @@ import {Notebook,Reminder,Tag,- TrashItem,- isDeleted+ TrashItem} from "../types";-import { isUnencryptedContent } from "../collections/content";+import { isFalse } from "../database";+import { sql } from "kysely";export default class Lookup {constructor(private readonly db: Database) {}- async notes(notes: Note[], query: string) {- const contents = await this.db.content.multi(- notes.map((note) => note.contentId || "")- );-- return search(notes, query, (note) => {- let text = note.title;- const noteContent = note.contentId ? contents[note.contentId] : "";- if (- !note.locked &&- noteContent &&- !isDeleted(noteContent) &&- isUnencryptedContent(noteContent)+ async notes(+ query: string,+ ids?: string[]+ ): Promise<Note & { rank: number }[]> {+ return (await this.db+ .sql()+ .with("matching", (eb) =>+ eb+ .selectFrom("content_fts")+ .where("data", "match", query)+ .select(["noteId as id", "rank"])+ .unionAll(+ eb+ .selectFrom("notes_fts")+ .where("title", "match", query)+ // add 10 weight to title+ .select(["id", sql.raw<number>(`rank * 10`).as("rank")])+ ))- text += noteContent.data;- return text;- });+ .selectFrom("notes")+ .$if(!!ids && ids.length > 0, (eb) => eb.where("id", "in", ids!))+ .where(isFalse("notes.deleted"))+ .where(isFalse("notes.dateDeleted"))+ .innerJoin("matching", (eb) => eb.onRef("notes.id", "==", "matching.id"))+ .orderBy("matching.rank")+ .selectAll()+ .execute()) as unknown as Note & { rank: number }[];}notebooks(array: Notebook[], query: string) {@@ -70,11 +80,7 @@ export default class Lookup {}attachments(array: Attachment[], query: string) {- return search(- array,- query,- (n) => `${n.metadata.filename} ${n.metadata.type} ${n.metadata.hash}`- );+ return search(array, query, (n) => `${n.filename} ${n.mimeType} ${n.hash}`);}private byTitle<T extends { title: string }>(array: T[], query: string) {@@ -82,14 +88,16 @@ export default class Lookup {}}+const uf = new uFuzzy();function search<T>(items: T[], query: string, selector: (item: T) => string) {try {- return filter(- parse(`text:"${query.toLowerCase()}"`),- items.map((item) => {- return { item, text: selector(item).toLowerCase() };- })- ).map((v) => v.item);+ const [_idxs, _info, order] = uf.search(items.map(selector), query, true);+ if (!order) return [];+ const filtered: T[] = [];+ for (const i of order) {+ filtered.push(items[i]);+ }+ return filtered;} catch (e) {return [];}diff --git a/packages/core/src/api/sync/__tests__/collector.test.js b/packages/core/src/api/sync/__tests__/collector.test.jsindex bb225985a..13441fd91 100644--- a/packages/core/src/api/sync/__tests__/collector.test.js+++ b/packages/core/src/api/sync/__tests__/collector.test.js@@ -26,7 +26,7 @@ import {import Collector from "../collector";import { test, expect } from "vitest";-test.only("newly created note should get included in collector", () =>+test("newly created note should get included in collector", () =>databaseTest().then(async (db) => {await loginFakeUser(db);const collector = new Collector(db);@@ -42,7 +42,7 @@ test.only("newly created note should get included in collector", () =>expect(items).toHaveLength(2);expect(items[0].type).toBe("content");- expect(items[0].items[0].id).toBe(db.notes.note(noteId).data.contentId);+ expect(items[0].items[0].id).toBe((await db.notes.note(noteId)).contentId);expect(items[1].items[0].id).toBe(noteId);expect(items[1].type).toBe("note");}));diff --git a/packages/core/src/api/sync/collector.ts b/packages/core/src/api/sync/collector.tsindex 204632984..7ffa6921f 100644--- a/packages/core/src/api/sync/collector.ts+++ b/packages/core/src/api/sync/collector.ts@@ -21,12 +21,14 @@ import { Cipher, SerializedKey } from "@notesnook/crypto";import Database from "..";import { CURRENT_DATABASE_VERSION, EV, EVENTS } from "../../common";import { logger } from "../../logger";-import { SYNC_COLLECTIONS_MAP, SyncItem, SyncTransferItem } from "./types";+import {+ SyncItem,+ SyncTransferItem,+ SYNC_COLLECTIONS_MAP,+ SYNC_ITEM_TYPES+} from "./types";import { Item, MaybeDeletedItem } from "../../types";-const ASYNC_COLLECTIONS_MAP = {- content: "content"-} as const;class Collector {logger = logger.scope("SyncCollector");constructor(private readonly db: Database) {}@@ -42,58 +44,22 @@ class Collector {throw new Error("User encryption key not generated. Please relogin.");}- const attachments = await this.prepareChunk(- this.db.attachments.syncable,- lastSyncedTimestamp,- isForceSync,- key- );- if (attachments) yield { items: attachments, type: "attachment" };-- for (const itemType in ASYNC_COLLECTIONS_MAP) {- const collectionKey =- ASYNC_COLLECTIONS_MAP[itemType as keyof typeof ASYNC_COLLECTIONS_MAP];+ for (const itemType of SYNC_ITEM_TYPES) {+ const collectionKey = SYNC_COLLECTIONS_MAP[itemType];const collection = this.db[collectionKey].collection;- for await (const chunk of collection.iterate(chunkSize)) {- const items = await this.prepareChunk(- chunk.map((item) => item[1]),- lastSyncedTimestamp,- isForceSync,- key- );+ for await (const chunk of collection.unsynced(+ isForceSync ? 0 : lastSyncedTimestamp,+ chunkSize+ )) {+ const items = await this.prepareChunk(chunk, key);if (!items) continue;- yield { items, type: itemType as keyof typeof ASYNC_COLLECTIONS_MAP };+ yield { items, type: itemType };}}-- // for (const itemType in SYNC_COLLECTIONS_MAP) {- // const collectionKey =- // SYNC_COLLECTIONS_MAP[itemType as keyof typeof SYNC_COLLECTIONS_MAP];- // const collection = this.db[collectionKey].collection;- // for (const chunk of collection.iterateSync(chunkSize)) {- // const items = await this.prepareChunk(- // chunk,- // lastSyncedTimestamp,- // isForceSync,- // key- // );- // if (!items) continue;- // yield { items, type: itemType as keyof typeof SYNC_COLLECTIONS_MAP };- // }- // }}- async prepareChunk(- chunk: MaybeDeletedItem<Item>[],- lastSyncedTimestamp: number,- isForceSync: boolean,- key: SerializedKey- ) {- const { ids, items } = filterSyncableItems(- chunk,- lastSyncedTimestamp,- isForceSync- );+ async prepareChunk(chunk: MaybeDeletedItem<Item>[], key: SerializedKey) {+ const { ids, items } = filterSyncableItems(chunk);if (!ids.length) return;const ciphers = await this.db.storage().encryptMulti(key, items);return toPushItem(ids, ciphers);@@ -114,39 +80,22 @@ function toPushItem(ids: string[], ciphers: Cipher<"base64">[]) {return items;}-function filterSyncableItems(- items: MaybeDeletedItem<Item>[],- lastSyncedTimestamp: number,- isForceSync = false-): { items: string[]; ids: string[] } {+function filterSyncableItems(items: MaybeDeletedItem<Item>[]): {+ items: string[];+ ids: string[];+} {if (!items || !items.length) return { items: [], ids: [] };const ids = [];const syncableItems = [];for (const item of items) {- if (!item) continue;+ // const isSyncable = !item.synced || isForceSync;- const isSyncable = !item.synced || isForceSync;- const isUnsynced = item.dateModified > lastSyncedTimestamp || isForceSync;-- // synced is a local only property+ // synced is a local only property. we don't want to sync it.delete item.synced;- if (isUnsynced && isSyncable) {- ids.push(item.id);- syncableItems.push(- JSON.stringify(- "localOnly" in item && item.localOnly- ? {- id: item.id,- deleted: true,- dateModified: item.dateModified,- deleteReason: "localOnly"- }- : item- )- );- }+ ids.push(item.id);+ syncableItems.push(JSON.stringify(item));}return { items: syncableItems, ids };}diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.tsindex 30b84e196..1dd72fddb 100644--- a/packages/core/src/api/sync/index.ts+++ b/packages/core/src/api/sync/index.ts@@ -37,18 +37,9 @@ import { Mutex } from "async-mutex";import Database from "..";import { migrateItem } from "../../migrations";import { SerializedKey } from "@notesnook/crypto";-import {- ItemMap,- MaybeDeletedItem,- Note,- Notebook,- TrashOrItem-} from "../../types";-import {- MERGE_COLLECTIONS_MAP,- SyncableItemType,- SyncTransferItem-} from "./types";+import { Item, MaybeDeletedItem } from "../../types";+import { SYNC_COLLECTIONS_MAP, SyncTransferItem } from "./types";+import { DownloadableFile } from "../../database/fs";export type SyncOptions = {type: "full" | "fetch" | "send";@@ -347,14 +338,13 @@ class Sync {* @private*/async uploadAttachments() {- const attachments = this.db.attachments.pending;+ const attachments = await this.db.attachments.pending.items();this.logger.info("Uploading attachments...", { total: attachments.length });await this.db.fs().queueUploads(- attachments.map((a) => ({- filename: a.metadata.hash,- chunkSize: a.chunkSize,- metadata: a.metadata+ attachments.map<DownloadableFile>((a) => ({+ filename: a.hash,+ chunkSize: a.chunkSize})),"sync-uploads");@@ -389,36 +379,29 @@ class Sync {));- let items: (- | MaybeDeletedItem<- ItemMap[SyncableItemType] | TrashOrItem<Note> | TrashOrItem<Notebook>- >- | undefined- )[] = [];+ const collectionType = SYNC_COLLECTIONS_MAP[itemType];+ const collection = this.db[collectionType].collection;+ const localItems = await collection.items(chunk.items.map((i) => i.id));+ let items: (MaybeDeletedItem<Item> | undefined)[] = [];if (itemType === "content") {- const localItems = await this.db.content.multi(- chunk.items.map((i) => i.id)- );items = await Promise.all(deserialized.map((item) =>this.merger.mergeContent(item, localItems[item.id], dbLastSynced)));} else {- items = this.merger.isSyncCollection(itemType)- ? deserialized.map((item) =>- this.merger.mergeItemSync(item, itemType, dbLastSynced)- )- : await Promise.all(- deserialized.map((item) =>- this.merger.mergeItem(item, itemType, dbLastSynced)+ items =+ itemType === "attachment"+ ? await Promise.all(+ deserialized.map((item) =>+ this.merger.mergeItemAsync(item, localItems[item.id], itemType)+ ))- );+ : deserialized.map((item) =>+ this.merger.mergeItemSync(item, localItems[item.id], itemType)+ );}- const collectionType = MERGE_COLLECTIONS_MAP[itemType];- await this.db[collectionType].collection.setItems(items as any);-if (notify &&(itemType === "note" || itemType === "content") &&@@ -428,6 +411,8 @@ class Sync {this.db.eventManager.publish(EVENTS.syncItemMerged, item));}++ await collection.put(items as any);}private async pushItem(item: SyncTransferItem, newLastSynced: number) {diff --git a/packages/core/src/api/sync/merger.ts b/packages/core/src/api/sync/merger.tsindex 7f27699b0..9fb5e7849 100644--- a/packages/core/src/api/sync/merger.ts+++ b/packages/core/src/api/sync/merger.ts@@ -20,26 +20,15 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { logger } from "../../logger";import { isHTMLEqual } from "../../utils/html-diff";import Database from "..";-import { SYNC_COLLECTIONS_MAP } from "./types";-import {- Attachment,- ContentItem,- Item,- ItemMap,- MaybeDeletedItem,- Note,- Notebook,- TrashOrItem,- isDeleted-} from "../../types";+import { ContentItem, Item, MaybeDeletedItem, isDeleted } from "../../types";class Merger {logger = logger.scope("Merger");constructor(private readonly db: Database) {}- isSyncCollection(type: string): type is keyof typeof SYNC_COLLECTIONS_MAP {- return type in SYNC_COLLECTIONS_MAP;- }+ // isSyncCollection(type: string): type is keyof typeof SYNC_COLLECTIONS_MAP {+ // return type in SYNC_COLLECTIONS_MAP;+ // }isConflicted(localItem: MaybeDeletedItem<Item>,@@ -79,12 +68,18 @@ class Merger {}}- mergeItemSync<TType extends keyof typeof SYNC_COLLECTIONS_MAP>(- remoteItem: MaybeDeletedItem<- ItemMap[TType] | TrashOrItem<Note> | TrashOrItem<Notebook>- >,- type: TType,- _lastSynced: number+ mergeItemSync(+ remoteItem: MaybeDeletedItem<Item>,+ localItem: MaybeDeletedItem<Item> | undefined,+ type:+ | "shortcut"+ | "reminder"+ | "tag"+ | "color"+ | "note"+ | "relation"+ | "notebook"+ | "settingitem") {switch (type) {case "shortcut":@@ -95,9 +90,6 @@ class Merger {case "relation":case "notebook":case "settingitem": {- const localItem = this.db[SYNC_COLLECTIONS_MAP[type]].collection.getRaw(- remoteItem.id- );if (!localItem || remoteItem.dateModified > localItem.dateModified) {return remoteItem;}@@ -107,8 +99,8 @@ class Merger {}async mergeContent(- remoteItem: MaybeDeletedItem<ContentItem>,- localItem: MaybeDeletedItem<ContentItem>,+ remoteItem: MaybeDeletedItem<Item>,+ localItem: MaybeDeletedItem<Item> | undefined,lastSynced: number) {if (localItem && "localOnly" in localItem && localItem.localOnly) return;@@ -120,63 +112,55 @@ class Merger {if (!localItem || conflicted === "merge") {return remoteItem;} else if (conflicted === "conflict") {- if (isDeleted(localItem) || isDeleted(remoteItem)) {+ if (+ isDeleted(localItem) ||+ isDeleted(remoteItem) ||+ remoteItem.type !== "tiptap" ||+ localItem.type !== "tiptap" ||+ localItem.locked ||+ remoteItem.locked ||+ !localItem.data ||+ !remoteItem.data ||+ isHTMLEqual(localItem.data, remoteItem.data)+ ) {if (remoteItem.dateModified > localItem.dateModified) return remoteItem;return;}- const note = this.db.notes.collection.get(localItem.noteId);- if (!note) return;-- // if hashes are equal do nothing- if (- !note.locked &&- (!remoteItem ||- !remoteItem ||- !localItem.data ||- !remoteItem.data ||- isHTMLEqual(localItem.data, remoteItem.data))- )- return;-- if (note.locked) {- // if note is locked or content is deleted we keep the most recent version.- if (remoteItem.dateModified > localItem.dateModified) return remoteItem;- } else {- // otherwise we trigger the conflicts- await this.db.notes.add({- id: localItem.noteId,- conflicted: true- });- await this.db.storage().write("hasConflicts", true);- return {- ...localItem,- conflicted: remoteItem- };- }+ // otherwise we trigger the conflicts+ await this.db.notes.add({+ id: localItem.noteId,+ conflicted: true+ });+ return {+ ...localItem,+ conflicted: remoteItem+ } as ContentItem;}}- async mergeItem(- remoteItem: MaybeDeletedItem<Attachment>,- type: "settings" | "attachment",- _lastSynced: number+ async mergeItemAsync(+ remoteItem: MaybeDeletedItem<Item>,+ localItem: MaybeDeletedItem<Item> | undefined,+ type: "attachment") {switch (type) {case "attachment": {- if (isDeleted(remoteItem)) return remoteItem;-- if (remoteItem.type !== "attachment") return;-- const localAttachment = this.db.attachments.attachment(- remoteItem.metadata.hash- );+ if (!localItem) return remoteItem;if (- localAttachment &&- localAttachment.dateUploaded !== remoteItem.dateUploaded+ isDeleted(localItem) ||+ isDeleted(remoteItem) ||+ remoteItem.type !== "attachment" ||+ localItem.type !== "attachment") {+ if (remoteItem.dateModified > localItem.dateModified)+ return remoteItem;+ return;+ }++ if (localItem.dateUploaded !== remoteItem.dateUploaded) {const isRemoved = await this.db.attachments.remove(- localAttachment.metadata.hash,+ localItem.hash,true);if (!isRemoved)diff --git a/packages/core/src/api/sync/types.ts b/packages/core/src/api/sync/types.tsindex 02cdda0c1..50ec5f2b6 100644--- a/packages/core/src/api/sync/types.ts+++ b/packages/core/src/api/sync/types.ts@@ -37,22 +37,46 @@ export type SyncItem = {v: number;} & Cipher<"base64">;+// export const SYNC_COLLECTIONS_MAP = {+// note: "notes",+// notebook: "notebooks",+// shortcut: "shortcuts",+// reminder: "reminders",+// relation: "relations",+// tag: "tags",+// color: "colors",+// settingitem: "settings"+// } as const;++// export const ASYNC_COLLECTIONS_MAP = {+// content: "content"+// } as const;+export const SYNC_COLLECTIONS_MAP = {- note: "notes",+ settingitem: "settings",+ attachment: "attachments",+ content: "content",notebook: "notebooks",shortcut: "shortcuts",reminder: "reminders",relation: "relations",tag: "tags",color: "colors",- settingitem: "settings"+ note: "notes"} as const;-export const MERGE_COLLECTIONS_MAP = {- ...SYNC_COLLECTIONS_MAP,- attachment: "attachments",- content: "content"-} as const;+export const SYNC_ITEM_TYPES = [+ "settingitem",+ "attachment",+ "content",+ "notebook",+ "shortcut",+ "reminder",+ "relation",+ "tag",+ "color",+ "note"+] as const;export type SyncTransferItem = {items: SyncItem[];diff --git a/packages/core/src/api/vault.ts b/packages/core/src/api/vault.tsindex 5abc4bd50..ece321bd4 100644--- a/packages/core/src/api/vault.ts+++ b/packages/core/src/api/vault.ts@@ -22,7 +22,7 @@ import Database from ".";import { CHECK_IDS, EV, EVENTS, checkIsUserPremium } from "../common";import { tinyToTiptap } from "../migrations";import { isCipher } from "../database/crypto";-import { EncryptedContentItem, Note, isDeleted } from "../types";+import { EncryptedContentItem, Note } from "../types";import {EMPTY_CONTENT,isEncryptedContent,@@ -102,14 +102,10 @@ export default class Vault {async changePassword(oldPassword: string, newPassword: string) {if (await this.unlock(oldPassword)) {const contentItems = [];- for (const note of this.db.notes.locked) {+ for await (const note of this.db.notes.locked) {if (!note.contentId) continue;- const encryptedContent = await this.db.content.raw(note.contentId);- if (- !encryptedContent ||- isDeleted(encryptedContent) ||- !isEncryptedContent(encryptedContent)- )+ const encryptedContent = await this.db.content.get(note.contentId);+ if (!encryptedContent || !isEncryptedContent(encryptedContent))continue;try {@@ -143,8 +139,7 @@ export default class Vault {async clear(password: string) {if (await this.unlock(password)) {- await this.db.notes.init();- for (const note of this.db.notes.locked) {+ for await (const note of this.db.notes.locked) {await this.unlockNote(note, password, true);}}@@ -152,10 +147,8 @@ export default class Vault {async delete(deleteAllLockedNotes = false) {if (deleteAllLockedNotes) {- await this.db.notes.init();- await this.db.notes.remove(- ...this.db.notes.locked.map((note) => note.id)- );+ const lockedIds = await this.db.notes.locked.ids();+ await this.db.notes.remove(...lockedIds);}await this.db.storage().remove("vaultKey");this.password = undefined;@@ -175,9 +168,9 @@ export default class Vault {* Permanently unlocks (remove from vault) a note*/async remove(noteId: string, password: string) {- const note = this.db.notes.note(noteId);+ const note = await this.db.notes.note(noteId);if (!note) return;- await this.unlockNote(note.data, password, true);+ await this.unlockNote(note, password, true);if (!(await this.exists())) await this.create(password);}@@ -186,10 +179,10 @@ export default class Vault {* Temporarily unlock (open) a note*/async open(noteId: string, password: string) {- const note = this.db.notes.note(noteId);+ const note = await this.db.notes.note(noteId);if (!note) return;- const unlockedNote = await this.unlockNote(note.data, password, false);+ const unlockedNote = await this.unlockNote(note, password, false);this.password = password;if (!(await this.exists())) await this.create(password);return unlockedNote;@@ -291,7 +284,7 @@ export default class Vault {const { id, content, sessionId, title } = item;let { type, data } = content || {};- const note = this.db.notes.note(id);+ const note = await this.db.notes.note(id);if (!note) return;const contentId = note.contentId;@@ -299,12 +292,8 @@ export default class Vault {// Case: when note is being newly lockedif (!note.locked && (!data || !type) && !!contentId) {- const rawContent = await this.db.content.raw(contentId);- if (- !rawContent ||- isDeleted(rawContent) ||- !isUnencryptedContent(rawContent)- )+ const rawContent = await this.db.content.get(contentId);+ if (!rawContent || !isUnencryptedContent(rawContent))return await this.db.notes.add({id,locked: true@@ -332,9 +321,9 @@ export default class Vault {locked: true,headline: "",title: title || note.title,- favorite: note.data.favorite,- localOnly: note.data.localOnly,- readonly: note.data.readonly,+ favorite: note.favorite,+ localOnly: note.localOnly,+ readonly: note.readonly,dateEdited: Date.now()});}@@ -342,13 +331,8 @@ export default class Vault {private async unlockNote(note: Note, password: string, perm = false) {if (!note.contentId) return;- const encryptedContent = await this.db.content.raw(note.contentId);- if (- !encryptedContent ||- isDeleted(encryptedContent) ||- !isEncryptedContent(encryptedContent)- )- return;+ const encryptedContent = await this.db.content.get(note.contentId);+ if (!encryptedContent || !isEncryptedContent(encryptedContent)) return;const content = await this.decryptContent(encryptedContent, password);if (perm) {diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.tsindex 4c8d04244..46dc3edb9 100644--- a/packages/core/src/collections/attachments.ts+++ b/packages/core/src/collections/attachments.ts@@ -33,7 +33,6 @@ import { Output } from "../interfaces";import { Attachment } from "../types";import Database from "../api";import { SQLCollection } from "../database/sql-collection";-import { isCipher } from "../database/crypto";export class Attachments implements ICollection {name = "attachments";@@ -113,12 +112,12 @@ export class Attachments implements ICollection {const id = oldAttachment?.id || getId();const encryptedKey = item.key- ? JSON.stringify(await this.encryptKey(item.key))- : oldAttachment?.encryptionKey;+ ? await this.encryptKey(item.key)+ : oldAttachment?.key;const attachment = {...oldAttachment,...item,- encryptionKey: encryptedKey+ key: encryptedKey};const {@@ -131,7 +130,7 @@ export class Attachments implements ICollection {mimeType,salt,chunkSize,- encryptionKey+ key} = attachment;if (@@ -144,7 +143,7 @@ export class Attachments implements ICollection {// !mimeType ||!salt ||!chunkSize ||- !encryptionKey+ !key) {console.error("Attachment is invalid because all properties are required:",@@ -161,7 +160,7 @@ export class Attachments implements ICollection {salt,size,alg,- encryptionKey,+ key,chunkSize,filename:@@ -188,9 +187,7 @@ export class Attachments implements ICollection {return await this.db.crypto().generateRandomKey();}- async decryptKey(keyJSON: string): Promise<SerializedKey | null> {- const key = JSON.parse(keyJSON);- if (!isCipher(key)) return null;+ async decryptKey(key: Cipher<"base64">): Promise<SerializedKey | null> {const encryptionKey = await this._getEncryptionKey();const plainData = await this.db.storage().decrypt(encryptionKey, key);if (!plainData) return null;@@ -263,7 +260,7 @@ export class Attachments implements ICollection {const attachment = await this.attachment(hash);if (!attachment) return;- const key = await this.decryptKey(attachment.encryptionKey);+ const key = await this.decryptKey(attachment.key);if (!key) return;const data = await this.db.fs().readEncrypted(attachment.hash, key, {chunkSize: attachment.chunkSize,@@ -361,27 +358,25 @@ export class Attachments implements ICollection {async cleanup() {const now = dayjs().unix();- for (const attachment of this.deleted) {- if (- !attachment.metadata ||- dayjs(attachment.dateDeleted).add(7, "days").unix() < now- )- continue;+ const ids: string[] = [];+ for await (const attachment of this.deleted) {+ if (dayjs(attachment.dateDeleted).add(7, "days").unix() < now) continue;- const isDeleted = await this.db.fs().deleteFile(attachment.metadata.hash);+ const isDeleted = await this.db.fs().deleteFile(attachment.hash);if (!isDeleted) continue;- await this.collection.softDelete(attachment.id);+ ids.push(attachment.id);}+ await this.collection.softDelete(ids);}- // get pending() {- // return this.all.filter(- // (attachment) =>- // (!attachment.dateUploaded || attachment.dateUploaded <= 0) &&- // this.db.relations.to(attachment, "note").length > 0- // );- // }+ get pending() {+ return this.collection.createFilter<Attachment>((qb) =>+ qb.where((eb) =>+ eb.or([eb("dateUploaded", "is", null), eb("dateUploaded", "<=", 0)])+ )+ );+ }// get uploaded() {// return this.all.filter((attachment) => !!attachment.dateUploaded);@@ -395,9 +390,11 @@ export class Attachments implements ICollection {// );// }- // get deleted() {- // return this.all.filter((attachment) => !!attachment.dateDeleted);- // }+ get deleted() {+ return this.collection.createFilter<Attachment>((qb) =>+ qb.where("dateDeleted", "is not", null)+ );+ }// get images() {// return this.all.filter((attachment) => isImage(attachment.metadata.type));diff --git a/packages/core/src/collections/colors.ts b/packages/core/src/collections/colors.tsindex edfc1c52e..df51019b5 100644--- a/packages/core/src/collections/colors.ts+++ b/packages/core/src/collections/colors.ts@@ -85,9 +85,11 @@ export class Colors implements ICollection {// return this.collection.raw();// }- // get all(): Color[] {- // return this.collection.items();- // }+ get all() {+ return this.collection.createFilter<Color>((qb) =>+ qb.where("deleted", "is", null)+ );+ }async remove(...ids: string[]) {await this.db.transaction(async () => {diff --git a/packages/core/src/collections/conflicts.ts b/packages/core/src/collections/conflicts.tsnew file mode 100644index 000000000..5b7e8a3af--- /dev/null+++ b/packages/core/src/collections/conflicts.ts@@ -0,0 +1,90 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { getId } from "../utils/id";+import { Tag } from "../types";+import Database from "../api";+import { ICollection } from "./collection";+import { SQLCollection } from "../database/sql-collection";++export class Tags implements ICollection {+ name = "tags";+ readonly collection: SQLCollection<"tags", Tag>;+ constructor(private readonly db: Database) {+ this.collection = new SQLCollection(db.sql, "tags", db.eventManager);+ }++ init() {+ return this.collection.init();+ }++ tag(id: string) {+ return this.collection.get(id);+ }++ // find(idOrTitle: string) {+ // return this.all.find(+ // (tag) => tag.title === idOrTitle || tag.id === idOrTitle+ // );+ // }++ async add(item: Partial<Tag>) {+ if (item.remote)+ throw new Error("Please use db.tags.merge to merge remote tags.");++ const id = item.id || getId(item.dateCreated);+ const oldTag = await this.tag(id);++ item.title = item.title ? Tags.sanitize(item.title) : item.title;+ if (!item.title && !oldTag?.title) throw new Error("Title is required.");++ await this.collection.upsert({+ id,+ dateCreated: item.dateCreated || oldTag?.dateCreated || Date.now(),+ dateModified: item.dateModified || oldTag?.dateModified || Date.now(),+ title: item.title || oldTag?.title || "",+ type: "tag",+ remote: false+ });+ return id;+ }++ // get raw() {+ // return this.collection.raw();+ // }++ // get all() {+ // return this.collection.items();+ // }++ async remove(...ids: string[]) {+ await this.db.transaction(async () => {+ await this.db.relations.unlinkOfType("tag", ids);+ await this.collection.softDelete(ids);+ });+ }++ exists(id: string) {+ return this.collection.exists(id);+ }++ static sanitize(title: string) {+ return title.replace(/^\s+|\s+$/gm, "");+ }+}diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.tsindex ec439ba14..4be3d9b97 100644--- a/packages/core/src/collections/content.ts+++ b/packages/core/src/collections/content.ts@@ -42,7 +42,8 @@ export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({id: getId(),localOnly: true,type: "tiptap",- data: "<p></p>"+ data: "<p></p>",+ locked: false});export class Content implements ICollection {@@ -71,59 +72,65 @@ export class Content implements ICollection {"Please use db.content.merge for merging remote content.");- const oldContent = content.id ? await this.raw(content.id) : undefined;- if (content.id && oldContent) {- content = {- ...oldContent,- ...content- };- }- if (!content.noteId) return;const id = content.id || getId();+ const oldContent = content.id ? await this.get(content.id) : undefined;+ const noteId = oldContent?.noteId || content.noteId;+ if (!noteId) throw new Error("No noteId found to link the content to.");++ const encryptedData = isCipher(content.data)+ ? content.data+ : oldContent && isCipher(oldContent.data)+ ? oldContent.data+ : null;++ const unencryptedData =+ typeof content.data === "string"+ ? content.data+ : oldContent && typeof oldContent.data === "string"+ ? oldContent.data+ : "<p></p>";const contentItem: ContentItem = {- noteId: content.noteId,+ type: "tiptap",+ noteId,id,- type: content.type || "tiptap",- data: content.data || "<p></p>",- dateEdited: content.dateEdited || Date.now(),- dateCreated: content.dateCreated || Date.now(),- dateModified: content.dateModified || Date.now(),- localOnly: !!content.localOnly,- conflicted: content.conflicted,- dateResolved: content.dateResolved++ dateEdited: content.dateEdited || oldContent?.dateEdited || Date.now(),+ dateCreated: content.dateCreated || oldContent?.dateCreated || Date.now(),+ dateModified: Date.now(),+ localOnly: content.localOnly || !!oldContent?.localOnly,++ conflicted: content.conflicted || oldContent?.conflicted,+ dateResolved: content.dateResolved || oldContent?.dateResolved,++ ...(encryptedData+ ? { locked: true, data: encryptedData }+ : { locked: false, data: unencryptedData })};+await this.collection.upsert(- isUnencryptedContent(contentItem)- ? await this.extractAttachments(contentItem)- : contentItem+ contentItem.locked+ ? contentItem+ : await this.extractAttachments(contentItem));- if (content.sessionId) {- await this.db.noteHistory?.add(- contentItem.noteId,- content.sessionId,- isCipher(contentItem.data),- {- data: contentItem.data,- type: contentItem.type- }- );- }+ if (content.sessionId)+ await this.db.noteHistory.add(content.sessionId, contentItem);+return id;}async get(id: string) {- const content = await this.raw(id);+ const content = await this.collection.get(id);if (!content || isDeleted(content)) return;return content;}- async raw(id: string) {- const content = await this.collection.get(id);- if (!content) return;- return content;- }+ // async raw(id: string) {+ // const content = await this.collection.get(id);+ // if (!content) return;+ // return content;+ // }remove(...ids: string[]) {return this.collection.softDelete(ids);@@ -147,6 +154,17 @@ export class Content implements ICollection {.execute();}+ async updateByNoteId(partial: Partial<ContentItem>, ...ids: string[]) {+ await this.db+ .sql()+ .updateTable("content")+ .where("noteId", "in", ids)+ .set({+ ...partial,+ dateModified: Date.now()+ })+ .execute();+ }// multi(ids: string[]) {// return this.collection.getItems(ids);// }@@ -228,9 +246,8 @@ export class Content implements ICollection {}async removeAttachments(id: string, hashes: string[]) {- const contentItem = await this.raw(id);- if (!contentItem || isDeleted(contentItem) || isCipher(contentItem.data))- return;+ const contentItem = await this.get(id);+ if (!contentItem || isCipher(contentItem.data)) return;const content = getContentFromData(contentItem.type, contentItem.data);if (!content) return;contentItem.data = content.removeAttachments(hashes);@@ -310,11 +327,11 @@ export class Content implements ICollection {export function isUnencryptedContent(content: ContentItem): content is UnencryptedContentItem {- return !isCipher(content.data);+ return content.locked === false;}export function isEncryptedContent(content: ContentItem): content is EncryptedContentItem {- return isCipher(content.data);+ return content.locked === true;}diff --git a/packages/core/src/collections/note-history.ts b/packages/core/src/collections/note-history.tsindex ee7b4effd..87769871d 100644--- a/packages/core/src/collections/note-history.ts+++ b/packages/core/src/collections/note-history.ts@@ -20,16 +20,16 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import Database from "../api";import { isCipher } from "../database/crypto";import { SQLCollection } from "../database/sql-collection";-import { HistorySession, isDeleted } from "../types";+import { ContentItem, HistorySession, isDeleted } from "../types";import { makeSessionContentId } from "../utils/id";import { ICollection } from "./collection";-import { SessionContent, NoteContent } from "./session-content";+import { SessionContent } from "./session-content";export class NoteHistory implements ICollection {name = "notehistory";versionsLimit = 100;sessionContent = new SessionContent(this.db);- private readonly collection: SQLCollection<"notehistory", HistorySession>;+ readonly collection: SQLCollection<"notehistory", HistorySession>;constructor(private readonly db: Database) {this.collection = new SQLCollection(db.sql, "notehistory", db.eventManager);}@@ -39,25 +39,29 @@ export class NoteHistory implements ICollection {await this.sessionContent.init();}- async get(noteId: string) {+ async get(noteId: string, order: "asc" | "desc" = "desc") {if (!noteId) return [];- const indices = this.collection.indexer.indices;- const sessionIds = indices.filter((id) => id.startsWith(noteId));- if (sessionIds.length === 0) return [];- const history = await this.getSessions(sessionIds);-- return history.sort(function (a, b) {- return b.dateModified - a.dateModified;- });+ // const indices = this.collection.indexer.indices;+ // const sessionIds = indices.filter((id) => id.startsWith(noteId));+ // if (sessionIds.length === 0) return [];+ // const history = await this.getSessions(sessionIds);++ // return history.sort(function (a, b) {+ // return b.dateModified - a.dateModified;+ // });+ const history = await this.db+ .sql()+ .selectFrom("notehistory")+ .where("noteId", "==", noteId)+ .orderBy(`dateModified ${order}`)+ .selectAll()+ .execute();+ return history as HistorySession[];}- async add(- noteId: string,- sessionId: string,- locked: boolean,- content: NoteContent<boolean>- ) {+ async add(sessionId: string, content: ContentItem) {+ const { noteId, locked } = content;sessionId = `${noteId}_${sessionId}`;const oldSession = await this.collection.get(sessionId);@@ -82,13 +86,9 @@ export class NoteHistory implements ICollection {}private async cleanup(noteId: string, limit = this.versionsLimit) {- const history = await this.get(noteId);+ const history = await this.get(noteId, "asc");if (history.length === 0 || history.length < limit) return;- history.sort(function (a, b) {- return a.dateModified - b.dateModified;- });const deleteCount = history.length - limit;-for (let i = 0; i < deleteCount; i++) {const session = history[i];await this._remove(session);@@ -108,14 +108,31 @@ export class NoteHistory implements ICollection {}async clearSessions(...noteIds: string[]) {- const history = await this.get(noteId);- for (const item of history) {- await this._remove(item);- }+ await this.db.transaction(async () => {+ const deletedIds = await this.db+ .sql()+ .deleteFrom("notehistory")+ .where("noteId", "in", noteIds)+ .returning("sessionContentId as sessionContentId")+ .execute();+ await this.db+ .sql()+ .deleteFrom("sessioncontent")+ .where(+ "id",+ "in",+ deletedIds.reduce((arr, item) => {+ if (item.sessionContentId && !arr.includes(item.sessionContentId))+ arr.push(item.sessionContentId);+ return arr;+ }, [] as string[])+ )+ .execute();+ });}private async _remove(session: HistorySession) {- await this.collection.delete(session.id);+ await this.collection.delete([session.id]);await this.sessionContent.remove(session.sessionContentId);}diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.tsindex a9ac92fbc..96668c9d8 100644--- a/packages/core/src/collections/notebooks.ts+++ b/packages/core/src/collections/notebooks.ts@@ -77,15 +77,20 @@ export class Notebooks implements ICollection {// return this.collection.raw();// }- // get all() {- // return this.collection.items((note) =>- // isTrashItem(note) ? undefined : note- // ) as Notebook[];- // }+ get all() {+ return this.collection.createFilter<Notebook>((qb) =>+ qb.where("dateDeleted", "is", null).where("deleted", "is", null)+ );+ }- // get pinned() {- // return this.all.filter((item) => item.pinned === true);- // }+ get pinned() {+ return this.collection.createFilter<Notebook>((qb) =>+ qb+ .where("dateDeleted", "is", null)+ .where("deleted", "is", null)+ .where("pinned", "==", true)+ );+ }// get trashed() {// return this.raw.filter((item) =>@@ -93,12 +98,8 @@ export class Notebooks implements ICollection {// ) as BaseTrashItem<Notebook>[];// }- async pin(...ids: string[]) {- await this.collection.update(ids, { pinned: true });- }-- async unpin(...ids: string[]) {- await this.collection.update(ids, { pinned: false });+ async pin(state: boolean, ...ids: string[]) {+ await this.collection.update(ids, { pinned: state });}async totalNotes(id: string) {@@ -114,7 +115,7 @@ export class Notebooks implements ICollection {.where("toType", "==", "notebook").where("fromType", "==", "notebook").whereRef("fromId", "==", "subNotebooks.id")- .where("toId", "not in", this.db.trash.cache)+ .where("toId", "not in", this.db.trash.cache.notebooks).$narrowType<{ id: string }>()))@@ -124,7 +125,7 @@ export class Notebooks implements ICollection {.where("fromId", "in", (eb) =>eb.selectFrom("subNotebooks").select("subNotebooks.id"))- .where("toId", "not in", this.db.trash.cache)+ .where("toId", "not in", this.db.trash.cache.notes).select((eb) => eb.fn.count<number>("id").as("totalNotes")).executeTakeFirst();@@ -142,11 +143,11 @@ export class Notebooks implements ICollection {return this.collection.exists(id);}- async remove(...ids: string[]) {+ async moveToTrash(...ids: string[]) {await this.db.trash.add("notebook", ids);}- async delete(...ids: string[]) {+ async remove(...ids: string[]) {await this.db.transaction(async () => {await this.db.relations.unlinkOfType("notebook", ids);await this.collection.softDelete(ids);diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.tsindex 2a668a000..f318773c4 100644--- a/packages/core/src/collections/notes.ts+++ b/packages/core/src/collections/notes.ts@@ -147,11 +147,11 @@ export class Notes implements ICollection {// return this.collection.raw();// }- // get all() {- // return this.collection.items((note) =>- // isTrashItem(note) ? undefined : note- // ) as Note[];- // }+ get all() {+ return this.collection.createFilter<Note>((qb) =>+ qb.where("dateDeleted", "is", null).where("deleted", "is", null)+ );+ }// isTrashed(id: string) {// return this.raw.find((item) => item.id === id && isTrashItem(item));@@ -163,29 +163,42 @@ export class Notes implements ICollection {// ) as BaseTrashItem<Note>[];// }- // get pinned() {- // return this.all.filter((item) => item.pinned === true);- // }+ get pinned() {+ return this.collection.createFilter<Note>((qb) =>+ qb+ .where("dateDeleted", "is", null)+ .where("deleted", "is", null)+ .where("pinned", "==", true)+ );+ }// get conflicted() {// return this.all.filter((item) => item.conflicted === true);// }- // get favorites() {- // return this.all.filter((item) => item.favorite === true);- // }+ get favorites() {+ return this.collection.createFilter<Note>((qb) =>+ qb+ .where("dateDeleted", "is", null)+ .where("deleted", "is", null)+ .where("favorite", "==", true)+ );+ }- // get locked(): Note[] {- // return this.all.filter(- // (item) => !isTrashItem(item) && item.locked === true- // ) as Note[];- // }+ get locked() {+ return this.collection.createFilter<Note>((qb) =>+ qb+ .where("dateDeleted", "is", null)+ .where("deleted", "is", null)+ .where("locked", "==", true)+ );+ }exists(id: string) {return this.collection.exists(id);}- delete(...ids: string[]) {+ moveToTrash(...ids: string[]) {return this._delete(true, ...ids);}@@ -193,6 +206,22 @@ export class Notes implements ICollection {return this._delete(false, ...ids);}+ pin(state: boolean, ...ids: string[]) {+ return this.collection.update(ids, { pinned: state });+ }+ favorite(state: boolean, ...ids: string[]) {+ return this.collection.update(ids, { favorite: state });+ }+ readonly(state: boolean, ...ids: string[]) {+ return this.collection.update(ids, { readonly: state });+ }+ async localOnly(state: boolean, ...ids: string[]) {+ await this.db.transaction(async () => {+ await this.collection.update(ids, { localOnly: state });+ await this.db.content.updateByNoteId({ localOnly: state }, ...ids);+ });+ }+async export(id: string, options: ExportOptions) {const { format, rawContent } = options;if (format !== "txt" && !(await checkIsUserPremium(CHECK_IDS.noteExport)))@@ -203,13 +232,9 @@ export class Notes implements ICollection {if (!options.contentItem) {const rawContent = note.contentId- ? await this.db.content.raw(note.contentId)+ ? await this.db.content.get(note.contentId): undefined;- if (- rawContent &&- (isDeleted(rawContent) || !isUnencryptedContent(rawContent))- )- return false;+ if (rawContent && !isUnencryptedContent(rawContent)) return false;options.contentItem = rawContent || EMPTY_CONTENT(note.id);}@@ -246,7 +271,7 @@ export class Notes implements ICollection {if (!note) continue;const content = note.contentId- ? await this.db.content.raw(note.contentId)+ ? await this.db.content.get(note.contentId): undefined;if (content && (isDeleted(content) || !isUnencryptedContent(content)))throw new Error("Cannot duplicate a locked or deleted note.");diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.tsindex 6f4095752..6bce13b00 100644--- a/packages/core/src/collections/relations.ts+++ b/packages/core/src/collections/relations.ts@@ -59,26 +59,14 @@ export class Relations implements ICollection {reference: ItemReference,type: TType) {- return new RelationsArray(- this.db.sql,- this.db.trash.cache,- reference,- type,- "from"- );+ return new RelationsArray(this.db, reference, type, "from");}to<TType extends keyof RelatableTable>(reference: ItemReference,type: TType) {- return new RelationsArray(- this.db.sql,- this.db.trash.cache,- reference,- type,- "to"- );+ return new RelationsArray(this.db, reference, type, "to");}// get raw() {@@ -161,15 +149,15 @@ class RelationsArray<TType extends keyof RelatableTable> {private table: ValueOf<RelatableTable> = TABLE_MAP[this.type];constructor(- private readonly sql: DatabaseAccessor,- private readonly trashIds: string[],+ private readonly db: Database,private readonly reference: ItemReference,private readonly type: TType,private readonly direction: "from" | "to") {}async resolve(limit?: number): Promise<ItemMap[TType][]> {- const items = await this.sql()+ const items = await this.db+ .sql().selectFrom(this.table).where("id", "in", (b) =>b@@ -189,7 +177,8 @@ class RelationsArray<TType extends keyof RelatableTable> {}async unlink() {- await this.sql()+ await this.db+ .sql().replaceInto("relations").columns(["id", "dateModified", "deleted"]).expression((eb) =>@@ -207,7 +196,8 @@ class RelationsArray<TType extends keyof RelatableTable> {}async get() {- const ids = await this.sql()+ const ids = await this.db+ .sql().selectFrom("relations").$call(this.buildRelationsQuery()).execute();@@ -215,7 +205,8 @@ class RelationsArray<TType extends keyof RelatableTable> {}async count() {- const result = await this.sql()+ const result = await this.db+ .sql().selectFrom("relations").$call(this.buildRelationsQuery()).clearSelect()@@ -226,7 +217,8 @@ class RelationsArray<TType extends keyof RelatableTable> {}async has(id: string) {- const result = await this.sql()+ const result = await this.db+ .sql().selectFrom("relations").$call(this.buildRelationsQuery()).clearSelect()@@ -252,9 +244,13 @@ class RelationsArray<TType extends keyof RelatableTable> {.where("toType", "==", this.reference.type).where("toId", "==", this.reference.id).$if(- (this.type === "note" || this.type === "notebook") &&- this.trashIds.length > 0,- (b) => b.where("fromId", "not in", this.trashIds)+ this.type === "note" && this.db.trash.cache.notes.length > 0,+ (b) => b.where("fromId", "not in", this.db.trash.cache.notes)+ )+ .$if(+ this.type === "notebook" &&+ this.db.trash.cache.notebooks.length > 0,+ (b) => b.where("fromId", "not in", this.db.trash.cache.notebooks)).select("relations.fromId as id").$narrowType<{ id: string }>();@@ -264,9 +260,13 @@ class RelationsArray<TType extends keyof RelatableTable> {.where("fromType", "==", this.reference.type).where("fromId", "==", this.reference.id).$if(- (this.type === "note" || this.type === "notebook") &&- this.trashIds.length > 0,- (b) => b.where("toId", "not in", this.trashIds)+ this.type === "note" && this.db.trash.cache.notes.length > 0,+ (b) => b.where("toId", "not in", this.db.trash.cache.notes)+ )+ .$if(+ this.type === "notebook" &&+ this.db.trash.cache.notebooks.length > 0,+ (b) => b.where("toId", "not in", this.db.trash.cache.notebooks)).select("relations.toId as id").$narrowType<{ id: string }>();diff --git a/packages/core/src/collections/session-content.ts b/packages/core/src/collections/session-content.tsindex 3c3d90263..6fae38a99 100644--- a/packages/core/src/collections/session-content.ts+++ b/packages/core/src/collections/session-content.ts@@ -33,10 +33,7 @@ export type NoteContent<TLocked extends boolean> = {export class SessionContent implements ICollection {name = "sessioncontent";- private readonly collection: SQLCollection<- "sessioncontent",- SessionContentItem- >;+ readonly collection: SQLCollection<"sessioncontent", SessionContentItem>;constructor(private readonly db: Database) {this.collection = new SQLCollection(db.sql,@@ -102,7 +99,7 @@ export class SessionContent implements ICollection {}async remove(sessionContentId: string) {- await this.collection.delete(sessionContentId);+ await this.collection.delete([sessionContentId]);}// async all() {diff --git a/packages/core/src/collections/shortcuts.ts b/packages/core/src/collections/shortcuts.tsindex aa4269761..a99c22801 100644--- a/packages/core/src/collections/shortcuts.ts+++ b/packages/core/src/collections/shortcuts.ts@@ -59,20 +59,19 @@ export class Shortcuts implements ICollection {...shortcut};- if (!shortcut.item)+ if (!shortcut.itemId || !shortcut.itemType)throw new Error("Cannot create a shortcut without an item.");- const id = shortcut.id || shortcut.item.id;+ const id = shortcut.id || shortcut.itemId;await this.collection.upsert({id,type: "shortcut",itemId: shortcut.itemId,itemType: shortcut.itemType,- // item: shortcut.item,dateCreated: shortcut.dateCreated || Date.now(),dateModified: shortcut.dateModified || Date.now(),- sortIndex: await this.collection.count()+ sortIndex: -1 // await this.collection.count()});return id;}@@ -81,9 +80,11 @@ export class Shortcuts implements ICollection {// return this.collection.raw();// }- // get all() {- // return this.collection.items();- // }+ get all() {+ return this.collection.createFilter<Shortcut>((qb) =>+ qb.where("deleted", "is", null)+ );+ }async get() {// return this.all.reduce((prev, shortcut) => {@@ -106,7 +107,7 @@ export class Shortcuts implements ICollection {// }, [] as (Notebook | Topic | Tag)[]);}- exists(id: string) {+ async exists(id: string) {return this.collection.exists(id);}diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.tsindex 5b7e8a3af..804a2f906 100644--- a/packages/core/src/collections/tags.ts+++ b/packages/core/src/collections/tags.ts@@ -69,9 +69,11 @@ export class Tags implements ICollection {// return this.collection.raw();// }- // get all() {- // return this.collection.items();- // }+ get all() {+ return this.collection.createFilter<Tag>((qb) =>+ qb.where("deleted", "is", null)+ );+ }async remove(...ids: string[]) {await this.db.transaction(async () => {diff --git a/packages/core/src/collections/trash.ts b/packages/core/src/collections/trash.tsindex a277e6207..994bb282f 100644--- a/packages/core/src/collections/trash.ts+++ b/packages/core/src/collections/trash.ts@@ -19,10 +19,19 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import dayjs from "dayjs";import Database from "../api";+import { deleteItems } from "../utils/array";+import { FilteredSelector } from "../database/sql-collection";+import { TrashItem } from "../types";export default class Trash {collections = ["notes", "notebooks"] as const;- cache: string[] = [];+ cache: {+ notes: string[];+ notebooks: string[];+ } = {+ notebooks: [],+ notes: []+ };constructor(private readonly db: Database) {}async init() {@@ -34,19 +43,19 @@ export default class Trash {.selectFrom("notes").where("type", "==", "trash").select("id")- .as("id"),+ .as("noteId"),eb.selectFrom("notebooks").where("type", "==", "trash").select("id")- .as("id")+ .as("notebookId")]).execute();- this.cache = result.reduce((ids, item) => {- if (item.id) ids.push(item.id);- return ids;- }, [] as string[]);+ for (const { noteId, notebookId } of result) {+ if (noteId) this.cache.notes.push(noteId);+ else if (notebookId) this.cache.notebooks.push(notebookId);+ }}async cleanup() {@@ -79,8 +88,8 @@ export default class Trash {},{ noteIds: [] as string[], notebookIds: [] as string[] });- await this.delete("note", noteIds);- await this.delete("note", notebookIds);++ await this._delete(noteIds, notebookIds);}async add(type: "note" | "notebook", ids: string[]) {@@ -90,50 +99,85 @@ export default class Trash {itemType: "note",dateDeleted: Date.now()});+ this.cache.notes.push(...ids);} else if (type === "notebook") {await this.db.notebooks.collection.update(ids, {type: "trash",itemType: "notebook",dateDeleted: Date.now()});+ this.cache.notebooks.push(...ids);}- this.cache.push(...ids);}- async delete(type: "note" | "notebook", ids: string[]) {- if (type === "note") {- await this.db.content.removeByNoteId(...ids);- await this.db.noteHistory.clearSessions(...ids);- await this.db.notes.delete(...ids);- } else if (type === "notebook") {- await this.db.relations.unlinkOfType("notebook", ids);- await this.db.notebooks.delete(...ids);+ async delete(...items: { id: string; type: "note" | "notebook" }[]) {+ if (items.length <= 0) return;++ const noteIds = [];+ const notebookIds = [];+ for (const item of items) {+ if (item.type === "note") {+ noteIds.push(item.id);+ this.cache.notes.splice(this.cache.notes.indexOf(item.id), 1);+ } else if (item.type === "notebook") {+ notebookIds.push(item.id);+ this.cache.notebooks.splice(this.cache.notebooks.indexOf(item.id), 1);+ }+ }++ await this._delete(noteIds, notebookIds);+ }++ private async _delete(noteIds: string[], notebookIds: string[]) {+ if (noteIds.length > 0) {+ await this.db.content.removeByNoteId(...noteIds);+ await this.db.noteHistory.clearSessions(...noteIds);+ await this.db.notes.remove(...noteIds);+ deleteItems(this.cache.notes, ...noteIds);+ }++ if (notebookIds.length > 0) {+ await this.db.relations.unlinkOfType("notebook", notebookIds);+ await this.db.notebooks.remove(...notebookIds);+ deleteItems(this.cache.notebooks, ...notebookIds);}- ids.forEach((id) => this.cache.splice(this.cache.indexOf(id), 1));}- async restore(type: "note" | "notebook", ids: string[]) {- if (type === "note") {- await this.db.notes.collection.update(ids, {+ async restore(...items: { id: string; type: "note" | "notebook" }[]) {+ if (items.length <= 0) return;++ const noteIds = [];+ const notebookIds = [];+ for (const item of items) {+ if (item.type === "note") {+ noteIds.push(item.id);+ this.cache.notes.splice(this.cache.notes.indexOf(item.id), 1);+ } else if (item.type === "notebook") {+ notebookIds.push(item.id);+ this.cache.notebooks.splice(this.cache.notebooks.indexOf(item.id), 1);+ }+ }++ if (noteIds.length > 0) {+ await this.db.notes.collection.update(noteIds, {type: "note",dateDeleted: null,itemType: null});- } else {- await this.db.notebooks.collection.update(ids, {+ }++ if (notebookIds.length > 0) {+ await this.db.notebooks.collection.update(notebookIds, {type: "notebook",dateDeleted: null,itemType: null});}- ids.forEach((id) => this.cache.splice(this.cache.indexOf(id), 1));}async clear() {- // for (const item of this.all) {- // await this.delete(item.id);- // }- this.cache = [];+ await this._delete(this.cache.notes, this.cache.notebooks);+ this.cache = { notebooks: [], notes: [] };}// synced(id: string) {@@ -144,11 +188,31 @@ export default class Trash {// } else return true;// }+ async all() {+ const trashedNotes = await this.db+ .sql()+ .selectFrom("notes")+ .where("type", "==", "trash")+ .where("id", "in", this.cache.notes)+ .selectAll()+ .execute();++ const trashedNotebooks = await this.db+ .sql()+ .selectFrom("notebooks")+ .where("type", "==", "trash")+ .where("id", "in", this.cache.notebooks)+ .selectAll()+ .execute();++ return [...trashedNotes, ...trashedNotebooks] as TrashItem[];+ }+/**** @param {string} id*/exists(id: string) {- return this.cache.includes(id);+ return this.cache.notebooks.includes(id) || this.cache.notes.includes(id);}}diff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.tsindex 5d55dd965..7284dc2f3 100644--- a/packages/core/src/database/backup.ts+++ b/packages/core/src/database/backup.ts@@ -21,20 +21,11 @@ import SparkMD5 from "spark-md5";import { CURRENT_DATABASE_VERSION } from "../common.js";import Migrator from "./migrator.js";import Database from "../api/index.js";-import {- Item,- MaybeDeletedItem,- Note,- Notebook,- ValueOf,- isDeleted-} from "../types.js";-import { Cipher } from "@notesnook/crypto";+import { Item, MaybeDeletedItem, Note, Notebook, isDeleted } from "../types.js";+import { Cipher, SerializedKey } from "@notesnook/crypto";import { isCipher } from "./crypto.js";-import { toChunks } from "../utils/array";import { migrateItem } from "../migrations";-import Indexer from "./indexer";-import { set } from "../utils/set.js";+import { DatabaseCollection } from "./index.js";type BackupDataItem = MaybeDeletedItem<Item> | string[];type BackupPlatform = "web" | "mobile" | "node";@@ -74,6 +65,15 @@ type EncryptedBackupFile = BaseBackupFile & {type BackupFile = UnencryptedBackupFile | EncryptedBackupFile;type LegacyBackupFile = LegacyUnencryptedBackupFile | LegacyEncryptedBackupFile;+type BackupState = {+ buffer: string[];+ bufferLength: number;+ chunkIndex: number;+ key?: SerializedKey;+ encrypt: boolean;+ type: BackupPlatform;+};+function isEncryptedBackup(backup: LegacyBackupFile | BackupFile): backup is EncryptedBackupFile | LegacyEncryptedBackupFile {@@ -86,6 +86,7 @@ function isLegacyBackupFile(return backup.version <= 5.8;}+const MAX_CHUNK_SIZE = 10 * 1024 * 1024;const COLORS = ["red","orange",@@ -133,11 +134,12 @@ const itemTypeToCollectionKey = {relation: "relations",reminder: "reminders",sessioncontent: "sessioncontent",- session: "notehistory",+ session: "noteHistory",notehistory: "notehistory",content: "content",shortcut: "shortcuts",- settingitem: "settingsv2",+ settingitem: "settings",+ settings: "settings",// to make ts happytopic: "topics"@@ -170,81 +172,85 @@ export default class Backup {data: ""};- const keys = await this.db.storage().getAllKeys();- const chunks = toChunks(keys, 20);- let buffer: string[] = [];- let bufferLength = 0;- const MAX_CHUNK_SIZE = 10 * 1024 * 1024;- let chunkIndex = 0;-- while (chunks.length > 0) {- const chunk = chunks.pop();- if (!chunk) break;-- const items = await this.db.storage().readMulti(chunk);- items.forEach(([id, item]) => {- const isDeleted =- item &&- typeof item === "object" &&- "deleted" in item &&- !("type" in item);-- if (- !item ||- invalidKeys.includes(id) ||- isDeleted ||- id.startsWith("_uk_")- )- return;-- const data = JSON.stringify(item);- buffer.push(data);- bufferLength += data.length;- });+ const backupState: BackupState = {+ buffer: [] as string[],+ bufferLength: 0,+ chunkIndex: 0,+ key,+ encrypt,+ type+ };- if (bufferLength >= MAX_CHUNK_SIZE || chunks.length === 0) {- let itemsJSON = `[${buffer.join(",")}]`;+ yield* this.backupCollection(this.db.notes.collection, backupState);+ yield* this.backupCollection(this.db.notebooks.collection, backupState);+ yield* this.backupCollection(this.db.content.collection, backupState);+ yield* this.backupCollection(this.db.noteHistory.collection, backupState);+ yield* this.backupCollection(+ this.db.noteHistory.sessionContent.collection,+ backupState+ );+ yield* this.backupCollection(this.db.colors.collection, backupState);+ yield* this.backupCollection(this.db.tags.collection, backupState);+ yield* this.backupCollection(this.db.settings.collection, backupState);+ yield* this.backupCollection(this.db.shortcuts.collection, backupState);+ yield* this.backupCollection(this.db.reminders.collection, backupState);+ yield* this.backupCollection(this.db.relations.collection, backupState);+ yield* this.backupCollection(this.db.attachments.collection, backupState);- buffer = [];- bufferLength = 0;+ if (backupState.buffer.length > 0) yield* this.bufferToFile(backupState);- itemsJSON = await this.db.compressor().compress(itemsJSON);+ await this.updateBackupTime();+ }- const hash = SparkMD5.hash(itemsJSON);+ private async *backupCollection<T, B extends boolean>(+ collection: DatabaseCollection<T, B>,+ state: BackupState+ ) {+ for await (const item of collection.stream()) {+ const data = JSON.stringify(item);+ state.buffer.push(data);+ state.bufferLength += data.length;- if (encrypt && key)- itemsJSON = JSON.stringify(- await this.db.storage().encrypt(key, itemsJSON)- );- else itemsJSON = JSON.stringify(itemsJSON);-- yield {- path: `${chunkIndex++}-${encrypt ? "encrypted" : "plain"}-${hash}`,- data: `{- "version": ${CURRENT_DATABASE_VERSION},- "type": "${type}",- "date": ${Date.now()},- "data": ${itemsJSON},- "hash": "${hash}",- "hash_type": "md5",- "compressed": true,- "encrypted": ${encrypt ? "true" : "false"}- }`- };+ if (state.bufferLength >= MAX_CHUNK_SIZE) {+ yield* this.bufferToFile(state);}}+ }- if (bufferLength > 0 || buffer.length > 0)- throw new Error("Buffer not empty.");+ private async *bufferToFile(state: BackupState) {+ let itemsJSON = `[${state.buffer.join(",")}]`;- await this.updateBackupTime();+ state.buffer = [];+ state.bufferLength = 0;++ itemsJSON = await this.db.compressor().compress(itemsJSON);++ const hash = SparkMD5.hash(itemsJSON);++ if (state.encrypt && state.key)+ itemsJSON = JSON.stringify(+ await this.db.storage().encrypt(state.key, itemsJSON)+ );+ else itemsJSON = JSON.stringify(itemsJSON);++ yield {+ path: `${state.chunkIndex++}-${+ state.encrypt ? "encrypted" : "plain"+ }-${hash}`,+ data: `{+"version": ${CURRENT_DATABASE_VERSION},+"type": "${state.type}",+"date": ${Date.now()},+"data": ${itemsJSON},+"hash": "${hash}",+"hash_type": "md5",+"compressed": true,+"encrypted": ${state.encrypt ? "true" : "false"}+}`+ };}- async import(- backup: LegacyBackupFile | BackupFile,- password?: string,- encryptionKey?: string- ) {+ async import(backup: LegacyBackupFile | BackupFile, password?: string, encryptionKey?: string) {if (!this.validate(backup)) throw new Error("Invalid backup.");backup = this.migrateBackup(backup);@@ -325,105 +331,102 @@ export default class Backup {}private async migrateData(data: BackupDataItem[], version: number) {- const toAdd: Partial<- Record<- ValueOf<typeof itemTypeToCollectionKey>,- [string, MaybeDeletedItem<Item>][]- >- > = {};- for (let item of data) {- // we do not want to restore deleted items- if (- !item ||- typeof item !== "object" ||- Array.isArray(item) ||- isDeleted(item)- )- continue;- // in v5.6 of the database, we did not set note history session's type- if ("sessionContentId" in item && item.type !== "session")- (item as any).type = "notehistory";-- await migrateItem(- item,- version,- CURRENT_DATABASE_VERSION,- item.type,- this.db,- "backup"- );- // since items in trash can have their own set of migrations,- // we have to run the migration again to account for that.- if (item.type === "trash" && item.itemType)+ await this.db.transaction(async () => {+ for (let item of data) {+ // we do not want to restore deleted items+ if (+ !item ||+ typeof item !== "object" ||+ Array.isArray(item) ||+ isDeleted(item)+ )+ continue;+ // in v5.6 of the database, we did not set note history session's type+ if ("sessionContentId" in item && item.type !== "session")+ (item as any).type = "notehistory";+await migrateItem(- item as unknown as Note | Notebook,+ item,version,CURRENT_DATABASE_VERSION,- item.itemType,+ item.type,this.db,"backup");+ // since items in trash can have their own set of migrations,+ // we have to run the migration again to account for that.+ if (item.type === "trash" && item.itemType)+ await migrateItem(+ item as unknown as Note | Notebook,+ version,+ CURRENT_DATABASE_VERSION,+ item.itemType,+ this.db,+ "backup"+ );- if (item.type === "attachment" && item.metadata && item.metadata.hash) {- const attachment = this.db.attachments.attachment(item.metadata.hash);- if (attachment) {- const isNewGeneric =- item.metadata.type === "application/octet-stream";- const isOldGeneric =- attachment.metadata.type === "application/octet-stream";- item = {- ...attachment,- metadata: {- ...attachment.metadata,- type:+ const itemType =+ // colors are naively of type "tag" instead of "color" so we have to fix that.+ item.type === "tag" && COLORS.includes(item.title.toLowerCase())+ ? "color"+ : "itemType" in item && item.itemType+ ? item.itemType+ : item.type;++ if (!itemType || itemType === "topic" || itemType === "settings")+ continue;++ if (item.type === "attachment" && (item.hash || item.metadata?.hash)) {+ const attachment = await this.db.attachments.attachment(+ item.metadata?.hash || item.hash+ );+ if (attachment) {+ const isNewGeneric =+ item.metadata?.type === "application/octet-stream" ||+ item.mimeType === "application/octet-stream";+ const isOldGeneric =+ attachment.mimeType === "application/octet-stream";+ item = {+ ...attachment,+ mimeType:// we keep whichever mime type is more specificisNewGeneric && !isOldGeneric- ? attachment.metadata.type- : item.metadata.type,+ ? attachment.mimeType+ : item.metadata?.type || item.mimeType,filename:// we keep the filename based on which item's mime type we keptisNewGeneric && !isOldGeneric- ? attachment.metadata.filename- : item.metadata.filename- },- noteIds: set.union(attachment.noteIds, item.noteIds)- };- } else {- item.dateUploaded = undefined;- item.failed = undefined;+ ? attachment.filename+ : item.metadata?.filename || item.filename+ };+ for (const noteId of item.noteIds || []) {+ await this.db.relations.add(+ {+ id: noteId,+ type: "note"+ },+ attachment+ );+ }+ } else {+ delete item.dateUploaded;+ delete item.failed;+ }}- }- // items should sync immediately after getting restored- item.dateModified = Date.now();- item.synced = false;-- if (item.type === "settings")- await this.db.storage().write("settings", item);- else {- const itemType =- // colors are naively of type "tag" instead of "color" so we have to fix that.- item.type === "tag" && COLORS.includes(item.title.toLowerCase())- ? "color"- : "itemType" in item- ? item.itemType- : item.type;const collectionKey = itemTypeToCollectionKey[itemType];- if (collectionKey) {- toAdd[collectionKey] = toAdd[collectionKey] || [];- toAdd[collectionKey]?.push([item.id, item]);- }- }- }+ const collection =+ collectionKey === "sessioncontent"+ ? this.db.noteHistory.sessionContent.collection+ : this.db[collectionKey].collection;- for (const collectionKey in toAdd) {- const items =- toAdd[collectionKey as ValueOf<typeof itemTypeToCollectionKey>];- if (!items) continue;- const indexer = new Indexer(this.db.storage, collectionKey);- await indexer.init();- await indexer.writeMulti(items);- }+ // items should sync immediately after getting restored+ item.dateModified = Date.now();+ item.synced = false;++ await collection.upsert(item as any);+ }+ });}private validate(backup: LegacyBackupFile | BackupFile) {diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.tsindex 83c8c8d94..6fc478d08 100644--- a/packages/core/src/database/index.ts+++ b/packages/core/src/database/index.ts@@ -35,13 +35,18 @@ import {ValueNode,PrimitiveValueListNode,Transaction,- ColumnType+ ColumnType,+ ExpressionBuilder,+ ReferenceExpression} from "kysely";import {Attachment,Color,ContentItem,+ GroupOptions,HistorySession,+ ItemType,+ MaybeDeletedItem,Note,Notebook,Relation,@@ -67,8 +72,10 @@ export type SQLiteItem<T> = {[P in keyof T]?: T[P] | null;} & { id: string };+export type SQLiteItemWithRowID<T> = SQLiteItem<T> & { rowid: number };+export interface DatabaseSchema {- notes: SQLiteItem<TrashOrItem<Note>>; //| SQLiteItem<BaseTrashItem<Note>>;+ notes: SQLiteItem<TrashOrItem<Note>>;content: SQLiteItem<ContentItem>;relations: SQLiteItem<Relation>;notebooks: SQLiteItem<TrashOrItem<Notebook>>;@@ -82,36 +89,77 @@ export interface DatabaseSchema {shortcuts: SQLiteItem<Shortcut>;}+export type DatabaseSchemaWithFTS = DatabaseSchema & {+ notes_fts: SQLiteItemWithRowID<{+ notes_fts: string;+ title: string;+ rank: number;+ }>;+ content_fts: SQLiteItemWithRowID<{+ content_fts: string;+ data: string;+ rank: number;+ noteId: string;+ }>;+};+type AsyncOrSyncResult<Async extends boolean, Response> = Async extends true? Promise<Response>: Response;-export interface DatabaseCollection<T, Async extends boolean> {+export interface DatabaseCollection<T, IsAsync extends boolean> {clear(): Promise<void>;init(): Promise<void>;upsert(item: T): Promise<void>;softDelete(ids: string[]): Promise<void>;delete(ids: string[]): Promise<void>;- exists(id: string): AsyncOrSyncResult<Async, boolean>;- count(): AsyncOrSyncResult<Async, number>;- get(id: string): AsyncOrSyncResult<Async, T | undefined>;+ exists(id: string): AsyncOrSyncResult<IsAsync, boolean>;+ count(): AsyncOrSyncResult<IsAsync, number>;+ get(id: string): AsyncOrSyncResult<IsAsync, T | undefined>;put(items: (T | undefined)[]): Promise<void>;update(ids: string[], partial: Partial<T>): Promise<void>;+ ids(options: GroupOptions): AsyncOrSyncResult<IsAsync, string[]>;+ items(+ ids: string[],+ sortOptions?: GroupOptions+ ): AsyncOrSyncResult<+ IsAsync,+ Record<string, MaybeDeletedItem<T> | undefined>+ >;+ unsynced(+ after: number,+ chunkSize: number+ ): IsAsync extends true+ ? AsyncIterableIterator<MaybeDeletedItem<T>[]>+ : IterableIterator<MaybeDeletedItem<T>[]>;+ stream(): IsAsync extends true+ ? AsyncIterableIterator<T>+ : IterableIterator<T>;}export type DatabaseAccessor = () =>| Kysely<DatabaseSchema>| Transaction<DatabaseSchema>;-type FilterBooleanProperties<T> = keyof {- [K in keyof T as T[K] extends boolean | undefined | null ? K : never]: T[K];+type FilterBooleanProperties<T, Type> = keyof {+ [K in keyof T as T[K] extends Type ? K : never]: T[K];};type BooleanFields = ValueOf<{- [D in keyof DatabaseSchema]: FilterBooleanProperties<DatabaseSchema[D]>;+ [D in keyof DatabaseSchema]: FilterBooleanProperties<+ DatabaseSchema[D],+ boolean | undefined | null+ >;}>;-const BooleanProperties: BooleanFields[] = [+// type ObjectFields = ValueOf<{+// [D in keyof DatabaseSchema]: FilterBooleanProperties<+// DatabaseSchema[D],+// object | undefined | null+// >;+// }>;++const BooleanProperties: Set<BooleanFields> = new Set(["compressed","conflicted","deleted",@@ -124,7 +172,31 @@ const BooleanProperties: BooleanFields[] = ["readonly","remote","synced"-];+]);++const DataMappers: Partial<Record<ItemType, (row: any) => void>> = {+ reminder: (row) => {+ if (row.selectedDays) row.selectedDays = JSON.parse(row.selectedDays);+ },+ settingitem: (row) => {+ if (+ row.value &&+ (row.key.startsWith("groupOptions") ||+ row.key.startsWith("toolbarConfig"))+ )+ row.value = JSON.parse(row.value);+ },+ tiptap: (row) => {+ if (row.conflicted) row.conflicted = JSON.parse(row.conflicted);+ if (row.locked && row.data) row.data = JSON.parse(row.data);+ },+ sessioncontent: (row) => {+ if (row.locked && row.data) row.data = JSON.parse(row.data);+ },+ attachment: (row) => {+ if (row.key) row.key = JSON.parse(row.key);+ }+};export async function createDatabase(driver: Driver) {const db = new Kysely<DatabaseSchema>({@@ -150,6 +222,13 @@ export async function createDatabase(driver: Driver) {return db;}+export function isFalse<TB extends keyof DatabaseSchema>(+ column: ReferenceExpression<DatabaseSchema, TB>+) {+ return (eb: ExpressionBuilder<DatabaseSchema, TB>) =>+ eb.or([eb(column, "is", eb.lit(null)), eb(column, "==", eb.lit(0))]);+}+export class SqliteBooleanPlugin implements KyselyPlugin {readonly #transformer = new SqliteBooleanTransformer();@@ -161,9 +240,15 @@ export class SqliteBooleanPlugin implements KyselyPlugin {args: PluginTransformResultArgs): Promise<QueryResult<UnknownRow>> {for (const row of args.result.rows) {- for (const key of BooleanProperties) {- const value = row[key];- row[key] = value === 1 ? true : false;+ for (const key in row) {+ if (BooleanProperties.has(key as BooleanFields)) {+ row[key] = row[key] === 1 ? true : false;+ }+ }++ const mapper = DataMappers[row.type as ItemType];+ if (row.type && mapper) {+ mapper(row);}}return Promise.resolve(args.result);@@ -174,7 +259,7 @@ class SqliteBooleanTransformer extends OperationNodeTransformer {transformValue(node: ValueNode): ValueNode {return {...super.transformValue(node),- value: typeof node.value === "boolean" ? (node.value ? 1 : 0) : node.value+ value: this.serialize(node.value)};}@@ -183,9 +268,17 @@ class SqliteBooleanTransformer extends OperationNodeTransformer {): PrimitiveValueListNode {return {...super.transformPrimitiveValueList(node),- values: node.values.map((value) =>- typeof value === "boolean" ? (value ? 1 : 0) : value- )+ values: node.values.map((value) => this.serialize(value))};}++ private serialize(value: unknown) {+ return typeof value === "boolean"+ ? value+ ? 1+ : 0+ : typeof value === "object" && value !== null+ ? JSON.stringify(value)+ : value;+ }}diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.tsindex d4f0d3b89..8189e0990 100644--- a/packages/core/src/database/migrations.ts+++ b/packages/core/src/database/migrations.ts@@ -17,7 +17,13 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { CreateTableBuilder, Migration, MigrationProvider, sql } from "kysely";+import {+ CreateTableBuilder,+ Kysely,+ Migration,+ MigrationProvider,+ sql+} from "kysely";export class NNMigrationProvider implements MigrationProvider {async getMigrations(): Promise<Record<string, Migration>> {@@ -26,8 +32,9 @@ export class NNMigrationProvider implements MigrationProvider {async up(db) {await db.schema.createTable("notes")- .modifyEnd(sql`without rowid`)+ // .modifyEnd(sql`without rowid`).$call(addBaseColumns)+ .$call(addTrashColumns).addColumn("title", "text").addColumn("headline", "text").addColumn("contentId", "text")@@ -38,34 +45,56 @@ export class NNMigrationProvider implements MigrationProvider {.addColumn("conflicted", "boolean").addColumn("readonly", "boolean").addColumn("dateEdited", "integer")- .addColumn("dateDeleted", "integer")- .addColumn("itemType", "text")- .addForeignKeyConstraint(- "note_has_content",- ["contentId"],- "content",- ["id"],- (b) => b.onDelete("restrict").onUpdate("restrict")- ).execute();+ await createFTS5Table(db, "notes", ["title"]);await db.schema.createTable("content")- .modifyEnd(sql`without rowid`)+ // .modifyEnd(sql`without rowid`).$call(addBaseColumns).addColumn("noteId", "text").addColumn("data", "text")+ .addColumn("locked", "boolean").addColumn("localOnly", "boolean").addColumn("conflicted", "text").addColumn("sessionId", "text").addColumn("dateEdited", "integer").addColumn("dateResolved", "integer").execute();+ await createFTS5Table(+ db,+ "content",+ ["data"],+ ["noteId"],+ ["(new.locked is null or new.locked == 0)"]+ );++ await db.schema+ .createTable("notehistory")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("noteId", "text")+ .addColumn("sessionContentId", "text")+ .addColumn("localOnly", "boolean")+ .addColumn("locked", "boolean")+ .execute();++ await db.schema+ .createTable("sessioncontent")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("data", "text")+ .addColumn("contentType", "text")+ .addColumn("locked", "boolean")+ .addColumn("compressed", "boolean")+ .addColumn("localOnly", "boolean")+ .execute();await db.schema.createTable("notebooks").modifyEnd(sql`without rowid`).$call(addBaseColumns)+ .$call(addTrashColumns).addColumn("title", "text").addColumn("description", "text").addColumn("dateEdited", "text")@@ -116,7 +145,7 @@ export class NNMigrationProvider implements MigrationProvider {.addColumn("date", "integer").addColumn("mode", "text").addColumn("recurringMode", "text")- .addColumn("selectedDays", "blob")+ .addColumn("selectedDays", "text").addColumn("localOnly", "boolean").addColumn("disabled", "boolean").addColumn("snoozeUntil", "integer")@@ -141,6 +170,20 @@ export class NNMigrationProvider implements MigrationProvider {.addColumn("failed", "text").execute();+ await db.schema+ .createTable("settings")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("key", "text")+ .addColumn("value", "text")+ .execute();++ await db.schema+ .createIndex("notehistory_noteid")+ .on("notehistory")+ .column("noteId")+ .execute();+await db.schema.createIndex("relation_from_general").on("relations")@@ -216,3 +259,63 @@ const addBaseColumns = <T extends string, C extends string = never>(.addColumn("synced", "boolean").addColumn("deleted", "boolean");};++const addTrashColumns = <T extends string, C extends string = never>(+ builder: CreateTableBuilder<T, C>+) => {+ return builder++ .addColumn("dateDeleted", "integer")+ .addColumn("itemType", "text");+};++async function createFTS5Table(+ db: Kysely<any>,+ table: string,+ indexedColumns: string[],+ unindexedColumns: string[] = [],+ insertConditions: string[] = []+) {+ const ref = sql.raw(table);+ const ref_fts = sql.raw(table + "_fts");+ const ref_ai = sql.raw(table + "_ai");+ const ref_ad = sql.raw(table + "_ad");+ const ref_au = sql.raw(table + "_au");++ const indexed_cols = sql.raw(indexedColumns.join(", "));+ const unindexed_cols =+ unindexedColumns.length > 0+ ? sql.raw(unindexedColumns.join(" UNINDEXED,") + " UNINDEXED,")+ : sql.raw("");+ const new_indexed_cols = sql.raw(indexedColumns.join(", new."));+ const old_indexed_cols = sql.raw(indexedColumns.join(", old."));++ await sql`CREATE VIRTUAL TABLE ${ref_fts} USING fts5(+ id UNINDEXED, ${unindexed_cols} ${indexed_cols}, content='${sql.raw(table)}'+ )`.execute(db);++ insertConditions = [+ "(new.deleted is null or new.deleted == 0)",+ ...insertConditions+ ];+ await sql`CREATE TRIGGER ${ref_ai} AFTER INSERT ON ${ref} WHEN ${sql.raw(+ insertConditions.join(" AND ")+ )}+ BEGIN+ INSERT INTO ${ref_fts}(rowid, id, ${indexed_cols}) VALUES (new.rowid, new.id, new.${new_indexed_cols});+ END;`.execute(db);++ await sql`CREATE TRIGGER ${ref_ad} AFTER DELETE ON ${ref}+ BEGIN+ INSERT INTO ${ref_fts} (${ref_fts}, rowid, id, ${indexed_cols})+ VALUES ('delete', old.rowid, old.id, old.${old_indexed_cols});+ END;`.execute(db);++ await sql`CREATE TRIGGER ${ref_au} AFTER UPDATE ON ${ref}+ BEGIN+ INSERT INTO ${ref_fts} (${ref_fts}, rowid, id, ${indexed_cols})+ VALUES ('delete', old.rowid, old.id, old.${old_indexed_cols});+ INSERT INTO ${ref_fts} (rowid, id, ${indexed_cols})+ VALUES (new.rowid, new.id, new.${new_indexed_cols});+ END;`.execute(db);+}diff --git a/packages/core/src/database/sql-cached-collection.ts b/packages/core/src/database/sql-cached-collection.tsindex 46b315b95..da1ef920a 100644--- a/packages/core/src/database/sql-cached-collection.ts+++ b/packages/core/src/database/sql-cached-collection.ts@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { MaybeDeletedItem, isDeleted } from "../types";+import { GroupOptions, MaybeDeletedItem, isDeleted } from "../types";import EventManager from "../utils/event-manager";import { DatabaseAccessor, DatabaseCollection, DatabaseSchema } from ".";import { SQLCollection } from "./sql-collection";@@ -110,6 +110,44 @@ export class SQLCachedCollection<}}+ ids(_options: GroupOptions): string[] {+ return Array.from(this.cache.keys());+ }++ items(+ ids: string[],+ _sortOptions?: GroupOptions+ ): Record<string, MaybeDeletedItem<T> | undefined> {+ const items: Record<string, MaybeDeletedItem<T> | undefined> = {};+ for (const id of ids) {+ items[id] = this.cache.get(id);+ }+ return items;+ }++ *unsynced(+ after: number,+ chunkSize: number+ ): IterableIterator<MaybeDeletedItem<T>[]> {+ let chunk: MaybeDeletedItem<T>[] = [];+ for (const [_key, value] of this.cache) {+ if (value.dateModified && value.dateModified > after) {+ chunk.push(value);+ if (chunk.length === chunkSize) {+ yield chunk;+ chunk = [];+ }+ }+ }+ if (chunk.length > 0) yield chunk;+ }++ *stream(): IterableIterator<T> {+ for (const [_key, value] of this.cache) {+ if (!value.deleted) yield value as T;+ }+ }+// has(id: string) {// return this.cache.has(id);// }diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.tsindex a77382288..c6e804a27 100644--- a/packages/core/src/database/sql-collection.ts+++ b/packages/core/src/database/sql-collection.ts@@ -18,14 +18,16 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { EVENTS } from "../common";-import { isDeleted } from "../types";+import { GroupOptions, MaybeDeletedItem, isDeleted } from "../types";import EventManager from "../utils/event-manager";import {DatabaseAccessor,DatabaseCollection,DatabaseSchema,- SQLiteItem+ SQLiteItem,+ isFalse} from ".";+import { ExpressionOrFactory, SelectQueryBuilder, SqlBool } from "kysely";export class SQLCollection<TCollectionType extends keyof DatabaseSchema,@@ -91,6 +93,7 @@ export class SQLCollection<.selectFrom<keyof DatabaseSchema>(this.type).select((a) => a.fn.count<number>("id").as("count")).where("id", "==", id)+ .where("deleted", "is", null).limit(1).executeTakeFirst()) || {};@@ -145,4 +148,173 @@ export class SQLCollection<}).execute();}++ async ids(sortOptions: GroupOptions): Promise<string[]> {+ const ids = await this.db()+ .selectFrom<keyof DatabaseSchema>(this.type)+ .select("id")+ .where("deleted", "is", null)+ .$if(this.type === "notes" || this.type === "notebooks", (eb) =>+ eb.where("dateDeleted", "is", null)+ )+ .orderBy(sortOptions.sortBy, sortOptions.sortDirection)+ .execute();+ return ids.map((id) => id.id);+ }++ async items(+ ids: string[],+ sortOptions?: GroupOptions+ ): Promise<Record<string, MaybeDeletedItem<T> | undefined>> {+ const results = await this.db()+ .selectFrom<keyof DatabaseSchema>(this.type)+ .selectAll()+ .where("id", "in", ids)+ .$if(!!sortOptions, (eb) =>+ eb.orderBy(sortOptions!.sortBy, sortOptions!.sortDirection)+ )+ .execute();+ const items: Record<string, MaybeDeletedItem<T>> = {};+ for (const item of results) {+ items[item.id] = item as MaybeDeletedItem<T>;+ }+ return items;+ }++ async *unsynced(+ after: number,+ chunkSize: number+ ): AsyncIterableIterator<MaybeDeletedItem<T>[]> {+ let index = 0;+ while (true) {+ const rows = await this.db()+ .selectFrom<keyof DatabaseSchema>(this.type)+ .selectAll()+ .orderBy("dateModified", "asc")+ .$if(after > 0, (eb) =>+ eb+ .where("dateModified", ">", after)+ .where((eb) =>+ eb.or([eb("synced", "is", null), eb("synced", "==", false)])+ )+ )+ .$if(this.type === "attachments", (eb) =>+ eb.where("dateUploaded", ">", 0)+ )+ .offset(index)+ .limit(chunkSize)+ .execute();+ if (rows.length === 0) break;+ index += chunkSize;+ yield rows as MaybeDeletedItem<T>[];+ }+ }++ async *stream(): AsyncIterableIterator<T> {+ let index = 0;+ const chunkSize = 50;+ while (true) {+ const rows = await this.db()+ .selectFrom<keyof DatabaseSchema>(this.type)+ .where(isFalse("deleted"))+ .orderBy("dateModified desc")+ .selectAll()+ .offset(index)+ .limit(chunkSize)+ .execute();+ if (rows.length === 0) break;+ index += chunkSize;+ for (const row of rows) {+ yield row as T;+ }+ }+ }++ createFilter<T>(+ selector: (+ qb: SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, unknown>+ ) => SelectQueryBuilder<DatabaseSchema, keyof DatabaseSchema, unknown>,+ batchSize = 50+ ) {+ return new FilteredSelector<T>(+ this.db().selectFrom<keyof DatabaseSchema>(this.type).$call(selector),+ batchSize+ );+ }+}++export class FilteredSelector<T> {+ constructor(+ readonly filter: SelectQueryBuilder<+ DatabaseSchema,+ keyof DatabaseSchema,+ unknown+ >,+ readonly batchSize: number+ ) {}++ async ids() {+ return (await this.filter.select("id").execute()).map((i) => i.id);+ }++ async items(ids?: string[]) {+ return (await this.filter+ .$if(!!ids && ids.length > 0, (eb) => eb.where("id", "in", ids!))+ .selectAll()+ .execute()) as T[];+ }++ async has(id: string) {+ const { count } =+ (await this.filter+ .where("id", "==", id)+ .limit(1)+ .select((a) => a.fn.count<number>("id").as("count"))+ .executeTakeFirst()) || {};+ return count !== undefined && count > 0;+ }++ async count() {+ const { count } =+ (await this.filter+ .select((a) => a.fn.count<number>("id").as("count"))+ .executeTakeFirst()) || {};+ return count || 0;+ }++ async find(+ filter: ExpressionOrFactory<DatabaseSchema, keyof DatabaseSchema, SqlBool>+ ) {+ const item = await this.filter+ .where(filter)+ .limit(1)+ .selectAll()+ .executeTakeFirst();+ return item as T | undefined;+ }++ async *map<TReturnType>(+ fn: (item: T) => TReturnType+ ): AsyncIterableIterator<TReturnType> {+ for await (const item of this) {+ yield fn(item);+ }+ }++ async *[Symbol.asyncIterator]() {+ let index = 0;+ while (true) {+ const rows = await this.filter+ .selectAll()+ .orderBy("dateModified asc")+ .offset(index)+ .limit(this.batchSize)+ .execute();+ if (rows.length === 0) break;+ index += this.batchSize;+ for (const row of rows) {+ yield row as T;+ }+ }+ }}diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.tsindex 4cd65c0d6..d2e53fdb7 100644--- a/packages/core/src/migrations.ts+++ b/packages/core/src/migrations.ts@@ -131,11 +131,8 @@ const migrations: Migration[] = [for (const pin of item.pins) {if (!pin.data) continue;await db.shortcuts.add({- item: {- type: pin.type,- id: pin.data.id,- notebookId: pin.data.notebookId- }+ itemId: pin.data.id,+ itemType: pin.type === "topic" ? "notebook" : pin.type});}delete item.pins;@@ -156,7 +153,7 @@ const migrations: Migration[] = [return oldType !== item.type;},shortcut: (item) => {- if (item.id === item.item.id) return false;+ if (!item.item || item.id === item.item.id) return false;item.id = item.item.id;return true;},@@ -192,12 +189,12 @@ const migrations: Migration[] = [const alias = db.legacySettings.getAlias(item.id);if (!alias &&- (db.tags.all.find(- (t) => item.title === t.title && t.id !== oldTagId- ) ||- db.colors.all.find(- (t) => item.title === t.title && t.id !== oldTagId- ))+ (db.legacyTags+ .items()+ .find((t) => item.title === t.title && t.id !== oldTagId) ||+ db.legacyColors+ .items()+ .find((t) => item.title === t.title && t.id !== oldTagId)))return false;@@ -218,10 +215,10 @@ const migrations: Migration[] = [note: async (item, db) => {for (const tag of item.tags || []) {const oldTagId = makeId(tag);- const oldTag = db.tags.tag(oldTagId);+ const oldTag = db.legacyTags.get(oldTagId);const alias = db.legacySettings.getAlias(oldTagId);- const newTag = db.tags.all.find(- (t) => [alias, tag].includes(t.title) && t.id !== oldTagId+ const newTag = await db.tags.all.find((eb) =>+ eb.or([eb("title", "in", [alias, tag])]));const newTagId =@@ -234,16 +231,17 @@ const migrations: Migration[] = [}));if (!newTagId) continue;await db.relations.add({ type: "tag", id: newTagId }, item);- await db.tags.delete(oldTagId);+ await db.legacyTags.delete(oldTagId);}if (item.color) {const oldColorId = makeId(item.color);- const oldColor = db.colors.color(oldColorId);+ const oldColor = db.legacyColors.get(oldColorId);const alias = db.legacySettings.getAlias(oldColorId);- const newColor = db.colors.all.find(- (t) => [alias, item.color].includes(t.title) && t.id !== oldColorId+ const newColor = await db.colors.all.find((eb) =>+ eb.or([eb("title", "in", [alias, item.color])]));+const newColorId =newColor?.id ||(await db.colors.add({@@ -255,7 +253,7 @@ const migrations: Migration[] = [}));if (newColorId) {await db.relations.add({ type: "color", id: newColorId }, item);- await db.colors.delete(oldColorId);+ await db.legacyColors.delete(oldColorId);}}@@ -279,6 +277,18 @@ const migrations: Migration[] = [{ type: "note", id: noteId });}++ if (item.metadata) {+ item.hash = item.metadata.hash;+ item.mimeType = item.metadata.type;+ item.hashType = item.metadata.hashType;+ item.filename = item.metadata.filename;+ }++ if (item.length) item.size = item.length;++ delete item.length;+ delete item.metadata;delete item.noteIds;return true;},@@ -353,7 +363,12 @@ const migrations: Migration[] = [delete item.to;delete item.from;return true;- }+ },+ tiptap: (item) => {+ item.locked = isCipher(item.data);+ return true;+ },+ all: () => true}},{@@ -394,7 +409,10 @@ export async function migrateItem<TItemType extends MigrationItemType>(const itemMigrator = migration.items[type];if (!itemMigrator) continue;- if (await itemMigrator(item, database, migrationType)) count++;+ if (await itemMigrator(item, database, migrationType)) {+ if (item.type && item.type !== type) type = item.type as TItemType;+ count++;+ }}return count > 0;diff --git a/packages/core/src/types.ts b/packages/core/src/types.tsindex 3f9f11b5e..80792b0bc 100644--- a/packages/core/src/types.ts+++ b/packages/core/src/types.ts@@ -224,12 +224,9 @@ export interface Attachment extends BaseItem<"attachment"> {hash: string;hashType: string;mimeType: string;- encryptionKey: string;- /**- * @deprecated only kept here for migration purposes- */- key?: Cipher<"base64">;+ key: Cipher<"base64">;+/*** @deprecated only kept here for migration purposes*/@@ -335,24 +332,27 @@ export interface Reminder extends BaseItem<"reminder"> {}export type ContentType = "tiptap" | "tiny";-export interface ContentItem extends BaseItem<ContentType> {+export interface BaseContentItem extends BaseItem<ContentType> {noteId: string;- data: string | Cipher<"base64">;dateEdited: number;localOnly: boolean;- conflicted?: ContentItem;dateResolved?: number;sessionId?: string;+ conflicted?: UnencryptedContentItem;}-export type UnencryptedContentItem = Omit<ContentItem, "data"> & {+export type UnencryptedContentItem = BaseContentItem & {data: string;+ locked: false;};-export type EncryptedContentItem = Omit<ContentItem, "data"> & {+export type EncryptedContentItem = BaseContentItem & {data: Cipher<"base64">;+ locked: true;};+export type ContentItem = EncryptedContentItem | UnencryptedContentItem;+export interface HistorySession extends BaseItem<"session"> {sessionContentId: string;noteId: string;diff --git a/packages/core/src/utils/array.ts b/packages/core/src/utils/array.tsindex 9f20fd5b0..fa92c2955 100644--- a/packages/core/src/utils/array.ts+++ b/packages/core/src/utils/array.ts@@ -72,3 +72,20 @@ export function* chunkedIterate<T>(array: T[], chunkSize: number) {yield chunk;}}++export async function* chunkify<T>(+ iterator: AsyncIterableIterator<T> | IterableIterator<T>,+ chunkSize: number+) {+ let chunk: T[] = [];+ for await (const item of iterator) {+ chunk.push(item);+ if (chunk.length === chunkSize) {+ yield chunk;+ chunk = [];+ }+ }+ if (chunk.length > 0) {+ yield chunk;+ }+}