Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: add tests for migrations

Abdullah Atta86652230bcce
diff --git a/packages/core/__tests__/migrations.test.ts b/packages/core/__tests__/migrations.test.ts
new file mode 100644
index 000000000..2c0ea3666
--- /dev/null
+++ b/packages/core/__tests__/migrations.test.ts
@@ -0,0 +1,663 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { test, expect, describe } from "vitest";
+import { migrateItem } from "../src/migrations";
+import { databaseTest } from "./utils";
+import { getId, makeId } from "../src/utils/id";
+import { LegacySettingsItem } from "../src/types";
+
+describe("[5.2] replace date edited with date modified", () => {
+ const itemsWithDateEdited = ["note", "notebook", "trash", "tiny"] as const;
+ const itemsWithoutDateEdited = ["tag", "attachment", "settings"] as const;
+ for (const type of itemsWithDateEdited) {
+ test(type, () =>
+ databaseTest().then(async (db) => {
+ const item = { type, dateEdited: Date.now() };
+ expect(await migrateItem(item, 5.2, 5.3, type, db, "local")).toBe(true);
+ expect(item.dateModified).toBeDefined();
+ expect(item.dateEdited).toBeDefined();
+ })
+ );
+ }
+
+ for (const type of itemsWithoutDateEdited) {
+ test(type, () =>
+ databaseTest().then(async (db) => {
+ const item = { type, dateEdited: Date.now() };
+ expect(await migrateItem(item, 5.2, 5.3, type, db, "local")).toBe(true);
+ expect(item.dateModified).toBeDefined();
+ expect(item.dateEdited).toBeUndefined();
+ })
+ );
+ }
+});
+
+test("[5.2] remove tox class from checklist", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ type: "tiny",
+ dateEdited: Date.now(),
+ data: `<p>hello</p><ul class="tox-checklist"><li class="tox-checklist--checked">world</li></ul>`
+ };
+ expect(await migrateItem(item, 5.2, 5.3, "tiny", db, "local")).toBe(true);
+ expect(item.data).toBe(
+ `<p>hello</p><ul class="checklist"><li class="checked">world</li></ul>`
+ );
+ }));
+
+test("[5.2] wrap tables with div", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ type: "tiny",
+ dateEdited: Date.now(),
+ data: `<p>hello</p><table></table>`
+ };
+ expect(await migrateItem(item, 5.2, 5.3, "tiny", db, "local")).toBe(true);
+ expect(item.data).toBe(
+ `<p>hello</p><div class="table-container" contenteditable="false"><table contenteditable="true"></table></div>`
+ );
+ }));
+
+test("[5.3] decode wrapped table html entities", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ type: "tiny",
+ dateEdited: Date.now(),
+ data: `<p>hello</p>&lt;div class="table-container" contenteditable="false"&gt;&lt;table contenteditable="true"&gt;&lt;/table&gt;&lt;/div&gt;`
+ };
+ expect(await migrateItem(item, 5.3, 5.4, "tiny", db, "local")).toBe(true);
+ expect(item.data).toBe(
+ `<p>hello</p><div class="table-container" contenteditable="false"><table contenteditable="true"></table></div>`
+ );
+ }));
+
+describe("[5.4] convert tiny to tiptap", () => {
+ const cases = [
+ {
+ name: "preserve newlines in code blocks",
+ from: `<p>Function</p>\n<p>Hello</p>\n<pre class="hljs language-javascript" spellcheck="false"><span class="hljs-keyword">function</span> <span class="hljs-title function_">google</span>() {<br><span class="hljs-keyword">var</span> <span class="hljs-keyword">function</span> <span class="hljs-title function_">google</span>();<br><span class="hljs-title class_">Function</span> <span class="hljs-title function_">google</span>();<br>}</pre>\n<p>Hh</p>\n<p>&nbsp;</p>`,
+ to: `<p>Function</p><br><p>Hello</p><br><pre class="hljs language-javascript" spellcheck="false">function google() {<br>var function google();<br>Function google();<br>}</pre><br><p>Hh</p><br><p>&nbsp;</p>`
+ },
+ {
+ name: "replace table container with table",
+ from: `<p>hello</p><div class="table-container" contenteditable="false"><table contenteditable="true"></table></div>`,
+ to: `<p>hello</p><table></table>`
+ },
+ {
+ name: "move images out of paragraphs",
+ from: `<p><img src="hello.jpg" /></p>`,
+ to: `<img src="hello.jpg">`
+ },
+ {
+ name: "replace [data-mce-bogus] elements",
+ from: `<p><br data-mce-bogus="" /></p><br data-mce-bogus="" />`,
+ to: `<p></p>`
+ },
+ {
+ name: "remove [data-mce-href] & [data-mce-flag] attributes",
+ from: `<p>Hello <a data-mce-href="#" href="#">world</a><span data-mce-flag="true">2</span></p>`,
+ to: `<p>Hello <a href="#">world</a><span>2</span></p>`
+ }
+ ];
+
+ for (const testCase of cases) {
+ test(testCase.name, () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ type: "tiny",
+ dateEdited: Date.now(),
+ data: testCase.from
+ };
+ expect(await migrateItem(item, 5.4, 5.5, "tiny", db, "local")).toBe(
+ true
+ );
+ expect(item.data).toBe(testCase.to);
+ })
+ );
+ }
+});
+
+test("[5.6] remove note ids from topics", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ type: "notebook",
+ topics: [{ notes: ["helloworld"] }, { notes: ["helloworld2"] }]
+ };
+ expect(await migrateItem(item, 5.6, 5.7, "notebook", db, "local")).toBe(
+ true
+ );
+ expect(item.topics.every((t) => !t.notes)).toBe(true);
+ }));
+
+test("[5.6] move pins to shortcuts", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ type: "settings",
+ pins: [
+ {
+ type: "topic",
+ data: {
+ id: "hello",
+ notebookId: "world"
+ }
+ },
+ {
+ type: "notebook",
+ data: { id: "world" }
+ },
+ {
+ type: "tag",
+ data: { id: "tag" }
+ }
+ ]
+ };
+ expect(await migrateItem(item, 5.6, 5.7, "settings", db, "local")).toBe(
+ true
+ );
+ expect(item.pins).toBeUndefined();
+ expect(
+ db.shortcuts.all.find(
+ (s) =>
+ s.item.type === "topic" &&
+ s.item.id === "hello" &&
+ s.item.notebookId === "world"
+ )
+ ).toBeDefined();
+ expect(
+ db.shortcuts.all.find(
+ (s) => s.item.type === "notebook" && s.item.id === "world"
+ )
+ ).toBeDefined();
+ expect(
+ db.shortcuts.all.find((s) => s.item.type === "tag" && s.item.id === "tag")
+ ).toBeDefined();
+ }));
+
+test("[5.7] change session content type from tiny to tiptap", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ id: "hello_content",
+ type: "tiny",
+ data: "<p>hello world</p>"
+ };
+ expect(await migrateItem(item, 5.7, 5.8, "tiny", db, "local")).toBe(true);
+ expect(item.type).toBe("sessioncontent");
+ expect(item.contentType).toBe("tiptap");
+ }));
+
+test("[5.7] change content item type to tiptap", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ id: "hello_content",
+ type: "content",
+ data: "<p>hello world</p>"
+ };
+ expect(await migrateItem(item, 5.7, 5.8, "content", db, "local")).toBe(
+ true
+ );
+ expect(item.type).toBe("tiptap");
+ }));
+
+test("[5.7] change shortcut item id to be same as its reference id", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ id: "something",
+ type: "shortcut",
+ item: {
+ type: "notebook",
+ id: "world"
+ }
+ };
+ expect(await migrateItem(item, 5.7, 5.8, "shortcut", db, "local")).toBe(
+ true
+ );
+ expect(item.id).toBe("world");
+ }));
+
+test("[5.7] add type to session content", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ id: "hello_content",
+ type: "tiptap",
+ data: "<p>hello world</p>"
+ };
+ expect(await migrateItem(item, 5.7, 5.8, "tiptap", db, "local")).toBe(true);
+ expect(item.type).toBe("sessioncontent");
+ expect(item.contentType).toBe("tiptap");
+ }));
+
+test("[5.7] change notehistory type to session", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ type: "notehistory"
+ };
+ expect(await migrateItem(item, 5.7, 5.8, "notehistory", db, "local")).toBe(
+ true
+ );
+ expect(item.type).toBe("session");
+ }));
+
+test("[5.8] remove remote property from items", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ type: "note",
+ remote: true
+ };
+ expect(await migrateItem(item, 5.8, 5.9, "note", db, "local")).toBe(true);
+ expect(item.remote).toBeUndefined();
+ }));
+
+test("[5.8] do nothing if backup type is not local", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ type: "note",
+ remote: true
+ };
+ expect(await migrateItem(item, 5.8, 5.9, "note", db, "backup")).toBe(false);
+ expect(await migrateItem(item, 5.8, 5.9, "note", db, "sync")).toBe(false);
+ }));
+
+describe("[5.9] make tags syncable", () => {
+ test("create tags inside notes & link to them using relations", () =>
+ databaseTest().then(async (db) => {
+ const noteId = getId();
+ const tags = ["hello", "world", "i am here"];
+ await db.notes.collection.add({
+ type: "note",
+ title: "I am a note",
+ tags,
+ id: noteId
+ });
+
+ for (const tag of tags) {
+ await db.tags.collection.add({
+ id: makeId(tag),
+ noteIds: [noteId],
+ type: "tag",
+ title: tag
+ });
+ }
+ await db.storage().write("settings", {
+ ...db.legacySettings.raw,
+ aliases: {
+ [makeId(tags[1])]: "I AM GOOD!"
+ }
+ });
+ await db.legacySettings.init();
+
+ const note = db.notes.note(noteId);
+ if (!note) throw new Error("Failed to find note.");
+
+ expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(
+ true
+ );
+
+ const resolvedTags = db.relations
+ .to({ type: "note", id: noteId }, "tag")
+ .resolved()
+ .sort((a, b) => a.title.localeCompare(b.title));
+
+ expect(note.data.tags).toBeUndefined();
+ expect(db.tags.all).toHaveLength(3);
+ expect(resolvedTags).toHaveLength(3);
+ expect(resolvedTags[0].title).toBe("hello");
+ expect(resolvedTags[1].title).toBe("I AM GOOD!");
+ expect(resolvedTags[2].title).toBe("i am here");
+ expect(
+ tags.every((t) => !db.tags.collection.exists(makeId(t)))
+ ).toBeTruthy();
+ }));
+
+ test("migrate old tag item to new one", () =>
+ databaseTest().then(async (db) => {
+ const tag = {
+ id: makeId("oldone"),
+ noteIds: [],
+ type: "tag",
+ title: "oldone"
+ };
+ expect(await migrateItem(tag, 5.9, 6.0, "tag", db, "backup")).toBe(true);
+ expect(tag.id).not.toBe(makeId("oldone"));
+ expect(tag.noteIds).toBeUndefined();
+ expect(tag.alias).toBeUndefined();
+ expect(tag.title).toBe("oldone");
+ }));
+
+ test("migrate old tag item with alias to new one", () =>
+ databaseTest().then(async (db) => {
+ await db.storage().write("settings", {
+ ...db.legacySettings.raw,
+ aliases: {
+ [makeId("oldone")]: "alias"
+ }
+ });
+ await db.legacySettings.init();
+
+ const tag = {
+ id: makeId("oldone"),
+ noteIds: [],
+ type: "tag",
+ title: "oldone"
+ };
+ expect(await migrateItem(tag, 5.9, 6.0, "tag", db, "backup")).toBe(true);
+ expect(tag.id).not.toBe(makeId("oldone"));
+ expect(tag.noteIds).toBeUndefined();
+ expect(tag.alias).toBeUndefined();
+ expect(tag.title).toBe("alias");
+ }));
+
+ test("migrate tags before notes", () =>
+ databaseTest().then(async (db) => {
+ const noteId = getId();
+ const tags = ["hello", "world", "i am here"];
+ await db.notes.collection.add({
+ type: "note",
+ title: "I am a note",
+ tags,
+ id: noteId
+ });
+ await db.storage().write("settings", {
+ ...db.legacySettings.raw,
+ aliases: {
+ [makeId(tags[1])]: "I AM GOOD!"
+ }
+ });
+ await db.legacySettings.init();
+ for (const tag of tags) {
+ const item = {
+ id: makeId(tag),
+ noteIds: [noteId],
+ type: "tag",
+ title: tag
+ };
+ await migrateItem(item, 5.9, 6.0, "tag", db, "backup");
+ await db.tags.collection.add(item);
+ }
+
+ const note = db.notes.note(noteId);
+ if (!note) throw new Error("Failed to find note.");
+
+ expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(
+ true
+ );
+
+ const resolvedTags = db.relations
+ .to({ type: "note", id: noteId }, "tag")
+ .resolved()
+ .sort((a, b) => a.title.localeCompare(b.title));
+
+ expect(note.data.tags).toBeUndefined();
+ expect(db.tags.all).toHaveLength(3);
+ expect(resolvedTags).toHaveLength(3);
+ expect(resolvedTags[0].title).toBe("hello");
+ expect(resolvedTags[1].title).toBe("I AM GOOD!");
+ expect(resolvedTags[2].title).toBe("i am here");
+ expect(
+ tags.every((t) => !db.tags.collection.exists(makeId(t)))
+ ).toBeTruthy();
+ }));
+});
+
+describe("[5.9] make colors syncable", () => {
+ test("create colors from notes & link to them using relations", () =>
+ databaseTest().then(async (db) => {
+ const noteId = getId();
+ await db.notes.collection.add({
+ type: "note",
+ title: "I am a note",
+ color: "blue",
+ id: noteId
+ });
+
+ await db.colors.collection.add({
+ id: makeId("blue"),
+ noteIds: [noteId],
+ type: "tag",
+ title: "blue"
+ });
+
+ const note = db.notes.note(noteId);
+ if (!note) throw new Error("Failed to find note.");
+
+ expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(
+ true
+ );
+
+ const resolvedColors = db.relations
+ .to({ type: "note", id: noteId }, "color")
+ .resolved();
+
+ expect(note.data.color).toBeUndefined();
+ expect(db.colors.all).toHaveLength(1);
+ expect(resolvedColors).toHaveLength(1);
+ expect(resolvedColors[0].title).toBe("blue");
+ expect(resolvedColors[0].colorCode).toBe("#2196F3");
+ expect(db.colors.collection.exists(makeId("blue"))).toBeFalsy();
+ }));
+
+ test("migrate old color item to new one", () =>
+ databaseTest().then(async (db) => {
+ const color = {
+ id: makeId("blue"),
+ noteIds: [],
+ type: "tag",
+ title: "blue"
+ };
+ expect(await migrateItem(color, 5.9, 6.0, "tag", db, "backup")).toBe(
+ true
+ );
+ expect(color.id).not.toBe(makeId("oldone"));
+ expect(color.noteIds).toBeUndefined();
+ expect(color.alias).toBeUndefined();
+ expect(color.title).toBe("blue");
+ expect(color.type).toBe("color");
+ expect(color.colorCode).toBe("#2196F3");
+ }));
+
+ test("migrate old color item with alias to new one", () =>
+ databaseTest().then(async (db) => {
+ await db.storage().write("settings", {
+ ...db.legacySettings.raw,
+ aliases: {
+ [makeId("blue")]: "very important"
+ }
+ });
+ await db.legacySettings.init();
+
+ const color = {
+ id: makeId("blue"),
+ noteIds: [],
+ type: "tag",
+ title: "blue"
+ };
+ expect(await migrateItem(color, 5.9, 6.0, "tag", db, "backup")).toBe(
+ true
+ );
+ expect(color.id).not.toBe(makeId("oldone"));
+ expect(color.noteIds).toBeUndefined();
+ expect(color.alias).toBeUndefined();
+ expect(color.title).toBe("very important");
+ expect(color.type).toBe("color");
+ expect(color.colorCode).toBe("#2196F3");
+ }));
+
+ test("migrate color before notes", () =>
+ databaseTest().then(async (db) => {
+ const noteId = getId();
+ await db.notes.collection.add({
+ type: "note",
+ title: "I am a note",
+ color: "blue",
+ id: noteId
+ });
+ await db.storage().write("settings", {
+ ...db.legacySettings.raw,
+ aliases: {
+ [makeId("blue")]: "I AM GOOD!"
+ }
+ });
+ await db.legacySettings.init();
+
+ const color = {
+ id: makeId("blue"),
+ noteIds: [noteId],
+ type: "tag",
+ title: "blue"
+ };
+ await migrateItem(color, 5.9, 6.0, "tag", db, "backup");
+ await db.colors.collection.add(color);
+
+ const note = db.notes.note(noteId);
+ if (!note) throw new Error("Failed to find note.");
+
+ expect(await migrateItem(note.data, 5.9, 6.0, "note", db, "backup")).toBe(
+ true
+ );
+
+ const resolvedColors = db.relations
+ .to({ type: "note", id: noteId }, "color")
+ .resolved();
+
+ expect(note.data.color).toBeUndefined();
+ expect(db.colors.all).toHaveLength(1);
+ expect(resolvedColors).toHaveLength(1);
+ expect(resolvedColors[0].title).toBe("I AM GOOD!");
+ expect(resolvedColors[0].colorCode).toBe("#2196F3");
+ expect(db.colors.collection.exists(makeId("blue"))).toBeFalsy();
+ }));
+});
+
+test("[5.9] move attachments.noteIds to relations", () =>
+ databaseTest().then(async (db) => {
+ const attachment = {
+ id: "ATTACHMENT_ID",
+ type: "attachment",
+ noteIds: ["HELLO_NOTE_ID"]
+ };
+ await migrateItem(attachment, 5.9, 6.0, "attachment", db, "backup");
+
+ const linkedNotes = db.relations.from(
+ { type: "attachment", id: "ATTACHMENT_ID" },
+ "note"
+ );
+ expect(attachment.noteIds).toBeUndefined();
+ expect(linkedNotes).toHaveLength(1);
+ expect(linkedNotes[0].to.id).toBe("HELLO_NOTE_ID");
+ }));
+
+describe("[5.9] move topics out of notebooks & use relations", () => {
+ test("convert topics to subnotebooks", () =>
+ databaseTest().then(async (db) => {
+ const notebook = {
+ id: "parent_notebook",
+ type: "notebook",
+ topics: [
+ { id: "topics1", title: "Topic 1" },
+ { id: "topics2", title: "Topic 2" }
+ ]
+ };
+ await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup");
+
+ const linkedNotebooks = db.relations
+ .from({ type: "notebook", id: "parent_notebook" }, "notebook")
+ .sort((a, b) => a.to.id.localeCompare(b.to.id));
+ expect(notebook.topics).toBeUndefined();
+ expect(linkedNotebooks).toHaveLength(2);
+ expect(linkedNotebooks[0].to.id).toBe("topics1");
+ expect(linkedNotebooks[1].to.id).toBe("topics2");
+ expect(db.notebooks.all).toHaveLength(2);
+ expect(db.notebooks.notebook("topics1")).toBeDefined();
+ expect(db.notebooks.notebook("topics2")).toBeDefined();
+ }));
+
+ test("convert topic shortcuts to notebook shortcuts", () =>
+ databaseTest().then(async (db) => {
+ const shortcut = {
+ id: "shortcut1",
+ type: "shortcut",
+ item: {
+ type: "topic",
+ id: "topics1"
+ }
+ };
+ await migrateItem(shortcut, 5.9, 6.0, "shortcut", db, "backup");
+
+ expect(shortcut.item.type).toBe("notebook");
+ expect(shortcut.item.id).toBe("topics1");
+ }));
+
+ test("convert topic links in note to relations", () =>
+ databaseTest().then(async (db) => {
+ const note = {
+ id: "note1",
+ type: "note",
+ notebooks: [{ id: "notebook1", topics: ["topic1", "topic2"] }]
+ };
+ await migrateItem(note, 5.9, 6.0, "note", db, "backup");
+
+ const linkedNotebooks = db.relations
+ .to({ type: "note", id: "note1" }, "notebook")
+ .sort();
+ expect(note.notebooks).toBeUndefined();
+ expect(linkedNotebooks).toHaveLength(2);
+ expect(linkedNotebooks[0].from.id).toBe("topic1");
+ expect(linkedNotebooks[1].from.id).toBe("topic2");
+ }));
+});
+
+test("[5.9] migrate settings to its own collection", () =>
+ databaseTest().then(async (db) => {
+ const settings: LegacySettingsItem = {
+ type: "settings",
+ id: "settings",
+ dateCreated: Date.now(),
+ dateModified: Date.now(),
+ dateFormat: "CUSTOM_DATE_FORMAT!",
+ defaultNotebook: { id: "notebook1", topic: "topic1" },
+ groupOptions: {
+ favorites: {
+ groupBy: "abc",
+ sortBy: "dateCreated",
+ sortDirection: "asc"
+ }
+ },
+ timeFormat: "24-hour",
+ titleFormat: "I AM TITLE FORMAT",
+ toolbarConfig: { desktop: { preset: "custom" } },
+ trashCleanupInterval: 365
+ };
+ await migrateItem(settings, 5.9, 6.0, "settings", db, "backup");
+
+ expect(db.settings.getDateFormat()).toBe(settings.dateFormat);
+ expect(db.settings.getDefaultNotebook()).toBe(
+ settings.defaultNotebook?.topic
+ );
+ expect(db.settings.getGroupOptions("favorites")).toMatchObject(
+ settings.groupOptions?.favorites || {}
+ );
+ expect(db.settings.getTitleFormat()).toBe(settings.titleFormat);
+ expect(db.settings.getTimeFormat()).toBe(settings.timeFormat);
+ expect(db.settings.getToolbarConfig("desktop")).toMatchObject(
+ settings.toolbarConfig?.desktop || {}
+ );
+ expect(db.settings.getTrashCleanupInterval()).toBe(
+ settings.trashCleanupInterval
+ );
+ }));
diff --git a/packages/core/src/api/migrations.ts b/packages/core/src/api/migrations.ts
index bebbf104f..da88de8d5 100644
--- a/packages/core/src/api/migrations.ts
+++ b/packages/core/src/api/migrations.ts
@@ -45,6 +45,14 @@ class Migrations {
await this.db.notes.init();
const collections: MigratableCollections = [
+ {
+ items: () => [this.db.legacySettings.raw],
+ type: "settings"
+ },
+ {
+ items: () => this.db.settings.raw,
+ type: "settingsv2"
+ },
{
items: () => this.db.attachments.all,
type: "attachments"
@@ -65,10 +73,6 @@ class Migrations {
iterate: true,
type: "content"
},
- {
- items: () => [this.db.settings.raw],
- type: "settings"
- },
{
items: () => this.db.shortcuts.raw,
type: "shortcuts"
diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.ts
index 713ede51e..0a1c4d159 100644
--- a/packages/core/src/api/sync/index.ts
+++ b/packages/core/src/api/sync/index.ts
@@ -19,6 +19,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import {
checkSyncStatus,
+ CURRENT_DATABASE_VERSION,
EV,
EVENTS,
sendSyncProgressEvent,
@@ -43,19 +44,11 @@ import {
Notebook,
TrashOrItem
} from "../../types";
-import { SyncableItemType, SyncTransferItem } from "./types";
-
-const ITEM_TYPE_TO_COLLECTION_TYPE = {
- note: "notes",
- notebook: "notebooks",
- content: "content",
- attachment: "attachments",
- relation: "relations",
- reminder: "reminders",
- shortcut: "shortcuts",
- tag: "tags",
- color: "colors"
-};
+import {
+ MERGE_COLLECTIONS_MAP,
+ SyncableItemType,
+ SyncTransferItem
+} from "./types";
export type SyncOptions = {
type: "full" | "fetch" | "send";
@@ -388,6 +381,9 @@ class Sync {
dbLastSynced: number,
notify = false
) {
+ const itemType = chunk.type;
+ if (itemType === "settings") return;
+
const decrypted = await this.db.storage().decryptMulti(key, chunk.items);
const deserialized = await Promise.all(
@@ -396,7 +392,6 @@ class Sync {
)
);
- const itemType = chunk.type;
let items: (
| MaybeDeletedItem<
ItemMap[SyncableItemType] | TrashOrItem<Note> | TrashOrItem<Notebook>
@@ -412,9 +407,6 @@ class Sync {
this.merger.mergeContent(item, localItems[item.id], dbLastSynced)
)
);
- } else if (itemType === "settings") {
- await this.merger.mergeItem(deserialized[0], itemType, dbLastSynced);
- return;
} else {
items = this.merger.isSyncCollection(itemType)
? deserialized.map((item) =>
@@ -427,7 +419,7 @@ class Sync {
);
}
- const collectionType = ITEM_TYPE_TO_COLLECTION_TYPE[itemType];
+ const collectionType = MERGE_COLLECTIONS_MAP[itemType];
await this.db[collectionType].collection.setItems(items as any);
if (
@@ -498,6 +490,7 @@ async function deserializeItem(
await migrateItem(
deserialized,
version,
+ CURRENT_DATABASE_VERSION,
deserialized.type,
database,
"sync"
diff --git a/packages/core/src/api/sync/types.ts b/packages/core/src/api/sync/types.ts
index 1f0ef4b60..02cdda0c1 100644
--- a/packages/core/src/api/sync/types.ts
+++ b/packages/core/src/api/sync/types.ts
@@ -48,6 +48,12 @@ export const SYNC_COLLECTIONS_MAP = {
settingitem: "settings"
} as const;
+export const MERGE_COLLECTIONS_MAP = {
+ ...SYNC_COLLECTIONS_MAP,
+ attachment: "attachments",
+ content: "content"
+} as const;
+
export type SyncTransferItem = {
items: SyncItem[];
type: SyncableItemType;
diff --git a/packages/core/src/collections/settings.ts b/packages/core/src/collections/settings.ts
index 08278a39e..6cb0042fb 100644
--- a/packages/core/src/collections/settings.ts
+++ b/packages/core/src/collections/settings.ts
@@ -20,7 +20,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { makeId } from "../utils/id";
import Database from "../api";
import {
- DefaultNotebook,
GroupOptions,
GroupingKey,
SettingItem,
@@ -138,7 +137,7 @@ export class Settings implements ICollection {
return this.get("trashCleanupInterval");
}
- setDefaultNotebook(item: DefaultNotebook | undefined) {
+ setDefaultNotebook(item: string | undefined) {
return this.set("defaultNotebook", item);
}
diff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.ts
index a540851da..5d55dd965 100644
--- a/packages/core/src/database/backup.ts
+++ b/packages/core/src/database/backup.ts
@@ -137,6 +137,7 @@ const itemTypeToCollectionKey = {
notehistory: "notehistory",
content: "content",
shortcut: "shortcuts",
+ settingitem: "settingsv2",
// to make ts happy
topic: "topics"
@@ -343,13 +344,21 @@ export default class Backup {
if ("sessionContentId" in item && item.type !== "session")
(item as any).type = "notehistory";
- await migrateItem(item, version, item.type, this.db, "backup");
+ await migrateItem(
+ item,
+ version,
+ CURRENT_DATABASE_VERSION,
+ item.type,
+ this.db,
+ "backup"
+ );
// since items in trash can have their own set of migrations,
// we have to run the migration again to account for that.
if (item.type === "trash" && item.itemType)
await migrateItem(
item as unknown as Note | Notebook,
version,
+ CURRENT_DATABASE_VERSION,
item.itemType,
this.db,
"backup"
diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.ts
index 75a9108d9..cabeba08d 100644
--- a/packages/core/src/database/migrator.ts
+++ b/packages/core/src/database/migrator.ts
@@ -18,7 +18,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import Database from "../api";
-import { sendMigrationProgressEvent } from "../common";
+import {
+ CURRENT_DATABASE_VERSION,
+ sendMigrationProgressEvent
+} from "../common";
import { migrateCollection, migrateItem } from "../migrations";
import {
CollectionType,
@@ -99,17 +102,31 @@ class Migrator {
}
const itemId = item.id;
- const migrated = await migrateItem(
+ let migrated = await migrateItem(
item,
version,
+ CURRENT_DATABASE_VERSION,
item.type || type,
db,
"local"
);
+ // trash item is also a notebook or a note so we have to migrate it separately.
+ if (isTrashItem(item)) {
+ migrated = await migrateItem(
+ item as any,
+ version,
+ CURRENT_DATABASE_VERSION,
+ item.itemType,
+ db,
+ "local"
+ );
+ }
+
if (migrated) {
- if (item.type === "settings") {
- await db.settings.merge(item, Infinity);
+ if (item.type !== "settings") {
+ // we are removing the old settings.
+ await db.storage().remove("settings");
} else toAdd.push(item);
// if id changed after migration, we need to delete the old one.
diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.ts
index b5b38151e..c14e8aaac 100644
--- a/packages/core/src/migrations.ts
+++ b/packages/core/src/migrations.ts
@@ -76,7 +76,7 @@ const migrations: Migration[] = [
notebook: replaceDateEditedWithDateModified(false),
tag: replaceDateEditedWithDateModified(true),
attachment: replaceDateEditedWithDateModified(true),
- trash: replaceDateEditedWithDateModified(),
+ trash: replaceDateEditedWithDateModified(false),
tiny: (item) => {
replaceDateEditedWithDateModified(false)(item);
@@ -147,12 +147,10 @@ const migrations: Migration[] = [
version: 5.7,
items: {
tiny: (item) => {
- if (!item.data || isCipher(item.data)) return false;
item.type = "tiptap";
return changeSessionContentType(item);
},
content: (item) => {
- if (!item.data || isCipher(item.data)) return false;
const oldType = item.type;
item.type = "tiptap";
return oldType !== item.type;
@@ -190,9 +188,18 @@ const migrations: Migration[] = [
version: 5.9,
items: {
tag: async (item, db) => {
+ const oldTagId = makeId(item.title);
const alias = db.legacySettings.getAlias(item.id);
- item.title = alias || item.title;
- item.id = getId(item.dateCreated);
+ if (
+ !alias &&
+ (db.tags.all.find(
+ (t) => item.title === t.title && t.id !== oldTagId
+ ) ||
+ db.colors.all.find(
+ (t) => item.title === t.title && t.id !== oldTagId
+ ))
+ )
+ return false;
const colorCode = ColorToHexCode[item.title];
if (colorCode) {
@@ -200,6 +207,9 @@ const migrations: Migration[] = [
(item as unknown as Color).colorCode = colorCode;
}
+ item.title = alias || item.title;
+ item.id = getId(item.dateCreated);
+
delete item.localOnly;
delete item.noteIds;
delete item.alias;
@@ -229,9 +239,9 @@ const migrations: Migration[] = [
if (item.color) {
const oldColorId = makeId(item.color);
- const oldColor = db.tags.tag(oldColorId);
+ const oldColor = db.colors.color(oldColorId);
const alias = db.legacySettings.getAlias(oldColorId);
- const newColor = db.tags.all.find(
+ const newColor = db.colors.all.find(
(t) => [alias, item.color].includes(t.title) && t.id !== oldColorId
);
const newColorId =
@@ -257,6 +267,7 @@ const migrations: Migration[] = [
}
}
+ delete item.notebooks;
delete item.tags;
delete item.color;
return true;
@@ -296,7 +307,11 @@ const migrations: Migration[] = [
if (item.trashCleanupInterval)
await db.settings.setTrashCleanupInterval(item.trashCleanupInterval);
if (item.defaultNotebook)
- await db.settings.setDefaultNotebook(item.defaultNotebook);
+ await db.settings.setDefaultNotebook(
+ item.defaultNotebook
+ ? item.defaultNotebook.topic || item.defaultNotebook.id
+ : undefined
+ );
if (item.titleFormat)
await db.settings.setTitleFormat(item.titleFormat);
@@ -332,15 +347,18 @@ const migrations: Migration[] = [
export async function migrateItem<TItemType extends MigrationItemType>(
item: MigrationItemMap[TItemType],
- version: number,
+ itemVersion: number,
+ databaseVersion: number,
type: TItemType,
database: Database,
migrationType: MigrationType
) {
- let migrationStartIndex = migrations.findIndex((m) => m.version === version);
+ let migrationStartIndex = migrations.findIndex(
+ (m) => m.version === itemVersion
+ );
if (migrationStartIndex <= -1) {
throw new Error(
- version > CURRENT_DATABASE_VERSION
+ itemVersion > databaseVersion
? `Please update the app to the latest version.`
: `You seem to be on a very outdated version. Please update the app to the latest version.`
);
@@ -349,7 +367,7 @@ export async function migrateItem<TItemType extends MigrationItemType>(
let count = 0;
for (; migrationStartIndex < migrations.length; ++migrationStartIndex) {
const migration = migrations[migrationStartIndex];
- if (migration.version === CURRENT_DATABASE_VERSION) break;
+ if (migration.version === databaseVersion) break;
if (
migration.items.all &&
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
index 3acbe55d7..8618a2109 100644
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -86,7 +86,7 @@ export type GroupableItem = ValueOf<
| "session"
| "sessioncontent"
| "settings"
- | "settingsv2"
+ | "settingitem"
>
>;
@@ -324,7 +324,7 @@ export type DefaultNotebook = { id: string; topic?: string };
*/
export interface LegacySettingsItem extends BaseItem<"settings"> {
groupOptions?: Partial<Record<GroupingKey, GroupOptions>>;
- toolbarConfig?: Record<ToolbarConfigPlatforms, ToolbarConfig>;
+ toolbarConfig?: Partial<Record<ToolbarConfigPlatforms, ToolbarConfig>>;
trashCleanupInterval?: TrashCleanupInterval;
titleFormat?: string;
timeFormat?: TimeFormat;
@@ -350,7 +350,7 @@ export type SettingItemMap = {
titleFormat: string;
timeFormat: TimeFormat;
dateFormat: string;
- defaultNotebook: DefaultNotebook | undefined;
+ defaultNotebook: string | undefined;
} & Record<`groupOptions:${GroupingKey}`, GroupOptions> &
Record<`toolbarConfig:${ToolbarConfigPlatforms}`, ToolbarConfig | undefined>;
@@ -389,9 +389,7 @@ export function isDeleted<T extends BaseItem<ItemType>>(
return "deleted" in item;
}
-export function isTrashItem(
- item: MaybeDeletedItem<TrashOrItem<BaseItem<"note" | "notebook">>>
-): item is TrashItem {
+export function isTrashItem(item: MaybeDeletedItem<Item>): item is TrashItem {
return !isDeleted(item) && item.type === "trash";
}