A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: fix database migration
Abdullah Atta44846cc7b54c
diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.tsindex 85755b5ab..132ce6140 100644--- a/packages/core/__tests__/notes.test.ts+++ b/packages/core/__tests__/notes.test.ts@@ -23,7 +23,6 @@ import { groupArray } from "../src/utils/grouping";import {databaseTest,noteTest,- groupedTest,TEST_NOTE,TEST_NOTEBOOK,IMG_CONTENT,@@ -394,17 +393,20 @@ test("get grouped notes by abc", () =>title: `Conflicted`,conflicted: true});- const { grouping, ids } = await db.notes.all.grouped({+ const grouping = await db.notes.all.grouped({groupBy: "abc",sortDirection: "asc",sortBy: "title"});- expect((await grouping.item(ids[0]))?.group?.title).toBe("Conflicted");- expect((await grouping.item(ids[1]))?.group?.title).toBe("Pinned");+ expect((await grouping.item(grouping.ids[0]))?.group?.title).toBe(+ "Conflicted"+ );+ expect((await grouping.item(grouping.ids[1]))?.group?.title).toBe("Pinned");for (let i = 0; i < alphabet.length; ++i) {expect(- (await grouping.item(ids[i * alphabet.length + 2]))?.group?.title+ (await grouping.item(grouping.ids[i * alphabet.length + 2]))?.group+ ?.title).toBe(alphabet[i]);}}));@@ -421,16 +423,16 @@ test("get grouped notes by month", () =>}}- const { grouping, ids } = await db.notes.all.grouped({+ const grouping = await db.notes.all.grouped({groupBy: "month",sortDirection: "desc",sortBy: "dateCreated"});for (let month = 11; month >= 0; --month) {- expect((await grouping.item(ids[(11 - month) * 5]))?.group?.title).toBe(- MONTHS_FULL[month]- );+ expect(+ (await grouping.item(grouping.ids[(11 - month) * 5]))?.group?.title+ ).toBe(MONTHS_FULL[month]);}}));@@ -446,16 +448,16 @@ test("get grouped notes by year", () =>}}- const { grouping, ids } = await db.notes.all.grouped({+ const grouping = await db.notes.all.grouped({groupBy: "year",sortDirection: "desc",sortBy: "dateCreated"});for (let year = 2020; year <= 2025; ++year) {- expect((await grouping.item(ids[(2025 - year) * 5]))?.group?.title).toBe(- year.toString()- );+ expect(+ (await grouping.item(grouping.ids[(2025 - year) * 5]))?.group?.title+ ).toBe(year.toString());}}));@@ -477,7 +479,7 @@ test("get grouped notes by week", () =>}}- const { grouping, ids } = await db.notes.all.grouped({+ const grouping = await db.notes.all.grouped({groupBy: "week",sortDirection: "desc",sortBy: "dateCreated"@@ -491,7 +493,7 @@ test("get grouped notes by week", () =>"20 - 26 Feb, 2023"];for (let i = 1; i <= 5; ++i) {- expect((await grouping.item(ids[i * 4]))?.group?.title).toBe(+ expect((await grouping.item(grouping.ids[i * 4]))?.group?.title).toBe(weeks[i - 1]);}@@ -516,7 +518,7 @@ test("get grouped notes default", () =>}}- const { grouping, ids } = await db.notes.all.grouped({+ const grouping = await db.notes.all.grouped({groupBy: "default",sortDirection: "desc",sortBy: "dateCreated"@@ -524,7 +526,9 @@ test("get grouped notes default", () =>let i = 0;for (const key in ranges) {- expect((await grouping.item(ids[i * 7]))?.group?.title).toBe(key);+ expect((await grouping.item(grouping.ids[i * 7]))?.group?.title).toBe(+ key+ );++i;}}));diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.tsindex d3f261953..d9029b815 100644--- a/packages/core/src/api/index.ts+++ b/packages/core/src/api/index.ts@@ -23,8 +23,8 @@ import { FileStorage, FileStorageAccessor } from "../database/fs";import { Notebooks } from "../collections/notebooks";import Trash from "../collections/trash";import Sync, { SyncOptions } from "./sync";-import { Tags } from "../collections/tags";-import { Colors, } from "../collections/colors";+import { Tags } from "../collections/tags";+import { Colors } from "../collections/colors";import Vault from "./vault";import Lookup from "./lookup";import { Content } from "../collections/content";diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.tsindex 723ed53c6..c5d153875 100644--- a/packages/core/src/api/lookup.ts+++ b/packages/core/src/api/lookup.ts@@ -28,8 +28,8 @@ import {Tag,TrashItem} from "../types";-import { isFalse } from "../database";-import { sql } from "kysely";+import { DatabaseSchemaWithFTS, isFalse } from "../database";+import { Kysely, sql } from "kysely";export default class Lookup {constructor(private readonly db: Database) {}@@ -38,8 +38,8 @@ export default class Lookup {query: string,ids?: string[]): Promise<Note & { rank: number }[]> {- return (await this.db- .sql()+ const db = this.db.sql() as Kysely<DatabaseSchemaWithFTS>;+ return (await db.with("matching", (eb) =>eb.selectFrom("content_fts")diff --git a/packages/core/src/api/migrations.ts b/packages/core/src/api/migrations.tsindex da88de8d5..42da09168 100644--- a/packages/core/src/api/migrations.ts+++ b/packages/core/src/api/migrations.ts@@ -21,6 +21,61 @@ import Database from ".";import { CURRENT_DATABASE_VERSION } from "../common";import Migrator, { MigratableCollections } from "../database/migrator";+const collections: MigratableCollections = [+ {+ name: "settings",+ table: "settings"+ },+ {+ name: "settingsv2",+ table: "settings"+ },+ {+ name: "attachments",+ table: "attachments"+ },+ {+ name: "notebooks",+ table: "notebooks"+ },+ {+ name: "tags",+ table: "tags"+ },+ {+ name: "colors",+ table: "colors"+ },+ {+ name: "content",+ table: "content"+ },+ {+ name: "shortcuts",+ table: "shortcuts"+ },+ {+ name: "reminders",+ table: "reminders"+ },+ {+ name: "relations",+ table: "relations"+ },+ {+ name: "notehistory",+ table: "relations"+ },+ {+ name: "sessioncontent",+ table: "sessioncontent"+ },+ {+ name: "notes",+ table: "notes"+ }+];+class Migrations {private readonly migrator = new Migrator();private migrating = false;@@ -44,61 +99,6 @@ class Migrations {await this.db.notes.init();- const collections: MigratableCollections = [- {- items: () => [this.db.legacySettings.raw],- type: "settings"- },- {- items: () => this.db.settings.raw,- type: "settingsv2"- },- {- items: () => this.db.attachments.all,- type: "attachments"- },- {- items: () => this.db.notebooks.raw,- type: "notebooks"- },- {- items: () => this.db.tags.raw,- type: "tags"- },- {- items: () => this.db.colors.raw,- type: "colors"- },- {- iterate: true,- type: "content"- },- {- items: () => this.db.shortcuts.raw,- type: "shortcuts"- },- {- items: () => this.db.reminders.raw,- type: "reminders"- },- {- items: () => this.db.relations.raw,- type: "relations"- },- {- iterate: true,- type: "notehistory"- },- {- iterate: true,- type: "sessioncontent"- },- {- items: () => this.db.notes.raw,- type: "notes"- }- ];-await this.migrator.migrate(this.db, collections, this.version);await this.db.storage().write("v", CURRENT_DATABASE_VERSION);this.version = CURRENT_DATABASE_VERSION;diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.tsindex 978daeba5..ece0d4bb9 100644--- a/packages/core/src/api/sync/index.ts+++ b/packages/core/src/api/sync/index.ts@@ -59,7 +59,7 @@ export default class SyncManager {} catch (e) {const isHubException = (e as Error).message.includes("HubException:");if (isHubException) {- var actualError = /HubException: (.*)/gm.exec((e as Error).message);+ const actualError = /HubException: (.*)/gm.exec((e as Error).message);const errorText =actualError && actualError.length > 1? actualError[1]diff --git a/packages/core/src/collections/note-history.ts b/packages/core/src/collections/note-history.tsindex 87769871d..404b53f35 100644--- a/packages/core/src/collections/note-history.ts+++ b/packages/core/src/collections/note-history.ts@@ -141,7 +141,7 @@ export class NoteHistory implements ICollection {if (!session || isDeleted(session)) return;const content = await this.sessionContent.get(session.sessionContentId);- const note = this.db.notes.note(session.noteId);+ const note = await this.db.notes.note(session.noteId);if (!note || !content) return;if (session.locked && isCipher(content.data)) {diff --git a/packages/core/src/content-types/tiptap.ts b/packages/core/src/content-types/tiptap.tsindex b584208b8..620fa8702 100644--- a/packages/core/src/content-types/tiptap.ts+++ b/packages/core/src/content-types/tiptap.ts@@ -181,9 +181,9 @@ export class Tiptap {const images: Record<string, string | false> = {};for (const image of sources) {try {- const { data, mime } = dataurl.toObject(image.src);+ const { data, mimeType } = dataurl.toObject(image.src);if (!data) continue;- const hash = await store(data, mime, image.filename);+ const hash = await store(data, mimeType, image.filename);if (!hash) continue;images[image.id] = hash;diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.tsindex cabeba08d..8a0ab4a41 100644--- a/packages/core/src/database/migrator.ts+++ b/packages/core/src/database/migrator.ts@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/+import { DatabaseSchema } from ".";import Database from "../api";import {CURRENT_DATABASE_VERSION,@@ -32,12 +33,14 @@ import {isTrashItem} from "../types";import { IndexedCollection } from "./indexed-collection";+import { SQLCollection } from "./sql-collection";export type RawItem = MaybeDeletedItem<Item>;type MigratableCollection = {- iterate?: boolean;- items?: () => (RawItem | undefined)[];- type: CollectionType;+ // iterate?: boolean;+ // items?: () => (RawItem | undefined)[];+ name: CollectionType;+ table: keyof DatabaseSchema;};export type MigratableCollections = MigratableCollection[];@@ -48,35 +51,31 @@ class Migrator {version: number) {for (const collection of collections) {- sendMigrationProgressEvent(db.eventManager, collection.type, 0, 0);+ sendMigrationProgressEvent(db.eventManager, collection.name, 0, 0);const indexedCollection = new IndexedCollection(db.storage,- collection.type,+ collection.name,+ db.eventManager+ );+ const table = new SQLCollection(+ db.sql,+ collection.table,db.eventManager);await migrateCollection(indexedCollection, version);- if (collection.items) {+ await indexedCollection.init();+ await table.init();+ for await (const entries of indexedCollection.iterate(100)) {await this.migrateItems(db,- collection.type,- indexedCollection,- collection.items(),+ table,+ collection.name,+ entries.map((i) => i[1]),version);- } else if (collection.iterate) {- await indexedCollection.init();- for await (const entries of indexedCollection.iterate(100)) {- await this.migrateItems(- db,- collection.type,- indexedCollection,- entries.map((i) => i[1]),- version- );- }}}await db.initCollections();@@ -85,8 +84,8 @@ class Migrator {async migrateItems(db: Database,+ table: SQLCollection<keyof DatabaseSchema>,type: keyof Collections,- collection: IndexedCollection,items: (RawItem | undefined)[],version: number) {@@ -131,13 +130,14 @@ class Migrator {// if id changed after migration, we need to delete the old one.if (item.id !== itemId) {- await collection.deleteItem(itemId);+ // await collection.deleteItem(itemId);}}}if (toAdd.length > 0) {- await collection.setItems(toAdd);+ await table.put(toAdd as any);+ // await collection.setItems(toAdd);sendMigrationProgressEvent(db.eventManager,type,diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.tsindex b3be4baab..347a08c4e 100644--- a/packages/core/src/database/sql-collection.ts+++ b/packages/core/src/database/sql-collection.ts@@ -308,25 +308,22 @@ export class FilteredSelector<T extends Item> {async grouped(options: GroupOptions) {const ids = await this.ids(options);- return {+ return new VirtualizedGrouping<T>(ids,- grouping: new VirtualizedGrouping<T>(- ids,- this.batchSize,- async (ids) => {- const results = await this.filter- .where("id", "in", ids)- .selectAll()- .execute();- const items: Record<string, T> = {};- for (const item of results) {- items[item.id] = item as T;- }- return items;- },- (ids, items) => groupArray(ids, items, options)- )- };+ this.batchSize,+ async (ids) => {+ const results = await this.filter+ .where("id", "in", ids)+ .selectAll()+ .execute();+ const items: Record<string, T> = {};+ for (const item of results) {+ items[item.id] = item as T;+ }+ return items;+ },+ (ids, items) => groupArray(ids, items, options)+ );}private buildSortExpression(options: GroupOptions) {diff --git a/packages/core/src/models/note.ts b/packages/core/src/models/note.tsindex f1d8e7a27..68914a28f 100644--- a/packages/core/src/models/note.ts+++ b/packages/core/src/models/note.ts@@ -26,7 +26,7 @@ export function createNoteModel(note: Note, db: Database) {data: note,async content() {if (!note.contentId) return null;- const content = await db.content.raw(note.contentId);+ const content = await db.content.get(note.contentId);return content && !isDeleted(content) ? content.data : null;},synced() {diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.tsindex c748c253d..afdbdbc38 100644--- a/packages/core/src/utils/virtualized-grouping.ts+++ b/packages/core/src/utils/virtualized-grouping.ts@@ -26,7 +26,7 @@ export class VirtualizedGrouping<T> {private groups: Map<number, VirtualizedGroupHeader[]> = new Map();constructor(- private ids: string[],+ public ids: string[],private readonly batchSize: number,private readonly fetchItems: (ids: string[]) => Promise<Record<string, T>>,private readonly groupItems: (