Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: fix database migration

Abdullah Atta44846cc7b54c
diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.ts
index 85755b5ab..132ce6140 100644
--- a/packages/core/__tests__/notes.test.ts
+++ b/packages/core/__tests__/notes.test.ts
@@ -23,7 +23,6 @@ import { groupArray } from "../src/utils/grouping";
import {
databaseTest,
noteTest,
- groupedTest,
TEST_NOTE,
TEST_NOTEBOOK,
IMG_CONTENT,
@@ -394,17 +393,20 @@ test("get grouped notes by abc", () =>
title: `Conflicted`,
conflicted: true
});
- const { grouping, ids } = await db.notes.all.grouped({
+ const grouping = await db.notes.all.grouped({
groupBy: "abc",
sortDirection: "asc",
sortBy: "title"
});
- expect((await grouping.item(ids[0]))?.group?.title).toBe("Conflicted");
- expect((await grouping.item(ids[1]))?.group?.title).toBe("Pinned");
+ expect((await grouping.item(grouping.ids[0]))?.group?.title).toBe(
+ "Conflicted"
+ );
+ expect((await grouping.item(grouping.ids[1]))?.group?.title).toBe("Pinned");
for (let i = 0; i < alphabet.length; ++i) {
expect(
- (await grouping.item(ids[i * alphabet.length + 2]))?.group?.title
+ (await grouping.item(grouping.ids[i * alphabet.length + 2]))?.group
+ ?.title
).toBe(alphabet[i]);
}
}));
@@ -421,16 +423,16 @@ test("get grouped notes by month", () =>
}
}
- const { grouping, ids } = await db.notes.all.grouped({
+ const grouping = await db.notes.all.grouped({
groupBy: "month",
sortDirection: "desc",
sortBy: "dateCreated"
});
for (let month = 11; month >= 0; --month) {
- expect((await grouping.item(ids[(11 - month) * 5]))?.group?.title).toBe(
- MONTHS_FULL[month]
- );
+ expect(
+ (await grouping.item(grouping.ids[(11 - month) * 5]))?.group?.title
+ ).toBe(MONTHS_FULL[month]);
}
}));
@@ -446,16 +448,16 @@ test("get grouped notes by year", () =>
}
}
- const { grouping, ids } = await db.notes.all.grouped({
+ const grouping = await db.notes.all.grouped({
groupBy: "year",
sortDirection: "desc",
sortBy: "dateCreated"
});
for (let year = 2020; year <= 2025; ++year) {
- expect((await grouping.item(ids[(2025 - year) * 5]))?.group?.title).toBe(
- year.toString()
- );
+ expect(
+ (await grouping.item(grouping.ids[(2025 - year) * 5]))?.group?.title
+ ).toBe(year.toString());
}
}));
@@ -477,7 +479,7 @@ test("get grouped notes by week", () =>
}
}
- const { grouping, ids } = await db.notes.all.grouped({
+ const grouping = await db.notes.all.grouped({
groupBy: "week",
sortDirection: "desc",
sortBy: "dateCreated"
@@ -491,7 +493,7 @@ test("get grouped notes by week", () =>
"20 - 26 Feb, 2023"
];
for (let i = 1; i <= 5; ++i) {
- expect((await grouping.item(ids[i * 4]))?.group?.title).toBe(
+ expect((await grouping.item(grouping.ids[i * 4]))?.group?.title).toBe(
weeks[i - 1]
);
}
@@ -516,7 +518,7 @@ test("get grouped notes default", () =>
}
}
- const { grouping, ids } = await db.notes.all.grouped({
+ const grouping = await db.notes.all.grouped({
groupBy: "default",
sortDirection: "desc",
sortBy: "dateCreated"
@@ -524,7 +526,9 @@ test("get grouped notes default", () =>
let i = 0;
for (const key in ranges) {
- expect((await grouping.item(ids[i * 7]))?.group?.title).toBe(key);
+ expect((await grouping.item(grouping.ids[i * 7]))?.group?.title).toBe(
+ key
+ );
++i;
}
}));
diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.ts
index d3f261953..d9029b815 100644
--- a/packages/core/src/api/index.ts
+++ b/packages/core/src/api/index.ts
@@ -23,8 +23,8 @@ import { FileStorage, FileStorageAccessor } from "../database/fs";
import { Notebooks } from "../collections/notebooks";
import Trash from "../collections/trash";
import Sync, { SyncOptions } from "./sync";
-import { Tags } from "../collections/tags";
-import { Colors, } from "../collections/colors";
+import { Tags } from "../collections/tags";
+import { Colors } from "../collections/colors";
import Vault from "./vault";
import Lookup from "./lookup";
import { Content } from "../collections/content";
diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.ts
index 723ed53c6..c5d153875 100644
--- a/packages/core/src/api/lookup.ts
+++ b/packages/core/src/api/lookup.ts
@@ -28,8 +28,8 @@ import {
Tag,
TrashItem
} from "../types";
-import { isFalse } from "../database";
-import { sql } from "kysely";
+import { DatabaseSchemaWithFTS, isFalse } from "../database";
+import { Kysely, sql } from "kysely";
export default class Lookup {
constructor(private readonly db: Database) {}
@@ -38,8 +38,8 @@ export default class Lookup {
query: string,
ids?: string[]
): Promise<Note & { rank: number }[]> {
- return (await this.db
- .sql()
+ const db = this.db.sql() as Kysely<DatabaseSchemaWithFTS>;
+ return (await db
.with("matching", (eb) =>
eb
.selectFrom("content_fts")
diff --git a/packages/core/src/api/migrations.ts b/packages/core/src/api/migrations.ts
index da88de8d5..42da09168 100644
--- a/packages/core/src/api/migrations.ts
+++ b/packages/core/src/api/migrations.ts
@@ -21,6 +21,61 @@ import Database from ".";
import { CURRENT_DATABASE_VERSION } from "../common";
import Migrator, { MigratableCollections } from "../database/migrator";
+const collections: MigratableCollections = [
+ {
+ name: "settings",
+ table: "settings"
+ },
+ {
+ name: "settingsv2",
+ table: "settings"
+ },
+ {
+ name: "attachments",
+ table: "attachments"
+ },
+ {
+ name: "notebooks",
+ table: "notebooks"
+ },
+ {
+ name: "tags",
+ table: "tags"
+ },
+ {
+ name: "colors",
+ table: "colors"
+ },
+ {
+ name: "content",
+ table: "content"
+ },
+ {
+ name: "shortcuts",
+ table: "shortcuts"
+ },
+ {
+ name: "reminders",
+ table: "reminders"
+ },
+ {
+ name: "relations",
+ table: "relations"
+ },
+ {
+ name: "notehistory",
+ table: "relations"
+ },
+ {
+ name: "sessioncontent",
+ table: "sessioncontent"
+ },
+ {
+ name: "notes",
+ table: "notes"
+ }
+];
+
class Migrations {
private readonly migrator = new Migrator();
private migrating = false;
@@ -44,61 +99,6 @@ class Migrations {
await this.db.notes.init();
- const collections: MigratableCollections = [
- {
- items: () => [this.db.legacySettings.raw],
- type: "settings"
- },
- {
- items: () => this.db.settings.raw,
- type: "settingsv2"
- },
- {
- items: () => this.db.attachments.all,
- type: "attachments"
- },
- {
- items: () => this.db.notebooks.raw,
- type: "notebooks"
- },
- {
- items: () => this.db.tags.raw,
- type: "tags"
- },
- {
- items: () => this.db.colors.raw,
- type: "colors"
- },
- {
- iterate: true,
- type: "content"
- },
- {
- items: () => this.db.shortcuts.raw,
- type: "shortcuts"
- },
- {
- items: () => this.db.reminders.raw,
- type: "reminders"
- },
- {
- items: () => this.db.relations.raw,
- type: "relations"
- },
- {
- iterate: true,
- type: "notehistory"
- },
- {
- iterate: true,
- type: "sessioncontent"
- },
- {
- items: () => this.db.notes.raw,
- type: "notes"
- }
- ];
-
await this.migrator.migrate(this.db, collections, this.version);
await this.db.storage().write("v", CURRENT_DATABASE_VERSION);
this.version = CURRENT_DATABASE_VERSION;
diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.ts
index 978daeba5..ece0d4bb9 100644
--- a/packages/core/src/api/sync/index.ts
+++ b/packages/core/src/api/sync/index.ts
@@ -59,7 +59,7 @@ export default class SyncManager {
} catch (e) {
const isHubException = (e as Error).message.includes("HubException:");
if (isHubException) {
- var actualError = /HubException: (.*)/gm.exec((e as Error).message);
+ const actualError = /HubException: (.*)/gm.exec((e as Error).message);
const errorText =
actualError && actualError.length > 1
? actualError[1]
diff --git a/packages/core/src/collections/note-history.ts b/packages/core/src/collections/note-history.ts
index 87769871d..404b53f35 100644
--- a/packages/core/src/collections/note-history.ts
+++ b/packages/core/src/collections/note-history.ts
@@ -141,7 +141,7 @@ export class NoteHistory implements ICollection {
if (!session || isDeleted(session)) return;
const content = await this.sessionContent.get(session.sessionContentId);
- const note = this.db.notes.note(session.noteId);
+ const note = await this.db.notes.note(session.noteId);
if (!note || !content) return;
if (session.locked && isCipher(content.data)) {
diff --git a/packages/core/src/content-types/tiptap.ts b/packages/core/src/content-types/tiptap.ts
index b584208b8..620fa8702 100644
--- a/packages/core/src/content-types/tiptap.ts
+++ b/packages/core/src/content-types/tiptap.ts
@@ -181,9 +181,9 @@ export class Tiptap {
const images: Record<string, string | false> = {};
for (const image of sources) {
try {
- const { data, mime } = dataurl.toObject(image.src);
+ const { data, mimeType } = dataurl.toObject(image.src);
if (!data) continue;
- const hash = await store(data, mime, image.filename);
+ const hash = await store(data, mimeType, image.filename);
if (!hash) continue;
images[image.id] = hash;
diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.ts
index cabeba08d..8a0ab4a41 100644
--- a/packages/core/src/database/migrator.ts
+++ b/packages/core/src/database/migrator.ts
@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
+import { DatabaseSchema } from ".";
import Database from "../api";
import {
CURRENT_DATABASE_VERSION,
@@ -32,12 +33,14 @@ import {
isTrashItem
} from "../types";
import { IndexedCollection } from "./indexed-collection";
+import { SQLCollection } from "./sql-collection";
export type RawItem = MaybeDeletedItem<Item>;
type MigratableCollection = {
- iterate?: boolean;
- items?: () => (RawItem | undefined)[];
- type: CollectionType;
+ // iterate?: boolean;
+ // items?: () => (RawItem | undefined)[];
+ name: CollectionType;
+ table: keyof DatabaseSchema;
};
export type MigratableCollections = MigratableCollection[];
@@ -48,35 +51,31 @@ class Migrator {
version: number
) {
for (const collection of collections) {
- sendMigrationProgressEvent(db.eventManager, collection.type, 0, 0);
+ sendMigrationProgressEvent(db.eventManager, collection.name, 0, 0);
const indexedCollection = new IndexedCollection(
db.storage,
- collection.type,
+ collection.name,
+ db.eventManager
+ );
+ const table = new SQLCollection(
+ db.sql,
+ collection.table,
db.eventManager
);
await migrateCollection(indexedCollection, version);
- if (collection.items) {
+ await indexedCollection.init();
+ await table.init();
+ for await (const entries of indexedCollection.iterate(100)) {
await this.migrateItems(
db,
- collection.type,
- indexedCollection,
- collection.items(),
+ table,
+ collection.name,
+ entries.map((i) => i[1]),
version
);
- } else if (collection.iterate) {
- await indexedCollection.init();
- for await (const entries of indexedCollection.iterate(100)) {
- await this.migrateItems(
- db,
- collection.type,
- indexedCollection,
- entries.map((i) => i[1]),
- version
- );
- }
}
}
await db.initCollections();
@@ -85,8 +84,8 @@ class Migrator {
async migrateItems(
db: Database,
+ table: SQLCollection<keyof DatabaseSchema>,
type: keyof Collections,
- collection: IndexedCollection,
items: (RawItem | undefined)[],
version: number
) {
@@ -131,13 +130,14 @@ class Migrator {
// if id changed after migration, we need to delete the old one.
if (item.id !== itemId) {
- await collection.deleteItem(itemId);
+ // await collection.deleteItem(itemId);
}
}
}
if (toAdd.length > 0) {
- await collection.setItems(toAdd);
+ await table.put(toAdd as any);
+ // await collection.setItems(toAdd);
sendMigrationProgressEvent(
db.eventManager,
type,
diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.ts
index b3be4baab..347a08c4e 100644
--- a/packages/core/src/database/sql-collection.ts
+++ b/packages/core/src/database/sql-collection.ts
@@ -308,25 +308,22 @@ export class FilteredSelector<T extends Item> {
async grouped(options: GroupOptions) {
const ids = await this.ids(options);
- return {
+ return new VirtualizedGrouping<T>(
ids,
- grouping: new VirtualizedGrouping<T>(
- ids,
- this.batchSize,
- async (ids) => {
- const results = await this.filter
- .where("id", "in", ids)
- .selectAll()
- .execute();
- const items: Record<string, T> = {};
- for (const item of results) {
- items[item.id] = item as T;
- }
- return items;
- },
- (ids, items) => groupArray(ids, items, options)
- )
- };
+ this.batchSize,
+ async (ids) => {
+ const results = await this.filter
+ .where("id", "in", ids)
+ .selectAll()
+ .execute();
+ const items: Record<string, T> = {};
+ for (const item of results) {
+ items[item.id] = item as T;
+ }
+ return items;
+ },
+ (ids, items) => groupArray(ids, items, options)
+ );
}
private buildSortExpression(options: GroupOptions) {
diff --git a/packages/core/src/models/note.ts b/packages/core/src/models/note.ts
index f1d8e7a27..68914a28f 100644
--- a/packages/core/src/models/note.ts
+++ b/packages/core/src/models/note.ts
@@ -26,7 +26,7 @@ export function createNoteModel(note: Note, db: Database) {
data: note,
async content() {
if (!note.contentId) return null;
- const content = await db.content.raw(note.contentId);
+ const content = await db.content.get(note.contentId);
return content && !isDeleted(content) ? content.data : null;
},
synced() {
diff --git a/packages/core/src/utils/virtualized-grouping.ts b/packages/core/src/utils/virtualized-grouping.ts
index c748c253d..afdbdbc38 100644
--- a/packages/core/src/utils/virtualized-grouping.ts
+++ b/packages/core/src/utils/virtualized-grouping.ts
@@ -26,7 +26,7 @@ export class VirtualizedGrouping<T> {
private groups: Map<number, VirtualizedGroupHeader[]> = new Map();
constructor(
- private ids: string[],
+ public ids: string[],
private readonly batchSize: number,
private readonly fetchItems: (ids: string[]) => Promise<Record<string, T>>,
private readonly groupItems: (