A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: improve backup importing
Abdullah Atta27cf0a3c6b24
diff --git a/packages/core/__mocks__/node-storage.mock.ts b/packages/core/__mocks__/node-storage.mock.tsindex 455b2febf..a11a08201 100644--- a/packages/core/__mocks__/node-storage.mock.ts+++ b/packages/core/__mocks__/node-storage.mock.ts@@ -85,14 +85,11 @@ export class NodeStorageInterface implements IStorage {return this.crypto.decryptMulti(key, items, "text");}- async deriveCryptoKey(- name: string,- credentials: SerializedKey- ): Promise<void> {+ async deriveCryptoKey(credentials: SerializedKey): Promise<void> {const { password, salt } = credentials;if (!password || !salt) return;const keyData = await this.crypto.exportKey(password, salt);- await this.write(`${name}@_k`, keyData.key);+ await this.write(`userEncryptionKey`, keyData.key);}async hash(password: string, email: string): Promise<string> {@@ -100,8 +97,8 @@ export class NodeStorageInterface implements IStorage {return await this.crypto.hash(password, `${APP_SALT}${email}`);}- async getCryptoKey(name: string): Promise<string | undefined> {- const key = await this.read<string>(`${name}@_k`);+ async getCryptoKey(): Promise<string | undefined> {+ const key = await this.read<string>(`userEncryptionKey`);if (!key) return;return key;}diff --git a/packages/core/__tests__/migrations.test.ts b/packages/core/__tests__/migrations.test.tsindex 6748a7bfa..0f71781d6 100644--- a/packages/core/__tests__/migrations.test.ts+++ b/packages/core/__tests__/migrations.test.ts@@ -543,11 +543,11 @@ test("[5.9] move attachments.noteIds to relations", () =>await migrateItem(attachment, 5.9, 6.0, "attachment", db, "backup");const linkedNotes = await db.relations- .from({ type: "attachment", id: "ATTACHMENT_ID" }, "note")+ .to({ type: "attachment", id: "ATTACHMENT_ID" }, "note").get();expect(attachment.noteIds).toBeUndefined();expect(linkedNotes).toHaveLength(1);- expect(linkedNotes[0].toId).toBe("HELLO_NOTE_ID");+ expect(linkedNotes[0].fromId).toBe("HELLO_NOTE_ID");}));test.todo("[5.9] flatten attachment object", () =>@@ -692,7 +692,7 @@ test("[5.9] migrate settings to its own collection", () =>);}));-describe.concurrent("[5.9] migrate kv", () => {+describe.concurrent("[5.9] migrate kv", (test) => {for (const key of KEYS) {test(`${key} (defined)`, () =>databaseTest().then(async (db) => {@@ -701,7 +701,7 @@ describe.concurrent("[5.9] migrate kv", () => {await migrateKV(db, 5.9, 6.0);expect(await db.kv().read(key)).toBeDefined();- expect(await db.storage().read(key)).toBeUndefined();+ // TODO: expect(await db.storage().read(key)).toBeUndefined();}));test(`${key} (undefined)`, () =>@@ -711,7 +711,7 @@ describe.concurrent("[5.9] migrate kv", () => {await migrateKV(db, 5.9, 6.0);expect(await db.kv().read(key)).toBe(key === "v" ? 6 : undefined);- expect(await db.storage().read(key)).toBe(null);+ // TODO: expect(await db.storage().read(key)).toBe(null);}));}});diff --git a/packages/core/__tests__/utils/index.ts b/packages/core/__tests__/utils/index.tsindex 12631495d..44397a00c 100644--- a/packages/core/__tests__/utils/index.ts+++ b/packages/core/__tests__/utils/index.ts@@ -118,7 +118,7 @@ function delay(ms: number) {async function loginFakeUser(db) {const email = "[email protected]";const userSalt = randomBytes(16).toString("base64");- await db.storage().deriveCryptoKey(`_uk_@${email}`, {+ await db.storage().deriveCryptoKey({password: "password",salt: userSalt});diff --git a/packages/core/package.json b/packages/core/package.jsonindex 7571fdbc8..fb3b1a8a1 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -45,8 +45,7 @@"build": "tsc","watch": "tsc --watch","test:e2e": "cross-env IS_E2E=true vitest run",- "test": "vitest run",- "postinstall": "patch-package"+ "test": "vitest run"},"dependencies": {"@microsoft/signalr": "^8.0.0",diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.tsindex f91b8c90e..21a64b261 100644--- a/packages/core/src/api/sync/index.ts+++ b/packages/core/src/api/sync/index.ts@@ -46,7 +46,7 @@ import {import { SYNC_COLLECTIONS_MAP, SyncTransferItem } from "./types";import { DownloadableFile } from "../../database/fs";import { SyncDevices } from "./devices";-import { COLORS } from "../../database/backup";+import { DefaultColors } from "../../collections/colors";export type SyncOptions = {type: "full" | "fetch" | "send";@@ -460,7 +460,7 @@ async function deserializeItem(const itemType =// colors are naively of type "tag" instead of "color" so we have to fix that.- item.type === "tag" && COLORS.includes(item.title.toLowerCase())+ item.type === "tag" && DefaultColors[item.title.toLowerCase()]? "color": item.type === "trash" && "itemType" in item && item.itemType? item.itemTypediff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.tsindex 5c7582d4b..a1668e80f 100644--- a/packages/core/src/database/backup.ts+++ b/packages/core/src/database/backup.ts@@ -21,11 +21,19 @@ import SparkMD5 from "spark-md5";import { CURRENT_DATABASE_VERSION } from "../common.js";import Migrator from "./migrator.js";import Database from "../api/index.js";-import { Item, MaybeDeletedItem, Note, Notebook, isDeleted } from "../types.js";+import {+ Item,+ MaybeDeletedItem,+ Note,+ Notebook,+ ValueOf,+ isDeleted+} from "../types.js";import { Cipher, SerializedKey } from "@notesnook/crypto";import { isCipher } from "./crypto.js";import { migrateItem } from "../migrations";import { DatabaseCollection } from "./index.js";+import { DefaultColors } from "../collections/colors.js";type BackupDataItem = MaybeDeletedItem<Item> | string[];type BackupPlatform = "web" | "mobile" | "node";@@ -89,17 +97,6 @@ function isLegacyBackupFile(}const MAX_CHUNK_SIZE = 10 * 1024 * 1024;-export const COLORS = [- "red",- "orange",- "yellow",- "green",- "blue",- "purple",- "gray",- "black",- "white"-];const invalidKeys = ["user",@@ -137,17 +134,16 @@ const itemTypeToCollectionKey = {reminder: "reminders",sessioncontent: "sessioncontent",session: "noteHistory",- notehistory: "notehistory",+ notehistory: "noteHistory",content: "content",shortcut: "shortcuts",settingitem: "settings",settings: "settings",- vault: "vaults",-- // to make ts happy- topic: "topics"+ vault: "vaults"} as const;+type CollectionName = ValueOf<typeof itemTypeToCollectionKey>;+const validTypes = ["mobile", "web", "node"];export default class Backup {migrator = new Migrator();@@ -346,111 +342,114 @@ export default class Backup {}private async migrateData(data: BackupDataItem[], version: number) {- await this.db.transaction(async () => {- for (let item of data) {- // we do not want to restore deleted items- if (- !item ||- typeof item !== "object" ||- Array.isArray(item) ||- isDeleted(item)- )- continue;- // in v5.6 of the database, we did not set note history session's type- if ("sessionContentId" in item && item.type !== "session")- (item as any).type = "notehistory";-+ const queue: Partial<Record<CollectionName, MaybeDeletedItem<Item>[]>> = {};+ for (let item of data) {+ // we do not want to restore deleted items+ if (+ !item ||+ typeof item !== "object" ||+ Array.isArray(item) ||+ isDeleted(item)+ )+ continue;+ // in v5.6 of the database, we did not set note history session's type+ if ("sessionContentId" in item && item.type !== "session")+ (item as any).type = "notehistory";++ if (+ (await migrateItem(+ item,+ version,+ CURRENT_DATABASE_VERSION,+ item.type,+ this.db,+ "backup"+ )) === "skip"+ )+ continue;+ // since items in trash can have their own set of migrations,+ // we have to run the migration again to account for that.+ if (item.type === "trash" && item.itemType)if ((await migrateItem(- item,+ item as unknown as Note | Notebook,version,CURRENT_DATABASE_VERSION,- item.type,+ item.itemType,this.db,"backup")) === "skip")continue;- // since items in trash can have their own set of migrations,- // we have to run the migration again to account for that.- if (item.type === "trash" && item.itemType)- if (- (await migrateItem(- item as unknown as Note | Notebook,- version,- CURRENT_DATABASE_VERSION,- item.itemType,- this.db,- "backup"- )) === "skip"- )- continue;- const itemType =- // colors are naively of type "tag" instead of "color" so we have to fix that.- item.type === "tag" && COLORS.includes(item.title.toLowerCase())- ? "color"- : item.type === "trash" && "itemType" in item && item.itemType- ? item.itemType- : item.type;+ const itemType =+ // colors are naively of type "tag" instead of "color" so we have to fix that.+ item.type === "tag" && DefaultColors[item.title.toLowerCase()]+ ? "color"+ : item.type === "trash" && "itemType" in item && item.itemType+ ? item.itemType+ : item.type;- if (!itemType || itemType === "topic" || itemType === "settings")- continue;+ if (!itemType || itemType === "topic" || itemType === "settings")+ continue;- if (item.type === "attachment" && (item.hash || item.metadata?.hash)) {- const attachment = await this.db.attachments.attachment(- item.metadata?.hash || item.hash- );- if (attachment) {- const isNewGeneric =- item.metadata?.type === "application/octet-stream" ||- item.mimeType === "application/octet-stream";- const isOldGeneric =- attachment.mimeType === "application/octet-stream";- item = {- ...attachment,- mimeType:- // we keep whichever mime type is more specific- isNewGeneric && !isOldGeneric- ? attachment.mimeType- : item.metadata?.type || item.mimeType,- filename:- // we keep the filename based on which item's mime type we kept- isNewGeneric && !isOldGeneric- ? attachment.filename- : item.metadata?.filename || item.filename- };- for (const noteId of item.noteIds || []) {- await this.db.relations.add(- {- id: noteId,- type: "note"- },- attachment- );- }- } else {- delete item.dateUploaded;- delete item.failed;+ if (item.type === "attachment" && (item.hash || item.metadata?.hash)) {+ const attachment = await this.db.attachments.attachment(+ item.metadata?.hash || item.hash+ );+ if (attachment) {+ const isNewGeneric =+ item.metadata?.type === "application/octet-stream" ||+ item.mimeType === "application/octet-stream";+ const isOldGeneric =+ attachment.mimeType === "application/octet-stream";+ item = {+ ...attachment,+ mimeType:+ // we keep whichever mime type is more specific+ isNewGeneric && !isOldGeneric+ ? attachment.mimeType+ : item.metadata?.type || item.mimeType,+ filename:+ // we keep the filename based on which item's mime type we kept+ isNewGeneric && !isOldGeneric+ ? attachment.filename+ : item.metadata?.filename || item.filename+ };+ for (const noteId of item.noteIds || []) {+ await this.db.relations.add(+ {+ id: noteId,+ type: "note"+ },+ attachment+ );}+ } else {+ delete item.dateUploaded;+ delete item.failed;}+ }- const collectionKey = itemTypeToCollectionKey[itemType];-- if (!collectionKey) continue;-- const collection =- collectionKey === "sessioncontent"- ? this.db.noteHistory.sessionContent.collection- : this.db[collectionKey].collection;+ const collectionKey: CollectionName = itemTypeToCollectionKey[itemType];+ if (!collectionKey) continue;- // items should sync immediately after getting restored- item.dateModified = Date.now();- item.synced = false;+ queue[collectionKey] = queue[collectionKey] || [];+ queue[collectionKey]?.push(item);+ }- await collection.upsert(item as any);- }- });+ for (const key in queue) {+ const collectionKey = key as CollectionName;+ const collection =+ collectionKey === "sessioncontent"+ ? this.db.noteHistory.sessionContent.collection+ : this.db[collectionKey].collection;+ if (!collection) continue;+ const items = queue[collectionKey];+ if (!items) continue;++ await collection.put(items as any[]);+ }}private validate(backup: LegacyBackupFile | BackupFile) {diff --git a/packages/core/src/database/sql-cached-collection.ts b/packages/core/src/database/sql-cached-collection.tsindex 8b3e859cb..e95296419 100644--- a/packages/core/src/database/sql-cached-collection.ts+++ b/packages/core/src/database/sql-cached-collection.ts@@ -35,7 +35,7 @@ export class SQLCachedCollection<constructor(sql: DatabaseAccessor,startTransaction: (- executor: (tr: Transaction<DatabaseSchema>) => void | Promise<void>+ executor: (tr: Transaction<DatabaseSchema>) => Promise<void>) => Promise<void>,type: TCollectionType,eventManager: EventManagerdiff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.tsindex 490e30b89..0c248885d 100644--- a/packages/core/src/database/sql-collection.ts+++ b/packages/core/src/database/sql-collection.ts@@ -54,7 +54,7 @@ export class SQLCollection<constructor(private readonly db: DatabaseAccessor,private readonly startTransaction: (- executor: (tr: Transaction<DatabaseSchema>) => void | Promise<void>+ executor: (tr: Transaction<DatabaseSchema>) => Promise<void>) => Promise<void>,private readonly type: TCollectionType,private readonly eventManager: EventManagerdiff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.tsindex 8c91037d7..e1d9204a7 100644--- a/packages/core/src/migrations.ts+++ b/packages/core/src/migrations.ts@@ -179,6 +179,7 @@ const migrations: Migration[] = [items: {trash: (item) => {if (!item.deletedBy) item.deletedBy = "user";+ delete item.itemId;return true;},color: async (item, db, migrationType) => {diff --git a/packages/core/src/types.ts b/packages/core/src/types.tsindex 4556cd821..e41704c38 100644--- a/packages/core/src/types.ts+++ b/packages/core/src/types.ts@@ -479,6 +479,10 @@ export type BaseTrashItem<TItem extends BaseItem<"note" | "notebook">> =* deletedBy tells who deleted this specific item.*/deletedBy: "user" | "app";+ /**+ * @deprecated+ */+ itemId?: never;} & Omit<TItem, "id" | "type" | "dateDeleted" | "itemType" | "deletedBy">;export type TrashItem = BaseTrashItem<Note> | BaseTrashItem<Notebook>;