A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: migrate vault to a database collection
Abdullah Attadc3caa16d0ef
diff --git a/packages/core/__e2e__/__snapshots__/pricing.test.js.snap b/packages/core/__e2e__/__snapshots__/pricing.test.js.snapindex 49e608e4c..b92262c58 100644--- a/packages/core/__e2e__/__snapshots__/pricing.test.js.snap+++ b/packages/core/__e2e__/__snapshots__/pricing.test.js.snap@@ -72,6 +72,15 @@ exports[`get web pricing tier > get yearly web tier > yearly-web-pricing 1`] = `}`;+exports[`get yearly price > monthly-pricing 1`] = `+{+ "country": Any<String>,+ "countryCode": Any<String>,+ "discount": Any<Number>,+ "price": Any<Number>,+}+`;+exports[`get yearly price > yearly-pricing 1`] = `{"country": Any<String>,diff --git a/packages/core/__tests__/vault.test.js b/packages/core/__tests__/vault.test.jsindex e23a7c9c7..baa5076a7 100644--- a/packages/core/__tests__/vault.test.js+++ b/packages/core/__tests__/vault.test.js@@ -17,17 +17,18 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { databaseTest, noteTest, TEST_NOTE } from "./utils";+import { VAULT_ERRORS } from "../src/api/vault";+import { databaseTest, delay, noteTest, TEST_NOTE } from "./utils";import { test, expect } from "vitest";test("create vault", () =>databaseTest().then(async (db) => {await expect(db.vault.create("password")).resolves.toBe(true);- const vaultKey = await db.storage().read("vaultKey");- expect(vaultKey).toBeDefined();- expect(vaultKey.iv).toBeDefined();- expect(vaultKey.cipher).toBeDefined();- expect(vaultKey.length).toBeDefined();+ const vault = await db.vaults.default();+ expect(vault).toBeDefined();+ expect(vault.key.iv).toBeDefined();+ expect(vault.key.cipher).toBeDefined();+ expect(vault.key.length).toBeDefined();}));test("unlock vault", () =>@@ -57,15 +58,20 @@ test("lock a note when no vault has been created", () =>test("lock a note", () =>noteTest().then(async ({ db, id }) => {await db.vault.create("password");+await db.vault.add(id);+const note = await db.notes.note(id);+ const content = await db.content.get(note.contentId);+ const vault = await db.vaults.default();expect(note.headline).toBe("");- const content = await db.content.get(note.contentId);expect(content.noteId).toBeDefined();expect(content.data.iv).toBeDefined();expect(content.data.cipher).toBeDefined();++ expect(await db.relations.from(vault, "note").has(id)).toBe(true);}));test("locked note is not favorited", () =>@@ -81,23 +87,32 @@ test("unlock a note", () =>noteTest().then(async ({ db, id }) => {await db.vault.create("password");await db.vault.add(id);+const note = await db.vault.open(id, "password");++ const vault = await db.vaults.default();expect(note.id).toBe(id);expect(note.content.data).toBeDefined();expect(note.content.type).toBe(TEST_NOTE.content.type);+ expect(await db.relations.from(vault, "note").has(id)).toBe(true);}));test("unlock a note permanently", () =>noteTest().then(async ({ db, id }) => {await db.vault.create("password");await db.vault.add(id);+await db.vault.remove(id, "password");+const note = await db.notes.note(id);+ const content = await db.content.get(note.contentId);+ const vault = await db.vaults.default();+expect(note.id).toBe(id);expect(note.headline).not.toBe("");- const content = await db.content.get(note.contentId);expect(content.data).toBeDefined();expect(typeof content.data).toBe("string");+ expect(await db.relations.from(vault, "note").has(id)).toBe(false);}));test("save a locked note", () =>@@ -142,3 +157,75 @@ test("change vault password", () =>await expect(db.vault.open(id, "password")).rejects.toThrow();await expect(db.vault.open(id, "newPassword")).resolves.toBeDefined();}));++test("changing vault password without a vault should throw", () =>+ noteTest().then(async ({ db }) => {+ await expect(+ db.vault.changePassword("password", "newPassword")+ ).rejects.toThrow(VAULT_ERRORS.noVault);+ }));++test("clear vault", () =>+ noteTest().then(async ({ db, id }) => {+ await db.vault.create("password");+ await db.vault.add(id);++ await db.vault.clear("password");++ const vault = await db.vaults.default();+ expect(await db.relations.from(vault, "note").has(id)).toBe(false);+ }));++test("delete vault without deleting all locked notes", () =>+ noteTest().then(async ({ db, id }) => {+ await db.vault.create("password");+ await db.vault.add(id);+ const vault = await db.vaults.default();++ await db.vault.delete();++ expect(await db.relations.from(vault, "note").has(id)).toBe(false);+ expect(await db.vaults.default()).toBeUndefined();+ }));++test("delete vault and delete all locked notes", () =>+ noteTest().then(async ({ db, id }) => {+ await db.vault.create("password");+ await db.vault.add(id);+ const vault = await db.vaults.default();++ await db.vault.delete(true);++ expect(await db.relations.from(vault, "note").has(id)).toBe(false);+ expect(await db.notes.exists(id)).toBe(false);+ expect(await db.vaults.default()).toBeUndefined();+ }));++test("vault password is cleared after specified time", () =>+ databaseTest().then(async (db) => {+ await expect(db.vault.create("password")).resolves.toBe(true);+ db.vault.eraseTime = 1000;+ await expect(db.vault.unlock("password")).resolves.toBe(true);+ expect(db.vault.unlocked).toBe(true);+ await delay(1500);+ expect(db.vault.unlocked).toBe(false);+ }));++// test("remove note from vault if it isn't encrypted", () =>+// noteTest().then(async ({ db, id }) => {+// await db.vault.create("password");+// const vault = await db.vaults.default();++// await db.relations.add(vault, { id, type: "note" });++// const decryptedContent = await db.vault.decryptContent(+// {+// data: "<p>hello world</p>",+// type: "tiptap"+// },+// "password"+// );++// expect(await db.relations.from(vault, "note").has(id)).toBe(false);+// expect(decryptedContent.data).toBe("<p>hello world</p>");+// }));diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.tsindex 9a2d60481..c32e6a168 100644--- a/packages/core/src/api/index.ts+++ b/packages/core/src/api/index.ts@@ -68,6 +68,7 @@ import {} from "../database";import { Kysely, Transaction, sql } from "kysely";import { CachedCollection } from "../database/cached-collection";+import { Vaults } from "../collections/vaults";type EventSourceConstructor = new (uri: string,@@ -183,6 +184,7 @@ class Database {reminders = new Reminders(this);relations = new Relations(this);notes = new Notes(this);+ vaults = new Vaults(this);/*** @deprecated only kept here for migration purposes@@ -255,7 +257,6 @@ class Database {);EV.subscribe(EVENTS.attachmentDeleted, async (attachment: Attachment) => {await this.fs().cancel(attachment.hash);- await this.fs().cancel(attachment.hash);});EV.subscribe(EVENTS.userLoggedOut, async () => {await this.monographs.clear();diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.tsindex 4e0762cf9..0101824ce 100644--- a/packages/core/src/api/sync/index.ts+++ b/packages/core/src/api/sync/index.ts@@ -34,7 +34,7 @@ import { AutoSync } from "./auto-sync";import { logger } from "../../logger";import { Mutex } from "async-mutex";import Database from "..";-import { migrateItem } from "../../migrations";+import { migrateItem, migrateVaultKey } from "../../migrations";import { SerializedKey } from "@notesnook/crypto";import { Item, MaybeDeletedItem, Note, Notebook } from "../../types";import { SYNC_COLLECTIONS_MAP, SyncTransferItem } from "./types";@@ -189,6 +189,26 @@ class Sync {let count = 0;this.connection?.off("SendItems");+ this.connection?.off("SendVaultKey");++ this.connection?.on("SendVaultKey", async (vaultKey) => {+ if (this.connection?.state !== signalr.HubConnectionState.Connected)+ return false;++ if (+ vaultKey &&+ vaultKey.cipher !== null &&+ vaultKey.iv !== null &&+ vaultKey.salt !== null &&+ vaultKey.length > 0+ ) {+ const vault = await this.db.vaults.default();+ if (!vault) await migrateVaultKey(this.db, vaultKey, 5.9);+ }++ return true;+ });+this.connection?.on("SendItems", async (chunk) => {if (this.connection?.state !== signalr.HubConnectionState.Connected)return false;@@ -200,39 +220,17 @@ class Sync {return true;});- const serverResponse = await this.connection?.invoke(- "RequestFetch",- deviceId- );-- if (- serverResponse.vaultKey &&- serverResponse.vaultKey.cipher !== null &&- serverResponse.vaultKey.iv !== null &&- serverResponse.vaultKey.salt !== null &&- serverResponse.vaultKey.length > 0- ) {- await this.db.vault.setKey(serverResponse.vaultKey);- }+ await this.connection?.invoke("RequestFetch", deviceId);this.connection?.off("SendItems");+ this.connection?.off("SendVaultKey");}async send(deviceId: string, isForceSync?: boolean) {await this.uploadAttachments();- let isSyncInitialized = false;let done = 0;for await (const item of this.collector.collect(100, isForceSync)) {- if (!isSyncInitialized) {- const vaultKey = await this.db.vault.getKey();- await this.connection?.send("InitializePush", {- vaultKey,- synced: false- });- isSyncInitialized = true;- }-const result = await this.pushItem(deviceId, item);if (result) {done += item.items.length;@@ -245,7 +243,6 @@ class Sync {);}}- if (!isSyncInitialized) return false;await this.connection?.send("PushCompleted");return true;}diff --git a/packages/core/src/api/vault.ts b/packages/core/src/api/vault.tsindex 3f33beead..dd00d2d0e 100644--- a/packages/core/src/api/vault.ts+++ b/packages/core/src/api/vault.ts@@ -22,11 +22,6 @@ import Database from ".";import { CHECK_IDS, EV, EVENTS, checkIsUserPremium } from "../common";import { tinyToTiptap } from "../migrations";import { isCipher } from "../database/crypto";-import { Note } from "../types";-import {- isEncryptedContent,- isUnencryptedContent-} from "../collections/content";import { NoteContent } from "../collections/session-content";export const VAULT_ERRORS = {@@ -35,8 +30,8 @@ export const VAULT_ERRORS = {wrongPassword: "ERR_WRONG_PASSWORD"};-const ERASE_TIME = 1000 * 60 * 30;export default class Vault {+ eraseTime = 1000 * 60 * 30;private vaultPassword?: string;private erasureTimeout = 0;private key = "svvaads1212#2123";@@ -57,7 +52,7 @@ export default class Vault {this.erasureTimeout = setTimeout(() => {this.password = undefined;EV.publish(EVENTS.vaultLocked);- }, ERASE_TIME) as unknown as number;+ }, this.eraseTime) as unknown as number;}constructor(private readonly db: Database) {@@ -99,63 +94,73 @@ export default class Vault {}async changePassword(oldPassword: string, newPassword: string) {+ const vault = await this.db.vaults.default();+ if (!vault) throw new Error(VAULT_ERRORS.noVault);+if (await this.unlock(oldPassword)) {- const contentItems = [];- for await (const note of this.db.notes.locked) {- if (!note.contentId) continue;- const encryptedContent = await this.db.content.get(note.contentId);- if (!encryptedContent || !isEncryptedContent(encryptedContent))+ const relations = await this.db.relations.from(vault, "note").get();+ for (const { toId: noteId } of relations) {+ const content = await this.db.content.findByNoteId(noteId);+ if (!content || !content.locked) {+ await this.db.relations.unlink(vault, { id: noteId, type: "note" });continue;+ }try {- const content = await this.decryptContent(- encryptedContent,- note.id,+ const decryptedContent = await this.decryptContent(+ content,oldPassword);- contentItems.push({- ...content,- id: note.contentId,- noteId: note.id- });++ await this.encryptContent(+ decryptedContent,+ noteId,+ newPassword,+ `${Date.now()}`+ );} catch (e) {console.error(e);throw new Error(- `Could not decrypt content of note ${note.id}. Error: ${+ `Could not decrypt content of note ${noteId}. Error: ${(e as Error).message}`);}}- for (const content of contentItems) {- await this.encryptContent(- content,- content.noteId,- newPassword,- content.id- );- }-- await this.db.storage().remove("vaultKey");- await this.create(newPassword);+ await this.db.vaults.add({+ id: vault.id,+ key: await this.db+ .storage()+ .encrypt({ password: newPassword }, this.key)+ });}}async clear(password: string) {+ const vault = await this.db.vaults.default();+ if (!vault) return;+if (await this.unlock(password)) {- for await (const note of this.db.notes.locked) {- await this.unlockNote(note, password, true);+ const relations = await this.db.relations.from(vault, "note").get();+ for (const { toId: noteId } of relations) {+ await this.unlockNote(noteId, password, true);+ await this.db.relations.unlink(vault, { id: noteId, type: "note" });}}}async delete(deleteAllLockedNotes = false) {+ const vault = await this.db.vaults.default();+ if (!vault) return;+if (deleteAllLockedNotes) {- const lockedIds = await this.db.notes.locked.ids();+ const relations = await this.db.relations.from(vault, "note").get();+ const lockedIds = relations.map((r) => r.toId);await this.db.notes.remove(...lockedIds);}- await this.db.storage().remove("vaultKey");++ await this.db.vaults.remove(vault.id);this.password = undefined;}@@ -173,11 +178,11 @@ export default class Vault {* Permanently unlocks (remove from vault) a note*/async remove(noteId: string, password: string) {- const note = await this.db.notes.note(noteId);- if (!note) return;- await this.unlockNote(note, password, true);+ await this.unlockNote(noteId, password, true);- if (!(await this.exists())) await this.create(password);+ const vault = await this.db.vaults.default();+ if (!vault) await this.create(password);+ else await this.db.relations.unlink(vault, { id: noteId, type: "note" });}/**@@ -187,23 +192,22 @@ export default class Vault {const note = await this.db.notes.note(noteId);if (!note) return;- const unlockedNote = await this.unlockNote(note, password, false);+ const content = await this.unlockNote(noteId, password, false);if (password) {this.password = password;if (!(await this.exists())) await this.create(password);}-- return unlockedNote;+ return { ...note, ...content };}/*** Saves a note in the vault*/- async save(- note: Partial<Note & { content: NoteContent<false>; sessionId: string }> & {- id: string;- }- ) {+ async save(note: {+ content?: NoteContent<false>;+ sessionId?: string;+ id: string;+ }) {if (!note) return;// roll over erase timerthis.startEraser();@@ -233,7 +237,6 @@ export default class Vault {content: NoteContent<false>,noteId: string,password: string,- contentId?: string,sessionId?: string) {const encryptedContent = await this.db@@ -241,7 +244,6 @@ export default class Vault {.encrypt({ password }, JSON.stringify(content.data));await this.db.content.add({- id: contentId,noteId,sessionId,data: encryptedContent,@@ -249,23 +251,11 @@ export default class Vault {});}- async decryptContent(- encryptedContent: NoteContent<true>,- noteId: string,- password?: string- ) {+ async decryptContent(encryptedContent: NoteContent<true>, password?: string) {if (!password) password = await this.getVaultPassword();- if (- typeof encryptedContent.data !== "object" &&- !isCipher(encryptedContent.data)- ) {- await this.db.notes.add({- id: noteId,- locked: false- });- return { data: encryptedContent.data, type: encryptedContent.type };- }+ if (!isCipher(encryptedContent.data))+ return encryptedContent as unknown as NoteContent<false>;const decryptedContent = await this.db.storage()@@ -286,33 +276,30 @@ export default class Vault {}private async lockNote(- item: Partial<Note & { content: NoteContent<false>; sessionId: string }> & {- id: string;- },+ item: { content?: NoteContent<false>; sessionId?: string; id: string },password: string) {- const { id, content, sessionId, title } = item;- let { type, data } = content || {};+ const vault = await this.db.vaults.default();+ if (!vault) throw new Error(VAULT_ERRORS.noVault);- const note = await this.db.notes.note(id);- if (!note) return;+ const { id, content, sessionId } = item;+ let { type, data } = content || {};- const contentId = note.contentId;- // if (!contentId) throw new Error("Cannot lock note because it is empty.");+ const locked = await this.db.relations.from(vault, "note").has(id);// Case: when note is being newly locked- if (!note.locked && (!data || !type) && !!contentId) {- const rawContent = await this.db.content.get(contentId);- if (!rawContent || !isUnencryptedContent(rawContent))- return await this.db.notes.add({+ if (!locked && (!data || !type)) {+ const content = await this.db.content.findByNoteId(id);+ if (!content || content.locked)+ return await this.db.relations.add(vault, {id,- locked: true+ type: "note"});// NOTE:// At this point, the note already has all the attachments extracted// so we should just encrypt it as normal.- data = rawContent.data;- type = rawContent.type;+ data = content.data;+ type = content.type;} else if (data && type) {data = await this.db.content.extractAttachments({data,@@ -322,60 +309,47 @@ export default class Vault {}if (data && type)- await this.encryptContent(- { data, type },- id,- password,- contentId,- sessionId- );-- return await this.db.notes.add({+ await this.encryptContent({ data, type }, id, password, sessionId);++ await this.db.notes.add({id,- locked: true,- headline: "",- title: title || note.title,- favorite: note.favorite,- localOnly: note.localOnly,- readonly: note.readonly,- dateEdited: Date.now()+ headline: ""});- }- private async unlockNote(note: Note, password?: string, perm = false) {- if (!note.contentId) return;+ await this.db.relations.add(vault, {+ id,+ type: "note"+ });+ }- const encryptedContent = await this.db.content.get(note.contentId);- if (!encryptedContent || !isEncryptedContent(encryptedContent)) return;- const content = await this.decryptContent(- encryptedContent,- note.id,- password- );+ private async unlockNote(noteId: string, password?: string, perm = false) {+ const content = await this.db.content.findByNoteId(noteId);+ if (!content || !content.locked) return;+ const decryptedContent = await this.decryptContent(content, password);if (perm) {await this.db.notes.add({- id: note.id,- locked: false,- headline: note.headline,- contentId: note.contentId,- content+ id: noteId,+ contentId: content.id,+ content: decryptedContent});// await this.db.content.add({ id: note.contentId, data: content });return;}return {- ...note,content};}async getKey() {- return await this.db.storage().read<Cipher<"base64">>("vaultKey");+ const vault = await this.db.vaults.default();+ return vault?.key;}async setKey(vaultKey: Cipher<"base64">) {- await this.db.storage().write("vaultKey", vaultKey);+ const vault = await this.db.vaults.default();+ if (vault) return;+ await this.db.vaults.add({ title: "Default", key: vaultKey });}}diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.tsindex 698f6893d..504a9d243 100644--- a/packages/core/src/collections/content.ts+++ b/packages/core/src/collections/content.ts@@ -200,13 +200,17 @@ export class Content implements ICollection {.execute();}- // async findByNoteId(noteId: string) {- // await this.db- // .sql()- // .selectFrom("content")- // .where("noteId", "==", noteId)- // .execute();- // }+ async findByNoteId(noteId: string) {+ const content = (await this.db+ .sql()+ .selectFrom("content")+ .where("noteId", "==", noteId)+ .selectAll()+ .executeTakeFirst()) as ContentItem;+ if (!content || isDeleted(content)) return;+ return content;+ }+// multi(ids: string[]) {// return this.collection.getItems(ids);// }diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.tsindex f13372d2a..aba891ba3 100644--- a/packages/core/src/collections/notes.ts+++ b/packages/core/src/collections/notes.ts@@ -121,7 +121,6 @@ export class Notes implements ICollection {contentId,pinned: item.pinned,- locked: item.locked,favorite: item.favorite,localOnly: item.localOnly,conflicted: item.conflicted,@@ -228,17 +227,6 @@ export class Notes implements ICollection {);}- get locked() {- return this.collection.createFilter<Note>(- (qb) =>- qb- .where(isFalse("dateDeleted"))- .where(isFalse("deleted"))- .where("locked", "==", true),- this.db.options?.batchSize- );- }-exists(id: string) {return this.collection.exists(id);}diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.tsindex 80fb6a366..81471731a 100644--- a/packages/core/src/collections/relations.ts+++ b/packages/core/src/collections/relations.ts@@ -194,7 +194,8 @@ const TABLE_MAP = {reminder: "reminders",tag: "tags",color: "colors",- attachment: "attachments"+ attachment: "attachments",+ vault: "vaults"} as const;type RelatableTable = typeof TABLE_MAP;diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.tsindex 59a351f9e..833c0d74f 100644--- a/packages/core/src/collections/tags.ts+++ b/packages/core/src/collections/tags.ts@@ -49,8 +49,6 @@ export class Tags implements ICollection {}async add(item: Partial<Tag>) {- if (item.remote)- throw new Error("Please use db.tags.merge to merge remote tags.");item.title = item.title ? Tags.sanitize(item.title) : item.title;const id = item.id || getId(item.dateCreated);@@ -68,8 +66,7 @@ export class Tags implements ICollection {dateCreated: item.dateCreated || oldTag?.dateCreated || Date.now(),dateModified: item.dateModified || oldTag?.dateModified || Date.now(),title: item.title || oldTag?.title || "",- type: "tag",- remote: false+ type: "tag"});return id;}diff --git a/packages/core/src/collections/vaults.ts b/packages/core/src/collections/vaults.tsnew file mode 100644index 000000000..468b9c718--- /dev/null+++ b/packages/core/src/collections/vaults.ts@@ -0,0 +1,83 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import Database from "../api";+import { Vault } from "../types";+import { ICollection } from "./collection";+import { SQLCollection } from "../database/sql-collection";+import { getId } from "../utils/id";+import { isFalse } from "../database";++export class Vaults implements ICollection {+ name = "vaults";+ readonly collection: SQLCollection<"vaults", Vault>;+ constructor(private readonly db: Database) {+ this.collection = new SQLCollection(+ db.sql,+ db.transaction,+ "vaults",+ db.eventManager+ );+ }++ async init() {}++ async add(item: Partial<Vault>) {+ const id = item.id || getId();+ const oldVault = item.id ? await this.vault(item.id) : undefined;++ if (!item.title && !oldVault?.title) throw new Error("Title is required.");++ await this.collection.upsert({+ id,+ dateCreated: item.dateCreated || oldVault?.dateCreated || Date.now(),+ dateModified: item.dateModified || oldVault?.dateModified || Date.now(),+ title: item.title || oldVault?.title || "",+ key: item.key,+ type: "vault"+ });+ return id;+ }++ async remove(id: string) {+ await this.db.transaction(async () => {+ await this.db.relations.unlinkOfType("vault", [id]);+ await this.collection.softDelete([id]);+ });+ }++ vault(id: string) {+ return this.collection.get(id);+ }++ /**+ * This is temporary until we add proper support for multiple vaults+ * @deprecated+ */+ async default() {+ return (await this.all.items()).at(0);+ }++ get all() {+ return this.collection.createFilter<Vault>(+ (qb) => qb.where(isFalse("deleted")),+ this.db.options?.batchSize+ );+ }+}diff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.tsindex 4f376f93b..9a42b4781 100644--- a/packages/core/src/database/backup.ts+++ b/packages/core/src/database/backup.ts@@ -142,6 +142,7 @@ const itemTypeToCollectionKey = {shortcut: "shortcuts",settingitem: "settings",settings: "settings",+ vault: "vaults",// to make ts happytopic: "topics"@@ -198,6 +199,7 @@ export default class Backup {yield* this.backupCollection(this.db.reminders.collection, backupState);yield* this.backupCollection(this.db.relations.collection, backupState);yield* this.backupCollection(this.db.attachments.collection, backupState);+ yield* this.backupCollection(this.db.vaults.collection, backupState);if (backupState.buffer.length > 0) yield* this.bufferToFile(backupState);diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.tsindex 424f70f0a..40dfb065b 100644--- a/packages/core/src/database/index.ts+++ b/packages/core/src/database/index.ts@@ -53,7 +53,8 @@ import {Shortcut,Tag,TrashOrItem,- ValueOf+ ValueOf,+ Vault} from "../types";import { NNMigrationProvider } from "./migrations";import { createTriggers } from "./triggers";@@ -85,6 +86,7 @@ export interface DatabaseSchema {notehistory: SQLiteItem<HistorySession>;sessioncontent: SQLiteItem<SessionContentItem>;shortcuts: SQLiteItem<Shortcut>;+ vaults: SQLiteItem<Vault>;}export type DatabaseSchemaWithFTS = DatabaseSchema & {@@ -194,6 +196,9 @@ const DataMappers: Partial<Record<ItemType, (row: any) => void>> = {},attachment: (row) => {if (row.key) row.key = JSON.parse(row.key);+ },+ vault: (row) => {+ if (row.key) row.key = JSON.parse(row.key);}};diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.tsindex b028870d3..133385622 100644--- a/packages/core/src/database/migrations.ts+++ b/packages/core/src/database/migrations.ts@@ -123,6 +123,14 @@ export class NNMigrationProvider implements MigrationProvider {.addColumn("colorCode", "text").execute();+ await db.schema+ .createTable("vaults")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("title", "text", COLLATE_NOCASE)+ .addColumn("key", "text")+ .execute();+await db.schema.createTable("relations").modifyEnd(sql`without rowid`)diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.tsindex aa796f09b..3708f6a90 100644--- a/packages/core/src/database/migrator.ts+++ b/packages/core/src/database/migrator.ts@@ -17,13 +17,14 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/+import { Cipher } from "@notesnook/crypto";import { DatabaseSchema } from ".";import Database from "../api";import {CURRENT_DATABASE_VERSION,sendMigrationProgressEvent} from "../common";-import { migrateCollection, migrateItem } from "../migrations";+import { migrateCollection, migrateItem, migrateVaultKey } from "../migrations";import {CollectionType,Collections,@@ -50,6 +51,9 @@ class Migrator {collections: MigratableCollections,version: number) {+ const vaultKey = await db.storage().read<Cipher<"base64">>("vaultKey");+ if (vaultKey) await migrateVaultKey(db, vaultKey, version);+for (const collection of collections) {sendMigrationProgressEvent(db.eventManager, collection.name, 0, 0);@@ -79,6 +83,7 @@ class Migrator {);}}+await db.initCollections();return true;}diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.tsindex 8a3211709..e11c057b2 100644--- a/packages/core/src/migrations.ts+++ b/packages/core/src/migrations.ts@@ -34,18 +34,8 @@ import {} from "./types";import { isCipher } from "./database/crypto";import { IndexedCollection } from "./database/indexed-collection";--const ColorToHexCode: Record<string, string> = {- red: "#f44336",- orange: "#FF9800",- yellow: "#FFD600",- green: "#4CAF50",- blue: "#2196F3",- purple: "#673AB7",- gray: "#9E9E9E",- black: "#000000",- white: "#ffffff"-};+import { DefaultColors } from "./collections/colors";+import { Cipher } from "@notesnook/crypto";type MigrationType = "local" | "sync" | "backup";type MigrationItemType = ItemType | "notehistory" | "content" | "all";@@ -64,6 +54,7 @@ type Migration = {) => "skip" | boolean | Promise<boolean | "skip"> | void;};collection?: (collection: IndexedCollection) => Promise<void> | void;+ vaultKey?: (db: Database, key: Cipher<"base64">) => Promise<void> | void;};const migrations: Migration[] = [@@ -198,7 +189,7 @@ const migrations: Migration[] = [)return "skip";- const colorCode = ColorToHexCode[item.title];+ const colorCode = DefaultColors[item.title];if (colorCode) {const newColor = await db.colors.all.find((eb) =>eb.or([eb("title", "in", [alias, item.title])])@@ -258,7 +249,7 @@ const migrations: Migration[] = [dateCreated: oldColor?.dateCreated,dateModified: oldColor?.dateModified,title: alias || item.color,- colorCode: ColorToHexCode[item.color],+ colorCode: DefaultColors[item.color],type: "color"}));if (newColorId) {@@ -275,6 +266,13 @@ const migrations: Migration[] = [}}+ if (item.locked) {+ const vault = await db.vaults.default();+ if (vault)+ await db.relations.add({ type: "vault", id: vault.id }, item);+ }++ delete item.locked;delete item.notebooks;delete item.tags;delete item.color;@@ -380,6 +378,10 @@ const migrations: Migration[] = [return true;},all: () => true+ },+ async vaultKey(db, key) {+ await db.vaults.add({ title: "Default", key });+ await db.storage().remove("vaultKey");}},{@@ -453,6 +455,29 @@ export async function migrateCollection(}}+export async function migrateVaultKey(+ db: Database,+ vaultKey: Cipher<"base64">,+ version: number+) {+ let migrationStartIndex = migrations.findIndex((m) => m.version === version);+ if (migrationStartIndex <= -1) {+ throw new Error(+ version > CURRENT_DATABASE_VERSION+ ? `Please update the app to the latest version.`+ : `You seem to be on a very outdated version. Please update the app to the latest version.`+ );+ }++ for (; migrationStartIndex < migrations.length; ++migrationStartIndex) {+ const migration = migrations[migrationStartIndex];+ if (migration.version === CURRENT_DATABASE_VERSION) break;++ if (!migration.vaultKey) continue;+ await migration.vaultKey(db, vaultKey);+ }+}+function replaceDateEditedWithDateModified(removeDateEditedProperty = false) {return function (item: any) {item.dateModified = item.dateEdited;diff --git a/packages/core/src/types.ts b/packages/core/src/types.tsindex ee7cfe32d..82f44044c 100644--- a/packages/core/src/types.ts+++ b/packages/core/src/types.ts@@ -73,6 +73,7 @@ export type Collections = {notehistory: "session";sessioncontent: "sessioncontent";settingsv2: "settingitem";+ vaults: "vault";/*** @deprecated only kept here for migration purposes@@ -102,6 +103,7 @@ export type GroupableItem = ValueOf<| "sessioncontent"| "settings"| "settingitem"+ | "vault">>;@@ -121,6 +123,7 @@ export type ItemMap = {session: HistorySession;sessioncontent: SessionContentItem;settingitem: SettingItem;+ vault: Vault;/*** @deprecated only kept here for migration purposes@@ -171,9 +174,12 @@ export interface Note extends BaseItem<"note"> {* @deprecated only kept here for migration purposes.*/notebooks?: NotebookReference[];+ /**+ * @deprecated+ */+ locked?: boolean;pinned: boolean;- locked: boolean;favorite: boolean;localOnly: boolean;conflicted: boolean;@@ -432,6 +438,11 @@ export interface SettingItem<value: SettingItemMap[TKey];}+export interface Vault extends BaseItem<"vault"> {+ title: string;+ key: Cipher<"base64">;+}+export interface DeletedItem {id: string;deleted: true;diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.tsindex 3a4fb3a65..0d6334888 100644--- a/packages/core/vitest.config.ts+++ b/packages/core/vitest.config.ts@@ -25,7 +25,9 @@ export default defineConfig({test: {setupFiles: ["./globals.setup.js"],coverage: {- reporter: ["text", "html"]+ reporter: ["text", "html"],+ exclude: ["src/utils/templates/html/languages/*.js"],+ include: ["src/**/*.ts"]},retry: 1,exclude: ["__benches__/**/*.bench.ts"],