A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: migrate key value storage to sqlite
Abdullah Atta28470d60c6c8
diff --git a/packages/core/__tests__/migrations.test.ts b/packages/core/__tests__/migrations.test.tsindex e62094b88..6748a7bfa 100644--- a/packages/core/__tests__/migrations.test.ts+++ b/packages/core/__tests__/migrations.test.ts@@ -18,12 +18,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { test, expect, describe } from "vitest";-import { migrateItem } from "../src/migrations";+import { migrateItem, migrateKV, migrateVaultKey } from "../src/migrations";import { databaseTest } from "./utils";import { getId, makeId } from "../src/utils/id";import { LegacySettingsItem } from "../src/types";+import { KEYS } from "../src/database/kv";-describe("[5.2] replace date edited with date modified", () => {+describe.concurrent("[5.2] replace date edited with date modified", () => {const itemsWithDateEdited = ["note", "notebook", "trash", "tiny"] as const;const itemsWithoutDateEdited = ["tag", "attachment", "settings"] as const;for (const type of itemsWithDateEdited) {@@ -88,7 +89,7 @@ test("[5.3] decode wrapped table html entities", () =>);}));-describe("[5.4] convert tiny to tiptap", () => {+describe.concurrent("[5.4] convert tiny to tiptap", () => {const cases = [{name: "preserve newlines in code blocks",@@ -268,7 +269,7 @@ test("[5.8] do nothing if backup type is not local", () =>expect(await migrateItem(item, 5.8, 5.9, "note", db, "sync")).toBe(false);}));-describe("[5.9] make tags syncable", () => {+describe.concurrent("[5.9] make tags syncable", () => {test("create tags inside notes & link to them using relations", () =>databaseTest().then(async (db) => {const noteId = getId();@@ -403,7 +404,7 @@ describe("[5.9] make tags syncable", () => {}));});-describe("[5.9] make colors syncable", () => {+describe.concurrent("[5.9] make colors syncable", () => {test("create colors from notes & link to them using relations", () =>databaseTest().then(async (db) => {const noteId = getId();@@ -566,65 +567,90 @@ test.todo("[5.9] flatten attachment object", () =>}));-describe("[5.9] move topics out of notebooks & use relations", () => {- test("convert topics to subnotebooks", () =>- databaseTest().then(async (db) => {- const notebook = {- id: "parent_notebook",- type: "notebook",- topics: [- { id: "topics1", title: "Topic 1" },- { id: "topics2", title: "Topic 2" }- ]- };- await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup");-- const linkedNotebooks = await db.relations- .from({ type: "notebook", id: "parent_notebook" }, "notebook")- .get();- expect(notebook.topics).toBeUndefined();- expect(linkedNotebooks).toHaveLength(2);- expect(linkedNotebooks.some((a) => a.toId === "topics1")).toBeTruthy();- expect(linkedNotebooks.some((a) => a.toId === "topics2")).toBeTruthy();- expect(await db.notebooks.all.count()).toBe(2);- expect(await db.notebooks.notebook("topics1")).toBeDefined();- expect(await db.notebooks.notebook("topics2")).toBeDefined();- }));+describe.concurrent(+ "[5.9] move topics out of notebooks & use relations",+ () => {+ test("move topics of deleted notebook to trash after migration", () =>+ databaseTest().then(async (db) => {+ const notebook = {+ id: "parent_notebook",+ type: "trash",+ itemType: "notebook",+ dateDeleted: Date.now(),+ topics: [+ { id: "topics1", title: "Topic 1" },+ { id: "topics2", title: "Topic 2" }+ ]+ };- test("convert topic shortcuts to notebook shortcuts", () =>- databaseTest().then(async (db) => {- const shortcut = {- id: "shortcut1",- type: "shortcut",- item: {- type: "topic",- id: "topics1"- }- };- await migrateItem(shortcut, 5.9, 6.0, "shortcut", db, "backup");+ await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup");- expect(shortcut.itemType).toBe("notebook");- expect(shortcut.itemId).toBe("topics1");- }));+ const trash = await db.trash.all();+ expect(await db.notebooks.notebook("topics1")).toBeUndefined();+ expect(await db.notebooks.notebook("topics2")).toBeUndefined();+ expect(trash.find((t) => t.title === "Topic 1")).toBeDefined();+ expect(trash.find((t) => t.title === "Topic 2")).toBeDefined();+ }));- test("convert topic links in note to relations", () =>- databaseTest().then(async (db) => {- const note = {- id: "note1",- type: "note",- notebooks: [{ id: "notebook1", topics: ["topic1", "topic2"] }]- };- await migrateItem(note, 5.9, 6.0, "note", db, "backup");-- const linkedNotebooks = await db.relations- .to({ type: "note", id: "note1" }, "notebook")- .get();- expect(note.notebooks).toBeUndefined();- expect(linkedNotebooks).toHaveLength(2);- expect(linkedNotebooks.some((a) => a.fromId === "topic1")).toBeTruthy();- expect(linkedNotebooks.some((a) => a.fromId === "topic2")).toBeTruthy();- }));-});+ test("convert topics to subnotebooks", () =>+ databaseTest().then(async (db) => {+ const notebook = {+ id: "parent_notebook",+ type: "notebook",+ topics: [+ { id: "topics1", title: "Topic 1" },+ { id: "topics2", title: "Topic 2" }+ ]+ };+ await migrateItem(notebook, 5.9, 6.0, "notebook", db, "backup");++ const linkedNotebooks = await db.relations+ .from({ type: "notebook", id: "parent_notebook" }, "notebook")+ .get();+ expect(notebook.topics).toBeUndefined();+ expect(linkedNotebooks).toHaveLength(2);+ expect(linkedNotebooks.some((a) => a.toId === "topics1")).toBeTruthy();+ expect(linkedNotebooks.some((a) => a.toId === "topics2")).toBeTruthy();+ expect(await db.notebooks.all.count()).toBe(2);+ expect(await db.notebooks.notebook("topics1")).toBeDefined();+ expect(await db.notebooks.notebook("topics2")).toBeDefined();+ }));++ test("convert topic shortcuts to notebook shortcuts", () =>+ databaseTest().then(async (db) => {+ const shortcut = {+ id: "shortcut1",+ type: "shortcut",+ item: {+ type: "topic",+ id: "topics1"+ }+ };+ await migrateItem(shortcut, 5.9, 6.0, "shortcut", db, "backup");++ expect(shortcut.itemType).toBe("notebook");+ expect(shortcut.itemId).toBe("topics1");+ }));++ test("convert topic links in note to relations", () =>+ databaseTest().then(async (db) => {+ const note = {+ id: "note1",+ type: "note",+ notebooks: [{ id: "notebook1", topics: ["topic1", "topic2"] }]+ };+ await migrateItem(note, 5.9, 6.0, "note", db, "backup");++ const linkedNotebooks = await db.relations+ .to({ type: "note", id: "note1" }, "notebook")+ .get();+ expect(note.notebooks).toBeUndefined();+ expect(linkedNotebooks).toHaveLength(2);+ expect(linkedNotebooks.some((a) => a.fromId === "topic1")).toBeTruthy();+ expect(linkedNotebooks.some((a) => a.fromId === "topic2")).toBeTruthy();+ }));+ }+);test("[5.9] migrate settings to its own collection", () =>databaseTest().then(async (db) => {@@ -665,3 +691,40 @@ test("[5.9] migrate settings to its own collection", () =>settings.trashCleanupInterval);}));++describe.concurrent("[5.9] migrate kv", () => {+ for (const key of KEYS) {+ test(`${key} (defined)`, () =>+ databaseTest().then(async (db) => {+ await db.storage().write(key, "test");++ await migrateKV(db, 5.9, 6.0);++ expect(await db.kv().read(key)).toBeDefined();+ expect(await db.storage().read(key)).toBeUndefined();+ }));++ test(`${key} (undefined)`, () =>+ databaseTest().then(async (db) => {+ await db.storage().write(key, null);++ await migrateKV(db, 5.9, 6.0);++ expect(await db.kv().read(key)).toBe(key === "v" ? 6 : undefined);+ expect(await db.storage().read(key)).toBe(null);+ }));+ }+});++test("[5.9] migrate vaultKey", () =>+ databaseTest().then(async (db) => {+ const key = await db.storage().encrypt({ password: "hello" }, "world");+ await db.storage().write("vaultKey", key);++ await migrateVaultKey(db, key, 5.9, 6.0);++ expect(await db.storage().read("vaultKey")).toBeUndefined();+ expect(await db.vaults.default()).toBeDefined();+ expect((await db.vaults.default())?.key).toStrictEqual(key);+ expect((await db.vaults.default())?.key).toStrictEqual(key);+ }));diff --git a/packages/core/__tests__/trash.test.ts b/packages/core/__tests__/trash.test.tsindex ab3e91bfe..60ae8fe59 100644--- a/packages/core/__tests__/trash.test.ts+++ b/packages/core/__tests__/trash.test.ts@@ -245,10 +245,11 @@ test("deleting a notebook should not re-delete already deleted subnotebooks", ()await db.notebooks.moveToTrash(child3);await db.notebooks.moveToTrash(parent);- expect((await db.trash.all()).some((a) => a.id === child3)).toBe(true);- expect((await db.trash.all()).some((a) => a.id === parent)).toBe(true);- expect((await db.trash.all()).some((a) => a.id === child2)).toBe(false);- expect((await db.trash.all()).some((a) => a.id === child)).toBe(false);+ const trash = await db.trash.all("user");+ expect(trash.some((a) => a.id === child3)).toBe(true);+ expect(trash.some((a) => a.id === parent)).toBe(true);+ expect(trash.some((a) => a.id === child2)).toBe(false);+ expect(trash.some((a) => a.id === child)).toBe(false);}));test("restoring a deleted notebook should also restore all its subnotebooks", () =>@@ -355,8 +356,9 @@ test("permanently deleting a notebook should not delete independently deleted suawait db.trash.delete(parent);- expect((await db.trash.all()).some((a) => a.id === child3)).toBe(true);- expect((await db.trash.all()).some((a) => a.id === parent)).toBe(false);- expect((await db.trash.all()).some((a) => a.id === child2)).toBe(false);- expect((await db.trash.all()).some((a) => a.id === child)).toBe(false);+ const trash = await db.trash.all("user");+ expect(trash.some((a) => a.id === child3)).toBe(true);+ expect(trash.some((a) => a.id === parent)).toBe(false);+ expect(trash.some((a) => a.id === child2)).toBe(false);+ expect(trash.some((a) => a.id === child)).toBe(false);}));diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.tsindex c32e6a168..65033c550 100644--- a/packages/core/src/api/index.ts+++ b/packages/core/src/api/index.ts@@ -29,7 +29,6 @@ import Vault from "./vault";import Lookup from "./lookup";import { Content } from "../collections/content";import Backup from "../database/backup";-import Session from "./session";import Hosts from "../utils/constants";import { EV, EVENTS } from "../common";import { LegacySettings } from "../collections/legacy-settings";@@ -55,6 +54,7 @@ import {ICompressor,IFileStorage,IStorage,+ KVStorageAccessor,StorageAccessor} from "../interfaces";import TokenManager from "./token-manager";@@ -69,6 +69,7 @@ import {import { Kysely, Transaction, sql } from "kysely";import { CachedCollection } from "../database/cached-collection";import { Vaults } from "../collections/vaults";+import { KVStorage } from "../database/kv";type EventSourceConstructor = new (uri: string,@@ -103,7 +104,7 @@ class Database {throw new Error("Database not initialized. Did you forget to call db.setup()?");- return new FileStorage(this.options.fs, this.storage);+ return new FileStorage(this.options.fs, this.tokenManager);};crypto: CryptoAccessor = () => {@@ -133,6 +134,9 @@ class Database {return this._sql;};+ private _kv?: KVStorage;+ kv: KVStorageAccessor = () => this._kv || new KVStorage(this.sql);+private _transaction?: Transaction<DatabaseSchema>;private transactionMutex = new Mutex();transaction = (@@ -156,9 +160,8 @@ class Database {EventSource?: EventSourceConstructor;eventSource?: EventSource | null;- session = new Session(this.storage);- mfa = new MFAManager(this.storage);- tokenManager = new TokenManager(this.storage);+ tokenManager = new TokenManager(this.kv);+ mfa = new MFAManager(this.tokenManager);subscriptions = new Subscriptions(this.tokenManager);offers = new Offers();debug = new Debug();@@ -189,19 +192,15 @@ class Database {/*** @deprecated only kept here for migration purposes*/- legacyTags = new CachedCollection(this.storage, "tags", this.eventManager);+ legacyTags = new CachedCollection(this.storage, "tags");/*** @deprecated only kept here for migration purposes*/- legacyColors = new CachedCollection(- this.storage,- "colors",- this.eventManager- );+ legacyColors = new CachedCollection(this.storage, "colors");/*** @deprecated only kept here for migration purposes*/- legacyNotes = new CachedCollection(this.storage, "notes", this.eventManager);+ legacyNotes = new CachedCollection(this.storage, "notes");/*** @deprecated only kept here for migration purposes*/@@ -216,15 +215,6 @@ class Database {this.options = options;}- async _validate() {- if (!(await this.session.valid())) {- throw new Error(- "Your system clock is not setup correctly. Please adjust your date and time and then retry."- );- }- await this.session.set();- }-async reset() {await this.storage().clear();@@ -269,8 +259,6 @@ class Database {this.options.sqliteOptions)) as unknown as Kysely<DatabaseSchema>;- await this._validate();-await this.initCollections();await this.migrations.init();@@ -383,7 +371,11 @@ class Database {}async lastSynced() {- return (await this.storage().read<number | undefined>("lastSynced")) || 0;+ return (await this.kv().read("lastSynced")) || 0;+ }++ setLastSynced(lastSynced: number) {+ return this.kv().write("lastSynced", lastSynced);}sync(options: SyncOptions) {diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.tsindex 5dd3fbfa9..49f519827 100644--- a/packages/core/src/api/lookup.ts+++ b/packages/core/src/api/lookup.ts@@ -20,7 +20,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { match } from "fuzzyjs";import Database from ".";import { Item, Note, TrashItem } from "../types";-import { DatabaseSchema, DatabaseSchemaWithFTS, isFalse } from "../database";+import { DatabaseSchema, RawDatabaseSchema, isFalse } from "../database";import { AnyColumnWithTable, Kysely, sql } from "kysely";import { FilteredSelector } from "../database/sql-collection";import { VirtualizedGrouping } from "../utils/virtualized-grouping";@@ -43,7 +43,7 @@ export default class Lookup {return this.toSearchResults(async (limit) => {if (query.length <= 3) return [];- const db = this.db.sql() as Kysely<DatabaseSchemaWithFTS>;+ const db = this.db.sql() as Kysely<RawDatabaseSchema>;query = query.replace(/"/, '""');const result = await db.with("matching", (eb) =>diff --git a/packages/core/src/api/mfa-manager.ts b/packages/core/src/api/mfa-manager.tsindex f9c2890d9..cb8bfda73 100644--- a/packages/core/src/api/mfa-manager.ts+++ b/packages/core/src/api/mfa-manager.ts@@ -20,7 +20,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import http from "../utils/http";import constants from "../utils/constants";import TokenManager from "./token-manager";-import { StorageAccessor } from "../interfaces";const ENDPOINTS = {setup: "/mfa",@@ -31,10 +30,7 @@ const ENDPOINTS = {};class MFAManager {- tokenManager: TokenManager;- constructor(private readonly storage: StorageAccessor) {- this.tokenManager = new TokenManager(storage);- }+ constructor(private readonly tokenManager: TokenManager) {}async setup(type: "app" | "sms" | "email", phoneNumber?: string) {const token = await this.tokenManager.getAccessToken();diff --git a/packages/core/src/api/migrations.ts b/packages/core/src/api/migrations.tsindex 42da09168..af772f4bd 100644--- a/packages/core/src/api/migrations.ts+++ b/packages/core/src/api/migrations.ts@@ -85,7 +85,7 @@ class Migrations {async init() {this.version =(await this.db.storage().read("v")) || CURRENT_DATABASE_VERSION;- this.db.storage().write("v", this.version);+ this.db.kv().write("v", this.version);}required() {@@ -100,7 +100,7 @@ class Migrations {await this.db.notes.init();await this.migrator.migrate(this.db, collections, this.version);- await this.db.storage().write("v", CURRENT_DATABASE_VERSION);+ await this.db.kv().write("v", CURRENT_DATABASE_VERSION);this.version = CURRENT_DATABASE_VERSION;} finally {this.migrating = false;diff --git a/packages/core/src/api/monographs.ts b/packages/core/src/api/monographs.tsindex 13b18636f..2070189c7 100644--- a/packages/core/src/api/monographs.ts+++ b/packages/core/src/api/monographs.ts@@ -46,7 +46,7 @@ export class Monographs {async clear() {this.monographs = [];- await this.db.storage().write("monographs", this.monographs);+ await this.db.kv().write("monographs", this.monographs);}async refresh() {@@ -59,7 +59,7 @@ export class Monographs {`${Constants.API_HOST}/monographs`,token);- await this.db.storage().write("monographs", monographs);+ await this.db.kv().write("monographs", monographs);if (monographs) this.monographs = monographs;} catch (e) {console.error(e);diff --git a/packages/core/src/api/session.ts b/packages/core/src/api/session.tsdeleted file mode 100644index 6cb989a7a..000000000--- a/packages/core/src/api/session.ts+++ /dev/null@@ -1,38 +0,0 @@-/*-This file is part of the Notesnook project (https://notesnook.com/)--Copyright (C) 2023 Streetwriters (Private) Limited--This program is free software: you can redistribute it and/or modify-it under the terms of the GNU General Public License as published by-the Free Software Foundation, either version 3 of the License, or-(at your option) any later version.--This program is distributed in the hope that it will be useful,-but WITHOUT ANY WARRANTY; without even the implied warranty of-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-GNU General Public License for more details.--You should have received a copy of the GNU General Public License-along with this program. If not, see <http://www.gnu.org/licenses/>.-*/--import { StorageAccessor } from "../interfaces";--class Session {- constructor(private readonly storage: StorageAccessor) {}-- get() {- return this.storage().read<number>("t");- }-- set() {- return this.storage().write("t", Date.now());- }-- async valid() {- const t = await this.get();- return !t || t < Date.now();- }-}-export default Session;diff --git a/packages/core/src/api/sync/devices.ts b/packages/core/src/api/sync/devices.tsindex 449340815..20e55e912 100644--- a/packages/core/src/api/sync/devices.ts+++ b/packages/core/src/api/sync/devices.ts@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { StorageAccessor } from "../../interfaces";+import { KVStorageAccessor } from "../../interfaces";import hosts from "../../utils/constants";import http from "../../utils/http";import { getId } from "../../utils/id";@@ -25,7 +25,7 @@ import TokenManager from "../token-manager";export class SyncDevices {constructor(- private readonly storage: StorageAccessor,+ private readonly kv: KVStorageAccessor,private readonly tokenManager: TokenManager) {}@@ -35,20 +35,18 @@ export class SyncDevices {const token = await this.tokenManager.getAccessToken();return http.post(url, null, token)- .then(() => this.storage().write("deviceId", deviceId));+ .then(() => this.kv().write("deviceId", deviceId));}async unregister() {- const deviceId = await this.storage().read("deviceId");+ const deviceId = await this.kv().read("deviceId");if (!deviceId) return;const url = `${hosts.API_HOST}/devices?deviceId=${deviceId}`;const token = await this.tokenManager.getAccessToken();- return http- .delete(url, token)- .then(() => this.storage().remove("deviceId"));+ return http.delete(url, token).then(() => this.kv().delete("deviceId"));}get() {- return this.storage().read<string>("deviceId");+ return this.kv().read("deviceId");}}diff --git a/packages/core/src/api/sync/index.ts b/packages/core/src/api/sync/index.tsindex 2d2ea1d08..5e5c38a29 100644--- a/packages/core/src/api/sync/index.ts+++ b/packages/core/src/api/sync/index.ts@@ -113,7 +113,7 @@ class Sync {logger = logger.scope("Sync");syncConnectionMutex = new Mutex();connection?: signalr.HubConnection;- devices = new SyncDevices(this.db.storage, this.db.tokenManager);+ devices = new SyncDevices(this.db.kv, this.db.tokenManager);constructor(private readonly db: Database) {EV.subscribe(EVENTS.userLoggedOut, async () => {@@ -209,7 +209,13 @@ class Sync {vaultKey.length > 0) {const vault = await this.db.vaults.default();- if (!vault) await migrateVaultKey(this.db, vaultKey, 5.9);+ if (!vault)+ await migrateVaultKey(+ this.db,+ vaultKey,+ 5.9,+ CURRENT_DATABASE_VERSION+ );}return true;@@ -258,7 +264,7 @@ class Sync {await this.db.monographs.refresh();this.logger.info("Stopping sync");- await this.db.storage().write("lastSynced", Date.now());+ await this.db.setLastSynced(Date.now());this.db.eventManager.publish(EVENTS.syncCompleted);}@@ -345,7 +351,7 @@ class Sync {private createConnection() {if (this.connection) return;- const tokenManager = new TokenManager(this.db.storage);+ const tokenManager = new TokenManager(this.db.kv);this.connection = new signalr.HubConnectionBuilder().withUrl(`${Constants.API_HOST}/hubs/sync/v2`, {accessTokenFactory: async () => {diff --git a/packages/core/src/api/token-manager.ts b/packages/core/src/api/token-manager.tsindex 1d9acc9f4..de612bda3 100644--- a/packages/core/src/api/token-manager.ts+++ b/packages/core/src/api/token-manager.ts@@ -22,9 +22,9 @@ import constants from "../utils/constants";import { EV, EVENTS } from "../common";import { withTimeout, Mutex } from "async-mutex";import { logger } from "../logger";-import { StorageAccessor } from "../interfaces";+import { KVStorageAccessor } from "../interfaces";-type Token = {+export type Token = {access_token: string;t: number;expires_in: number;@@ -42,10 +42,10 @@ const ENDPOINTS = {class TokenManager {mutex = withTimeout(new Mutex(), 10 * 1000);logger = logger.scope("TokenManager");- constructor(private readonly storage: StorageAccessor) {}+ constructor(private readonly storage: KVStorageAccessor) {}async getToken(renew = true, forceRenew = false): Promise<Token | undefined> {- const token = await this.storage().read<Token>("token");+ const token = await this.storage().read("token");if (!token || !token.access_token) return;this.logger.info("Access token requested", {@@ -119,7 +119,7 @@ class TokenManager {if (!token) return;const { access_token } = token;- await this.storage().remove("token");+ await this.storage().delete("token");await http.post(`${constants.AUTH_HOST}${ENDPOINTS.logout}`,null,diff --git a/packages/core/src/api/user-manager.ts b/packages/core/src/api/user-manager.tsindex 04d9da4ea..8362a7c58 100644--- a/packages/core/src/api/user-manager.ts+++ b/packages/core/src/api/user-manager.ts@@ -68,7 +68,7 @@ const ENDPOINTS = {class UserManager {private tokenManager: TokenManager;constructor(private readonly db: Database) {- this.tokenManager = new TokenManager(this.db.storage);+ this.tokenManager = new TokenManager(this.db.kv);EV.subscribe(EVENTS.userUnauthorized, async (url: string) => {if (url.includes("/connect/token") || !(await HealthCheck.auth())) return;@@ -175,7 +175,7 @@ class UserManager {if (!user) throw new Error("Unauthorized.");if (!sessionExpired) {- await this.db.storage().write("lastSynced", 0);+ await this.db.setLastSynced(0);await this.db.syncer.devices.register();}@@ -228,7 +228,7 @@ class UserManager {password,salt: user.salt});- await this.db.storage().write("lastSynced", 0);+ await this.db.setLastSynced(0);await this.db.syncer.devices.register();EV.publish(EVENTS.userLoggedIn, user);@@ -276,11 +276,11 @@ class UserManager {}setUser(user: User) {- return this.db.storage().write("user", user);+ return this.db.kv().write("user", user);}getUser() {- return this.db.storage().read<User>("user");+ return this.db.kv().read("user");}async resetUser(removeAttachments = true) {diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.tsindex fc5f9851d..e538532a8 100644--- a/packages/core/src/collections/notebooks.ts+++ b/packages/core/src/collections/notebooks.ts@@ -24,6 +24,7 @@ import { ICollection } from "./collection";import { SQLCollection } from "../database/sql-collection";import { isFalse } from "../database";import { sql } from "kysely";+import { deleteItems } from "../utils/array";export class Notebooks implements ICollection {name = "notebooks";@@ -240,7 +241,7 @@ export class Notebooks implements ICollection {}async moveToTrash(...ids: string[]) {- this.db.transaction(async (tr) => {+ await this.db.transaction(async (tr) => {const query = tr.withRecursive(`subNotebooks(id)`, (eb) =>eb@@ -265,7 +266,9 @@ export class Notebooks implements ICollection {.select("id");const subNotebookIds = (await query.execute()).map((ref) => ref.id);- await this.db.trash.add("notebook", subNotebookIds, "app");+ deleteItems(subNotebookIds, ...ids);+ if (subNotebookIds.length > 0)+ await this.db.trash.add("notebook", subNotebookIds, "app");await this.db.trash.add("notebook", ids, "user");});}diff --git a/packages/core/src/collections/trash.ts b/packages/core/src/collections/trash.tsindex 9c67653a9..6efb5dbfb 100644--- a/packages/core/src/collections/trash.ts+++ b/packages/core/src/collections/trash.ts@@ -201,31 +201,37 @@ export default class Trash {// } else return true;// }- async all() {+ async all(deletedBy?: TrashItem["deletedBy"]) {return [- ...(await this.trashedNotes(this.cache.notes)),- ...(await this.trashedNotebooks(this.cache.notebooks))+ ...(await this.trashedNotes(this.cache.notes, deletedBy)),+ ...(await this.trashedNotebooks(this.cache.notebooks, deletedBy))] as TrashItem[];}- private async trashedNotes(ids: string[]) {+ private async trashedNotes(+ ids: string[],+ deletedBy?: TrashItem["deletedBy"]+ ) {return (await this.db.sql().selectFrom("notes").where("type", "==", "trash").where("id", "in", ids)- .where("deletedBy", "==", "user")+ .$if(!!deletedBy, (eb) => eb.where("deletedBy", "==", deletedBy)).selectAll().execute()) as TrashItem[];}- private async trashedNotebooks(ids: string[]) {+ private async trashedNotebooks(+ ids: string[],+ deletedBy?: TrashItem["deletedBy"]+ ) {return (await this.db.sql().selectFrom("notebooks").where("type", "==", "trash").where("id", "in", ids)- .where("deletedBy", "==", "user")+ .$if(!!deletedBy, (eb) => eb.where("deletedBy", "==", deletedBy)).selectAll().execute()) as TrashItem[];}@@ -249,10 +255,12 @@ export default class Trash {const items = [...(await this.trashedNotes(- this.cache.notes.slice(notesRange[0], notesRange[1])+ this.cache.notes.slice(notesRange[0], notesRange[1]),+ "user")),...(await this.trashedNotebooks(- this.cache.notebooks.slice(notebooksRange[0], notebooksRange[1])+ this.cache.notebooks.slice(notebooksRange[0], notebooksRange[1]),+ "user"))];items.sort(selector);@@ -314,6 +322,10 @@ export default class Trash {.selectFrom("subNotebooks").select("id").execute();- return ids.map((ref) => ref.id);++ return deleteItems(+ ids.map((ref) => ref.id),+ ...notebookIds+ );}}diff --git a/packages/core/src/database/backup.ts b/packages/core/src/database/backup.tsindex 9a42b4781..0de75a9de 100644--- a/packages/core/src/database/backup.ts+++ b/packages/core/src/database/backup.ts@@ -154,11 +154,11 @@ export default class Backup {constructor(private readonly db: Database) {}lastBackupTime() {- return this.db.storage().read<number>("lastBackupTime");+ return this.db.kv().read("lastBackupTime");}async updateBackupTime() {- await this.db.storage().write("lastBackupTime", Date.now());+ await this.db.kv().write("lastBackupTime", Date.now());}async *export(type: BackupPlatform, encrypt = false) {diff --git a/packages/core/src/database/cached-collection.ts b/packages/core/src/database/cached-collection.tsindex 796ae609c..b04143fb6 100644--- a/packages/core/src/database/cached-collection.ts+++ b/packages/core/src/database/cached-collection.ts@@ -26,8 +26,6 @@ import {isDeleted} from "../types";import { StorageAccessor } from "../interfaces";-import EventManager from "../utils/event-manager";-import { chunkedIterate } from "../utils/array";/*** @deprecated only kept here for migration purposes@@ -40,12 +38,8 @@ export class CachedCollection<private cache = new Map<string, MaybeDeletedItem<T>>();private cachedItems?: T[];- constructor(- storage: StorageAccessor,- type: TCollectionType,- eventManager: EventManager- ) {- this.collection = new IndexedCollection(storage, type, eventManager);+ constructor(storage: StorageAccessor, type: TCollectionType) {+ this.collection = new IndexedCollection(storage, type);}async init() {@@ -68,28 +62,6 @@ export class CachedCollection<this.invalidateCache();}- async update(item: T) {- await this.collection.updateItem(item);- this.cache.set(item.id, item);- this.invalidateCache();- }-- async delete(id: string) {- this.cache.delete(id);- await this.collection.deleteItem(id);- this.invalidateCache();- }-- async remove(id: string) {- this.cache.set(id, {- id,- deleted: true,- dateModified: Date.now()- });- await this.collection.removeItem(id);- this.invalidateCache();- }-exists(id: string) {const item = this.cache.get(id);return this.collection.exists(id) && !!item && !isDeleted(item);@@ -103,19 +75,10 @@ export class CachedCollection<return this.cache.size;}- get(id: string) {- const item = this.cache.get(id);- if (!item || isDeleted(item)) return;- return item;- }-- getRaw(id: string) {- const item = this.cache.get(id);- return item;- }-- raw() {- return Array.from(this.cache.values());+ async delete(id: string) {+ this.cache.delete(id);+ await this.collection.deleteItem(id);+ this.invalidateCache();}items(map?: (item: T) => T | undefined) {@@ -133,19 +96,10 @@ export class CachedCollection<return this.cachedItems;}- async setItems(items: (MaybeDeletedItem<T> | undefined)[]) {- await this.collection.setItems(items);- for (const item of items) {- if (item) {- this.cache.set(item.id, item);- }- }-- this.invalidateCache();- }-- *iterateSync(chunkSize: number) {- yield* chunkedIterate(Array.from(this.cache.values()), chunkSize);+ get(id: string) {+ const item = this.cache.get(id);+ if (!item || isDeleted(item)) return;+ return item;}invalidateCache() {diff --git a/packages/core/src/database/fs.ts b/packages/core/src/database/fs.tsindex f635d1d4a..d7bea463f 100644--- a/packages/core/src/database/fs.ts+++ b/packages/core/src/database/fs.ts@@ -21,8 +21,7 @@ import hosts from "../utils/constants";import TokenManager from "../api/token-manager";import {FileEncryptionMetadataWithOutputType,- IFileStorage,- StorageAccessor+ IFileStorage} from "../interfaces";import { DataFormat, SerializedKey } from "@notesnook/crypto/dist/src/types";import { EV, EVENTS } from "../common";@@ -38,12 +37,12 @@ export type QueueItem = DownloadableFile & {};export class FileStorage {- private readonly tokenManager: TokenManager;downloads = new Map<string, QueueItem[]>();uploads = new Map<string, QueueItem[]>();- constructor(private readonly fs: IFileStorage, storage: StorageAccessor) {- this.tokenManager = new TokenManager(storage);- }+ constructor(+ private readonly fs: IFileStorage,+ private readonly tokenManager: TokenManager+ ) {}async queueDownloads(files: DownloadableFile[],diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.tsindex 40dfb065b..95a86ab71 100644--- a/packages/core/src/database/index.ts+++ b/packages/core/src/database/index.ts@@ -89,7 +89,13 @@ export interface DatabaseSchema {vaults: SQLiteItem<Vault>;}-export type DatabaseSchemaWithFTS = DatabaseSchema & {+export type RawDatabaseSchema = DatabaseSchema & {+ kv: {+ key: string;+ value?: string | null;+ dateModified?: number | null;+ };+notes_fts: SQLiteItemWithRowID<{notes_fts: string;title: string;@@ -136,9 +142,9 @@ export interface DatabaseCollection<T, IsAsync extends boolean> {): IsAsync extends true ? AsyncIterableIterator<T> : IterableIterator<T>;}-export type DatabaseAccessor = () =>- | Kysely<DatabaseSchema>- | Transaction<DatabaseSchema>;+export type DatabaseAccessor<TSchema = DatabaseSchema> = () =>+ | Kysely<TSchema>+ | Transaction<TSchema>;type FilterBooleanProperties<T, Type> = keyof {[K in keyof T as T[K] extends Type ? K : never]: T[K];@@ -212,7 +218,7 @@ export type SQLiteOptions = {pageSize?: number;};export async function createDatabase(name: string, options: SQLiteOptions) {- const db = new Kysely<DatabaseSchemaWithFTS>({+ const db = new Kysely<RawDatabaseSchema>({dialect: options.dialect(name),plugins: [new SqliteBooleanPlugin()]});diff --git a/packages/core/src/database/indexed-collection.ts b/packages/core/src/database/indexed-collection.tsindex 99168e291..aa60e4928 100644--- a/packages/core/src/database/indexed-collection.ts+++ b/packages/core/src/database/indexed-collection.ts@@ -17,30 +17,26 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { EVENTS } from "../common";import { toChunks } from "../utils/array";import { StorageAccessor } from "../interfaces";import {CollectionType,Collections,ItemMap,- MaybeDeletedItem,- isDeleted+ MaybeDeletedItem} from "../types";-import EventManager from "../utils/event-manager";import Indexer from "./indexer";+/**+ * @deprecated only kept here for migration purposes+ */export class IndexedCollection<TCollectionType extends CollectionType = CollectionType,T extends ItemMap[Collections[TCollectionType]] = ItemMap[Collections[TCollectionType]]> {readonly indexer: Indexer<T>;- constructor(- storage: StorageAccessor,- type: TCollectionType,- private readonly eventManager: EventManager- ) {+ constructor(storage: StorageAccessor, type: TCollectionType) {this.indexer = new Indexer(storage, type);}@@ -48,82 +44,24 @@ export class IndexedCollection<return this.indexer.clear();}+ async deleteItem(id: string) {+ await this.indexer.deindex(id);+ return await this.indexer.remove(id);+ }+async init() {await this.indexer.init();}async addItem(item: MaybeDeletedItem<T>) {- if (!item.id) throw new Error("The item must contain the id field.");-- const exists = this.exists(item.id);- if (!exists && !isDeleted(item))- item.dateCreated = item.dateCreated || Date.now();- await this.updateItem(item);- if (!exists) {- await this.indexer.index(item.id);- }- }-- async updateItem(item: MaybeDeletedItem<T>) {- if (!item.id) throw new Error("The item must contain the id field.");- this.eventManager.publish(EVENTS.databaseUpdated, item.id, item);-- // if item is newly synced, remote will be true.- if (!item.remote) {- item.dateModified = Date.now();- item.synced = false;- }- // the item has become local now, so remove the flags- delete item.remote;await this.indexer.write(item.id, item);- }-- removeItem(id: string) {- this.eventManager.publish(EVENTS.databaseUpdated, id);- return this.indexer.write(id, {- id,- deleted: true,- dateModified: Date.now()- });- }-- async deleteItem(id: string) {- this.eventManager.publish(EVENTS.databaseUpdated, id);- await this.indexer.deindex(id);- return await this.indexer.remove(id);+ await this.indexer.index(item.id);}exists(id: string) {return this.indexer.exists(id);}- async getItem(id: string) {- const item = await this.indexer.read(id);- if (!item) return;- return item;- }-- async getItems(indices: string[]) {- const data = await this.indexer.readMulti(indices);- return Object.fromEntries(data);- }-- setItems(items: (MaybeDeletedItem<T> | undefined)[]) {- const entries = items.reduce((array, item) => {- if (!item) return array;-- if (!item.remote) {- item.dateModified = Date.now();- item.synced = false;- }- delete item.remote;-- array.push([item.id, item]);- return array;- }, [] as [string, MaybeDeletedItem<T>][]);- return this.indexer.writeMulti(entries);- }-async *iterate(chunkSize: number) {const chunks = toChunks(this.indexer.indices, chunkSize);for (const chunk of chunks) {diff --git a/packages/core/src/database/kv.ts b/packages/core/src/database/kv.tsnew file mode 100644index 000000000..7dbb305bd--- /dev/null+++ b/packages/core/src/database/kv.ts@@ -0,0 +1,75 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { DatabaseAccessor, RawDatabaseSchema } from ".";+import { Token } from "../api/token-manager";+import { User } from "../api/user-manager";++interface KV {+ v: number;+ lastSynced: number;+ user: User;+ token: Token;+ monographs: string[];+ deviceId: string;+ lastBackupTime: number;+}++export const KEYS: (keyof KV)[] = [+ "v",+ "lastSynced",+ "user",+ "token",+ "monographs",+ "deviceId",+ "lastBackupTime"+];++export class KVStorage {+ private readonly db: DatabaseAccessor<RawDatabaseSchema>;+ constructor(db: DatabaseAccessor) {+ this.db = db as unknown as DatabaseAccessor<RawDatabaseSchema>;+ }++ async read<T extends keyof KV>(key: T): Promise<KV[T] | undefined> {+ const result = await this.db()+ .selectFrom("kv")+ .where("key", "==", key)+ .select("value")+ .limit(1)+ .executeTakeFirst();+ if (!result?.value) return;+ return JSON.parse(result.value) as KV[T];+ }++ async write<T extends keyof KV>(key: T, value: KV[T]) {+ await this.db()+ .replaceInto("kv")+ .values({+ key,+ value: JSON.stringify(value),+ dateModified: Date.now()+ })+ .execute();+ }++ async delete<T extends keyof KV>(key: T) {+ await this.db().deleteFrom("kv").where("key", "==", key).execute();+ }+}diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.tsindex 7949816cb..087f2bbe6 100644--- a/packages/core/src/database/migrations.ts+++ b/packages/core/src/database/migrations.ts@@ -33,6 +33,14 @@ export class NNMigrationProvider implements MigrationProvider {return {"1": {async up(db) {+ await db.schema+ .createTable("kv")+ .modifyEnd(sql`without rowid`)+ .addColumn("key", "text", (c) => c.primaryKey().unique().notNull())+ .addColumn("value", "text")+ .addColumn("dateModified", "integer")+ .execute();+await db.schema.createTable("notes")// .modifyEnd(sql`without rowid`)diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.tsindex 3708f6a90..707ff64f5 100644--- a/packages/core/src/database/migrator.ts+++ b/packages/core/src/database/migrator.ts@@ -24,7 +24,12 @@ import {CURRENT_DATABASE_VERSION,sendMigrationProgressEvent} from "../common";-import { migrateCollection, migrateItem, migrateVaultKey } from "../migrations";+import {+ migrateCollection,+ migrateItem,+ migrateKV,+ migrateVaultKey+} from "../migrations";import {CollectionType,Collections,@@ -52,15 +57,16 @@ class Migrator {version: number) {const vaultKey = await db.storage().read<Cipher<"base64">>("vaultKey");- if (vaultKey) await migrateVaultKey(db, vaultKey, version);+ if (vaultKey)+ await migrateVaultKey(db, vaultKey, version, CURRENT_DATABASE_VERSION);+ await migrateKV(db, version, CURRENT_DATABASE_VERSION);for (const collection of collections) {sendMigrationProgressEvent(db.eventManager, collection.name, 0, 0);const indexedCollection = new IndexedCollection(db.storage,- collection.name,- db.eventManager+ collection.name);const table = new SQLCollection(db.sql,diff --git a/packages/core/src/database/triggers.ts b/packages/core/src/database/triggers.tsindex 284dc89b6..4e70d5466 100644--- a/packages/core/src/database/triggers.ts+++ b/packages/core/src/database/triggers.ts@@ -18,9 +18,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { Kysely, sql } from "kysely";-import { DatabaseSchemaWithFTS } from ".";+import { RawDatabaseSchema } from ".";-export async function createTriggers(db: Kysely<DatabaseSchemaWithFTS>) {+export async function createTriggers(db: Kysely<RawDatabaseSchema>) {// content triggersawait db.schema.createTrigger("content_after_insert_content_fts")diff --git a/packages/core/src/interfaces.ts b/packages/core/src/interfaces.tsindex 2b5c47dac..5d792e312 100644--- a/packages/core/src/interfaces.ts+++ b/packages/core/src/interfaces.ts@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";+import { KVStorage } from "./database/kv";export type Output<TOutputFormat extends DataFormat> =TOutputFormat extends Omit<DataFormat, "uint8array"> ? string : Uint8Array;@@ -111,4 +112,5 @@ export interface IFileStorage {}export type StorageAccessor = () => IStorage;+export type KVStorageAccessor = () => KVStorage;export type CompressorAccessor = () => ICompressor;diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.tsindex c74a07546..407698948 100644--- a/packages/core/src/migrations.ts+++ b/packages/core/src/migrations.ts@@ -36,6 +36,7 @@ import { isCipher } from "./database/crypto";import { IndexedCollection } from "./database/indexed-collection";import { DefaultColors } from "./collections/colors";import { Cipher } from "@notesnook/crypto";+import { KEYS } from "./database/kv";type MigrationType = "local" | "sync" | "backup";type MigrationItemType = ItemType | "notehistory" | "content" | "all";@@ -55,6 +56,7 @@ type Migration = {};collection?: (collection: IndexedCollection) => Promise<void> | void;vaultKey?: (db: Database, key: Cipher<"base64">) => Promise<void> | void;+ kv?: (db: Database) => Promise<void> | void;};const migrations: Migration[] = [@@ -317,7 +319,7 @@ const migrations: Migration[] = [await db.relations.add(item, { id: subNotebookId, type: "notebook" });// if the parent notebook is deleted, we should delete the newly// created notebooks too- if (item.dateDeleted !== null) {+ if (item.dateDeleted) {await db.trash.add("notebook", [subNotebookId], "app");}}@@ -391,6 +393,14 @@ const migrations: Migration[] = [async vaultKey(db, key) {await db.vaults.add({ title: "Default", key });await db.storage().remove("vaultKey");+ },+ async kv(db) {+ for (const key of KEYS) {+ const value = await db.storage().read(key);+ if (value === undefined || value === null) continue;+ await db.kv().write(key, value as any);+ await db.storage().remove(key);+ }}},{@@ -467,12 +477,13 @@ export async function migrateCollection(export async function migrateVaultKey(db: Database,vaultKey: Cipher<"base64">,- version: number+ version: number,+ databaseVersion: number) {let migrationStartIndex = migrations.findIndex((m) => m.version === version);if (migrationStartIndex <= -1) {throw new Error(- version > CURRENT_DATABASE_VERSION+ version > databaseVersion? `Please update the app to the latest version.`: `You seem to be on a very outdated version. Please update the app to the latest version.`);@@ -480,13 +491,36 @@ export async function migrateVaultKey(for (; migrationStartIndex < migrations.length; ++migrationStartIndex) {const migration = migrations[migrationStartIndex];- if (migration.version === CURRENT_DATABASE_VERSION) break;+ if (migration.version === databaseVersion) break;if (!migration.vaultKey) continue;await migration.vaultKey(db, vaultKey);}}+export async function migrateKV(+ db: Database,+ version: number,+ databaseVersion: number+) {+ let migrationStartIndex = migrations.findIndex((m) => m.version === version);+ if (migrationStartIndex <= -1) {+ throw new Error(+ version > databaseVersion+ ? `Please update the app to the latest version.`+ : `You seem to be on a very outdated version. Please update the app to the latest version.`+ );+ }++ for (; migrationStartIndex < migrations.length; ++migrationStartIndex) {+ const migration = migrations[migrationStartIndex];+ if (migration.version === databaseVersion) break;++ if (!migration.kv) continue;+ await migration.kv(db);+ }+}+function replaceDateEditedWithDateModified(removeDateEditedProperty = false) {return function (item: any) {item.dateModified = item.dateEdited;diff --git a/packages/core/src/utils/array.ts b/packages/core/src/utils/array.tsindex fa92c2955..7a9608193 100644--- a/packages/core/src/utils/array.ts+++ b/packages/core/src/utils/array.ts@@ -39,6 +39,7 @@ export function deleteItems<T>(array: T[], ...items: T[]) {for (const item of items) {deleteItem(array, item);}+ return array;}export function findById<T extends { id: string }>(array: T[], id: string) {