A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: migrate all database writes to sqlite
Abdullah Attaa91717aca3c7
diff --git a/packages/core/__benches__/relations.bench.ts b/packages/core/__benches__/relations.bench.tsnew file mode 100644index 000000000..e28e4f01d--- /dev/null+++ b/packages/core/__benches__/relations.bench.ts@@ -0,0 +1,57 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { bench, describe } from "vitest";+import { databaseTest } from "../__tests__/utils";++describe("relations", async () => {+ const db = await databaseTest();+ // const totalNotebooks = 10;++ // let parentNotebookId: string | undefined = undefined;+ // for (let i = 1; i <= 100; ++i) {+ // const id = await db.notebooks.add({ title: `notebook-somethign-${i}` });+ // if (parentNotebookId)+ // await db.relations.add(+ // { id: parentNotebookId, type: "notebook" },+ // { id, type: "notebook" }+ // );+ // parentNotebookId = id;+ // for (let j = 1; j <= 100; ++j) {+ // await db.relations.add(+ // { type: "notebook", id },+ // { id: `${j * i}-note`, type: "note" }+ // );+ // }+ // }++ //++ // const id2 = await db.notebooks.add({ title: `notebook-somethign` });++ console.log(await db.notebooks.totalNotes("6516a04a35a073f359e7e801"));++ // console.log(+ // await db.relations.from({ id: "8-note", type: "note" }, "notebook").unlink()+ // );++ // bench("get some relations from 10k relations", async () => {+ // await db.notebooks.totalNotes("6516a04a35a073f359e7e801");+ // });+});diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.tsindex dc62615af..31f31c7ed 100644--- a/packages/core/__tests__/notes.test.ts+++ b/packages/core/__tests__/notes.test.ts@@ -68,11 +68,12 @@ test("add invalid note", () =>expect(db.notes.add({ hello: "world" })).rejects.toThrow();}));-test("add note", () =>+test.only("add note", () =>noteTest().then(async ({ db, id }) => {- const note = db.notes.note(id);+ const note = await db.notes.note$(id);expect(note).toBeDefined();- expect(await note?.content()).toStrictEqual(TEST_NOTE.content.data);+ const content = await db.content.get(note!.contentId!);+ expect(content!.data).toStrictEqual(TEST_NOTE.content.data);}));test("get note content", () =>diff --git a/packages/core/nn.db b/packages/core/nn.dbnew file mode 100644index 000000000..2701cf90aBinary files /dev/null and b/packages/core/nn.db differdiff --git a/packages/core/package-lock.json b/packages/core/package-lock.jsonindex 94a35ef6e..2258b8bed 100644--- a/packages/core/package-lock.json+++ b/packages/core/package-lock.json@@ -20,6 +20,7 @@"html-to-text": "^9.0.5","htmlparser2": "^8.0.1","katex": "0.16.2",+ "kysely": "^0.26.3","linkedom": "^0.14.17","liqe": "^1.13.0","mime-db": "1.52.0",@@ -30,6 +31,7 @@},"devDependencies": {"@notesnook/crypto": "file:../crypto",+ "@types/better-sqlite3": "^7.6.5","@types/event-source-polyfill": "^1.0.1","@types/html-to-text": "^9.0.0","@types/katex": "^0.16.2",@@ -40,6 +42,7 @@"@types/ws": "^8.5.5","@vitest/coverage-v8": "^0.34.1","abortcontroller-polyfill": "^1.7.3",+ "better-sqlite3": "^8.6.0","cross-env": "^7.0.3","dotenv": "^16.0.1","event-source-polyfill": "^1.0.31",@@ -2173,6 +2176,15 @@"node": ">= 10"}},+ "node_modules/@types/better-sqlite3": {+ "version": "7.6.9",+ "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.9.tgz",+ "integrity": "sha512-FvktcujPDj9XKMJQWFcl2vVl7OdRIqsSRX9b0acWwTmwLK9CF2eqo/FRcmMLNpugKoX/avA6pb7TorDLmpgTnQ==",+ "dev": true,+ "dependencies": {+ "@types/node": "*"+ }+ },"node_modules/@types/chai": {"version": "4.3.14","resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.14.tgz",@@ -2444,6 +2456,57 @@"dev": true,"license": "MIT"},+ "node_modules/base64-js": {+ "version": "1.5.1",+ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",+ "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",+ "dev": true,+ "funding": [+ {+ "type": "github",+ "url": "https://github.com/sponsors/feross"+ },+ {+ "type": "patreon",+ "url": "https://www.patreon.com/feross"+ },+ {+ "type": "consulting",+ "url": "https://feross.org/support"+ }+ ]+ },+ "node_modules/better-sqlite3": {+ "version": "8.7.0",+ "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-8.7.0.tgz",+ "integrity": "sha512-99jZU4le+f3G6aIl6PmmV0cxUIWqKieHxsiF7G34CVFiE+/UabpYqkU0NJIkY/96mQKikHeBjtR27vFfs5JpEw==",+ "dev": true,+ "hasInstallScript": true,+ "dependencies": {+ "bindings": "^1.5.0",+ "prebuild-install": "^7.1.1"+ }+ },+ "node_modules/bindings": {+ "version": "1.5.0",+ "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",+ "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",+ "dev": true,+ "dependencies": {+ "file-uri-to-path": "1.0.0"+ }+ },+ "node_modules/bl": {+ "version": "4.1.0",+ "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",+ "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",+ "dev": true,+ "dependencies": {+ "buffer": "^5.5.0",+ "inherits": "^2.0.4",+ "readable-stream": "^3.4.0"+ }+ },"node_modules/boolbase": {"version": "1.0.0","license": "ISC"@@ -2457,6 +2520,30 @@"concat-map": "0.0.1"}},+ "node_modules/buffer": {+ "version": "5.7.1",+ "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",+ "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",+ "dev": true,+ "funding": [+ {+ "type": "github",+ "url": "https://github.com/sponsors/feross"+ },+ {+ "type": "patreon",+ "url": "https://www.patreon.com/feross"+ },+ {+ "type": "consulting",+ "url": "https://feross.org/support"+ }+ ],+ "dependencies": {+ "base64-js": "^1.3.1",+ "ieee754": "^1.1.13"+ }+ },"node_modules/cac": {"version": "6.7.14","resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz",@@ -2523,6 +2610,12 @@"node": "*"}},+ "node_modules/chownr": {+ "version": "1.1.4",+ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",+ "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",+ "dev": true+ },"node_modules/combined-stream": {"version": "1.0.8","dev": true,@@ -2667,6 +2760,21 @@"url": "https://github.com/sponsors/wooorm"}},+ "node_modules/decompress-response": {+ "version": "6.0.0",+ "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",+ "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",+ "dev": true,+ "dependencies": {+ "mimic-response": "^3.1.0"+ },+ "engines": {+ "node": ">=10"+ },+ "funding": {+ "url": "https://github.com/sponsors/sindresorhus"+ }+ },"node_modules/deep-eql": {"version": "4.1.3","resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.3.tgz",@@ -2679,6 +2787,15 @@"node": ">=6"}},+ "node_modules/deep-extend": {+ "version": "0.6.0",+ "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",+ "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",+ "dev": true,+ "engines": {+ "node": ">=4.0.0"+ }+ },"node_modules/deepmerge": {"version": "4.3.1","license": "MIT",@@ -2694,6 +2811,15 @@"node": ">=0.4.0"}},+ "node_modules/detect-libc": {+ "version": "2.0.2",+ "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz",+ "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==",+ "dev": true,+ "engines": {+ "node": ">=8"+ }+ },"node_modules/diff-sequences": {"version": "29.6.3","resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz",@@ -2773,6 +2899,15 @@"node": ">=12"}},+ "node_modules/end-of-stream": {+ "version": "1.4.4",+ "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",+ "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",+ "dev": true,+ "dependencies": {+ "once": "^1.4.0"+ }+ },"node_modules/entities": {"version": "4.5.0","license": "BSD-2-Clause",@@ -2840,6 +2975,15 @@"node": ">=12.0.0"}},+ "node_modules/expand-template": {+ "version": "2.0.3",+ "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",+ "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",+ "dev": true,+ "engines": {+ "node": ">=6"+ }+ },"node_modules/fetch-cookie": {"version": "2.1.0","license": "Unlicense",@@ -2848,6 +2992,12 @@"tough-cookie": "^4.0.0"}},+ "node_modules/file-uri-to-path": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",+ "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",+ "dev": true+ },"node_modules/form-data": {"version": "4.0.0","dev": true,@@ -2861,25 +3011,17 @@"node": ">= 6"}},+ "node_modules/fs-constants": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",+ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",+ "dev": true+ },"node_modules/fs.realpath": {"version": "1.0.0","dev": true,"license": "ISC"},- "node_modules/fsevents": {- "version": "2.3.3",- "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",- "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",- "dev": true,- "hasInstallScript": true,- "optional": true,- "os": [- "darwin"- ],- "engines": {- "node": "^8.16.0 || ^10.6.0 || >=11.0.0"- }- },"node_modules/get-func-name": {"version": "2.0.2","resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz",@@ -2889,6 +3031,12 @@"node": "*"}},+ "node_modules/github-from-package": {+ "version": "0.0.0",+ "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",+ "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",+ "dev": true+ },"node_modules/glob": {"version": "7.2.3","dev": true,@@ -3032,6 +3180,26 @@"node": ">=0.10.0"}},+ "node_modules/ieee754": {+ "version": "1.2.1",+ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",+ "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",+ "dev": true,+ "funding": [+ {+ "type": "github",+ "url": "https://github.com/sponsors/feross"+ },+ {+ "type": "patreon",+ "url": "https://www.patreon.com/feross"+ },+ {+ "type": "consulting",+ "url": "https://feross.org/support"+ }+ ]+ },"node_modules/inflight": {"version": "1.0.6","dev": true,@@ -3046,6 +3214,12 @@"dev": true,"license": "ISC"},+ "node_modules/ini": {+ "version": "1.3.8",+ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",+ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",+ "dev": true+ },"node_modules/is-alphabetical": {"version": "2.0.1","dev": true,@@ -3252,6 +3426,14 @@"node": ">= 12"}},+ "node_modules/kysely": {+ "version": "0.26.3",+ "resolved": "https://registry.npmjs.org/kysely/-/kysely-0.26.3.tgz",+ "integrity": "sha512-yWSgGi9bY13b/W06DD2OCDDHQmq1kwTGYlQ4wpZkMOJqMGCstVCFIvxCCVG4KfY1/3G0MhDAcZsip/Lw8/vJWw==",+ "engines": {+ "node": ">=14.0.0"+ }+ },"node_modules/leac": {"version": "0.6.0","license": "MIT",@@ -3360,6 +3542,18 @@"node": ">= 0.6"}},+ "node_modules/mimic-response": {+ "version": "3.1.0",+ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",+ "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",+ "dev": true,+ "engines": {+ "node": ">=10"+ },+ "funding": {+ "url": "https://github.com/sponsors/sindresorhus"+ }+ },"node_modules/minimatch": {"version": "3.1.2","dev": true,@@ -3371,6 +3565,21 @@"node": "*"}},+ "node_modules/minimist": {+ "version": "1.2.8",+ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",+ "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",+ "dev": true,+ "funding": {+ "url": "https://github.com/sponsors/ljharb"+ }+ },+ "node_modules/mkdirp-classic": {+ "version": "0.5.3",+ "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",+ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",+ "dev": true+ },"node_modules/mlly": {"version": "1.6.1","resolved": "https://registry.npmjs.org/mlly/-/mlly-1.6.1.tgz",@@ -3397,6 +3606,12 @@"dev": true,"license": "MIT"},+ "node_modules/napi-build-utils": {+ "version": "1.0.2",+ "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",+ "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==",+ "dev": true+ },"node_modules/nearley": {"version": "2.20.1","license": "MIT",@@ -3421,6 +3636,18 @@"version": "2.20.3","license": "MIT"},+ "node_modules/node-abi": {+ "version": "3.56.0",+ "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.56.0.tgz",+ "integrity": "sha512-fZjdhDOeRcaS+rcpve7XuwHBmktS1nS1gzgghwKUQQ8nTy2FdSDr6ZT8k6YhvlJeHmmQMYiT/IH9hfco5zeW2Q==",+ "dev": true,+ "dependencies": {+ "semver": "^7.3.5"+ },+ "engines": {+ "node": ">=10"+ }+ },"node_modules/node-fetch": {"version": "2.6.7","license": "MIT",@@ -3644,6 +3871,32 @@"node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"}},+ "node_modules/prebuild-install": {+ "version": "7.1.2",+ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz",+ "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==",+ "dev": true,+ "dependencies": {+ "detect-libc": "^2.0.0",+ "expand-template": "^2.0.3",+ "github-from-package": "0.0.0",+ "minimist": "^1.2.3",+ "mkdirp-classic": "^0.5.3",+ "napi-build-utils": "^1.0.1",+ "node-abi": "^3.3.0",+ "pump": "^3.0.0",+ "rc": "^1.2.7",+ "simple-get": "^4.0.0",+ "tar-fs": "^2.0.0",+ "tunnel-agent": "^0.6.0"+ },+ "bin": {+ "prebuild-install": "bin.js"+ },+ "engines": {+ "node": ">=10"+ }+ },"node_modules/pretty-format": {"version": "29.7.0","resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz",@@ -3678,6 +3931,16 @@"version": "1.9.0","license": "MIT"},+ "node_modules/pump": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",+ "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",+ "dev": true,+ "dependencies": {+ "end-of-stream": "^1.1.0",+ "once": "^1.3.1"+ }+ },"node_modules/punycode": {"version": "2.3.1","license": "MIT",@@ -3708,12 +3971,41 @@"node": ">=0.12"}},+ "node_modules/rc": {+ "version": "1.2.8",+ "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",+ "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",+ "dev": true,+ "dependencies": {+ "deep-extend": "^0.6.0",+ "ini": "~1.3.0",+ "minimist": "^1.2.0",+ "strip-json-comments": "~2.0.1"+ },+ "bin": {+ "rc": "cli.js"+ }+ },"node_modules/react-is": {"version": "18.2.0","resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz","integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==","dev": true},+ "node_modules/readable-stream": {+ "version": "3.6.2",+ "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",+ "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",+ "dev": true,+ "dependencies": {+ "inherits": "^2.0.3",+ "string_decoder": "^1.1.1",+ "util-deprecate": "^1.0.1"+ },+ "engines": {+ "node": ">= 6"+ }+ },"node_modules/refractor": {"version": "4.8.1","dev": true,@@ -3781,6 +4073,26 @@"dev": true,"license": "MIT"},+ "node_modules/safe-buffer": {+ "version": "5.2.1",+ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",+ "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",+ "dev": true,+ "funding": [+ {+ "type": "github",+ "url": "https://github.com/sponsors/feross"+ },+ {+ "type": "patreon",+ "url": "https://www.patreon.com/feross"+ },+ {+ "type": "consulting",+ "url": "https://feross.org/support"+ }+ ]+ },"node_modules/safer-buffer": {"version": "2.1.2","dev": true,@@ -3849,6 +4161,51 @@"dev": true,"license": "ISC"},+ "node_modules/simple-concat": {+ "version": "1.0.1",+ "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",+ "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",+ "dev": true,+ "funding": [+ {+ "type": "github",+ "url": "https://github.com/sponsors/feross"+ },+ {+ "type": "patreon",+ "url": "https://www.patreon.com/feross"+ },+ {+ "type": "consulting",+ "url": "https://feross.org/support"+ }+ ]+ },+ "node_modules/simple-get": {+ "version": "4.0.1",+ "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",+ "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",+ "dev": true,+ "funding": [+ {+ "type": "github",+ "url": "https://github.com/sponsors/feross"+ },+ {+ "type": "patreon",+ "url": "https://www.patreon.com/feross"+ },+ {+ "type": "consulting",+ "url": "https://feross.org/support"+ }+ ],+ "dependencies": {+ "decompress-response": "^6.0.0",+ "once": "^1.3.1",+ "simple-concat": "^1.0.0"+ }+ },"node_modules/source-map": {"version": "0.6.1","dev": true,@@ -3889,6 +4246,24 @@"dev": true,"license": "MIT"},+ "node_modules/string_decoder": {+ "version": "1.3.0",+ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",+ "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",+ "dev": true,+ "dependencies": {+ "safe-buffer": "~5.2.0"+ }+ },+ "node_modules/strip-json-comments": {+ "version": "2.0.1",+ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",+ "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",+ "dev": true,+ "engines": {+ "node": ">=0.10.0"+ }+ },"node_modules/strip-literal": {"version": "1.3.0","resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-1.3.0.tgz",@@ -3917,6 +4292,34 @@"dev": true,"license": "MIT"},+ "node_modules/tar-fs": {+ "version": "2.1.1",+ "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",+ "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",+ "dev": true,+ "dependencies": {+ "chownr": "^1.1.1",+ "mkdirp-classic": "^0.5.2",+ "pump": "^3.0.0",+ "tar-stream": "^2.1.4"+ }+ },+ "node_modules/tar-stream": {+ "version": "2.2.0",+ "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",+ "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",+ "dev": true,+ "dependencies": {+ "bl": "^4.0.3",+ "end-of-stream": "^1.4.1",+ "fs-constants": "^1.0.0",+ "inherits": "^2.0.3",+ "readable-stream": "^3.1.1"+ },+ "engines": {+ "node": ">=6"+ }+ },"node_modules/test-exclude": {"version": "6.0.0","dev": true,@@ -3992,6 +4395,18 @@"version": "2.4.1","license": "0BSD"},+ "node_modules/tunnel-agent": {+ "version": "0.6.0",+ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",+ "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",+ "dev": true,+ "dependencies": {+ "safe-buffer": "^5.0.1"+ },+ "engines": {+ "node": "*"+ }+ },"node_modules/type-detect": {"version": "4.0.8","resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz",@@ -4026,6 +4441,12 @@"requires-port": "^1.0.0"}},+ "node_modules/util-deprecate": {+ "version": "1.0.2",+ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",+ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",+ "dev": true+ },"node_modules/v8-to-istanbul": {"version": "9.2.0","dev": true,diff --git a/packages/core/package.json b/packages/core/package.jsonindex ef7814a58..c8137e13a 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -9,6 +9,7 @@},"devDependencies": {"@notesnook/crypto": "file:../crypto",+ "@types/better-sqlite3": "^7.6.5","@types/event-source-polyfill": "^1.0.1","@types/html-to-text": "^9.0.0","@types/katex": "^0.16.2",@@ -19,6 +20,7 @@"@types/ws": "^8.5.5","@vitest/coverage-v8": "^0.34.1","abortcontroller-polyfill": "^1.7.3",+ "better-sqlite3": "^8.6.0","cross-env": "^7.0.3","dotenv": "^16.0.1","event-source-polyfill": "^1.0.31",@@ -54,6 +56,7 @@"html-to-text": "^9.0.5","htmlparser2": "^8.0.1","katex": "0.16.2",+ "kysely": "^0.26.3","linkedom": "^0.14.17","liqe": "^1.13.0","mime-db": "1.52.0",diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.tsindex c41e4da85..ad92931de 100644--- a/packages/core/src/api/index.ts+++ b/packages/core/src/api/index.ts@@ -60,6 +60,9 @@ import {import TokenManager from "./token-manager";import { Attachment } from "../types";import { Settings } from "../collections/settings";+import { DatabaseAccessor, DatabaseSchema, createDatabase } from "../database";+import { Kysely, SqliteDriver, Transaction } from "kysely";+import BetterSQLite3 from "better-sqlite3";type EventSourceConstructor = new (uri: string,@@ -111,6 +114,32 @@ class Database {return this.options.compressor;};+ private _sql?: Kysely<DatabaseSchema>;+ sql: DatabaseAccessor = () => {+ if (this._transaction) return this._transaction;++ if (!this._sql)+ throw new Error(+ "Database not initialized. Did you forget to call db.init()?"+ );+ return this._sql;+ };++ private _transaction?: Transaction<DatabaseSchema>;+ transaction = (+ executor: (tr: Transaction<DatabaseSchema>) => void | Promise<void>+ ) => {+ if (this._transaction) return executor(this._transaction);+ return this.sql()+ .transaction()+ .execute(async (tr) => {+ this._transaction = tr;+ await executor(tr);+ this._transaction = undefined;+ })+ .finally(() => (this._transaction = undefined));+ };+private options?: Options;EventSource?: EventSourceConstructor;eventSource?: EventSource | null;@@ -144,6 +173,7 @@ class Database {reminders = new Reminders(this);relations = new Relations(this);notes = new Notes(this);+// constructor() {// this.sseMutex = new Mutex();// // this.lastHeartbeat = undefined; // { local: 0, server: 0 };@@ -170,8 +200,8 @@ class Database {this);EV.subscribe(EVENTS.attachmentDeleted, async (attachment: Attachment) => {- await this.fs().cancel(attachment.metadata.hash, "upload");- await this.fs().cancel(attachment.metadata.hash, "download");+ await this.fs().cancel(attachment.hash, "upload");+ await this.fs().cancel(attachment.hash, "download");});EV.subscribe(EVENTS.userLoggedOut, async () => {await this.monographs.clear();@@ -179,6 +209,10 @@ class Database {this.disconnectSSE();});+ this._sql = await createDatabase(+ new SqliteDriver({ database: BetterSQLite3("nn.db") })+ );+await this._validate();await this.initCollections();diff --git a/packages/core/src/api/sync/collector.ts b/packages/core/src/api/sync/collector.tsindex 199cbcc76..204632984 100644--- a/packages/core/src/api/sync/collector.ts+++ b/packages/core/src/api/sync/collector.ts@@ -66,21 +66,21 @@ class Collector {}}- for (const itemType in SYNC_COLLECTIONS_MAP) {- const collectionKey =- SYNC_COLLECTIONS_MAP[itemType as keyof typeof SYNC_COLLECTIONS_MAP];- const collection = this.db[collectionKey].collection;- for (const chunk of collection.iterateSync(chunkSize)) {- const items = await this.prepareChunk(- chunk,- lastSyncedTimestamp,- isForceSync,- key- );- if (!items) continue;- yield { items, type: itemType as keyof typeof SYNC_COLLECTIONS_MAP };- }- }+ // for (const itemType in SYNC_COLLECTIONS_MAP) {+ // const collectionKey =+ // SYNC_COLLECTIONS_MAP[itemType as keyof typeof SYNC_COLLECTIONS_MAP];+ // const collection = this.db[collectionKey].collection;+ // for (const chunk of collection.iterateSync(chunkSize)) {+ // const items = await this.prepareChunk(+ // chunk,+ // lastSyncedTimestamp,+ // isForceSync,+ // key+ // );+ // if (!items) continue;+ // yield { items, type: itemType as keyof typeof SYNC_COLLECTIONS_MAP };+ // }+ // }}async prepareChunk(diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.tsindex e95e0ada9..4c8d04244 100644--- a/packages/core/src/collections/attachments.ts+++ b/packages/core/src/collections/attachments.ts@@ -29,21 +29,18 @@ import {isWebClip} from "../utils/filename";import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";-import { CachedCollection } from "../database/cached-collection";import { Output } from "../interfaces";-import { Attachment, AttachmentMetadata, isDeleted } from "../types";+import { Attachment } from "../types";import Database from "../api";+import { SQLCollection } from "../database/sql-collection";+import { isCipher } from "../database/crypto";export class Attachments implements ICollection {name = "attachments";key: Cipher<"base64"> | null = null;- readonly collection: CachedCollection<"attachments", Attachment>;+ readonly collection: SQLCollection<"attachments", Attachment>;constructor(private readonly db: Database) {- this.collection = new CachedCollection(- db.storage,- "attachments",- db.eventManager- );+ this.collection = new SQLCollection(db.sql, "attachments", db.eventManager);this.key = null;EV.subscribe(@@ -60,15 +57,15 @@ export class Attachments implements ICollection {eventData: Record<string, unknown>;}) => {if (!success || !eventData || !eventData.readOnDownload) return;- const attachment = this.attachment(filename);- if (!attachment || !attachment.metadata) return;+ const attachment = await this.attachment(filename);+ if (!attachment) return;const src = await this.read(filename, getOutputType(attachment));if (!src) return;EV.publish(EVENTS.mediaAttachmentDownloaded, {groupId,- hash: attachment.metadata.hash,+ hash: attachment.hash,attachmentType: getAttachmentType(attachment),src});@@ -86,7 +83,7 @@ export class Attachments implements ICollection {filename: string;error: string;}) => {- const attachment = this.attachment(filename);+ const attachment = await this.attachment(filename);if (!attachment) return;if (success) await this.markAsUploaded(attachment.id);else@@ -104,54 +101,50 @@ export class Attachments implements ICollection {async add(item: Partial<- Omit<Attachment, "key" | "metadata"> & {+ Omit<Attachment, "key" | "encryptionKey"> & {key: SerializedKey;}- > & {- metadata: Partial<AttachmentMetadata> & { hash: string };- }+ >) {if (!item) return console.error("attachment cannot be undefined");- if (!item.metadata.hash) throw new Error("Please provide attachment hash.");+ if (!item.hash) throw new Error("Please provide attachment hash.");- const oldAttachment = this.all.find(- (a) => a.metadata.hash === item.metadata?.hash- );+ const oldAttachment = await this.attachment(item.hash);const id = oldAttachment?.id || getId();const encryptedKey = item.key- ? await this.encryptKey(item.key)- : oldAttachment?.key;+ ? JSON.stringify(await this.encryptKey(item.key))+ : oldAttachment?.encryptionKey;const attachment = {...oldAttachment,- ...oldAttachment?.metadata,...item,- key: encryptedKey+ encryptionKey: encryptedKey};const {iv,- length,+ size,alg,hash,hashType,filename,+ mimeType,salt,- type,chunkSize,- key+ encryptionKey} = attachment;if (!iv ||- !length ||+ !size ||!alg ||!hash ||!hashType ||- !filename ||+ // !filename ||+ // !mimeType ||!salt ||!chunkSize ||- !key+ !encryptionKey) {console.error("Attachment is invalid because all properties are required:",@@ -161,27 +154,33 @@ export class Attachments implements ICollection {return;}- return this.collection.add({+ await this.collection.upsert({type: "attachment",id,iv,salt,- length,+ size,alg,- key,+ encryptionKey,chunkSize,- metadata: {- hash,- hashType,- filename: getFileNameWithExtension(filename, type),- type: type || "application/octet-stream"- },++ filename:+ filename ||+ getFileNameWithExtension(+ filename || hash,+ mimeType || "application/octet-stream"+ ),+ hash,+ hashType,+ mimeType: mimeType || "application/octet-stream",+dateCreated: attachment.dateCreated || Date.now(),dateModified: attachment.dateModified || Date.now(),dateUploaded: attachment.dateUploaded,- dateDeleted: undefined,failed: attachment.failed});++ return id;}async generateKey() {@@ -189,7 +188,9 @@ export class Attachments implements ICollection {return await this.db.crypto().generateRandomKey();}- async decryptKey(key: Cipher<"base64">): Promise<SerializedKey | null> {+ async decryptKey(keyJSON: string): Promise<SerializedKey | null> {+ const key = JSON.parse(keyJSON);+ if (!isCipher(key)) return null;const encryptionKey = await this._getEncryptionKey();const plainData = await this.db.storage().decrypt(encryptionKey, key);if (!plainData) return null;@@ -197,8 +198,8 @@ export class Attachments implements ICollection {}async remove(hashOrId: string, localOnly: boolean) {- const attachment = this.attachment(hashOrId);- if (!attachment || !attachment.metadata) return false;+ const attachment = await this.attachment(hashOrId);+ if (!attachment) return false;if (!localOnly && !(await this.canDetach(attachment)))throw new Error("This attachment is inside a locked note.");@@ -206,116 +207,115 @@ export class Attachments implements ICollection {if (await this.db.fs()- .deleteFile(- attachment.metadata.hash,- localOnly || !attachment.dateUploaded- )+ .deleteFile(attachment.hash, localOnly || !attachment.dateUploaded)) {if (!localOnly) {await this.detach(attachment);}- await this.collection.remove(attachment.id);+ await this.collection.softDelete([attachment.id]);return true;}return false;}async detach(attachment: Attachment) {- for (const note of this.db.relations.from(attachment, "note").resolved()) {+ for (const note of await this.db.relations+ .from(attachment, "note")+ .resolve()) {if (!note || !note.contentId) continue;await this.db.content.removeAttachments(note.contentId, [- attachment.metadata.hash+ attachment.hash]);}}private async canDetach(attachment: Attachment) {- return this.db.relations- .from(attachment, "note")- .resolved()- .every((note) => !note.locked);+ return (await this.db.relations.from(attachment, "note").resolve()).every(+ (note) => !note.locked+ );}- ofNote(+ async ofNote(noteId: string,...types: ("files" | "images" | "webclips" | "all")[]- ): Attachment[] {- const noteAttachments = this.db.relations+ ): Promise<Attachment[]> {+ const noteAttachments = await this.db.relations.from({ type: "note", id: noteId }, "attachment")- .resolved();+ .resolve();if (types.includes("all")) return noteAttachments;return noteAttachments.filter((a) => {- if (isImage(a.metadata.type) && types.includes("images")) return true;- else if (isWebClip(a.metadata.type) && types.includes("webclips"))- return true;+ if (isImage(a.mimeType) && types.includes("images")) return true;+ else if (isWebClip(a.mimeType) && types.includes("webclips")) return true;else if (types.includes("files")) return true;});}- exists(hash: string) {- const attachment = this.all.find((a) => a.metadata.hash === hash);- return !!attachment;+ async exists(hash: string) {+ return !!(await this.attachment(hash));}async read<TOutputFormat extends DataFormat>(hash: string,outputType: TOutputFormat): Promise<Output<TOutputFormat> | undefined> {- const attachment = this.all.find((a) => a.metadata.hash === hash);+ const attachment = await this.attachment(hash);if (!attachment) return;- const key = await this.decryptKey(attachment.key);+ const key = await this.decryptKey(attachment.encryptionKey);if (!key) return;- const data = await this.db- .fs()- .readEncrypted(attachment.metadata.hash, key, {- chunkSize: attachment.chunkSize,- iv: attachment.iv,- salt: attachment.salt,- length: attachment.length,- alg: attachment.alg,- outputType- });+ const data = await this.db.fs().readEncrypted(attachment.hash, key, {+ chunkSize: attachment.chunkSize,+ iv: attachment.iv,+ salt: attachment.salt,+ size: attachment.size,+ alg: attachment.alg,+ outputType+ });if (!data) return;return (outputType === "base64"? dataurl.fromObject({- type: attachment.metadata.type,+ mimeType: attachment.mimeType,data}): data) as Output<TOutputFormat>;}- attachment(hashOrId: string) {- return this.all.find(- (a) => a.id === hashOrId || a.metadata?.hash === hashOrId- );+ async attachment(hashOrId: string): Promise<Attachment | undefined> {+ return await this.db+ .sql()+ .selectFrom("attachments")+ .selectAll()+ .where((eb) =>+ eb.or([eb("id", "==", hashOrId), eb("hash", "==", hashOrId)])+ )+ .where("deleted", "is", null)+ .$narrowType<Attachment>()+ .executeTakeFirst();}markAsUploaded(id: string) {- const attachment = this.attachment(id);- if (!attachment) return;- attachment.dateUploaded = Date.now();- attachment.failed = undefined;- return this.collection.update(attachment);+ return this.collection.update([id], {+ dateUploaded: Date.now(),+ failed: null+ });}reset(id: string) {- const attachment = this.attachment(id);- if (!attachment) return;- attachment.dateUploaded = undefined;- return this.collection.update(attachment);+ return this.collection.update([id], {+ dateUploaded: null+ });}markAsFailed(id: string, reason: string) {- const attachment = this.attachment(id);- if (!attachment) return;- attachment.failed = reason;- return this.collection.update(attachment);+ return this.collection.update([id], {+ dateUploaded: null,+ failed: reason+ });}async save(@@ -325,7 +325,7 @@ export class Attachments implements ICollection {): Promise<string | undefined> {const hashResult = await this.db.fs().hashBase64(data);if (!hashResult) return;- if (this.exists(hashResult.hash)) return hashResult.hash;+ if (await this.exists(hashResult.hash)) return hashResult.hash;const key = await this.generateKey();const { hash, hashType, ...encryptionMetadata } = await this.db@@ -335,27 +335,23 @@ export class Attachments implements ICollection {await this.add({...encryptionMetadata,key,- metadata: {- filename: filename || hash,- hash,- hashType,- type: mimeType || "application/octet-stream"- }++ filename: filename || hash,+ hash,+ hashType,+ mimeType: mimeType || "application/octet-stream"});return hash;}async downloadMedia(noteId: string, hashesToLoad?: string[]) {- let attachments = this.ofNote(noteId, "images", "webclips");+ let attachments = await this.ofNote(noteId, "images", "webclips");if (hashesToLoad)- attachments = attachments.filter((a) =>- hasItem(hashesToLoad, a.metadata.hash)- );+ attachments = attachments.filter((a) => hasItem(hashesToLoad, a.hash));await this.db.fs().queueDownloads(attachments.map((a) => ({- filename: a.metadata.hash,- metadata: a.metadata,+ filename: a.hash,chunkSize: a.chunkSize})),noteId,@@ -375,65 +371,60 @@ export class Attachments implements ICollection {const isDeleted = await this.db.fs().deleteFile(attachment.metadata.hash);if (!isDeleted) continue;- await this.collection.remove(attachment.id);+ await this.collection.softDelete(attachment.id);}}- get pending() {- return this.all.filter(- (attachment) => !attachment.dateUploaded || attachment.dateUploaded <= 0- );- }-- get uploaded() {- return this.all.filter((attachment) => !!attachment.dateUploaded);- }-- get syncable() {- return this.collection- .raw()- .filter(- (attachment) => isDeleted(attachment) || !!attachment.dateUploaded- );- }-- get deleted() {- return this.all.filter((attachment) => !!attachment.dateDeleted);- }-- get images() {- return this.all.filter(- (attachment) => attachment.metadata && isImage(attachment.metadata.type)- );- }-- get webclips() {- return this.all.filter(- (attachment) => attachment.metadata && isWebClip(attachment.metadata.type)- );- }-- get media() {- return this.all.filter(- (attachment) =>- attachment.metadata &&- (isImage(attachment.metadata.type) ||- isWebClip(attachment.metadata.type))- );- }-- get files() {- return this.all.filter(- (attachment) =>- attachment.metadata &&- !isImage(attachment.metadata.type) &&- !isWebClip(attachment.metadata.type)- );- }-- get all() {- return this.collection.items();- }+ // get pending() {+ // return this.all.filter(+ // (attachment) =>+ // (!attachment.dateUploaded || attachment.dateUploaded <= 0) &&+ // this.db.relations.to(attachment, "note").length > 0+ // );+ // }++ // get uploaded() {+ // return this.all.filter((attachment) => !!attachment.dateUploaded);+ // }++ // get syncable() {+ // return this.collection+ // .raw()+ // .filter(+ // (attachment) => isDeleted(attachment) || !!attachment.dateUploaded+ // );+ // }++ // get deleted() {+ // return this.all.filter((attachment) => !!attachment.dateDeleted);+ // }++ // get images() {+ // return this.all.filter((attachment) => isImage(attachment.metadata.type));+ // }++ // get webclips() {+ // return this.all.filter((attachment) => isWebClip(attachment.metadata.type));+ // }++ // get media() {+ // return this.all.filter(+ // (attachment) =>+ // isImage(attachment.metadata.type) || isWebClip(attachment.metadata.type)+ // );+ // }++ // get files() {+ // return this.all.filter(+ // (attachment) =>+ // !isImage(attachment.metadata.type) &&+ // !isWebClip(attachment.metadata.type)+ // );+ // }++ // get all() {+ // return this.collection.items();+ // }private async encryptKey(key: SerializedKey) {const encryptionKey = await this._getEncryptionKey();@@ -454,15 +445,15 @@ export class Attachments implements ICollection {}export function getOutputType(attachment: Attachment): DataFormat {- if (attachment.metadata.type === "application/vnd.notesnook.web-clip")+ if (attachment.mimeType === "application/vnd.notesnook.web-clip")return "text";- else if (attachment.metadata.type.startsWith("image/")) return "base64";+ else if (attachment.mimeType.startsWith("image/")) return "base64";return "uint8array";}function getAttachmentType(attachment: Attachment) {- if (attachment.metadata.type === "application/vnd.notesnook.web-clip")+ if (attachment.mimeType === "application/vnd.notesnook.web-clip")return "webclip";- else if (attachment.metadata.type.startsWith("image/")) return "image";+ else if (attachment.mimeType.startsWith("image/")) return "image";else return "generic";}diff --git a/packages/core/src/collections/colors.ts b/packages/core/src/collections/colors.tsindex ded823f59..edfc1c52e 100644--- a/packages/core/src/collections/colors.ts+++ b/packages/core/src/collections/colors.ts@@ -19,10 +19,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { ICollection } from "./collection";import { getId } from "../utils/id";-import { Color, MaybeDeletedItem } from "../types";+import { Color } from "../types";import Database from "../api";-import { CachedCollection } from "../database/cached-collection";import { Tags } from "./tags";+import { SQLCollection } from "../database/sql-collection";export const DefaultColors: Record<string, string> = {red: "#f44336",@@ -36,13 +36,9 @@ export const DefaultColors: Record<string, string> = {export class Colors implements ICollection {name = "colors";- readonly collection: CachedCollection<"colors", Color>;+ readonly collection: SQLCollection<"colors", Color>;constructor(private readonly db: Database) {- this.collection = new CachedCollection(- db.storage,- "colors",- db.eventManager- );+ this.collection = new SQLCollection(db.sql, "colors", db.eventManager);}init() {@@ -53,27 +49,27 @@ export class Colors implements ICollection {return this.collection.get(id);}- async merge(remoteColor: MaybeDeletedItem<Color>) {- if (!remoteColor) return;+ // async merge(remoteColor: MaybeDeletedItem<Color>) {+ // if (!remoteColor) return;- const localColor = this.collection.get(remoteColor.id);- if (!localColor || remoteColor.dateModified > localColor.dateModified)- await this.collection.add(remoteColor);- }+ // const localColor = this.collection.get(remoteColor.id);+ // if (!localColor || remoteColor.dateModified > localColor.dateModified)+ // await this.collection.add(remoteColor);+ // }async add(item: Partial<Color>) {if (item.remote)throw new Error("Please use db.colors.merge to merge remote colors.");const id = item.id || getId(item.dateCreated);- const oldColor = this.color(id);+ const oldColor = await this.color(id);item.title = item.title ? Tags.sanitize(item.title) : item.title;if (!item.title && !oldColor?.title) throw new Error("Title is required.");if (!item.colorCode && !oldColor?.colorCode)throw new Error("Color code is required.");- const color: Color = {+ await this.collection.upsert({id,dateCreated: item.dateCreated || oldColor?.dateCreated || Date.now(),dateModified: item.dateModified || oldColor?.dateModified || Date.now(),@@ -81,34 +77,35 @@ export class Colors implements ICollection {colorCode: item.colorCode || oldColor?.colorCode || "",type: "color",remote: false- };- await this.collection.add(color);- return color.id;+ });+ return id;}- get raw() {- return this.collection.raw();- }+ // get raw() {+ // return this.collection.raw();+ // }- get all(): Color[] {- return this.collection.items();- }+ // get all(): Color[] {+ // return this.collection.items();+ // }- async remove(id: string) {- await this.collection.remove(id);- await this.db.relations.cleanup();+ async remove(...ids: string[]) {+ await this.db.transaction(async () => {+ await this.db.relations.unlinkOfType("color", ids);+ await this.collection.softDelete(ids);+ });}- async delete(id: string) {- await this.collection.delete(id);- await this.db.relations.cleanup();- }+ // async delete(id: string) {+ // await this.collection.delete(id);+ // await this.db.relations.cleanup();+ // }exists(id: string) {return this.collection.exists(id);}- find(idOrTitle: string) {- return this.all.find((t) => t.title === idOrTitle || t.id === idOrTitle);- }+ // find(idOrTitle: string) {+ // return this.all.find((t) => t.title === idOrTitle || t.id === idOrTitle);+ // }}diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.tsindex 10b8fcc50..ec439ba14 100644--- a/packages/core/src/collections/content.ts+++ b/packages/core/src/collections/content.ts@@ -27,13 +27,12 @@ import {ContentItem,ContentType,EncryptedContentItem,- MaybeDeletedItem,UnencryptedContentItem,isDeleted} from "../types";-import { IndexedCollection } from "../database/indexed-collection";import Database from "../api";import { getOutputType } from "./attachments";+import { SQLCollection } from "../database/sql-collection";export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({noteId,@@ -48,27 +47,15 @@ export const EMPTY_CONTENT = (noteId: string): UnencryptedContentItem => ({export class Content implements ICollection {name = "content";- readonly collection: IndexedCollection<"content", ContentItem>;+ readonly collection: SQLCollection<"content", ContentItem>;constructor(private readonly db: Database) {- this.collection = new IndexedCollection(- db.storage,- "content",- db.eventManager- );+ this.collection = new SQLCollection(db.sql, "content", db.eventManager);}async init() {await this.collection.init();}- async merge(content: MaybeDeletedItem<ContentItem>) {- return await this.collection.addItem(- isDeleted(content) || !isUnencryptedContent(content)- ? content- : await this.extractAttachments(content)- );- }-async add(content: Partial<ContentItem>) {if (typeof content.data === "object") {if ("data" in content.data && typeof content.data.data === "string")@@ -106,7 +93,7 @@ export class Content implements ICollection {conflicted: content.conflicted,dateResolved: content.dateResolved};- await this.collection.addItem(+ await this.collection.upsert(isUnencryptedContent(contentItem)? await this.extractAttachments(contentItem): contentItem@@ -133,29 +120,46 @@ export class Content implements ICollection {}async raw(id: string) {- const content = await this.collection.getItem(id);+ const content = await this.collection.get(id);if (!content) return;return content;}- remove(id: string) {- if (!id) return;- return this.collection.removeItem(id);+ remove(...ids: string[]) {+ return this.collection.softDelete(ids);}- multi(ids: string[]) {- return this.collection.getItems(ids);+ removeByNoteId(...ids: string[]) {+ return this.db+ .sql()+ .replaceInto("content")+ .columns(["id", "dateModified", "deleted"])+ .expression((eb) =>+ eb+ .selectFrom("content")+ .where("noteId", "in", ids)+ .select((eb) => [+ "content.id",+ eb.lit(Date.now()).as("dateModified"),+ eb.lit(1).as("deleted")+ ])+ )+ .execute();}+ // multi(ids: string[]) {+ // return this.collection.getItems(ids);+ // }+exists(id: string) {return this.collection.exists(id);}- async all() {- return Object.values(- await this.collection.getItems(this.collection.indexer.indices)- );- }+ // async all() {+ // return Object.values(+ // await this.collection.getItems(this.collection.indexer.indices)+ // );+ // }insertMedia(contentItem: UnencryptedContentItem) {return this.insert(contentItem, async (hashes) => {@@ -183,17 +187,16 @@ export class Content implements ICollection {const content = getContentFromData(contentItem.type, contentItem.data);if (!content) return contentItem;contentItem.data = await content.insertMedia(async (hashes) => {- const attachments = hashes.reduce((attachments, hash) => {- const attachment = this.db.attachments.attachment(hash);- if (!attachment) return attachments;+ const attachments: Attachment[] = [];+ for (const hash of hashes) {+ const attachment = await this.db.attachments.attachment(hash);+ if (!attachment) continue;attachments.push(attachment);- return attachments;- }, [] as Attachment[]);+ }await this.db.fs().queueDownloads(attachments.map((a) => ({- filename: a.metadata.hash,- metadata: a.metadata,+ filename: a.hash,chunkSize: a.chunkSize})),groupId,@@ -203,11 +206,11 @@ export class Content implements ICollection {const sources: Record<string, string> = {};for (const attachment of attachments) {const src = await this.db.attachments.read(- attachment.metadata.hash,+ attachment.hash,getOutputType(attachment));if (!src) continue;- sources[attachment.metadata.hash] = src;+ sources[attachment.hash] = src;}return sources;});@@ -243,16 +246,16 @@ export class Content implements ICollection {this.db.attachments.save);- const noteAttachments = this.db.relations+ const noteAttachments = await this.db.relations.from({ type: "note", id: contentItem.noteId }, "attachment")- .resolved();+ .resolve();const toDelete = noteAttachments.filter((attachment) => {- return hashes.every((hash) => hash !== attachment.metadata.hash);+ return hashes.every((hash) => hash !== attachment.hash);});const toAdd = hashes.filter((hash) => {- return hash && noteAttachments.every((a) => hash !== a.metadata.hash);+ return hash && noteAttachments.every((a) => hash !== a.hash);});for (const attachment of toDelete) {@@ -266,7 +269,7 @@ export class Content implements ICollection {}for (const hash of toAdd) {- const attachment = this.db.attachments.attachment(hash);+ const attachment = await this.db.attachments.attachment(hash);if (!attachment) continue;await this.db.relations.add({@@ -284,24 +287,24 @@ export class Content implements ICollection {return contentItem;}- async cleanup() {- const indices = this.collection.indexer.indices;- await this.db.notes.init();- const notes = this.db.notes.all;- if (!notes.length && indices.length > 0) return [];- const ids = [];- for (const contentId of indices) {- const noteIndex = notes.findIndex((note) => note.contentId === contentId);- const isOrphaned = noteIndex === -1;- if (isOrphaned) {- ids.push(contentId);- await this.collection.deleteItem(contentId);- } else if (notes[noteIndex].localOnly) {- ids.push(contentId);- }- }- return ids;- }+ // async cleanup() {+ // const indices = this.collection.indexer.indices;+ // await this.db.notes.init();+ // const notes = this.db.notes.all;+ // if (!notes.length && indices.length > 0) return [];+ // const ids = [];+ // for (const contentId of indices) {+ // const noteIndex = notes.findIndex((note) => note.contentId === contentId);+ // const isOrphaned = noteIndex === -1;+ // if (isOrphaned) {+ // ids.push(contentId);+ // await this.collection.deleteItem(contentId);+ // } else if (notes[noteIndex].localOnly) {+ // ids.push(contentId);+ // }+ // }+ // return ids;+ // }}export function isUnencryptedContent(diff --git a/packages/core/src/collections/note-history.ts b/packages/core/src/collections/note-history.tsindex 2464e80be..ee7b4effd 100644--- a/packages/core/src/collections/note-history.ts+++ b/packages/core/src/collections/note-history.ts@@ -19,7 +19,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import Database from "../api";import { isCipher } from "../database/crypto";-import { IndexedCollection } from "../database/indexed-collection";+import { SQLCollection } from "../database/sql-collection";import { HistorySession, isDeleted } from "../types";import { makeSessionContentId } from "../utils/id";import { ICollection } from "./collection";@@ -29,13 +29,9 @@ export class NoteHistory implements ICollection {name = "notehistory";versionsLimit = 100;sessionContent = new SessionContent(this.db);- private readonly collection: IndexedCollection<"notehistory", HistorySession>;+ private readonly collection: SQLCollection<"notehistory", HistorySession>;constructor(private readonly db: Database) {- this.collection = new IndexedCollection(- db.storage,- "notehistory",- db.eventManager- );+ this.collection = new SQLCollection(db.sql, "notehistory", db.eventManager);}async init() {@@ -43,10 +39,6 @@ export class NoteHistory implements ICollection {await this.sessionContent.init();}- async merge(item: HistorySession) {- await this.collection.addItem(item);- }-async get(noteId: string) {if (!noteId) return [];@@ -67,7 +59,7 @@ export class NoteHistory implements ICollection {content: NoteContent<boolean>) {sessionId = `${noteId}_${sessionId}`;- const oldSession = await this.collection.getItem(sessionId);+ const oldSession = await this.collection.get(sessionId);if (oldSession && isDeleted(oldSession)) return;@@ -82,7 +74,7 @@ export class NoteHistory implements ICollection {locked};- await this.collection.addItem(session);+ await this.collection.upsert(session);await this.sessionContent.add(sessionId, content, locked);await this.cleanup(noteId);@@ -104,19 +96,18 @@ export class NoteHistory implements ICollection {}async content(sessionId: string) {- const session = await this.collection.getItem(sessionId);+ const session = await this.collection.get(sessionId);if (!session || isDeleted(session)) return;return await this.sessionContent.get(session.sessionContentId);}async remove(sessionId: string) {- const session = await this.collection.getItem(sessionId);+ const session = await this.collection.get(sessionId);if (!session || isDeleted(session)) return;await this._remove(session);}- async clearSessions(noteId: string) {- if (!noteId) return;+ async clearSessions(...noteIds: string[]) {const history = await this.get(noteId);for (const item of history) {await this._remove(item);@@ -124,12 +115,12 @@ export class NoteHistory implements ICollection {}private async _remove(session: HistorySession) {- await this.collection.deleteItem(session.id);+ await this.collection.delete(session.id);await this.sessionContent.remove(session.sessionContentId);}async restore(sessionId: string) {- const session = await this.collection.getItem(sessionId);+ const session = await this.collection.get(sessionId);if (!session || isDeleted(session)) return;const content = await this.sessionContent.get(session.sessionContentId);@@ -153,14 +144,14 @@ export class NoteHistory implements ICollection {}}- async all() {- return this.getSessions(this.collection.indexer.indices);- }+ // async all() {+ // return this.getSessions(this.collection.indexer.indices);+ // }- private async getSessions(sessionIds: string[]): Promise<HistorySession[]> {- const items = await this.collection.getItems(sessionIds);- return Object.values(items).filter(- (a) => !isDeleted(a)- ) as HistorySession[];- }+ // private async getSessions(sessionIds: string[]): Promise<HistorySession[]> {+ // const items = await this.collection.getItems(sessionIds);+ // return Object.values(items).filter(+ // (a) => !isDeleted(a)+ // ) as HistorySession[];+ // }}diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.tsindex 4c41a9391..a9ac92fbc 100644--- a/packages/core/src/collections/notebooks.ts+++ b/packages/core/src/collections/notebooks.ts@@ -17,25 +17,20 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { createNotebookModel } from "../models/notebook";import { getId } from "../utils/id";-import { CachedCollection } from "../database/cached-collection";import Database from "../api";-import { BaseTrashItem, Notebook, TrashOrItem, isTrashItem } from "../types";+import { Notebook, TrashOrItem, isTrashItem } from "../types";import { ICollection } from "./collection";+import { SQLCollection } from "../database/sql-collection";export class Notebooks implements ICollection {name = "notebooks";/*** @internal*/- collection: CachedCollection<"notebooks", TrashOrItem<Notebook>>;+ collection: SQLCollection<"notebooks", TrashOrItem<Notebook>>;constructor(private readonly db: Database) {- this.collection = new CachedCollection(- db.storage,- "notebooks",- db.eventManager- );+ this.collection = new SQLCollection(db.sql, "notebooks", db.eventManager);}init() {@@ -51,7 +46,7 @@ export class Notebooks implements ICollection {//TODO reliably and efficiently check for duplicates.const id = notebookArg.id || getId();- const oldNotebook = this.collection.get(id);+ const oldNotebook = await this.notebook(id);if (oldNotebook && isTrashItem(oldNotebook))throw new Error("Cannot modify trashed notebooks.");@@ -64,7 +59,7 @@ export class Notebooks implements ICollection {if (!mergedNotebook.title)throw new Error("Notebook must contain a title.");- const notebook: Notebook = {+ await this.collection.upsert({id,type: "notebook",title: mergedNotebook.title,@@ -74,79 +69,87 @@ export class Notebooks implements ICollection {dateCreated: mergedNotebook.dateCreated || Date.now(),dateModified: mergedNotebook.dateModified || Date.now(),dateEdited: Date.now()- };-- await this.collection.add(notebook);+ });return id;}- get raw() {- return this.collection.raw();- }+ // get raw() {+ // return this.collection.raw();+ // }- get all() {- return this.collection.items((note) =>- isTrashItem(note) ? undefined : note- ) as Notebook[];- }+ // get all() {+ // return this.collection.items((note) =>+ // isTrashItem(note) ? undefined : note+ // ) as Notebook[];+ // }- get pinned() {- return this.all.filter((item) => item.pinned === true);- }+ // get pinned() {+ // return this.all.filter((item) => item.pinned === true);+ // }- get trashed() {- return this.raw.filter((item) =>- isTrashItem(item)- ) as BaseTrashItem<Notebook>[];- }+ // get trashed() {+ // return this.raw.filter((item) =>+ // isTrashItem(item)+ // ) as BaseTrashItem<Notebook>[];+ // }async pin(...ids: string[]) {- for (const id of ids) {- if (!this.exists(id)) continue;- await this.add({ id, pinned: true });- }+ await this.collection.update(ids, { pinned: true });}async unpin(...ids: string[]) {- for (const id of ids) {- if (!this.exists(id)) continue;- await this.add({ id, pinned: false });- }+ await this.collection.update(ids, { pinned: false });}- totalNotes(id: string) {- let count = 0;- const subNotebooks = this.db.relations.from(- { type: "notebook", id },- "notebook"- );- for (const notebook of subNotebooks) {- count += this.totalNotes(notebook.to.id);- }- count += this.db.relations.from({ type: "notebook", id }, "note").length;- return count;+ async totalNotes(id: string) {+ const result = await this.db+ .sql()+ .withRecursive(`subNotebooks(id)`, (eb) =>+ eb+ .selectNoFrom((eb) => eb.val(id).as("id"))+ .unionAll((eb) =>+ eb+ .selectFrom(["relations", "subNotebooks"])+ .select("relations.toId as id")+ .where("toType", "==", "notebook")+ .where("fromType", "==", "notebook")+ .whereRef("fromId", "==", "subNotebooks.id")+ .where("toId", "not in", this.db.trash.cache)+ .$narrowType<{ id: string }>()+ )+ )+ .selectFrom("relations")+ .where("toType", "==", "note")+ .where("fromType", "==", "notebook")+ .where("fromId", "in", (eb) =>+ eb.selectFrom("subNotebooks").select("subNotebooks.id")+ )+ .where("toId", "not in", this.db.trash.cache)+ .select((eb) => eb.fn.count<number>("id").as("totalNotes"))+ .executeTakeFirst();++ if (!result) return 0;+ return result.totalNotes;}- notebook(idOrNotebook: string | Notebook) {- const notebook =- typeof idOrNotebook === "string"- ? this.collection.get(idOrNotebook)- : idOrNotebook;+ async notebook(id: string) {+ const notebook = await this.collection.get(id);if (!notebook || isTrashItem(notebook)) return;- return createNotebookModel(notebook, this.db);+ return notebook;}exists(id: string) {return this.collection.exists(id);}+ async remove(...ids: string[]) {+ await this.db.trash.add("notebook", ids);+ }+async delete(...ids: string[]) {- for (const id of ids) {- const notebook = this.collection.get(id);- if (!notebook || isTrashItem(notebook)) continue;- await this.collection.remove(id);- await this.db.shortcuts?.remove(id);- await this.db.trash?.add(notebook);- }+ await this.db.transaction(async () => {+ await this.db.relations.unlinkOfType("notebook", ids);+ await this.collection.softDelete(ids);+ });}}diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.tsindex 2d163d62a..2a668a000 100644--- a/packages/core/src/collections/notes.ts+++ b/packages/core/src/collections/notes.ts@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { createNoteModel } from "../models/note";import { getId } from "../utils/id";import { getContentFromData } from "../content-types";import { NEWLINE_STRIP_REGEX, formatTitle } from "../utils/title-format";@@ -26,17 +25,11 @@ import { Tiptap } from "../content-types/tiptap";import { EMPTY_CONTENT, isUnencryptedContent } from "./content";import { CHECK_IDS, checkIsUserPremium } from "../common";import { buildFromTemplate } from "../utils/templates";-import {- Note,- TrashOrItem,- isTrashItem,- isDeleted,- BaseTrashItem-} from "../types";+import { Note, TrashOrItem, isTrashItem, isDeleted } from "../types";import Database from "../api";-import { CachedCollection } from "../database/cached-collection";import { ICollection } from "./collection";import { NoteContent } from "./session-content";+import { SQLCollection } from "../database/sql-collection";type ExportOptions = {format: "html" | "md" | "txt" | "md-frontmatter";@@ -50,17 +43,15 @@ export class Notes implements ICollection {/*** @internal*/- collection: CachedCollection<"notes", TrashOrItem<Note>>;+ collection: SQLCollection<"notes", TrashOrItem<Note>>;+ totalNotes = 0;constructor(private readonly db: Database) {- this.collection = new CachedCollection(- db.storage,- "notes",- db.eventManager- );+ this.collection = new SQLCollection(db.sql, "notes", db.eventManager);}async init() {await this.collection.init();+ this.totalNotes = await this.collection.count();}async add(@@ -70,9 +61,7 @@ export class Notes implements ICollection {throw new Error("Please use db.notes.merge to merge remote notes.");const id = item.id || getId();- const oldNote = this.collection.get(id);- if (oldNote && isTrashItem(oldNote))- throw new Error("Cannot modify trashed notes.");+ const oldNote = await this.note(id);const note = {...oldNote,@@ -110,10 +99,10 @@ export class Notes implements ICollection {});}- const noteTitle = this.getNoteTitle(note, oldNote, note.headline);+ const noteTitle = await this.getNoteTitle(note, oldNote, note.headline);if (oldNote && oldNote.title !== noteTitle) note.dateEdited = Date.now();- await this.collection.add({+ await this.collection.upsert({id,contentId: note.contentId,type: "note",@@ -136,54 +125,61 @@ export class Notes implements ICollection {dateModified: note.dateModified || Date.now()});+ if (!oldNote) this.totalNotes++;return id;}- note(idOrNote: string | Note) {- if (!idOrNote) return;- const note =- typeof idOrNote === "object" ? idOrNote : this.collection.get(idOrNote);- if (!note || isTrashItem(note)) return;- return createNoteModel(note, this.db);+ async note(idOrNote: string) {+ const note = await this.collection.get(idOrNote);+ if (!note || isTrashItem(note) || isDeleted(note)) return;+ return note;}- get raw() {- return this.collection.raw();- }-- get all() {- return this.collection.items((note) =>- isTrashItem(note) ? undefined : note- ) as Note[];- }-- isTrashed(id: string) {- return this.raw.find((item) => item.id === id && isTrashItem(item));- }-- get trashed() {- return this.raw.filter((item) =>- isTrashItem(item)- ) as BaseTrashItem<Note>[];- }-- get pinned() {- return this.all.filter((item) => item.pinned === true);- }-- get conflicted() {- return this.all.filter((item) => item.conflicted === true);- }-- get favorites() {- return this.all.filter((item) => item.favorite === true);- }-- get locked(): Note[] {- return this.all.filter(- (item) => !isTrashItem(item) && item.locked === true- ) as Note[];- }+ // note(idOrNote: string | Note) {+ // if (!idOrNote) return;+ // const note =+ // typeof idOrNote === "object" ? idOrNote : this.collection.get(idOrNote);+ // if (!note || isTrashItem(note)) return;+ // return createNoteModel(note, this.db);+ // }++ // get raw() {+ // return this.collection.raw();+ // }++ // get all() {+ // return this.collection.items((note) =>+ // isTrashItem(note) ? undefined : note+ // ) as Note[];+ // }++ // isTrashed(id: string) {+ // return this.raw.find((item) => item.id === id && isTrashItem(item));+ // }++ // get trashed() {+ // return this.raw.filter((item) =>+ // isTrashItem(item)+ // ) as BaseTrashItem<Note>[];+ // }++ // get pinned() {+ // return this.all.filter((item) => item.pinned === true);+ // }++ // get conflicted() {+ // return this.all.filter((item) => item.conflicted === true);+ // }++ // get favorites() {+ // return this.all.filter((item) => item.favorite === true);+ // }++ // get locked(): Note[] {+ // return this.all.filter(+ // (item) => !isTrashItem(item) && item.locked === true+ // ) as Note[];+ // }exists(id: string) {return this.collection.exists(id);@@ -202,7 +198,7 @@ export class Notes implements ICollection {if (format !== "txt" && !(await checkIsUserPremium(CHECK_IDS.noteExport)))return false;- const note = this.note(id);+ const note = await this.note(id);if (!note) return false;if (!options.contentItem) {@@ -239,15 +235,15 @@ export class Notes implements ICollection {return options?.disableTemplate? contentString: buildFromTemplate(format, {- ...note.data,+ ...note,content: contentString});}async duplicate(...ids: string[]) {for (const id of ids) {- const note = this.collection.get(id);- if (!note || isTrashItem(note)) continue;+ const note = await this.note(id);+ if (!note) continue;const content = note.contentId? await this.db.content.raw(note.contentId)@@ -274,13 +270,16 @@ export class Notes implements ICollection {});if (!duplicateId) return;- for (const notebook of this.db.relations+ for (const notebook of await this.db.relations.to(note, "notebook")- .resolved()) {- await this.db.relations.add(notebook, {- id: duplicateId,- type: "note"- });+ .get()) {+ await this.db.relations.add(+ { type: "notebook", id: notebook },+ {+ id: duplicateId,+ type: "note"+ }+ );}return duplicateId;@@ -288,24 +287,17 @@ export class Notes implements ICollection {}private async _delete(moveToTrash = true, ...ids: string[]) {- for (const id of ids) {- const item = this.collection.get(id);- if (!item) continue;- const itemData = clone(item);-- if (moveToTrash && !isTrashItem(itemData))- await this.db.trash.add(itemData);- else {- await this.db.relations.unlinkAll(item, "tag");- await this.db.relations.unlinkAll(item, "color");- await this.db.relations.unlinkAll(item, "attachment");- await this.db.relations.unlinkAll(item, "notebook");-- await this.collection.remove(id);- if (itemData.contentId)- await this.db.content.remove(itemData.contentId);- }+ if (moveToTrash) {+ await this.db.trash.add("note", ids);+ } else {+ await this.db.transaction(async () => {+ await this.db.relations.unlinkOfType("note", ids);+ await this.collection.softDelete(ids);+ await this.db.content.removeByNoteId(...ids);+ });}++ this.totalNotes = Math.max(0, this.totalNotes - ids.length);}async addToNotebook(notebookId: string, ...noteIds: string[]) {@@ -318,24 +310,31 @@ export class Notes implements ICollection {}async removeFromNotebook(notebookId: string, ...noteIds: string[]) {- for (const noteId of noteIds) {- await this.db.relations.unlink(- { id: notebookId, type: "notebook" },- { type: "note", id: noteId }- );- }+ await this.db.transaction(async () => {+ for (const noteId of noteIds) {+ await this.db.relations.unlink(+ { id: notebookId, type: "notebook" },+ { type: "note", id: noteId }+ );+ }+ });}async removeFromAllNotebooks(...noteIds: string[]) {- for (const noteId of noteIds) {- await this.db.relations.unlinkAll(- { type: "note", id: noteId },- "notebook"- );- }+ await this.db.transaction(async () => {+ for (const noteId of noteIds) {+ await this.db.relations+ .to({ type: "note", id: noteId }, "notebook")+ .unlink();+ }+ });}- private getNoteTitle(note: Partial<Note>, oldNote?: Note, headline?: string) {+ private async getNoteTitle(+ note: Partial<Note>,+ oldNote?: Note,+ headline?: string+ ) {if (note.title && note.title.trim().length > 0) {return note.title.replace(NEWLINE_STRIP_REGEX, " ");} else if (@@ -352,7 +351,7 @@ export class Notes implements ICollection {this.db.settings.getDateFormat(),this.db.settings.getTimeFormat(),headline?.split(" ").splice(0, 10).join(" "),- this.collection.count()+ this.totalNotes);}}diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.tsindex 1f5fd388c..6f4095752 100644--- a/packages/core/src/collections/relations.ts+++ b/packages/core/src/collections/relations.ts@@ -17,197 +17,113 @@ You should have received a copy of the GNU General Public Licensealong with this program. If not, see <http://www.gnu.org/licenses/>.*/-import { CachedCollection } from "../database/cached-collection";import { makeId } from "../utils/id";import { ICollection } from "./collection";-import { Relation, ItemMap, ItemReference, MaybeDeletedItem } from "../types";+import {+ Relation,+ ItemMap,+ ItemReference,+ ValueOf,+ MaybeDeletedItem+} from "../types";import Database from "../api";--type RelationsArray<TType extends keyof ItemMap> = Relation[] & {- resolved: (limit?: number) => ItemMap[TType][];- has: (id: string) => boolean;-};+import { SQLCollection } from "../database/sql-collection";+import { DatabaseAccessor, DatabaseSchema } from "../database";+import { SelectQueryBuilder } from "kysely";export class Relations implements ICollection {name = "relations";- readonly collection: CachedCollection<"relations", Relation>;+ readonly collection: SQLCollection<"relations", Relation>;constructor(private readonly db: Database) {- this.collection = new CachedCollection(- db.storage,- "relations",- db.eventManager- );- }-- init() {- return this.collection.init();+ this.collection = new SQLCollection(db.sql, "relations", db.eventManager);}- async merge(relation: MaybeDeletedItem<Relation>) {- await this.collection.add(relation);+ async init() {+ // return this.collection.init();}async add(from: ItemReference, to: ItemReference) {- if (- this.all.find(- (a) =>- compareItemReference(a.from, from) && compareItemReference(a.to, to)- )- )- return;-- const relation: Relation = {+ await this.collection.upsert({id: generateId(from, to),type: "relation",dateCreated: Date.now(),dateModified: Date.now(),- from: { id: from.id, type: from.type },- to: { id: to.id, type: to.type }- };-- await this.collection.add(relation);+ fromId: from.id,+ fromType: from.type,+ toId: to.id,+ toType: to.type+ });}- from<TType extends keyof ItemMap>(+ from<TType extends keyof RelatableTable>(reference: ItemReference,type: TType- ): RelationsArray<TType> {- const relations =- type === "note" || type === "notebook"- ? this.all.filter(- (a) =>- compareItemReference(a.from, reference) &&- a.to.type === type &&- !this.db.trash.exists(a.to.id)- )- : this.all.filter(- (a) => compareItemReference(a.from, reference) && a.to.type === type- );-- Object.defineProperties(relations, {- resolved: {- writable: false,- enumerable: false,- configurable: false,- value: (limit?: number) =>- this.resolve(limit ? relations.slice(0, limit) : relations, "to")- },- has: {- writable: false,- enumerable: false,- configurable: false,- value: (id: string) => relations.some((rel) => rel.to.id === id)- }- });- return relations as RelationsArray<TType>;+ ) {+ return new RelationsArray(+ this.db.sql,+ this.db.trash.cache,+ reference,+ type,+ "from"+ );}- to<TType extends keyof ItemMap>(+ to<TType extends keyof RelatableTable>(reference: ItemReference,type: TType- ): RelationsArray<TType> {- const relations =- type === "note" || type === "notebook"- ? this.all.filter(- (a) =>- compareItemReference(a.to, reference) &&- a.from.type === type &&- !this.db.trash.exists(a.from.id)- )- : this.all.filter(- (a) => compareItemReference(a.to, reference) && a.from.type === type- );- Object.defineProperties(relations, {- resolved: {- writable: false,- enumerable: false,- configurable: false,- value: (limit?: number) =>- this.resolve(limit ? relations.slice(0, limit) : relations, "from")- },- has: {- writable: false,- enumerable: false,- configurable: false,- value: (id: string) => relations.some((rel) => rel.from.id === id)- }- });- return relations as RelationsArray<TType>;+ ) {+ return new RelationsArray(+ this.db.sql,+ this.db.trash.cache,+ reference,+ type,+ "to"+ );}- get raw() {- return this.collection.raw();- }+ // get raw() {+ // return this.collection.raw();+ // }- get all(): Relation[] {- return this.collection.items();- }+ // get all(): Relation[] {+ // return this.collection.items();+ // }- relation(id: string) {- return this.collection.get(id);- }+ // relation(id: string) {+ // return this.collection.get(id);+ // }async remove(...ids: string[]) {- for (const id of ids) {- await this.collection.remove(id);- }+ await this.collection.softDelete(ids);}- async unlink(from: ItemReference, to: ItemReference) {- const relation = this.all.find(- (a) =>- compareItemReference(a.from, from) && compareItemReference(a.to, to)- );- if (!relation) return;-- await this.remove(relation.id);- }-- async unlinkAll(to: ItemReference, type?: keyof ItemMap) {- for (const relation of this.all.filter(- (a) => compareItemReference(a.to, to) && (!type || a.from.type === type)- )) {- await this.remove(relation.id);- }+ unlink(from: ItemReference, to: ItemReference) {+ return this.remove(generateId(from, to));}- private resolve(relations: Relation[], resolveType: "from" | "to") {- const items = [];- for (const relation of relations) {- const reference = resolveType === "from" ? relation.from : relation.to;- let item = null;- switch (reference.type) {- case "tag":- item = this.db.tags.tag(reference.id);- break;- case "color":- item = this.db.colors.color(reference.id);- break;- case "reminder":- item = this.db.reminders.reminder(reference.id);- break;- case "note": {- const note = this.db.notes.note(reference.id);- if (!note) continue;- item = note.data;- break;- }- case "notebook": {- const notebook = this.db.notebooks.notebook(reference.id);- if (!notebook) continue;- item = notebook.data;- break;- }- case "attachment": {- const attachment = this.db.attachments.attachment(reference.id);- if (!attachment) continue;- item = attachment;- break;- }- }- if (item) items.push(item);- }- return items;+ async unlinkOfType(type: keyof RelatableTable, ids?: string[]) {+ await this.db+ .sql()+ .replaceInto("relations")+ .columns(["id", "dateModified", "deleted"])+ .expression((eb) =>+ eb+ .selectFrom("relations")+ .where((eb) =>+ eb.or([eb("fromType", "==", type), eb("toType", "==", type)])+ )+ .$if(ids !== undefined && ids.length > 0, (eb) =>+ eb.where((eb) =>+ eb.or([eb("fromId", "in", ids), eb("toId", "in", ids)])+ )+ )+ .select((eb) => [+ "relations.id",+ eb.lit(Date.now()).as("dateModified"),+ eb.lit(1).as("deleted")+ ])+ )+ .execute();}}@@ -229,3 +145,132 @@ function generateId(a: ItemReference, b: ItemReference) {const str = `${a.id}${b.id}${a.type}${b.type}`;return makeId(str);}++const TABLE_MAP = {+ note: "notes",+ notebook: "notebooks",+ reminder: "reminders",+ tag: "tags",+ color: "colors",+ attachment: "attachments"+} as const;++type RelatableTable = typeof TABLE_MAP;++class RelationsArray<TType extends keyof RelatableTable> {+ private table: ValueOf<RelatableTable> = TABLE_MAP[this.type];++ constructor(+ private readonly sql: DatabaseAccessor,+ private readonly trashIds: string[],+ private readonly reference: ItemReference,+ private readonly type: TType,+ private readonly direction: "from" | "to"+ ) {}++ async resolve(limit?: number): Promise<ItemMap[TType][]> {+ const items = await this.sql()+ .selectFrom(this.table)+ .where("id", "in", (b) =>+ b+ .selectFrom("relations")+ .$call((eb) =>+ this.buildRelationsQuery()(+ eb as SelectQueryBuilder<DatabaseSchema, "relations", unknown>+ )+ )+ )+ .$if(limit !== undefined && limit > 0, (b) => b.limit(limit!))+ .selectAll()+ // TODO: check if we need to index deleted field.+ .where("deleted", "is", null)+ .execute();+ return items as unknown as ItemMap[TType][];+ }++ async unlink() {+ await this.sql()+ .replaceInto("relations")+ .columns(["id", "dateModified", "deleted"])+ .expression((eb) =>+ eb+ .selectFrom("relations")+ .$call(this.buildRelationsQuery())+ .clearSelect()+ .select((eb) => [+ "relations.id",+ eb.lit(Date.now()).as("dateModified"),+ eb.lit(1).as("deleted")+ ])+ )+ .execute();+ }++ async get() {+ const ids = await this.sql()+ .selectFrom("relations")+ .$call(this.buildRelationsQuery())+ .execute();+ return ids.map((i) => i.id);+ }++ async count() {+ const result = await this.sql()+ .selectFrom("relations")+ .$call(this.buildRelationsQuery())+ .clearSelect()+ .select((b) => b.fn.count<number>("relations.id").as("count"))+ .executeTakeFirst();+ if (!result) return 0;+ return result.count;+ }++ async has(id: string) {+ const result = await this.sql()+ .selectFrom("relations")+ .$call(this.buildRelationsQuery())+ .clearSelect()+ .where(this.direction === "from" ? "toId" : "fromId", "==", id)+ .select((b) => b.fn.count<number>("id").as("count"))+ .executeTakeFirst();+ if (!result) return false;+ return result.count > 0;+ }++ /**+ * Build an optimized query for obtaining relations based on the given+ * parameters. The resulting query uses a covering index (the most+ * optimizable index) for obtaining relations.+ */+ private buildRelationsQuery() {+ return (+ builder: SelectQueryBuilder<DatabaseSchema, "relations", unknown>+ ) => {+ if (this.direction === "to") {+ return builder+ .where("fromType", "==", this.type)+ .where("toType", "==", this.reference.type)+ .where("toId", "==", this.reference.id)+ .$if(+ (this.type === "note" || this.type === "notebook") &&+ this.trashIds.length > 0,+ (b) => b.where("fromId", "not in", this.trashIds)+ )+ .select("relations.fromId as id")+ .$narrowType<{ id: string }>();+ } else {+ return builder+ .where("toType", "==", this.type)+ .where("fromType", "==", this.reference.type)+ .where("fromId", "==", this.reference.id)+ .$if(+ (this.type === "note" || this.type === "notebook") &&+ this.trashIds.length > 0,+ (b) => b.where("toId", "not in", this.trashIds)+ )+ .select("relations.toId as id")+ .$narrowType<{ id: string }>();+ }+ };+ }+}diff --git a/packages/core/src/collections/reminders.ts b/packages/core/src/collections/reminders.tsindex ff3e2e41b..e7c570c50 100644--- a/packages/core/src/collections/reminders.ts+++ b/packages/core/src/collections/reminders.ts@@ -25,9 +25,9 @@ import isYesterday from "dayjs/plugin/isYesterday";import { TimeFormat, formatDate } from "../utils/date";import { getId } from "../utils/id";import { ICollection } from "./collection";-import { CachedCollection } from "../database/cached-collection";import { Reminder } from "../types";import Database from "../api";+import { SQLCollection } from "../database/sql-collection";dayjs.extend(isTomorrow);dayjs.extend(isSameOrBefore);@@ -36,13 +36,9 @@ dayjs.extend(isToday);export class Reminders implements ICollection {name = "reminders";- readonly collection: CachedCollection<"reminders", Reminder>;+ readonly collection: SQLCollection<"reminders", Reminder>;constructor(private readonly db: Database) {- this.collection = new CachedCollection(- db.storage,- "reminders",- db.eventManager- );+ this.collection = new SQLCollection(db.sql, "reminders", db.eventManager);}async init() {@@ -65,7 +61,7 @@ export class Reminders implements ICollection {if (!reminder.date || !reminder.title)throw new Error("date and title are required in a reminder.");- await this.collection.add({+ await this.collection.upsert({id,type: "reminder",dateCreated: reminder.dateCreated || Date.now(),@@ -81,16 +77,16 @@ export class Reminders implements ICollection {disabled: reminder.disabled,snoozeUntil: reminder.snoozeUntil});- return reminder.id;+ return id;}- get raw() {- return this.collection.raw();- }+ // get raw() {+ // return this.collection.raw();+ // }- get all() {- return this.collection.items();- }+ // get all() {+ // return this.collection.items();+ // }exists(itemId: string) {return this.collection.exists(itemId);@@ -101,9 +97,7 @@ export class Reminders implements ICollection {}async remove(...reminderIds: string[]) {- for (const id of reminderIds) {- await this.collection.remove(id);- }+ await this.collection.softDelete(reminderIds);}}diff --git a/packages/core/src/collections/session-content.ts b/packages/core/src/collections/session-content.tsindex 0936ab6f7..3c3d90263 100644--- a/packages/core/src/collections/session-content.ts+++ b/packages/core/src/collections/session-content.ts@@ -22,9 +22,9 @@ import { tinyToTiptap } from "../migrations";import { makeSessionContentId } from "../utils/id";import { ICollection } from "./collection";import { isCipher } from "../database/crypto";-import { IndexedCollection } from "../database/indexed-collection";import Database from "../api";import { ContentType, SessionContentItem, isDeleted } from "../types";+import { SQLCollection } from "../database/sql-collection";export type NoteContent<TLocked extends boolean> = {data: TLocked extends true ? Cipher<"base64"> : string;@@ -33,13 +33,13 @@ export type NoteContent<TLocked extends boolean> = {export class SessionContent implements ICollection {name = "sessioncontent";- private readonly collection: IndexedCollection<+ private readonly collection: SQLCollection<"sessioncontent",SessionContentItem>;constructor(private readonly db: Database) {- this.collection = new IndexedCollection(- db.storage,+ this.collection = new SQLCollection(+ db.sql,"sessioncontent",db.eventManager);@@ -49,10 +49,6 @@ export class SessionContent implements ICollection {await this.collection.init();}- async merge(item: SessionContentItem) {- await this.collection.addItem(item);- }-async add<TLocked extends boolean>(sessionId: string,content: NoteContent<TLocked>,@@ -64,7 +60,7 @@ export class SessionContent implements ICollection {? content.data: await this.db.compressor().compress(content.data);- await this.collection.addItem({+ await this.collection.upsert({type: "sessioncontent",id: makeSessionContentId(sessionId),data,@@ -78,7 +74,7 @@ export class SessionContent implements ICollection {}async get(sessionContentId: string) {- const session = await this.collection.getItem(sessionContentId);+ const session = await this.collection.get(sessionContentId);if (!session || isDeleted(session)) return;if (@@ -93,7 +89,7 @@ export class SessionContent implements ICollection {tinyToTiptap(await this.db.compressor().decompress(session.data)));session.contentType = "tiptap";- await this.collection.addItem(session);+ await this.collection.upsert(session);}return {@@ -106,13 +102,13 @@ export class SessionContent implements ICollection {}async remove(sessionContentId: string) {- await this.collection.deleteItem(sessionContentId);+ await this.collection.delete(sessionContentId);}- async all() {- const indices = this.collection.indexer.indices;- const items = await this.collection.getItems(indices);+ // async all() {+ // const indices = this.collection.indexer.indices;+ // const items = await this.collection.getItems(indices);- return Object.values(items);- }+ // return Object.values(items);+ // }}diff --git a/packages/core/src/collections/settings.ts b/packages/core/src/collections/settings.tsindex 6cb0042fb..29d3367a4 100644--- a/packages/core/src/collections/settings.ts+++ b/packages/core/src/collections/settings.ts@@ -29,8 +29,8 @@ import {TrashCleanupInterval} from "../types";import { ICollection } from "./collection";-import { CachedCollection } from "../database/cached-collection";import { TimeFormat } from "../utils/date";+import { SQLCachedCollection } from "../database/sql-cached-collection";const DEFAULT_GROUP_OPTIONS = (key: GroupingKey) =>({@@ -66,12 +66,12 @@ const defaultSettings: SettingItemMap = {};export class Settings implements ICollection {- name = "settingsv2";- readonly collection: CachedCollection<"settingsv2", SettingItem>;+ name = "settings";+ readonly collection: SQLCachedCollection<"settings", SettingItem>;constructor(db: Database) {- this.collection = new CachedCollection(- db.storage,- "settingsv2",+ this.collection = new SQLCachedCollection(+ db.sql,+ "settings",db.eventManager);}@@ -80,19 +80,19 @@ export class Settings implements ICollection {return this.collection.init();}- get raw() {- return this.collection.raw();- }+ // get raw() {+ // return this.collection.raw();+ // }private async set<TKey extends keyof SettingItemMap>(key: TKey,value: SettingItemMap[TKey]) {const id = makeId(key);- const oldItem = this.collection.get(id);+ const oldItem = await this.collection.get(id);if (oldItem && oldItem.key !== key) throw new Error("Key conflict.");- await this.collection.add({+ await this.collection.upsert({id,key,value,diff --git a/packages/core/src/collections/shortcuts.ts b/packages/core/src/collections/shortcuts.tsindex d6cf7abb5..aa4269761 100644--- a/packages/core/src/collections/shortcuts.ts+++ b/packages/core/src/collections/shortcuts.ts@@ -18,20 +18,16 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import Database from "../api";-import { CachedCollection } from "../database/cached-collection";-import { Notebook, Shortcut, Tag, Topic } from "../types";+import { SQLCollection } from "../database/sql-collection";+import { Shortcut } from "../types";import { ICollection } from "./collection";const ALLOWED_SHORTCUT_TYPES = ["notebook", "topic", "tag"];export class Shortcuts implements ICollection {name = "shortcuts";- readonly collection: CachedCollection<"shortcuts", Shortcut>;+ readonly collection: SQLCollection<"shortcuts", Shortcut>;constructor(private readonly db: Database) {- this.collection = new CachedCollection(- db.storage,- "shortcuts",- db.eventManager- );+ this.collection = new SQLCollection(db.sql, "shortcuts", db.eventManager);}init() {@@ -45,11 +41,15 @@ export class Shortcuts implements ICollection {"Please use db.shortcuts.merge to merge remote shortcuts.");- if (shortcut.item && !ALLOWED_SHORTCUT_TYPES.includes(shortcut.item.type))+ if (+ shortcut.itemId &&+ shortcut.itemType &&+ !ALLOWED_SHORTCUT_TYPES.includes(shortcut.itemType)+ )throw new Error("Cannot create a shortcut for this type of item.");- const oldShortcut = shortcut.item- ? this.shortcut(shortcut.item.id)+ const oldShortcut = shortcut.itemId+ ? this.shortcut(shortcut.itemId): shortcut.id? this.shortcut(shortcut.id): null;@@ -64,65 +64,72 @@ export class Shortcuts implements ICollection {const id = shortcut.id || shortcut.item.id;- await this.collection.add({+ await this.collection.upsert({id,type: "shortcut",- item: shortcut.item,+ itemId: shortcut.itemId,+ itemType: shortcut.itemType,+ // item: shortcut.item,dateCreated: shortcut.dateCreated || Date.now(),dateModified: shortcut.dateModified || Date.now(),- sortIndex: this.collection.count()+ sortIndex: await this.collection.count()});return id;}- get raw() {- return this.collection.raw();- }-- get all() {- return this.collection.items();- }-- get resolved() {- return this.all.reduce((prev, shortcut) => {- const {- item: { id }- } = shortcut;-- let item: Notebook | Topic | Tag | null | undefined = null;- switch (shortcut.item.type) {- case "notebook": {- const notebook = this.db.notebooks.notebook(id);- item = notebook ? notebook.data : null;- break;- }- case "tag":- item = this.db.tags.tag(id);- break;- }- if (item) prev.push(item);- return prev;- }, [] as (Notebook | Topic | Tag)[]);+ // get raw() {+ // return this.collection.raw();+ // }++ // get all() {+ // return this.collection.items();+ // }++ async get() {+ // return this.all.reduce((prev, shortcut) => {+ // const {+ // item: { id }+ // } = shortcut;+ // let item: Notebook | Topic | Tag | null | undefined = null;+ // switch (shortcut.item.type) {+ // case "notebook": {+ // const notebook = this.db.notebooks.notebook(id);+ // item = notebook ? notebook.data : null;+ // break;+ // }+ // case "tag":+ // item = this.db.tags.tag(id);+ // break;+ // }+ // if (item) prev.push(item);+ // return prev;+ // }, [] as (Notebook | Topic | Tag)[]);}exists(id: string) {- return !!this.shortcut(id);+ return this.collection.exists(id);}shortcut(id: string) {- return this.all.find(- (shortcut) => shortcut.item.id === id || shortcut.id === id- );+ return this.collection.get(id);}async remove(...shortcutIds: string[]) {- const shortcuts = this.all.filter(- (shortcut) =>- shortcutIds.includes(shortcut.item.id) ||- shortcutIds.includes(shortcut.id)- );- for (const { id } of shortcuts) {- await this.collection.remove(id);- }+ await this.collection.softDelete(shortcutIds);+ // await this.db+ // .sql()+ // .deleteFrom("shortcuts")+ // .where((eb) =>+ // eb.or([eb("id", "in", shortcutIds), eb("itemId", "in", shortcutIds)])+ // )+ // .execute();+ // const shortcuts = this.all.filter(+ // (shortcut) =>+ // shortcutIds.includes(shortcut.item.id) ||+ // shortcutIds.includes(shortcut.id)+ // );+ // for (const { id } of shortcuts) {+ // await this.collection.remove(id);+ // }}}diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.tsindex 3a8ee704d..5b7e8a3af 100644--- a/packages/core/src/collections/tags.ts+++ b/packages/core/src/collections/tags.ts@@ -18,16 +18,16 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { getId } from "../utils/id";-import { CachedCollection } from "../database/cached-collection";-import { MaybeDeletedItem, Tag } from "../types";+import { Tag } from "../types";import Database from "../api";import { ICollection } from "./collection";+import { SQLCollection } from "../database/sql-collection";export class Tags implements ICollection {name = "tags";- readonly collection: CachedCollection<"tags", Tag>;+ readonly collection: SQLCollection<"tags", Tag>;constructor(private readonly db: Database) {- this.collection = new CachedCollection(db.storage, "tags", db.eventManager);+ this.collection = new SQLCollection(db.sql, "tags", db.eventManager);}init() {@@ -38,58 +38,46 @@ export class Tags implements ICollection {return this.collection.get(id);}- find(idOrTitle: string) {- return this.all.find(- (tag) => tag.title === idOrTitle || tag.id === idOrTitle- );- }-- async merge(remoteTag: MaybeDeletedItem<Tag>) {- if (!remoteTag) return;-- const localTag = this.collection.get(remoteTag.id);- if (!localTag || remoteTag.dateModified > localTag.dateModified)- await this.collection.add(remoteTag);- }+ // find(idOrTitle: string) {+ // return this.all.find(+ // (tag) => tag.title === idOrTitle || tag.id === idOrTitle+ // );+ // }async add(item: Partial<Tag>) {if (item.remote)throw new Error("Please use db.tags.merge to merge remote tags.");const id = item.id || getId(item.dateCreated);- const oldTag = this.tag(id);+ const oldTag = await this.tag(id);item.title = item.title ? Tags.sanitize(item.title) : item.title;if (!item.title && !oldTag?.title) throw new Error("Title is required.");- const tag: Tag = {+ await this.collection.upsert({id,dateCreated: item.dateCreated || oldTag?.dateCreated || Date.now(),dateModified: item.dateModified || oldTag?.dateModified || Date.now(),title: item.title || oldTag?.title || "",type: "tag",remote: false- };- await this.collection.add(tag);- return tag.id;+ });+ return id;}- get raw() {- return this.collection.raw();- }-- get all() {- return this.collection.items();- }+ // get raw() {+ // return this.collection.raw();+ // }- async remove(id: string) {- await this.collection.remove(id);- await this.db.relations.cleanup();- }+ // get all() {+ // return this.collection.items();+ // }- async delete(id: string) {- await this.collection.delete(id);- await this.db.relations.cleanup();+ async remove(...ids: string[]) {+ await this.db.transaction(async () => {+ await this.db.relations.unlinkOfType("tag", ids);+ await this.collection.softDelete(ids);+ });}exists(id: string) {diff --git a/packages/core/src/collections/trash.ts b/packages/core/src/collections/trash.tsindex dd78c1286..a277e6207 100644--- a/packages/core/src/collections/trash.ts+++ b/packages/core/src/collections/trash.ts@@ -19,23 +19,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import dayjs from "dayjs";import Database from "../api";-import {- BaseTrashItem,- Note,- Notebook,- TrashItem,- isTrashItem-} from "../types";--function toTrashItem<T extends Note | Notebook>(item: T): BaseTrashItem<T> {- return {- ...item,- id: item.id,- type: "trash",- itemType: item.type,- dateDeleted: Date.now()- };-}export default class Trash {collections = ["notes", "notebooks"] as const;@@ -44,96 +27,122 @@ export default class Trash {async init() {await this.cleanup();- this.cache = this.all.map((t) => t.id);+ const result = await this.db+ .sql()+ .selectNoFrom((eb) => [+ eb+ .selectFrom("notes")+ .where("type", "==", "trash")+ .select("id")+ .as("id"),+ eb+ .selectFrom("notebooks")+ .where("type", "==", "trash")+ .select("id")+ .as("id")+ ])+ .execute();++ this.cache = result.reduce((ids, item) => {+ if (item.id) ids.push(item.id);+ return ids;+ }, [] as string[]);}async cleanup() {- const now = dayjs().unix();const duration = this.db.settings.getTrashCleanupInterval();if (duration === -1 || !duration) return;- for (const item of this.all) {- if (- isTrashItem(item) &&- item.dateDeleted &&- dayjs(item.dateDeleted).add(duration, "days").unix() > now- )- continue;- await this.delete(item.id);- }- }- get all(): TrashItem[] {- const trashItems: TrashItem[] = [];- for (const key of this.collections) {- const collection = this.db[key];- trashItems.push(...collection.trashed);- }- return trashItems;+ const maxMs = dayjs().subtract(duration, "days").toDate().getTime();+ const expiredItems = await this.db+ .sql()+ .selectNoFrom((eb) => [+ eb+ .selectFrom("notes")+ .where("type", "==", "trash")+ .where("dateDeleted", "<=", maxMs)+ .select("id")+ .as("noteId"),+ eb+ .selectFrom("notebooks")+ .where("type", "==", "trash")+ .where("dateDeleted", "<=", maxMs)+ .select("id")+ .as("notebookId")+ ])+ .execute();+ const { noteIds, notebookIds } = expiredItems.reduce(+ (ids, item) => {+ if (item.noteId) ids.noteIds.push(item.noteId);+ if (item.notebookId) ids.notebookIds.push(item.notebookId);+ return ids;+ },+ { noteIds: [] as string[], notebookIds: [] as string[] }+ );+ await this.delete("note", noteIds);+ await this.delete("note", notebookIds);}- private getItem(id: string) {- for (const key of this.collections) {- const collection = this.db[key].collection;- const item = collection.get(id);- if (item && isTrashItem(item)) return [item, collection] as const;+ async add(type: "note" | "notebook", ids: string[]) {+ if (type === "note") {+ await this.db.notes.collection.update(ids, {+ type: "trash",+ itemType: "note",+ dateDeleted: Date.now()+ });+ } else if (type === "notebook") {+ await this.db.notebooks.collection.update(ids, {+ type: "trash",+ itemType: "notebook",+ dateDeleted: Date.now()+ });}- return [] as const;+ this.cache.push(...ids);}- async add(item: Note | Notebook) {- if (item.type === "note") {- await this.db.notes.collection.update(toTrashItem(item));- } else if (item.type === "notebook") {- await this.db.notebooks.collection.update(toTrashItem(item));+ async delete(type: "note" | "notebook", ids: string[]) {+ if (type === "note") {+ await this.db.content.removeByNoteId(...ids);+ await this.db.noteHistory.clearSessions(...ids);+ await this.db.notes.delete(...ids);+ } else if (type === "notebook") {+ await this.db.relations.unlinkOfType("notebook", ids);+ await this.db.notebooks.delete(...ids);}- this.cache.push(item.id);+ ids.forEach((id) => this.cache.splice(this.cache.indexOf(id), 1));}- async delete(...ids: string[]) {- for (const id of ids) {- const [item, collection] = this.getItem(id);- if (!item || !collection) continue;- if (item.itemType === "note") {- if (item.contentId) await this.db.content.remove(item.contentId);- await this.db.noteHistory.clearSessions(id);- } else if (item.itemType === "notebook") {- await this.db.relations.unlinkAll({ type: "notebook", id: item.id });- }- await collection.remove(id);- this.cache.splice(this.cache.indexOf(id), 1);- }- }-- async restore(...ids: string[]) {- for (const id of ids) {- const [item] = this.getItem(id);- if (!item) continue;- if (item.itemType === "note") {- await this.db.notes.collection.update({ ...item, type: "note" });- } else if (item.itemType === "notebook") {- await this.db.notebooks.collection.update({- ...item,- type: "notebook"- });- }- this.cache.splice(this.cache.indexOf(id), 1);+ async restore(type: "note" | "notebook", ids: string[]) {+ if (type === "note") {+ await this.db.notes.collection.update(ids, {+ type: "note",+ dateDeleted: null,+ itemType: null+ });+ } else {+ await this.db.notebooks.collection.update(ids, {+ type: "notebook",+ dateDeleted: null,+ itemType: null+ });}+ ids.forEach((id) => this.cache.splice(this.cache.indexOf(id), 1));}async clear() {- for (const item of this.all) {- await this.delete(item.id);- }+ // for (const item of this.all) {+ // await this.delete(item.id);+ // }this.cache = [];}- synced(id: string) {- const [item] = this.getItem(id);- if (item && item.itemType === "note") {- const { contentId } = item;- return !contentId || this.db.content.exists(contentId);- } else return true;- }+ // synced(id: string) {+ // // const [item] = this.getItem(id);+ // if (item && item.itemType === "note") {+ // const { contentId } = item;+ // return !contentId || this.db.content.exists(contentId);+ // } else return true;+ // }/***diff --git a/packages/core/src/database/fs.ts b/packages/core/src/database/fs.tsindex ae8ae0787..f635d1d4a 100644--- a/packages/core/src/database/fs.ts+++ b/packages/core/src/database/fs.ts@@ -25,13 +25,12 @@ import {StorageAccessor} from "../interfaces";import { DataFormat, SerializedKey } from "@notesnook/crypto/dist/src/types";-import { AttachmentMetadata } from "../types";import { EV, EVENTS } from "../common";export type FileStorageAccessor = () => FileStorage;export type DownloadableFile = {filename: string;- metadata: AttachmentMetadata;+ // metadata: AttachmentMetadata;chunkSize: number;};export type QueueItem = DownloadableFile & {@@ -57,7 +56,7 @@ export class FileStorage {this.downloads.set(groupId, files);for (const file of files as QueueItem[]) {- const { filename, metadata, chunkSize } = file;+ const { filename, chunkSize } = file;if (await this.exists(filename)) {current++;EV.publish(EVENTS.fileDownloaded, {@@ -71,7 +70,6 @@ export class FileStorage {const url = `${hosts.API_HOST}/s3?name=${filename}`;const { execute, cancel } = this.fs.downloadFile(filename, {- metadata,url,chunkSize,headers: { Authorization: `Bearer ${token}` }@@ -106,11 +104,10 @@ export class FileStorage {this.uploads.set(groupId, files);for (const file of files as QueueItem[]) {- const { filename, chunkSize, metadata } = file;+ const { filename, chunkSize } = file;const url = `${hosts.API_HOST}/s3?name=${filename}`;const { execute, cancel } = this.fs.uploadFile(filename, {chunkSize,- metadata,url,headers: { Authorization: `Bearer ${token}` }});@@ -142,21 +139,15 @@ export class FileStorage {this.uploads.delete(groupId);}- async downloadFile(- groupId: string,- filename: string,- chunkSize: number,- metadata: AttachmentMetadata- ) {+ async downloadFile(groupId: string, filename: string, chunkSize: number) {const url = `${hosts.API_HOST}/s3?name=${filename}`;const token = await this.tokenManager.getAccessToken();const { execute, cancel } = this.fs.downloadFile(filename, {- metadata,url,chunkSize,headers: { Authorization: `Bearer ${token}` }});- this.downloads.set(groupId, [{ cancel, filename, chunkSize, metadata }]);+ this.downloads.set(groupId, [{ cancel, filename, chunkSize }]);const result = await execute();this.downloads.delete(groupId);return result;diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.tsnew file mode 100644index 000000000..83c8c8d94--- /dev/null+++ b/packages/core/src/database/index.ts@@ -0,0 +1,191 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import {+ Migrator,+ Kysely,+ SqliteAdapter,+ SqliteIntrospector,+ SqliteQueryCompiler,+ sql,+ Driver,+ KyselyPlugin,+ PluginTransformQueryArgs,+ PluginTransformResultArgs,+ QueryResult,+ UnknownRow,+ RootOperationNode,+ OperationNodeTransformer,+ ValueNode,+ PrimitiveValueListNode,+ Transaction,+ ColumnType+} from "kysely";+import {+ Attachment,+ Color,+ ContentItem,+ HistorySession,+ Note,+ Notebook,+ Relation,+ Reminder,+ SessionContentItem,+ SettingItem,+ Shortcut,+ Tag,+ TrashOrItem,+ ValueOf+} from "../types";+import { NNMigrationProvider } from "./migrations";++// type FilteredKeys<T, U> = {+// [P in keyof T]: T[P] extends U ? P : never;+// }[keyof T];+type SQLiteValue<T> = T extends string | number | boolean | Array<number>+ ? T+ : T extends object | Array<any>+ ? ColumnType<T, string, string>+ : never;+export type SQLiteItem<T> = {+ [P in keyof T]?: T[P] | null;+} & { id: string };++export interface DatabaseSchema {+ notes: SQLiteItem<TrashOrItem<Note>>; //| SQLiteItem<BaseTrashItem<Note>>;+ content: SQLiteItem<ContentItem>;+ relations: SQLiteItem<Relation>;+ notebooks: SQLiteItem<TrashOrItem<Notebook>>;+ attachments: SQLiteItem<Attachment>;+ tags: SQLiteItem<Tag>;+ colors: SQLiteItem<Color>;+ reminders: SQLiteItem<Reminder>;+ settings: SQLiteItem<SettingItem>;+ notehistory: SQLiteItem<HistorySession>;+ sessioncontent: SQLiteItem<SessionContentItem>;+ shortcuts: SQLiteItem<Shortcut>;+}++type AsyncOrSyncResult<Async extends boolean, Response> = Async extends true+ ? Promise<Response>+ : Response;++export interface DatabaseCollection<T, Async extends boolean> {+ clear(): Promise<void>;+ init(): Promise<void>;+ upsert(item: T): Promise<void>;+ softDelete(ids: string[]): Promise<void>;+ delete(ids: string[]): Promise<void>;+ exists(id: string): AsyncOrSyncResult<Async, boolean>;+ count(): AsyncOrSyncResult<Async, number>;+ get(id: string): AsyncOrSyncResult<Async, T | undefined>;+ put(items: (T | undefined)[]): Promise<void>;+ update(ids: string[], partial: Partial<T>): Promise<void>;+}++export type DatabaseAccessor = () =>+ | Kysely<DatabaseSchema>+ | Transaction<DatabaseSchema>;++type FilterBooleanProperties<T> = keyof {+ [K in keyof T as T[K] extends boolean | undefined | null ? K : never]: T[K];+};++type BooleanFields = ValueOf<{+ [D in keyof DatabaseSchema]: FilterBooleanProperties<DatabaseSchema[D]>;+}>;++const BooleanProperties: BooleanFields[] = [+ "compressed",+ "conflicted",+ "deleted",+ "disabled",+ "favorite",+ "localOnly",+ "locked",+ "migrated",+ "pinned",+ "readonly",+ "remote",+ "synced"+];++export async function createDatabase(driver: Driver) {+ const db = new Kysely<DatabaseSchema>({+ dialect: {+ createAdapter: () => new SqliteAdapter(),+ createDriver: () => driver,+ createIntrospector: (db) => new SqliteIntrospector(db),+ createQueryCompiler: () => new SqliteQueryCompiler()+ },+ plugins: [new SqliteBooleanPlugin()]+ });++ const migrator = new Migrator({+ db,+ provider: new NNMigrationProvider()+ });++ await sql`PRAGMA journal_mode = WAL`.execute(db);+ await sql`PRAGMA synchronous = normal`.execute(db);++ await migrator.migrateToLatest();++ return db;+}++export class SqliteBooleanPlugin implements KyselyPlugin {+ readonly #transformer = new SqliteBooleanTransformer();++ transformQuery(args: PluginTransformQueryArgs): RootOperationNode {+ return this.#transformer.transformNode(args.node);+ }++ transformResult(+ args: PluginTransformResultArgs+ ): Promise<QueryResult<UnknownRow>> {+ for (const row of args.result.rows) {+ for (const key of BooleanProperties) {+ const value = row[key];+ row[key] = value === 1 ? true : false;+ }+ }+ return Promise.resolve(args.result);+ }+}++class SqliteBooleanTransformer extends OperationNodeTransformer {+ transformValue(node: ValueNode): ValueNode {+ return {+ ...super.transformValue(node),+ value: typeof node.value === "boolean" ? (node.value ? 1 : 0) : node.value+ };+ }++ protected transformPrimitiveValueList(+ node: PrimitiveValueListNode+ ): PrimitiveValueListNode {+ return {+ ...super.transformPrimitiveValueList(node),+ values: node.values.map((value) =>+ typeof value === "boolean" ? (value ? 1 : 0) : value+ )+ };+ }+}diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.tsnew file mode 100644index 000000000..d4f0d3b89--- /dev/null+++ b/packages/core/src/database/migrations.ts@@ -0,0 +1,218 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { CreateTableBuilder, Migration, MigrationProvider, sql } from "kysely";++export class NNMigrationProvider implements MigrationProvider {+ async getMigrations(): Promise<Record<string, Migration>> {+ return {+ "1": {+ async up(db) {+ await db.schema+ .createTable("notes")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("title", "text")+ .addColumn("headline", "text")+ .addColumn("contentId", "text")+ .addColumn("pinned", "boolean")+ .addColumn("locked", "boolean")+ .addColumn("favorite", "boolean")+ .addColumn("localOnly", "boolean")+ .addColumn("conflicted", "boolean")+ .addColumn("readonly", "boolean")+ .addColumn("dateEdited", "integer")+ .addColumn("dateDeleted", "integer")+ .addColumn("itemType", "text")+ .addForeignKeyConstraint(+ "note_has_content",+ ["contentId"],+ "content",+ ["id"],+ (b) => b.onDelete("restrict").onUpdate("restrict")+ )+ .execute();++ await db.schema+ .createTable("content")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("noteId", "text")+ .addColumn("data", "text")+ .addColumn("localOnly", "boolean")+ .addColumn("conflicted", "text")+ .addColumn("sessionId", "text")+ .addColumn("dateEdited", "integer")+ .addColumn("dateResolved", "integer")+ .execute();++ await db.schema+ .createTable("notebooks")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("title", "text")+ .addColumn("description", "text")+ .addColumn("dateEdited", "text")+ .addColumn("pinned", "boolean")+ .execute();++ await db.schema+ .createTable("tags")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("title", "text")+ .execute();++ await db.schema+ .createTable("colors")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("title", "text")+ .addColumn("colorCode", "text")+ .execute();++ await db.schema+ .createTable("relations")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("fromType", "text")+ .addColumn("fromId", "text")+ .addColumn("toType", "text")+ .addColumn("toId", "text")+ .execute();++ await db.schema+ .createTable("shortcuts")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("sortIndex", "integer")+ .addColumn("itemId", "text")+ .addColumn("itemType", "text")+ .execute();++ await db.schema+ .createTable("reminders")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("title", "text")+ .addColumn("description", "text")+ .addColumn("priority", "text")+ .addColumn("date", "integer")+ .addColumn("mode", "text")+ .addColumn("recurringMode", "text")+ .addColumn("selectedDays", "blob")+ .addColumn("localOnly", "boolean")+ .addColumn("disabled", "boolean")+ .addColumn("snoozeUntil", "integer")+ .execute();++ await db.schema+ .createTable("attachments")+ .modifyEnd(sql`without rowid`)+ .$call(addBaseColumns)+ .addColumn("iv", "text")+ .addColumn("salt", "text")+ .addColumn("size", "integer")+ .addColumn("alg", "text")+ .addColumn("encryptionKey", "text")+ .addColumn("chunkSize", "integer")+ .addColumn("hash", "text", (c) => c.unique())+ .addColumn("hashType", "text")+ .addColumn("mimeType", "text")+ .addColumn("filename", "text")+ .addColumn("dateDeleted", "integer")+ .addColumn("dateUploaded", "integer")+ .addColumn("failed", "text")+ .execute();++ await db.schema+ .createIndex("relation_from_general")+ .on("relations")+ .columns(["fromType", "toType", "fromId"])+ .where("toType", "!=", "note")+ .where("toType", "!=", "notebook")+ .execute();++ await db.schema+ .createIndex("relation_to_general")+ .on("relations")+ .columns(["fromType", "toType", "toId"])+ .where("fromType", "!=", "note")+ .where("fromType", "!=", "notebook")+ .execute();++ await db.schema+ .createIndex("relation_from_note_notebook")+ .on("relations")+ .columns(["fromType", "toType", "fromId", "toId"])+ .where((eb) =>+ eb.or([+ eb("toType", "==", "note"),+ eb("toType", "==", "notebook")+ ])+ )+ .execute();++ await db.schema+ .createIndex("relation_to_note_notebook")+ .on("relations")+ .columns(["fromType", "toType", "toId", "fromId"])+ .where((eb) =>+ eb.or([+ eb("fromType", "==", "note"),+ eb("fromType", "==", "notebook")+ ])+ )+ .execute();++ await db.schema+ .createIndex("note_type")+ .on("notes")+ .columns(["type"])+ .execute();++ await db.schema+ .createIndex("notebook_type")+ .on("notebooks")+ .columns(["type"])+ .execute();++ await db.schema+ .createIndex("attachment_hash")+ .on("attachments")+ .column("hash")+ .execute();+ },+ async down(db) {}+ }+ };+ }+}++const addBaseColumns = <T extends string, C extends string = never>(+ builder: CreateTableBuilder<T, C>+) => {+ return builder+ .addColumn("id", "text", (c) => c.primaryKey().unique().notNull())+ .addColumn("type", "text")+ .addColumn("dateModified", "integer")+ .addColumn("dateCreated", "integer")+ .addColumn("synced", "boolean")+ .addColumn("deleted", "boolean");+};diff --git a/packages/core/src/database/sql-cached-collection.ts b/packages/core/src/database/sql-cached-collection.tsnew file mode 100644index 000000000..46b315b95--- /dev/null+++ b/packages/core/src/database/sql-cached-collection.ts@@ -0,0 +1,169 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { MaybeDeletedItem, isDeleted } from "../types";+import EventManager from "../utils/event-manager";+import { DatabaseAccessor, DatabaseCollection, DatabaseSchema } from ".";+import { SQLCollection } from "./sql-collection";++export class SQLCachedCollection<+ TCollectionType extends keyof DatabaseSchema,+ T extends DatabaseSchema[TCollectionType] = DatabaseSchema[TCollectionType]+> implements DatabaseCollection<T, false>+{+ private collection: SQLCollection<TCollectionType, T>;+ private cache = new Map<string, MaybeDeletedItem<T>>();+ // private cachedItems?: T[];++ constructor(+ sql: DatabaseAccessor,+ type: TCollectionType,+ eventManager: EventManager+ ) {+ this.collection = new SQLCollection(sql, type, eventManager);+ }++ async init() {+ await this.collection.init();+ // const data = await this.collection.indexer.readMulti(+ // this.collection.indexer.indices+ // );+ // this.cache = new Map(data);+ }++ // async add(item: MaybeDeletedItem<T>) {+ // await this.collection.addItem(item);+ // this.cache.set(item.id, item);+ // this.invalidateCache();+ // }++ async clear() {+ await this.collection.clear();+ this.cache.clear();+ }++ async upsert(item: T) {+ await this.collection.upsert(item);+ this.cache.set(item.id, item);+ }++ async delete(ids: string[]) {+ ids.forEach((id) => this.cache.delete(id));+ await this.collection.delete(ids);+ }++ async softDelete(ids: string[]) {+ ids.forEach((id) =>+ this.cache.set(id, {+ id,+ deleted: true,+ dateModified: Date.now()+ })+ );+ await this.collection.softDelete(ids);+ }++ exists(id: string) {+ const item = this.cache.get(id);+ return !!item && !isDeleted(item);+ }++ count(): number {+ return this.cache.size;+ }++ get(id: string): T | undefined {+ const item = this.cache.get(id);+ if (!item || isDeleted(item)) return;+ return item;+ }++ async put(items: (T | undefined)[]): Promise<void> {+ await this.collection.put(items);+ for (const item of items) {+ if (item) this.cache.set(item.id, item);+ }+ }++ async update(ids: string[], partial: Partial<T>): Promise<void> {+ await this.collection.update(ids, partial);+ for (const id of ids) {+ const item = this.cache.get(id);+ if (!item) continue;+ this.cache.set(id, { ...item, ...partial, dateModified: Date.now() });+ }+ }++ // has(id: string) {+ // return this.cache.has(id);+ // }++ // count() {+ // return this.cache.size;+ // }++ // get(id: string) {+ // const item = this.cache.get(id);+ // if (!item || isDeleted(item)) return;+ // return item;+ // }++ // getRaw(id: string) {+ // const item = this.cache.get(id);+ // return item;+ // }++ // raw() {+ // return Array.from(this.cache.values());+ // }++ // items(map?: (item: T) => T | undefined) {+ // if (this.cachedItems && this.cachedItems.length === this.cache.size)+ // return this.cachedItems;++ // this.cachedItems = [];+ // this.cache.forEach((value) => {+ // if (isDeleted(value)) return;+ // const mapped = map ? map(value) : value;+ // if (!mapped) return;+ // this.cachedItems?.push(mapped);+ // });+ // this.cachedItems.sort((a, b) => b.dateCreated - a.dateCreated);+ // return this.cachedItems;+ // }++ // async setItems(items: (MaybeDeletedItem<T> | undefined)[]) {+ // await this.collection.setItems(items);+ // for (const item of items) {+ // if (item) {+ // this.cache.set(item.id, item);+ // }+ // }++ // this.invalidateCache();+ // }++ // *iterateSync(chunkSize: number) {+ // yield* chunkedIterate(Array.from(this.cache.values()), chunkSize);+ // }++ // invalidateCache() {+ // this.cachedItems = undefined;+ // }+}diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.tsnew file mode 100644index 000000000..a77382288--- /dev/null+++ b/packages/core/src/database/sql-collection.ts@@ -0,0 +1,148 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { EVENTS } from "../common";+import { isDeleted } from "../types";+import EventManager from "../utils/event-manager";+import {+ DatabaseAccessor,+ DatabaseCollection,+ DatabaseSchema,+ SQLiteItem+} from ".";++export class SQLCollection<+ TCollectionType extends keyof DatabaseSchema,+ T extends DatabaseSchema[TCollectionType] = DatabaseSchema[TCollectionType]+> implements DatabaseCollection<SQLiteItem<T>, true>+{+ constructor(+ private readonly db: DatabaseAccessor,+ private readonly type: TCollectionType,+ private readonly eventManager: EventManager+ ) {}++ async clear() {+ await this.db().deleteFrom(this.type).execute();+ }++ async init() {}++ async upsert(item: SQLiteItem<T>) {+ if (!item.id) throw new Error("The item must contain the id field.");+ if (!item.deleted) item.dateCreated = item.dateCreated || Date.now();+ this.eventManager.publish(EVENTS.databaseUpdated, item.id, item);++ // if item is newly synced, remote will be true.+ if (!item.remote) {+ item.dateModified = Date.now();+ item.synced = false;+ }+ // the item has become local now, so remove the flags+ delete item.remote;++ await this.db()+ .replaceInto<keyof DatabaseSchema>(this.type)+ .values(item)+ .execute();+ }++ async softDelete(ids: string[]) {+ this.eventManager.publish(EVENTS.databaseUpdated, ids);+ await this.db()+ .replaceInto<keyof DatabaseSchema>(this.type)+ .values(+ ids.map((id) => ({+ id,+ deleted: true,+ dateModified: Date.now()+ }))+ )+ .execute();+ }++ async delete(ids: string[]) {+ this.eventManager.publish(EVENTS.databaseUpdated, ids);+ await this.db()+ .deleteFrom<keyof DatabaseSchema>(this.type)+ .where("id", "in", ids)+ .execute();+ }++ async exists(id: string) {+ const { count } =+ (await this.db()+ .selectFrom<keyof DatabaseSchema>(this.type)+ .select((a) => a.fn.count<number>("id").as("count"))+ .where("id", "==", id)+ .limit(1)+ .executeTakeFirst()) || {};++ return count !== undefined && count > 0;+ }++ async count() {+ const { count } =+ (await this.db()+ .selectFrom<keyof DatabaseSchema>(this.type)+ .select((a) => a.fn.count<number>("id").as("count"))+ .where("deleted", "is", null)+ .executeTakeFirst()) || {};+ return count || 0;+ }++ async get(id: string) {+ const item = await this.db()+ .selectFrom<keyof DatabaseSchema>(this.type)+ .selectAll()+ .where("id", "==", id)+ .executeTakeFirst();+ if (!item || isDeleted(item)) return;+ return item as T;+ }++ async put(items: (SQLiteItem<T> | undefined)[]) {+ const entries = items.reduce((array, item) => {+ if (!item) return array;+ if (!item.remote) {+ item.dateModified = Date.now();+ item.synced = false;+ }+ delete item.remote;+ array.push(item);+ return array;+ }, [] as SQLiteItem<T>[]);++ await this.db()+ .replaceInto<keyof DatabaseSchema>(this.type)+ .values(entries)+ .execute();+ }++ async update(ids: string[], partial: Partial<SQLiteItem<T>>) {+ await this.db()+ .updateTable<keyof DatabaseSchema>(this.type)+ .where("id", "in", ids)+ .set({+ ...partial,+ dateModified: Date.now()+ })+ .execute();+ }+}diff --git a/packages/core/src/interfaces.ts b/packages/core/src/interfaces.tsindex e479ac37c..c6fbc13a6 100644--- a/packages/core/src/interfaces.ts+++ b/packages/core/src/interfaces.ts@@ -18,14 +18,13 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { Cipher, DataFormat, SerializedKey } from "@notesnook/crypto";-import { AttachmentMetadata } from "./types";export type Output<TOutputFormat extends DataFormat> =TOutputFormat extends Omit<DataFormat, "uint8array"> ? string : Uint8Array;export type FileEncryptionMetadata = {chunkSize: number;iv: string;- length: number;+ size: number;salt: string;alg: string;};@@ -74,7 +73,7 @@ export interface ICompressor {export type RequestOptions = {url: string;- metadata?: AttachmentMetadata;+ // metadata?: AttachmentMetadata;chunkSize: number;headers: { Authorization: string };};diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.tsindex c14e8aaac..4cd65c0d6 100644--- a/packages/core/src/migrations.ts+++ b/packages/core/src/migrations.ts@@ -298,10 +298,17 @@ const migrations: Migration[] = [return true;},shortcut: (item) => {- if (item.item.type === "topic") {+ if (item.item?.type === "topic") {item.item = { type: "notebook", id: item.item.id };- return true;}++ if (item.item) {+ item.itemId = item.item.id;+ item.itemType = item.item.type;+ }++ delete item.item;+ return true;},settings: async (item, db) => {if (item.trashCleanupInterval)@@ -336,6 +343,16 @@ const migrations: Migration[] = [}}return true;+ },+ relation: (item) => {+ item.fromId = item.from!.id;+ item.fromType = item.from!.type;+ item.toId = item.to!.id;+ item.toType = item.to!.type;++ delete item.to;+ delete item.from;+ return true;}}},diff --git a/packages/core/src/types.ts b/packages/core/src/types.tsindex 8618a2109..3f9f11b5e 100644--- a/packages/core/src/types.ts+++ b/packages/core/src/types.ts@@ -131,6 +131,7 @@ export interface BaseItem<TType extends ItemType> {migrated?: boolean;remote?: boolean;synced?: boolean;+ deleted?: boolean;}export type NotebookReference = {@@ -164,6 +165,9 @@ export interface Note extends BaseItem<"note"> {readonly: boolean;dateEdited: number;++ dateDeleted: null;+ itemType: null;}export interface Notebook extends BaseItem<"notebook"> {@@ -171,6 +175,10 @@ export interface Notebook extends BaseItem<"notebook"> {description?: string;dateEdited: number;pinned: boolean;++ dateDeleted: null;+ itemType: null;+/*** @deprecated only kept here for migration purposes.*/@@ -191,6 +199,9 @@ export interface Topic extends BaseItem<"topic"> {notes?: string[];}+/**+ * @deprecated only kept here for migration purposes+ */export type AttachmentMetadata = {hash: string;hashType: string;@@ -201,15 +212,32 @@ export type AttachmentMetadata = {export interface Attachment extends BaseItem<"attachment"> {iv: string;salt: string;- length: number;alg: string;- key: Cipher<"base64">;chunkSize: number;- metadata: AttachmentMetadata;+dateUploaded?: number;failed?: string;dateDeleted?: number;+ filename: string;+ size: number;+ hash: string;+ hashType: string;+ mimeType: string;+ encryptionKey: string;++ /**+ * @deprecated only kept here for migration purposes+ */+ key?: Cipher<"base64">;+ /**+ * @deprecated only kept here for migration purposes+ */+ length?: number;+ /**+ * @deprecated only kept here for migration purposes+ */+ metadata?: AttachmentMetadata;/*** @deprecated only kept here for migration purposes*/@@ -244,14 +272,31 @@ export type ItemReference = {};export interface Relation extends BaseItem<"relation"> {- from: ItemReference;- to: ItemReference;+ fromId: string;+ fromType: keyof ItemMap;+ toId: string;+ toType: keyof ItemMap;++ /**+ * @deprecated only kept here for migration purposes+ */+ from?: ItemReference;+ /**+ * @deprecated only kept here for migration purposes+ */+ to?: ItemReference;}+/**+ * @deprecated only kept here for migration purposes+ */type BaseShortcutReference = {id: string;};+/**+ * @deprecated only kept here for migration purposes+ */type TagNotebookShortcutReference = BaseShortcutReference & {type: "tag" | "notebook";};@@ -265,7 +310,14 @@ type TopicShortcutReference = BaseShortcutReference & {};export interface Shortcut extends BaseItem<"shortcut"> {- item: TopicShortcutReference | TagNotebookShortcutReference;+ itemId: string;+ itemType: "tag" | "notebook";++ /**+ * @deprecated only kept here for migration purposes+ */+ item?: TopicShortcutReference | TagNotebookShortcutReference;+sortIndex: number;}@@ -376,17 +428,14 @@ export type TrashOrItem<T extends BaseItem<"note" | "notebook">> =export type BaseTrashItem<TItem extends BaseItem<"note" | "notebook">> =BaseItem<"trash"> & {- title: string;itemType: TItem["type"];dateDeleted: number;- } & Omit<TItem, "id" | "type">;+ } & Omit<TItem, "id" | "type" | "dateDeleted" | "itemType">;export type TrashItem = BaseTrashItem<Note> | BaseTrashItem<Notebook>;-export function isDeleted<T extends BaseItem<ItemType>>(- item: MaybeDeletedItem<T>-): item is DeletedItem {- return "deleted" in item;+export function isDeleted(item: object): item is DeletedItem {+ return "deleted" in item && !!item.deleted;}export function isTrashItem(item: MaybeDeletedItem<Item>): item is TrashItem {diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.tsindex 98a695d14..c189446b4 100644--- a/packages/core/vitest.config.ts+++ b/packages/core/vitest.config.ts@@ -27,10 +27,14 @@ export default defineConfig({coverage: {reporter: ["text", "html"]},+ exclude: ["__benches__/**/*.bench.ts"],include: [...(IS_E2E ? ["__e2e__/**/*.test.{js,ts}"] : []),"__tests__/**/*.test.{js,ts}","src/**/*.test.{js,ts}"- ]+ ],+ benchmark: {+ include: ["__benches__/**/*.bench.ts"]+ }}});