A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: fix `table x has no column y` errors
Abdullah Atta85342e5f74c3
diff --git a/packages/core/__tests__/notes.test.ts b/packages/core/__tests__/notes.test.tsindex a1a065215..f6b5624ea 100644--- a/packages/core/__tests__/notes.test.ts+++ b/packages/core/__tests__/notes.test.ts@@ -161,7 +161,7 @@ test("update note", () =>title: "I am a new title",content: {type: TEST_NOTE.content.type,- data: "<p><br/></p>"+ data: '<p data-block-id="p1"><br/></p>'},pinned: true,favorite: truediff --git a/packages/core/__tests__/utils/index.ts b/packages/core/__tests__/utils/index.tsindex 7f22b889b..2018f46bf 100644--- a/packages/core/__tests__/utils/index.ts+++ b/packages/core/__tests__/utils/index.ts@@ -75,7 +75,7 @@ const notebookTest = (notebook = TEST_NOTEBOOK) =>const TEST_NOTE: { content: NoteContent<false> } = {content: {type: "tiptap",- data: `<p>Hello <span style="color:#f00">This is colorful</span></p>`+ data: `<p data-block-id="p1">Hello <span style="color:#f00">This is colorful</span></p>`}};diff --git a/packages/core/src/api/index.ts b/packages/core/src/api/index.tsindex 2f59fae13..2b9f1b18b 100644--- a/packages/core/src/api/index.ts+++ b/packages/core/src/api/index.ts@@ -72,6 +72,7 @@ import { CachedCollection } from "../database/cached-collection";import { Vaults } from "../collections/vaults";import { KVStorage } from "../database/kv";import { QueueValue } from "../utils/queue-value";+import { Sanitizer } from "../database/sanitizer";type EventSourceConstructor = new (uri: string,@@ -178,10 +179,10 @@ class Database {vault = new Vault(this);lookup = new Lookup(this);backup = new Backup(this);- settings = new Settings(this);migrations = new Migrations(this);monographs = new Monographs(this);trash = new Trash(this);+ sanitizer = new Sanitizer(this.sql);notebooks = new Notebooks(this);tags = new Tags(this);@@ -194,6 +195,7 @@ class Database {relations = new Relations(this);notes = new Notes(this);vaults = new Vaults(this);+ settings = new Settings(this);/*** @deprecated only kept here for migration purposes@@ -270,6 +272,8 @@ class Database {this.options.sqliteOptions)) as unknown as Kysely<DatabaseSchema>;+ await this.sanitizer.init();+await this.initCollections();await this.migrations.init();@@ -294,6 +298,7 @@ class Database {await this.reminders.init();await this.relations.init();await this.notes.init();+ await this.vaults.init();await this.trash.init();diff --git a/packages/core/src/collections/attachments.ts b/packages/core/src/collections/attachments.tsindex 6d15c3502..6ebef6b3b 100644--- a/packages/core/src/collections/attachments.ts+++ b/packages/core/src/collections/attachments.ts@@ -40,7 +40,8 @@ export class Attachments implements ICollection {db.sql,db.transaction,"attachments",- db.eventManager+ db.eventManager,+ db.sanitizer);this.key = null;diff --git a/packages/core/src/collections/colors.ts b/packages/core/src/collections/colors.tsindex 2a5e2a39b..4932af94e 100644--- a/packages/core/src/collections/colors.ts+++ b/packages/core/src/collections/colors.ts@@ -43,7 +43,8 @@ export class Colors implements ICollection {db.sql,db.transaction,"colors",- db.eventManager+ db.eventManager,+ db.sanitizer);}diff --git a/packages/core/src/collections/content.ts b/packages/core/src/collections/content.tsindex dde305c71..986002f8f 100644--- a/packages/core/src/collections/content.ts+++ b/packages/core/src/collections/content.ts@@ -56,7 +56,8 @@ export class Content implements ICollection {db.sql,db.transaction,"content",- db.eventManager+ db.eventManager,+ db.sanitizer);}diff --git a/packages/core/src/collections/note-history.ts b/packages/core/src/collections/note-history.tsindex bb1dc2729..116d18368 100644--- a/packages/core/src/collections/note-history.ts+++ b/packages/core/src/collections/note-history.ts@@ -35,7 +35,8 @@ export class NoteHistory implements ICollection {db.sql,db.transaction,"notehistory",- db.eventManager+ db.eventManager,+ db.sanitizer);}diff --git a/packages/core/src/collections/notebooks.ts b/packages/core/src/collections/notebooks.tsindex 4ca7df77e..abcc5109b 100644--- a/packages/core/src/collections/notebooks.ts+++ b/packages/core/src/collections/notebooks.ts@@ -37,7 +37,8 @@ export class Notebooks implements ICollection {db.sql,db.transaction,"notebooks",- db.eventManager+ db.eventManager,+ db.sanitizer);}diff --git a/packages/core/src/collections/notes.ts b/packages/core/src/collections/notes.tsindex 0d5ea11f2..17300f5de 100644--- a/packages/core/src/collections/notes.ts+++ b/packages/core/src/collections/notes.ts@@ -51,7 +51,8 @@ export class Notes implements ICollection {db.sql,db.transaction,"notes",- db.eventManager+ db.eventManager,+ db.sanitizer);}diff --git a/packages/core/src/collections/relations.ts b/packages/core/src/collections/relations.tsindex 81471731a..c094211b1 100644--- a/packages/core/src/collections/relations.ts+++ b/packages/core/src/collections/relations.ts@@ -34,13 +34,14 @@ export class Relations implements ICollection {db.sql,db.transaction,"relations",- db.eventManager+ db.eventManager,+ db.sanitizer);}async init() {// await this.buildCache();- // return this.collection.init();+ await this.collection.init();}async add(from: ItemReference, to: ItemReference) {diff --git a/packages/core/src/collections/reminders.ts b/packages/core/src/collections/reminders.tsindex 1e42bd793..e547112f8 100644--- a/packages/core/src/collections/reminders.ts+++ b/packages/core/src/collections/reminders.ts@@ -43,7 +43,8 @@ export class Reminders implements ICollection {db.sql,db.transaction,"reminders",- db.eventManager+ db.eventManager,+ db.sanitizer);}diff --git a/packages/core/src/collections/session-content.ts b/packages/core/src/collections/session-content.tsindex 78ec482b3..d8d3743f7 100644--- a/packages/core/src/collections/session-content.ts+++ b/packages/core/src/collections/session-content.ts@@ -39,7 +39,8 @@ export class SessionContent implements ICollection {db.sql,db.transaction,"sessioncontent",- db.eventManager+ db.eventManager,+ db.sanitizer);}diff --git a/packages/core/src/collections/settings.ts b/packages/core/src/collections/settings.tsindex 813ea1a3f..72faf6039 100644--- a/packages/core/src/collections/settings.ts+++ b/packages/core/src/collections/settings.ts@@ -82,7 +82,8 @@ export class Settings implements ICollection {db.sql,db.transaction,"settings",- db.eventManager+ db.eventManager,+ db.sanitizer);}diff --git a/packages/core/src/collections/shortcuts.ts b/packages/core/src/collections/shortcuts.tsindex 84bdef8a2..fea418be0 100644--- a/packages/core/src/collections/shortcuts.ts+++ b/packages/core/src/collections/shortcuts.ts@@ -38,7 +38,8 @@ export class Shortcuts implements ICollection {db.sql,db.transaction,"shortcuts",- db.eventManager+ db.eventManager,+ db.sanitizer);}diff --git a/packages/core/src/collections/tags.ts b/packages/core/src/collections/tags.tsindex 833c0d74f..45dad046c 100644--- a/packages/core/src/collections/tags.ts+++ b/packages/core/src/collections/tags.ts@@ -32,7 +32,8 @@ export class Tags implements ICollection {db.sql,db.transaction,"tags",- db.eventManager+ db.eventManager,+ db.sanitizer);}diff --git a/packages/core/src/collections/vaults.ts b/packages/core/src/collections/vaults.tsindex a8e761eb8..4c4f45d16 100644--- a/packages/core/src/collections/vaults.ts+++ b/packages/core/src/collections/vaults.ts@@ -32,11 +32,14 @@ export class Vaults implements ICollection {db.sql,db.transaction,"vaults",- db.eventManager+ db.eventManager,+ db.sanitizer);}- async init() {}+ async init() {+ await this.collection.init();+ }async add(item: Partial<Vault>) {const id = item.id || getId();diff --git a/packages/core/src/content-types/__tests__/tiptap.test.js b/packages/core/src/content-types/__tests__/tiptap.test.jsindex 63f1f347d..8c1405d5b 100644--- a/packages/core/src/content-types/__tests__/tiptap.test.js+++ b/packages/core/src/content-types/__tests__/tiptap.test.js@@ -26,7 +26,7 @@ import { test, expect } from "vitest";test("img src is empty after extract attachments", async () => {const tiptap = new Tiptap(IMG_CONTENT_WITHOUT_HASH);- const result = await tiptap.extractAttachments(async () => {+ const result = await tiptap.postProcess(async () => {return "helloworld";});expect(result.hashes).toHaveLength(1);@@ -37,7 +37,7 @@ test("img src is empty after extract attachments", async () => {test("img src is present after insert attachments", async () => {const tiptap = new Tiptap(IMG_CONTENT);- const result = await tiptap.extractAttachments(async () => {+ const result = await tiptap.postProcess(async () => {return { key: "hello", metadata: {} };});const tiptap2 = new Tiptap(result.data);diff --git a/packages/core/src/database/migrator.ts b/packages/core/src/database/migrator.tsindex 1fcb393d7..211c87eeb 100644--- a/packages/core/src/database/migrator.ts+++ b/packages/core/src/database/migrator.ts@@ -71,7 +71,8 @@ class Migrator {db.sql,db.transaction,collection.table,- db.eventManager+ db.eventManager,+ db.sanitizer);if (version <= 5.9) {if (collection.name === "settings") {diff --git a/packages/core/src/database/sanitizer.ts b/packages/core/src/database/sanitizer.tsnew file mode 100644index 000000000..0148836d7--- /dev/null+++ b/packages/core/src/database/sanitizer.ts@@ -0,0 +1,67 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { DatabaseAccessor } from ".";+import { logger } from "../logger";++export class Sanitizer {+ tables: Record<string, Set<string>> = {};+ logger = logger.scope("sanitizer");+ constructor(private readonly db: DatabaseAccessor) {}++ async init() {+ const metadata = await this.db().introspection.getTables({+ withInternalKyselyTables: false+ });+ for (const table of metadata) {+ this.tables[table.name] = new Set(table.columns.map((c) => c.name));+ }+ }++ /**+ * Sanitization is done based on the latest table schema in the database. All+ * unrecognized keys are removed+ */+ sanitize(table: string, item: any) {+ const schema = this.tables[table];+ if (!schema) {+ if (process.env.NODE_ENV === "test")+ throw new Error(+ `Invalid table: ${table} (expected one of ${Object.keys(+ this.tables+ ).join(", ")})`+ );+ return false;+ }++ for (const key in item) {+ if (schema.has(key)) continue;+ if (process.env.NODE_ENV === "test")+ throw new Error(`Found invalid key in item ${key} (${table})`);+ else+ this.logger.debug("Found invalid key in item", {+ table,+ key,+ value: item[key]+ });+ delete item[key];+ }+ return true;+ }+}diff --git a/packages/core/src/database/sql-cached-collection.ts b/packages/core/src/database/sql-cached-collection.tsindex ae87775c4..042dd9a07 100644--- a/packages/core/src/database/sql-cached-collection.ts+++ b/packages/core/src/database/sql-cached-collection.ts@@ -22,6 +22,7 @@ import EventManager from "../utils/event-manager";import { DatabaseAccessor, DatabaseCollection, DatabaseSchema } from ".";import { SQLCollection } from "./sql-collection";import { Transaction } from "kysely";+import { Sanitizer } from "./sanitizer";export class SQLCachedCollection<TCollectionType extends keyof DatabaseSchema,@@ -38,13 +39,15 @@ export class SQLCachedCollection<executor: (tr: Transaction<DatabaseSchema>) => Promise<void>) => Promise<void>,type: TCollectionType,- eventManager: EventManager+ eventManager: EventManager,+ sanitizer: Sanitizer) {this.collection = new SQLCollection(sql,startTransaction,type,- eventManager+ eventManager,+ sanitizer);}diff --git a/packages/core/src/database/sql-collection.ts b/packages/core/src/database/sql-collection.tsindex 8b15ece25..4fe7971af 100644--- a/packages/core/src/database/sql-collection.ts+++ b/packages/core/src/database/sql-collection.ts@@ -45,6 +45,7 @@ import {import { VirtualizedGrouping } from "../utils/virtualized-grouping";import { groupArray } from "../utils/grouping";import { toChunks } from "../utils/array";+import { Sanitizer } from "./sanitizer";const formats = {month: "%Y-%m",@@ -66,7 +67,8 @@ export class SQLCollection<executor: (tr: Transaction<DatabaseSchema>) => Promise<void>) => Promise<void>,private readonly type: TCollectionType,- private readonly eventManager: EventManager+ private readonly eventManager: EventManager,+ private readonly sanitizer: Sanitizer) {}async clear() {@@ -87,6 +89,8 @@ export class SQLCollection<// the item has become local now, so remove the flagsdelete item.remote;+ if (!this.sanitizer.sanitize(this.type, item)) return;+await this.db().replaceInto<keyof DatabaseSchema>(this.type).values(item)@@ -163,6 +167,9 @@ export class SQLCollection<item.synced = false;}delete item.remote;++ if (!this.sanitizer.sanitize(this.type, item)) return array;+array.push(item);return array;}, [] as SQLiteItem<T>[]);@@ -185,6 +192,7 @@ export class SQLCollection<partial: Partial<SQLiteItem<T>>,options: { sendEvent: boolean } = { sendEvent: true }) {+ if (!this.sanitizer.sanitize(this.type, partial)) return;await this.db().updateTable<keyof DatabaseSchema>(this.type).where("id", "in", ids)