A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: fix `database disk image malformed` error (#5525)
Abdullah Atta080101bc4570
diff --git a/packages/core/__tests__/fts-triggers.test.ts b/packages/core/__tests__/fts-triggers.test.tsindex 4f7fbf628..6a2590d11 100644--- a/packages/core/__tests__/fts-triggers.test.ts+++ b/packages/core/__tests__/fts-triggers.test.ts@@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.*/import { expect, test } from "vitest";-import { TEST_NOTE, databaseTest, noteTest } from "./utils";+import { databaseTest, noteTest } from "./utils";test("updating deleted content should not throw", () =>databaseTest().then(async (db) => {diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.tsindex d312ebd75..970fb6112 100644--- a/packages/core/src/api/lookup.ts+++ b/packages/core/src/api/lookup.ts@@ -25,6 +25,7 @@ import { AnyColumnWithTable, Kysely, sql } from "kysely";import { FilteredSelector } from "../database/sql-collection";import { VirtualizedGrouping } from "../utils/virtualized-grouping";import { logger } from "../logger";+import { rebuildSearchIndex } from "../database/fts";type SearchResults<T> = {sorted: (limit?: number) => Promise<VirtualizedGrouping<T>>;@@ -245,4 +246,9 @@ export default class Lookup {if (!ids.length) return [];return selector.items(ids);}++ async rebuild() {+ const db = this.db.sql() as unknown as Kysely<RawDatabaseSchema>;+ await rebuildSearchIndex(db);+ }}diff --git a/packages/core/src/database/fts.ts b/packages/core/src/database/fts.tsnew file mode 100644index 000000000..dc3ca49be--- /dev/null+++ b/packages/core/src/database/fts.ts@@ -0,0 +1,74 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { Kysely, sql } from "kysely";+import { RawDatabaseSchema } from ".";++export async function rebuildSearchIndex(db: Kysely<RawDatabaseSchema>) {+ await db.transaction().execute(async (tx) => {+ for (const query of [+ sql`INSERT INTO content_fts(content_fts) VALUES('delete-all')`,+ sql`INSERT INTO notes_fts(notes_fts) VALUES('delete-all')`+ ]) {+ await query.execute(tx);+ }++ await tx+ .insertInto("content_fts")+ .columns(["rowid", "id", "data", "noteId"])+ .expression((eb) =>+ eb+ .selectFrom("content")+ .where((eb) =>+ eb.and([+ eb("noteId", "is not", null),+ eb("data", "is not", null),+ eb("deleted", "is not", true)+ ])+ )+ .select([+ "rowid",+ "id",+ sql`IIF(locked == 1, '', data)`.as("data"),+ "noteId"+ ])+ )+ .execute();++ await tx+ .insertInto("notes_fts")+ .columns(["rowid", "id", "title"])+ .expression((eb) =>+ eb+ .selectFrom("notes")+ .where((eb) =>+ eb.and([eb("title", "is not", null), eb("deleted", "is not", true)])+ )+ .select(["rowid", "id", "title"])+ )+ .execute();++ for (const query of [+ sql`INSERT INTO content_fts(content_fts) VALUES('optimize')`,+ sql`INSERT INTO notes_fts(notes_fts) VALUES('optimize')`+ ]) {+ await query.execute(tx);+ }+ });+}diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.tsindex d06d4fecc..12fa7a161 100644--- a/packages/core/src/database/migrations.ts+++ b/packages/core/src/database/migrations.ts@@ -24,6 +24,7 @@ import {MigrationProvider,sql} from "kysely";+import { rebuildSearchIndex } from "./fts";const COLLATE_NOCASE: ColumnBuilderCallback = (col) =>col.modifyEnd(sql`collate nocase`);@@ -283,6 +284,11 @@ export class NNMigrationProvider implements MigrationProvider {.execute();},async down(db) {}+ },+ "2": {+ async up(db) {+ await rebuildSearchIndex(db);+ }}};}diff --git a/packages/core/src/database/triggers.ts b/packages/core/src/database/triggers.tsindex 295da9734..47222f554 100644--- a/packages/core/src/database/triggers.ts+++ b/packages/core/src/database/triggers.ts@@ -31,16 +31,16 @@ export async function createTriggers(db: Kysely<RawDatabaseSchema>) {.addEvent("insert").when((eb) =>eb.and([- eb("new.deleted", "is not", true),- eb("new.locked", "is not", true),- eb("new.data", "is not", null)+ eb("new.noteId", "is not", null),+ eb("new.data", "is not", null),+ eb("new.deleted", "is not", true)])).addQuery((c) =>c.insertInto("content_fts").values({rowid: sql`new.rowid`,id: sql`new.id`,- data: sql`new.data`,+ data: sql`IIF(new.locked == 1, '', new.data)`,noteId: sql`new.noteId`}))@@ -53,6 +53,13 @@ export async function createTriggers(db: Kysely<RawDatabaseSchema>) {.onTable("content", "main").after().addEvent("delete")+ .when((eb) =>+ eb.and([+ eb("old.noteId", "is not", null),+ eb("old.data", "is not", null),+ eb("old.deleted", "is not", true)+ ])+ ).addQuery((c) =>c.insertInto("content_fts").values({content_fts: sql.lit("delete"),@@ -73,9 +80,9 @@ export async function createTriggers(db: Kysely<RawDatabaseSchema>) {.addEvent("update").when((eb) =>eb.and([- eb("old.deleted", "is not", true),eb("old.noteId", "is not", null),- eb("old.data", "is not", null)+ eb("old.data", "is not", null),+ eb("old.deleted", "is not", true)])).addQuery((c) =>@@ -107,7 +114,8 @@ export async function createTriggers(db: Kysely<RawDatabaseSchema>) {.addEvent("insert").when((eb) =>eb.and([- eb.or([eb("new.deleted", "is", null), eb("new.deleted", "==", false)])+ eb("new.title", "is not", null),+ eb("new.deleted", "is not", true)])).addQuery((c) =>@@ -126,6 +134,12 @@ export async function createTriggers(db: Kysely<RawDatabaseSchema>) {.onTable("notes", "main").after().addEvent("delete")+ .when((eb) =>+ eb.and([+ eb("old.title", "is not", null),+ eb("old.deleted", "is not", true)+ ])+ ).addQuery((c) =>c.insertInto("notes_fts").values({notes_fts: sql.lit("delete"),