A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: make db migration more resilient and recoverable (#10037)
Ammar Ahmed48d88069c007
diff --git a/packages/core/__tests__/sql-migrations.test.ts b/packages/core/__tests__/sql-migrations.test.tsnew file mode 100644index 000000000..b621b913e--- /dev/null+++ b/packages/core/__tests__/sql-migrations.test.ts@@ -0,0 +1,133 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import {+ Kysely,+ Migration,+ MigrationProvider,+ SqliteDialect,+ sql+} from "@streetwriters/kysely";+import BetterSQLite3 from "better-sqlite3-multiple-ciphers";+import * as betterTrigram from "sqlite-better-trigram";+import * as fts5Html from "sqlite3-fts5-html";+import { getLoadablePath } from "sqlite-regex";+import { NNMigrationProvider } from "../src/database/migrations.js";+import {+ initializeDatabase,+ SqliteBooleanPlugin+} from "../src/database/index.js";+import { describe, expect, test } from "vitest";++function createSchemaTestDatabase() {+ const sqlite = BetterSQLite3(":memory:").unsafeMode(true);+ betterTrigram.load(sqlite);+ fts5Html.load(sqlite);+ sqlite.loadExtension(getLoadablePath());++ return new Kysely({+ dialect: new SqliteDialect({+ database: sqlite+ }),+ plugins: [new SqliteBooleanPlugin()]+ });+}++async function createMigrationProviderBefore(migrationName: string) {+ const migrations = await new NNMigrationProvider().getMigrations();+ const entries = Object.entries(migrations);+ const index = entries.findIndex(([name]) => name === migrationName);++ if (index === -1) {+ throw new Error(`Migration not found: ${migrationName}`);+ }++ return {+ migrations,+ provider: {+ async getMigrations() {+ return Object.fromEntries(entries.slice(0, index));+ }+ } satisfies MigrationProvider+ };+}++describe("Schema migration recovery", () => {+ test("replays a partially applied spellcheck migration", async () => {+ const db = createSchemaTestDatabase();++ try {+ const { migrations, provider } = await createMigrationProviderBefore(+ "a-2026-04-06"+ );++ await initializeDatabase(db, provider, "notesnook-test");+ await migrations["a-2026-04-06"]?.up(db);++ await initializeDatabase(db, new NNMigrationProvider(), "notesnook-test");++ const migrationRow = await sql<{ name: string }>`+ SELECT name+ FROM kysely_migration+ WHERE name = ${"a-2026-04-06"}+ `.execute(db);+ const column = await sql<{ name: string }>`+ SELECT name+ FROM pragma_table_info(${"notes"})+ WHERE name = ${"spellcheck"}+ `.execute(db);++ expect(migrationRow.rows).toHaveLength(1);+ expect(column.rows).toHaveLength(1);+ } finally {+ await db.destroy();+ }+ });++ test("replays a partially applied inbox items history migration", async () => {+ const db = createSchemaTestDatabase();++ try {+ const { migrations, provider } = await createMigrationProviderBefore(+ "a-2026-05-07"+ );++ await initializeDatabase(db, provider, "notesnook-test");+ await migrations["a-2026-05-07"]?.up(db);++ await initializeDatabase(db, new NNMigrationProvider(), "notesnook-test");++ const migrationRow = await sql<{ name: string }>`+ SELECT name+ FROM kysely_migration+ WHERE name = ${"a-2026-05-07"}+ `.execute(db);+ const table = await sql<{ name: string }>`+ SELECT name+ FROM sqlite_master+ WHERE type = 'table' AND name = ${"inboxitemshistory"}+ `.execute(db);++ expect(migrationRow.rows).toHaveLength(1);+ expect(table.rows).toHaveLength(1);+ } finally {+ await db.destroy();+ }+ });+});diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.tsindex 5290963e1..826bfcc98 100644--- a/packages/core/src/database/migrations.ts+++ b/packages/core/src/database/migrations.ts@@ -27,6 +27,7 @@ import {sql} from "@streetwriters/kysely";import { rebuildSearchIndex } from "./fts.js";+import type { RawDatabaseSchema } from "./index.js";const COLLATE_NOCASE: ColumnBuilderCallback = (col) =>col.modifyEnd(sql`collate nocase`);@@ -389,18 +390,22 @@ export class NNMigrationProvider implements MigrationProvider {},"8": {async up(db) {- await db.schema- .alterTable("notes")- .addColumn("isGeneratedTitle", "boolean")- .execute();+ await ensureColumn(db, "notes", "isGeneratedTitle", async () => {+ await db.schema+ .alterTable("notes")+ .addColumn("isGeneratedTitle", "boolean")+ .execute();+ });}},"9": {async up(db) {- await db.schema- .alterTable("notes")- .addColumn("archived", "boolean")- .execute();+ await ensureColumn(db, "notes", "archived", async () => {+ await db.schema+ .alterTable("notes")+ .addColumn("archived", "boolean")+ .execute();+ });}},// changing the migrations name scheme from here because@@ -434,10 +439,12 @@ export class NNMigrationProvider implements MigrationProvider {},"a-2026-01-07": {async up(db) {- await db.schema- .alterTable("notes")- .addColumn("expiryDate", "text")- .execute();+ await ensureColumn(db, "notes", "expiryDate", async () => {+ await db.schema+ .alterTable("notes")+ .addColumn("expiryDate", "text")+ .execute();+ });await db.schema.createIndex("note_expiry_date").ifNotExists()@@ -448,32 +455,39 @@ export class NNMigrationProvider implements MigrationProvider {},"a-2026-01-09": {async up(db) {- await db.schema- .alterTable("sessioncontent")- .addColumn("title", "text")- .execute();+ await ensureColumn(db, "sessioncontent", "title", async () => {+ await db.schema+ .alterTable("sessioncontent")+ .addColumn("title", "text")+ .execute();+ });}},"a-2026-02-11": {async up(db) {- await db.schema- .alterTable("monographs")- .addColumn("publishUrl", "text", COLLATE_NOCASE)- .execute();+ await ensureColumn(db, "monographs", "publishUrl", async () => {+ await db.schema+ .alterTable("monographs")+ .addColumn("publishUrl", "text", COLLATE_NOCASE)+ .execute();+ });}},"a-2026-04-06": {async up(db) {- await db.schema- .alterTable("notes")- .addColumn("spellcheck", "boolean", (c) => c.defaultTo(true))- .execute();+ await ensureColumn(db, "notes", "spellcheck", async () => {+ await db.schema+ .alterTable("notes")+ .addColumn("spellcheck", "boolean", (c) => c.defaultTo(true))+ .execute();+ });}},"a-2026-05-07": {async up(db) {await db.schema.createTable("inboxitemshistory")+ .ifNotExists().modifyEnd(sql`without rowid`).$call(addBaseColumns).addColumn("dateSynced", "integer")@@ -568,3 +582,27 @@ async function runFTSTablesMigrations(db: Kysely<any>) {});await rebuildSearchIndex(db);}++async function hasColumn(+ db: Kysely<RawDatabaseSchema>,+ tableName: string,+ columnName: string+) {+ const result = await sql<{ name: string }>`+ SELECT name+ FROM pragma_table_info(${tableName})+ WHERE name = ${columnName}+ `.execute(db);++ return result.rows.length > 0;+}++async function ensureColumn(+ db: Kysely<RawDatabaseSchema>,+ tableName: string,+ columnName: string,+ callback: () => Promise<void>+) {+ if (await hasColumn(db, tableName, columnName)) return;+ await callback();+}