Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: make db migration more resilient and recoverable (#10037)

Ammar Ahmed48d88069c007
diff --git a/packages/core/__tests__/sql-migrations.test.ts b/packages/core/__tests__/sql-migrations.test.ts
new file mode 100644
index 000000000..b621b913e
--- /dev/null
+++ b/packages/core/__tests__/sql-migrations.test.ts
@@ -0,0 +1,133 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import {
+ Kysely,
+ Migration,
+ MigrationProvider,
+ SqliteDialect,
+ sql
+} from "@streetwriters/kysely";
+import BetterSQLite3 from "better-sqlite3-multiple-ciphers";
+import * as betterTrigram from "sqlite-better-trigram";
+import * as fts5Html from "sqlite3-fts5-html";
+import { getLoadablePath } from "sqlite-regex";
+import { NNMigrationProvider } from "../src/database/migrations.js";
+import {
+ initializeDatabase,
+ SqliteBooleanPlugin
+} from "../src/database/index.js";
+import { describe, expect, test } from "vitest";
+
+function createSchemaTestDatabase() {
+ const sqlite = BetterSQLite3(":memory:").unsafeMode(true);
+ betterTrigram.load(sqlite);
+ fts5Html.load(sqlite);
+ sqlite.loadExtension(getLoadablePath());
+
+ return new Kysely({
+ dialect: new SqliteDialect({
+ database: sqlite
+ }),
+ plugins: [new SqliteBooleanPlugin()]
+ });
+}
+
+async function createMigrationProviderBefore(migrationName: string) {
+ const migrations = await new NNMigrationProvider().getMigrations();
+ const entries = Object.entries(migrations);
+ const index = entries.findIndex(([name]) => name === migrationName);
+
+ if (index === -1) {
+ throw new Error(`Migration not found: ${migrationName}`);
+ }
+
+ return {
+ migrations,
+ provider: {
+ async getMigrations() {
+ return Object.fromEntries(entries.slice(0, index));
+ }
+ } satisfies MigrationProvider
+ };
+}
+
+describe("Schema migration recovery", () => {
+ test("replays a partially applied spellcheck migration", async () => {
+ const db = createSchemaTestDatabase();
+
+ try {
+ const { migrations, provider } = await createMigrationProviderBefore(
+ "a-2026-04-06"
+ );
+
+ await initializeDatabase(db, provider, "notesnook-test");
+ await migrations["a-2026-04-06"]?.up(db);
+
+ await initializeDatabase(db, new NNMigrationProvider(), "notesnook-test");
+
+ const migrationRow = await sql<{ name: string }>`
+ SELECT name
+ FROM kysely_migration
+ WHERE name = ${"a-2026-04-06"}
+ `.execute(db);
+ const column = await sql<{ name: string }>`
+ SELECT name
+ FROM pragma_table_info(${"notes"})
+ WHERE name = ${"spellcheck"}
+ `.execute(db);
+
+ expect(migrationRow.rows).toHaveLength(1);
+ expect(column.rows).toHaveLength(1);
+ } finally {
+ await db.destroy();
+ }
+ });
+
+ test("replays a partially applied inbox items history migration", async () => {
+ const db = createSchemaTestDatabase();
+
+ try {
+ const { migrations, provider } = await createMigrationProviderBefore(
+ "a-2026-05-07"
+ );
+
+ await initializeDatabase(db, provider, "notesnook-test");
+ await migrations["a-2026-05-07"]?.up(db);
+
+ await initializeDatabase(db, new NNMigrationProvider(), "notesnook-test");
+
+ const migrationRow = await sql<{ name: string }>`
+ SELECT name
+ FROM kysely_migration
+ WHERE name = ${"a-2026-05-07"}
+ `.execute(db);
+ const table = await sql<{ name: string }>`
+ SELECT name
+ FROM sqlite_master
+ WHERE type = 'table' AND name = ${"inboxitemshistory"}
+ `.execute(db);
+
+ expect(migrationRow.rows).toHaveLength(1);
+ expect(table.rows).toHaveLength(1);
+ } finally {
+ await db.destroy();
+ }
+ });
+});
diff --git a/packages/core/src/database/migrations.ts b/packages/core/src/database/migrations.ts
index 5290963e1..826bfcc98 100644
--- a/packages/core/src/database/migrations.ts
+++ b/packages/core/src/database/migrations.ts
@@ -27,6 +27,7 @@ import {
sql
} from "@streetwriters/kysely";
import { rebuildSearchIndex } from "./fts.js";
+import type { RawDatabaseSchema } from "./index.js";
const COLLATE_NOCASE: ColumnBuilderCallback = (col) =>
col.modifyEnd(sql`collate nocase`);
@@ -389,18 +390,22 @@ export class NNMigrationProvider implements MigrationProvider {
},
"8": {
async up(db) {
- await db.schema
- .alterTable("notes")
- .addColumn("isGeneratedTitle", "boolean")
- .execute();
+ await ensureColumn(db, "notes", "isGeneratedTitle", async () => {
+ await db.schema
+ .alterTable("notes")
+ .addColumn("isGeneratedTitle", "boolean")
+ .execute();
+ });
}
},
"9": {
async up(db) {
- await db.schema
- .alterTable("notes")
- .addColumn("archived", "boolean")
- .execute();
+ await ensureColumn(db, "notes", "archived", async () => {
+ await db.schema
+ .alterTable("notes")
+ .addColumn("archived", "boolean")
+ .execute();
+ });
}
},
// changing the migrations name scheme from here because
@@ -434,10 +439,12 @@ export class NNMigrationProvider implements MigrationProvider {
},
"a-2026-01-07": {
async up(db) {
- await db.schema
- .alterTable("notes")
- .addColumn("expiryDate", "text")
- .execute();
+ await ensureColumn(db, "notes", "expiryDate", async () => {
+ await db.schema
+ .alterTable("notes")
+ .addColumn("expiryDate", "text")
+ .execute();
+ });
await db.schema
.createIndex("note_expiry_date")
.ifNotExists()
@@ -448,32 +455,39 @@ export class NNMigrationProvider implements MigrationProvider {
},
"a-2026-01-09": {
async up(db) {
- await db.schema
- .alterTable("sessioncontent")
- .addColumn("title", "text")
- .execute();
+ await ensureColumn(db, "sessioncontent", "title", async () => {
+ await db.schema
+ .alterTable("sessioncontent")
+ .addColumn("title", "text")
+ .execute();
+ });
}
},
"a-2026-02-11": {
async up(db) {
- await db.schema
- .alterTable("monographs")
- .addColumn("publishUrl", "text", COLLATE_NOCASE)
- .execute();
+ await ensureColumn(db, "monographs", "publishUrl", async () => {
+ await db.schema
+ .alterTable("monographs")
+ .addColumn("publishUrl", "text", COLLATE_NOCASE)
+ .execute();
+ });
}
},
"a-2026-04-06": {
async up(db) {
- await db.schema
- .alterTable("notes")
- .addColumn("spellcheck", "boolean", (c) => c.defaultTo(true))
- .execute();
+ await ensureColumn(db, "notes", "spellcheck", async () => {
+ await db.schema
+ .alterTable("notes")
+ .addColumn("spellcheck", "boolean", (c) => c.defaultTo(true))
+ .execute();
+ });
}
},
"a-2026-05-07": {
async up(db) {
await db.schema
.createTable("inboxitemshistory")
+ .ifNotExists()
.modifyEnd(sql`without rowid`)
.$call(addBaseColumns)
.addColumn("dateSynced", "integer")
@@ -568,3 +582,27 @@ async function runFTSTablesMigrations(db: Kysely<any>) {
});
await rebuildSearchIndex(db);
}
+
+async function hasColumn(
+ db: Kysely<RawDatabaseSchema>,
+ tableName: string,
+ columnName: string
+) {
+ const result = await sql<{ name: string }>`
+ SELECT name
+ FROM pragma_table_info(${tableName})
+ WHERE name = ${columnName}
+ `.execute(db);
+
+ return result.rows.length > 0;
+}
+
+async function ensureColumn(
+ db: Kysely<RawDatabaseSchema>,
+ tableName: string,
+ columnName: string,
+ callback: () => Promise<void>
+) {
+ if (await hasColumn(db, tableName, columnName)) return;
+ await callback();
+}