Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(search): repair a SQLite message index that no longer matches

Yudhi Armyndharis1705885a0dd3
diff --git a/src/database/migrations/1791600000000-RepairSqliteMessageFts.ts b/src/database/migrations/1791600000000-RepairSqliteMessageFts.ts
new file mode 100644
index 000000000..a641e5b18
--- /dev/null
+++ b/src/database/migrations/1791600000000-RepairSqliteMessageFts.ts
@@ -0,0 +1,22 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+import { repairSqliteMessageFts } from '../sqlite-message-fts';
+
+/**
+ * Rebuild the SQLite full-text index when it no longer matches `messages`. Rows that predate
+ * 1782400000000 with a NULL body, a synchronize-built index adopted by that migration's second backfill,
+ * and a table rebuild that renumbered rowids all leave an index that still answers, but for the wrong
+ * messages. A healthy index is left alone, so a fresh database pays one integrity check and nothing else.
+ * PostgreSQL maintains its `tsvector` column itself.
+ */
+export class RepairSqliteMessageFts1791600000000 implements MigrationInterface {
+ name = 'RepairSqliteMessageFts1791600000000';
+
+ async up(runner: QueryRunner): Promise<void> {
+ if (runner.dataSource.options.type !== 'better-sqlite3') return;
+ await repairSqliteMessageFts(runner);
+ }
+
+ async down(): Promise<void> {
+ // The previous inconsistent index is not restored.
+ }
+}
diff --git a/src/database/pg-boot-migrations.ts b/src/database/pg-boot-migrations.ts
index d91a12855..703ba7c8a 100644
--- a/src/database/pg-boot-migrations.ts
+++ b/src/database/pg-boot-migrations.ts
@@ -3,6 +3,7 @@ import { DataSource, DataSourceOptions } from 'typeorm';
import { createLogger } from '../common/services/logger.service';
import { assertDataConnectionUtc, postgresUtcExtra } from './postgres-utc';
import { DeduplicateTerminalWebhookFailures1787200000000 } from './migrations/1787200000000-DeduplicateTerminalWebhookFailures';
+import { repairSqliteMessageFts } from './sqlite-message-fts';
const logger = createLogger('PgBootMigrations');
@@ -68,6 +69,11 @@ export async function createBootDataSource(
await runner.release();
}
await dataSource.synchronize();
+ // synchronize rebuilds `messages` without its rowids, so the index it kept can now name other messages.
+ // Never fail boot over the index: search answers 501 until the next boot repairs it.
+ await repairSqliteMessageFts(dataSource).catch((error: unknown) =>
+ logger.error(`Failed to repair the SQLite message full-text index: ${String(error)}`),
+ );
return dataSource;
} catch (error) {
if (dataSource.isInitialized) await dataSource.destroy().catch(() => undefined);
diff --git a/src/database/sqlite-message-fts.spec.ts b/src/database/sqlite-message-fts.spec.ts
new file mode 100644
index 000000000..586c383f1
--- /dev/null
+++ b/src/database/sqlite-message-fts.spec.ts
@@ -0,0 +1,257 @@
+import 'reflect-metadata';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { DataSource, QueryRunner, TableColumn } from 'typeorm';
+import { Message, MessageDirection } from '../modules/message/entities/message.entity';
+import { Session } from '../modules/session/entities/session.entity';
+import { AddMessagesFts1782400000000 } from './migrations/1782400000000-AddMessagesFts';
+import { RepairSqliteMessageFts1791600000000 } from './migrations/1791600000000-RepairSqliteMessageFts';
+import { createBootDataSource } from './pg-boot-migrations';
+import { repairSqliteMessageFts } from './sqlite-message-fts';
+
+const row = (id: string, body: string | null) => ({
+ id,
+ sessionId: 's',
+ chatId: 'c',
+ from: 'sender',
+ to: 'recipient',
+ body: body as string,
+ type: 'text',
+ direction: MessageDirection.OUTGOING,
+});
+const matches = async (ds: DataSource, term: string): Promise<string[]> =>
+ (
+ await ds.query<Array<{ id: string }>>(
+ `SELECT m.id FROM messages_fts JOIN messages m ON m.rowid = messages_fts.rowid WHERE messages_fts MATCH ?`,
+ [term],
+ )
+ )
+ .map(message => message.id)
+ .sort();
+const integrity = (ds: DataSource) =>
+ ds.query(`INSERT INTO "messages_fts"("messages_fts", "rank") VALUES ('integrity-check', 1)`);
+
+describe('SQLite message full-text repair', () => {
+ let ds: DataSource;
+
+ beforeEach(async () => {
+ ds = await new DataSource({
+ type: 'better-sqlite3',
+ database: ':memory:',
+ entities: [Session, Message],
+ synchronize: true,
+ }).initialize();
+ });
+ afterEach(async () => {
+ jest.restoreAllMocks();
+ await ds.destroy();
+ });
+
+ const migrate = async (migration: { up(runner: QueryRunner): Promise<void> }) => {
+ const runner = ds.createQueryRunner();
+ try {
+ await runner.startTransaction();
+ await migration.up(runner);
+ await runner.commitTransaction();
+ } catch (error) {
+ if (runner.isTransactionActive) await runner.rollbackTransaction();
+ throw error;
+ } finally {
+ await runner.release();
+ }
+ };
+ /** The 1782400000000 state: rows that predate the index, `lead` first, and a NULL-body row the backfill skips. */
+ const legacy = async (tail: readonly (string | null)[]) => {
+ await ds
+ .getRepository(Message)
+ .insert([
+ row('lead', 'discarded'),
+ row('alpha', 'alpha survivor'),
+ ...tail.map((body, index) => row(`tail-${index}`, body)),
+ ]);
+ await migrate(new AddMessagesFts1782400000000());
+ };
+ /** A table rebuild: it copies the rows without their rowid, so the survivors of a deleted row move up. */
+ const rebuild = async () => {
+ await ds.getRepository(Message).delete({ id: 'lead' });
+ const runner = ds.createQueryRunner();
+ try {
+ await runner.addColumn('messages', new TableColumn({ name: 'rebuilt', type: 'text', isNullable: true }));
+ } finally {
+ await runner.release();
+ }
+ const rows = await ds.query<Array<{ id: string; rid: number }>>(
+ 'SELECT id, rowid AS rid FROM messages ORDER BY rowid',
+ );
+ expect(rows[0]).toEqual({ id: 'alpha', rid: 1 });
+ };
+
+ it.each([
+ ['one trailing NULL', [null]],
+ ['several trailing NULLs', [null, null, null]],
+ ['trailing empty body', ['']],
+ ['NULL then empty body', [null, '']],
+ ['nonempty tail', ['bravo tail']],
+ ] as const)('repairs rebuilt row identities with %s', async (_name, tail) => {
+ await legacy(tail);
+ expect(await matches(ds, 'alpha')).toEqual(['alpha']);
+ await rebuild();
+ await repairSqliteMessageFts(ds);
+ expect(await matches(ds, 'alpha')).toEqual(['alpha']);
+ await expect(integrity(ds)).resolves.toBeDefined();
+ });
+
+ it('repairs the index a second backfill doubled', async () => {
+ await ds.getRepository(Message).insert([row('alpha', 'alpha one'), row('bravo', 'bravo two')]);
+ await migrate(new AddMessagesFts1782400000000());
+ // A synchronize-built index adopted by the migration chain meets the backfill a second time.
+ await ds.query(`INSERT INTO "messages_fts"("rowid", "body") SELECT "rowid", "body" FROM "messages"`);
+ await expect(integrity(ds)).rejects.toMatchObject({ driverError: { code: 'SQLITE_CORRUPT_VTAB' } });
+
+ await repairSqliteMessageFts(ds);
+
+ await expect(integrity(ds)).resolves.toBeDefined();
+ expect(await matches(ds, 'alpha')).toEqual(['alpha']);
+ });
+
+ it('repairs a legacy NULL-body document through the migration', async () => {
+ await legacy([null]);
+ await expect(integrity(ds)).rejects.toMatchObject({ driverError: { code: 'SQLITE_CORRUPT_VTAB' } });
+
+ await migrate(new RepairSqliteMessageFts1791600000000());
+
+ await expect(integrity(ds)).resolves.toBeDefined();
+ await ds.getRepository(Message).update({ id: 'tail-0' }, { body: 'edited nullable' });
+ expect(await matches(ds, 'edited')).toEqual(['tail-0']);
+ await ds.getRepository(Message).delete({ id: 'tail-0' });
+ expect(await matches(ds, 'edited')).toEqual([]);
+ });
+
+ it('leaves a healthy index alone', async () => {
+ await legacy([null]);
+ await repairSqliteMessageFts(ds);
+ const query = jest.spyOn(ds, 'query');
+ await repairSqliteMessageFts(ds);
+ expect(query.mock.calls.filter(([sql]) => sql.includes("VALUES ('rebuild')"))).toEqual([]);
+ expect(await matches(ds, 'alpha')).toEqual(['alpha']);
+ });
+
+ it('does nothing when the full-text table is absent', async () => {
+ await migrate(new RepairSqliteMessageFts1791600000000());
+ await repairSqliteMessageFts(ds);
+ expect(await ds.query("SELECT name FROM sqlite_master WHERE name='messages_fts'")).toEqual([]);
+ });
+
+ it('does nothing on PostgreSQL', async () => {
+ const query = jest.fn();
+ const runner = { dataSource: { options: { type: 'postgres' } }, query } as unknown as QueryRunner;
+ await new RepairSqliteMessageFts1791600000000().up(runner);
+ expect(query).not.toHaveBeenCalled();
+ });
+});
+
+describe('SQLite message full-text repair failures', () => {
+ it.each(['SQLITE_BUSY', 'SQLITE_IOERR', 'SQLITE_ERROR', 'SQLITE_CORRUPT', undefined])(
+ 'propagates an unrelated %s error without rebuilding',
+ async code => {
+ const failure = Object.assign(new Error('database disk image is malformed'), { code });
+ const database = {
+ query: jest
+ .fn()
+ .mockResolvedValueOnce([{ name: 'messages_fts' }])
+ .mockRejectedValueOnce(failure),
+ };
+ await expect(repairSqliteMessageFts(database)).rejects.toBe(failure);
+ expect(database.query).toHaveBeenCalledTimes(2);
+ },
+ );
+
+ it('propagates a rebuild failure', async () => {
+ const mismatch = { driverError: { code: 'SQLITE_CORRUPT_VTAB' } };
+ const failure = Object.assign(new Error('database is busy'), { code: 'SQLITE_BUSY' });
+ const database = {
+ query: jest
+ .fn()
+ .mockResolvedValueOnce([{ name: 'messages_fts' }])
+ .mockRejectedValueOnce(mismatch)
+ .mockRejectedValueOnce(failure),
+ };
+ await expect(repairSqliteMessageFts(database)).rejects.toBe(failure);
+ expect(database.query).toHaveBeenCalledTimes(3);
+ });
+});
+
+describe('SQLite boot with synchronize', () => {
+ it('still boots when the repair itself fails', async () => {
+ const directory = await mkdtemp(join(tmpdir(), 'openwa-fts-sync-'));
+ const options = {
+ type: 'better-sqlite3' as const,
+ database: join(directory, 'data.sqlite'),
+ entities: [Session, Message],
+ synchronize: true,
+ };
+ let ds: DataSource | undefined;
+ try {
+ ds = await new DataSource(options).initialize();
+ const runner = ds.createQueryRunner();
+ try {
+ await new AddMessagesFts1782400000000().up(runner);
+ } finally {
+ await runner.release();
+ }
+ await ds.destroy();
+ ds = await createBootDataSource(options, {
+ createDataSource: built => {
+ const source = new DataSource(built);
+ const query = source.query.bind(source);
+ jest.spyOn(source, 'query').mockImplementation(async (sql, ...parameters) => {
+ if (sql.includes('integrity-check'))
+ throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' });
+ return query(sql, ...parameters);
+ });
+ return source;
+ },
+ });
+ expect(ds.isInitialized).toBe(true);
+ } finally {
+ if (ds?.isInitialized) await ds.destroy();
+ await rm(directory, { recursive: true, force: true });
+ }
+ });
+
+ it.each([false, true])('repairs the kept index after synchronize (table rebuild=%s)', async rebuilt => {
+ const directory = await mkdtemp(join(tmpdir(), 'openwa-fts-sync-'));
+ const options = {
+ type: 'better-sqlite3' as const,
+ database: join(directory, 'data.sqlite'),
+ entities: [Session, Message],
+ synchronize: true,
+ };
+ let ds: DataSource | undefined;
+ try {
+ ds = await new DataSource(options).initialize();
+ await ds
+ .getRepository(Message)
+ .insert([row('lead', 'discarded'), row('alpha', 'alpha survivor'), row('nullable', null)]);
+ const runner = ds.createQueryRunner();
+ try {
+ await new AddMessagesFts1782400000000().up(runner);
+ } finally {
+ await runner.release();
+ }
+ if (rebuilt) {
+ await ds.getRepository(Message).delete({ id: 'lead' });
+ await ds.query('ALTER TABLE messages ADD COLUMN legacy_extra text');
+ }
+ await ds.destroy();
+ ds = await createBootDataSource(options);
+ expect(ds.isInitialized).toBe(true);
+ expect(await matches(ds, 'alpha')).toEqual(['alpha']);
+ await expect(integrity(ds)).resolves.toBeDefined();
+ } finally {
+ if (ds?.isInitialized) await ds.destroy();
+ await rm(directory, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/src/database/sqlite-message-fts.ts b/src/database/sqlite-message-fts.ts
new file mode 100644
index 000000000..05e69a450
--- /dev/null
+++ b/src/database/sqlite-message-fts.ts
@@ -0,0 +1,27 @@
+/**
+ * Bring the SQLite `messages_fts` index back in line with `messages`.
+ *
+ * It is an external-content FTS5 table keyed on the implicit rowid, so it breaks silently: the 1782400000000
+ * backfill skips NULL bodies while the insert trigger indexes them, a repeated backfill doubles an
+ * index's postings, and a table rebuild (synchronize, `addColumn`) renumbers rowids around deleted rows
+ * and leaves every key naming another message. A rowid anti-join cannot see any of that; the rank=1
+ * integrity check recomputes every document from the content table and fails on all of it.
+ */
+export async function repairSqliteMessageFts(database: { query(sql: string): Promise<unknown> }): Promise<void> {
+ const tables = (await database.query(
+ `SELECT name FROM sqlite_master WHERE type='table' AND name='messages_fts'`,
+ )) as unknown[];
+ if (tables.length === 0) return;
+
+ const integrityCheck = `INSERT INTO "messages_fts"("messages_fts", "rank") VALUES ('integrity-check', 1)`;
+ try {
+ await database.query(integrityCheck);
+ } catch (error) {
+ const failure = error as { code?: string; driverError?: { code?: string } } | null;
+ // Any other failure (busy, I/O, a missing content table) says nothing about the index: rebuilding on it
+ // would only hide the cause.
+ if ((failure?.driverError?.code ?? failure?.code) !== 'SQLITE_CORRUPT_VTAB') throw error;
+ await database.query(`INSERT INTO "messages_fts"("messages_fts") VALUES ('rebuild')`);
+ await database.query(integrityCheck);
+ }
+}