Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(search): repair a SQLite message index that no longer matches
Yudhi Armyndharis1705885a0dd3
diff --git a/src/database/migrations/1791600000000-RepairSqliteMessageFts.ts b/src/database/migrations/1791600000000-RepairSqliteMessageFts.tsnew file mode 100644index 000000000..a641e5b18--- /dev/null+++ b/src/database/migrations/1791600000000-RepairSqliteMessageFts.ts@@ -0,0 +1,22 @@+import { MigrationInterface, QueryRunner } from 'typeorm';+import { repairSqliteMessageFts } from '../sqlite-message-fts';++/**+ * Rebuild the SQLite full-text index when it no longer matches `messages`. Rows that predate+ * 1782400000000 with a NULL body, a synchronize-built index adopted by that migration's second backfill,+ * and a table rebuild that renumbered rowids all leave an index that still answers, but for the wrong+ * messages. A healthy index is left alone, so a fresh database pays one integrity check and nothing else.+ * PostgreSQL maintains its `tsvector` column itself.+ */+export class RepairSqliteMessageFts1791600000000 implements MigrationInterface {+ name = 'RepairSqliteMessageFts1791600000000';++ async up(runner: QueryRunner): Promise<void> {+ if (runner.dataSource.options.type !== 'better-sqlite3') return;+ await repairSqliteMessageFts(runner);+ }++ async down(): Promise<void> {+ // The previous inconsistent index is not restored.+ }+}diff --git a/src/database/pg-boot-migrations.ts b/src/database/pg-boot-migrations.tsindex d91a12855..703ba7c8a 100644--- a/src/database/pg-boot-migrations.ts+++ b/src/database/pg-boot-migrations.ts@@ -3,6 +3,7 @@ import { DataSource, DataSourceOptions } from 'typeorm';import { createLogger } from '../common/services/logger.service';import { assertDataConnectionUtc, postgresUtcExtra } from './postgres-utc';import { DeduplicateTerminalWebhookFailures1787200000000 } from './migrations/1787200000000-DeduplicateTerminalWebhookFailures';+import { repairSqliteMessageFts } from './sqlite-message-fts';const logger = createLogger('PgBootMigrations');@@ -68,6 +69,11 @@ export async function createBootDataSource(await runner.release();}await dataSource.synchronize();+ // synchronize rebuilds `messages` without its rowids, so the index it kept can now name other messages.+ // Never fail boot over the index: search answers 501 until the next boot repairs it.+ await repairSqliteMessageFts(dataSource).catch((error: unknown) =>+ logger.error(`Failed to repair the SQLite message full-text index: ${String(error)}`),+ );return dataSource;} catch (error) {if (dataSource.isInitialized) await dataSource.destroy().catch(() => undefined);diff --git a/src/database/sqlite-message-fts.spec.ts b/src/database/sqlite-message-fts.spec.tsnew file mode 100644index 000000000..586c383f1--- /dev/null+++ b/src/database/sqlite-message-fts.spec.ts@@ -0,0 +1,257 @@+import 'reflect-metadata';+import { mkdtemp, rm } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { DataSource, QueryRunner, TableColumn } from 'typeorm';+import { Message, MessageDirection } from '../modules/message/entities/message.entity';+import { Session } from '../modules/session/entities/session.entity';+import { AddMessagesFts1782400000000 } from './migrations/1782400000000-AddMessagesFts';+import { RepairSqliteMessageFts1791600000000 } from './migrations/1791600000000-RepairSqliteMessageFts';+import { createBootDataSource } from './pg-boot-migrations';+import { repairSqliteMessageFts } from './sqlite-message-fts';++const row = (id: string, body: string | null) => ({+ id,+ sessionId: 's',+ chatId: 'c',+ from: 'sender',+ to: 'recipient',+ body: body as string,+ type: 'text',+ direction: MessageDirection.OUTGOING,+});+const matches = async (ds: DataSource, term: string): Promise<string[]> =>+ (+ await ds.query<Array<{ id: string }>>(+ `SELECT m.id FROM messages_fts JOIN messages m ON m.rowid = messages_fts.rowid WHERE messages_fts MATCH ?`,+ [term],+ )+ )+ .map(message => message.id)+ .sort();+const integrity = (ds: DataSource) =>+ ds.query(`INSERT INTO "messages_fts"("messages_fts", "rank") VALUES ('integrity-check', 1)`);++describe('SQLite message full-text repair', () => {+ let ds: DataSource;++ beforeEach(async () => {+ ds = await new DataSource({+ type: 'better-sqlite3',+ database: ':memory:',+ entities: [Session, Message],+ synchronize: true,+ }).initialize();+ });+ afterEach(async () => {+ jest.restoreAllMocks();+ await ds.destroy();+ });++ const migrate = async (migration: { up(runner: QueryRunner): Promise<void> }) => {+ const runner = ds.createQueryRunner();+ try {+ await runner.startTransaction();+ await migration.up(runner);+ await runner.commitTransaction();+ } catch (error) {+ if (runner.isTransactionActive) await runner.rollbackTransaction();+ throw error;+ } finally {+ await runner.release();+ }+ };+ /** The 1782400000000 state: rows that predate the index, `lead` first, and a NULL-body row the backfill skips. */+ const legacy = async (tail: readonly (string | null)[]) => {+ await ds+ .getRepository(Message)+ .insert([+ row('lead', 'discarded'),+ row('alpha', 'alpha survivor'),+ ...tail.map((body, index) => row(`tail-${index}`, body)),+ ]);+ await migrate(new AddMessagesFts1782400000000());+ };+ /** A table rebuild: it copies the rows without their rowid, so the survivors of a deleted row move up. */+ const rebuild = async () => {+ await ds.getRepository(Message).delete({ id: 'lead' });+ const runner = ds.createQueryRunner();+ try {+ await runner.addColumn('messages', new TableColumn({ name: 'rebuilt', type: 'text', isNullable: true }));+ } finally {+ await runner.release();+ }+ const rows = await ds.query<Array<{ id: string; rid: number }>>(+ 'SELECT id, rowid AS rid FROM messages ORDER BY rowid',+ );+ expect(rows[0]).toEqual({ id: 'alpha', rid: 1 });+ };++ it.each([+ ['one trailing NULL', [null]],+ ['several trailing NULLs', [null, null, null]],+ ['trailing empty body', ['']],+ ['NULL then empty body', [null, '']],+ ['nonempty tail', ['bravo tail']],+ ] as const)('repairs rebuilt row identities with %s', async (_name, tail) => {+ await legacy(tail);+ expect(await matches(ds, 'alpha')).toEqual(['alpha']);+ await rebuild();+ await repairSqliteMessageFts(ds);+ expect(await matches(ds, 'alpha')).toEqual(['alpha']);+ await expect(integrity(ds)).resolves.toBeDefined();+ });++ it('repairs the index a second backfill doubled', async () => {+ await ds.getRepository(Message).insert([row('alpha', 'alpha one'), row('bravo', 'bravo two')]);+ await migrate(new AddMessagesFts1782400000000());+ // A synchronize-built index adopted by the migration chain meets the backfill a second time.+ await ds.query(`INSERT INTO "messages_fts"("rowid", "body") SELECT "rowid", "body" FROM "messages"`);+ await expect(integrity(ds)).rejects.toMatchObject({ driverError: { code: 'SQLITE_CORRUPT_VTAB' } });++ await repairSqliteMessageFts(ds);++ await expect(integrity(ds)).resolves.toBeDefined();+ expect(await matches(ds, 'alpha')).toEqual(['alpha']);+ });++ it('repairs a legacy NULL-body document through the migration', async () => {+ await legacy([null]);+ await expect(integrity(ds)).rejects.toMatchObject({ driverError: { code: 'SQLITE_CORRUPT_VTAB' } });++ await migrate(new RepairSqliteMessageFts1791600000000());++ await expect(integrity(ds)).resolves.toBeDefined();+ await ds.getRepository(Message).update({ id: 'tail-0' }, { body: 'edited nullable' });+ expect(await matches(ds, 'edited')).toEqual(['tail-0']);+ await ds.getRepository(Message).delete({ id: 'tail-0' });+ expect(await matches(ds, 'edited')).toEqual([]);+ });++ it('leaves a healthy index alone', async () => {+ await legacy([null]);+ await repairSqliteMessageFts(ds);+ const query = jest.spyOn(ds, 'query');+ await repairSqliteMessageFts(ds);+ expect(query.mock.calls.filter(([sql]) => sql.includes("VALUES ('rebuild')"))).toEqual([]);+ expect(await matches(ds, 'alpha')).toEqual(['alpha']);+ });++ it('does nothing when the full-text table is absent', async () => {+ await migrate(new RepairSqliteMessageFts1791600000000());+ await repairSqliteMessageFts(ds);+ expect(await ds.query("SELECT name FROM sqlite_master WHERE name='messages_fts'")).toEqual([]);+ });++ it('does nothing on PostgreSQL', async () => {+ const query = jest.fn();+ const runner = { dataSource: { options: { type: 'postgres' } }, query } as unknown as QueryRunner;+ await new RepairSqliteMessageFts1791600000000().up(runner);+ expect(query).not.toHaveBeenCalled();+ });+});++describe('SQLite message full-text repair failures', () => {+ it.each(['SQLITE_BUSY', 'SQLITE_IOERR', 'SQLITE_ERROR', 'SQLITE_CORRUPT', undefined])(+ 'propagates an unrelated %s error without rebuilding',+ async code => {+ const failure = Object.assign(new Error('database disk image is malformed'), { code });+ const database = {+ query: jest+ .fn()+ .mockResolvedValueOnce([{ name: 'messages_fts' }])+ .mockRejectedValueOnce(failure),+ };+ await expect(repairSqliteMessageFts(database)).rejects.toBe(failure);+ expect(database.query).toHaveBeenCalledTimes(2);+ },+ );++ it('propagates a rebuild failure', async () => {+ const mismatch = { driverError: { code: 'SQLITE_CORRUPT_VTAB' } };+ const failure = Object.assign(new Error('database is busy'), { code: 'SQLITE_BUSY' });+ const database = {+ query: jest+ .fn()+ .mockResolvedValueOnce([{ name: 'messages_fts' }])+ .mockRejectedValueOnce(mismatch)+ .mockRejectedValueOnce(failure),+ };+ await expect(repairSqliteMessageFts(database)).rejects.toBe(failure);+ expect(database.query).toHaveBeenCalledTimes(3);+ });+});++describe('SQLite boot with synchronize', () => {+ it('still boots when the repair itself fails', async () => {+ const directory = await mkdtemp(join(tmpdir(), 'openwa-fts-sync-'));+ const options = {+ type: 'better-sqlite3' as const,+ database: join(directory, 'data.sqlite'),+ entities: [Session, Message],+ synchronize: true,+ };+ let ds: DataSource | undefined;+ try {+ ds = await new DataSource(options).initialize();+ const runner = ds.createQueryRunner();+ try {+ await new AddMessagesFts1782400000000().up(runner);+ } finally {+ await runner.release();+ }+ await ds.destroy();+ ds = await createBootDataSource(options, {+ createDataSource: built => {+ const source = new DataSource(built);+ const query = source.query.bind(source);+ jest.spyOn(source, 'query').mockImplementation(async (sql, ...parameters) => {+ if (sql.includes('integrity-check'))+ throw Object.assign(new Error('database is locked'), { code: 'SQLITE_BUSY' });+ return query(sql, ...parameters);+ });+ return source;+ },+ });+ expect(ds.isInitialized).toBe(true);+ } finally {+ if (ds?.isInitialized) await ds.destroy();+ await rm(directory, { recursive: true, force: true });+ }+ });++ it.each([false, true])('repairs the kept index after synchronize (table rebuild=%s)', async rebuilt => {+ const directory = await mkdtemp(join(tmpdir(), 'openwa-fts-sync-'));+ const options = {+ type: 'better-sqlite3' as const,+ database: join(directory, 'data.sqlite'),+ entities: [Session, Message],+ synchronize: true,+ };+ let ds: DataSource | undefined;+ try {+ ds = await new DataSource(options).initialize();+ await ds+ .getRepository(Message)+ .insert([row('lead', 'discarded'), row('alpha', 'alpha survivor'), row('nullable', null)]);+ const runner = ds.createQueryRunner();+ try {+ await new AddMessagesFts1782400000000().up(runner);+ } finally {+ await runner.release();+ }+ if (rebuilt) {+ await ds.getRepository(Message).delete({ id: 'lead' });+ await ds.query('ALTER TABLE messages ADD COLUMN legacy_extra text');+ }+ await ds.destroy();+ ds = await createBootDataSource(options);+ expect(ds.isInitialized).toBe(true);+ expect(await matches(ds, 'alpha')).toEqual(['alpha']);+ await expect(integrity(ds)).resolves.toBeDefined();+ } finally {+ if (ds?.isInitialized) await ds.destroy();+ await rm(directory, { recursive: true, force: true });+ }+ });+});diff --git a/src/database/sqlite-message-fts.ts b/src/database/sqlite-message-fts.tsnew file mode 100644index 000000000..05e69a450--- /dev/null+++ b/src/database/sqlite-message-fts.ts@@ -0,0 +1,27 @@+/**+ * Bring the SQLite `messages_fts` index back in line with `messages`.+ *+ * It is an external-content FTS5 table keyed on the implicit rowid, so it breaks silently: the 1782400000000+ * backfill skips NULL bodies while the insert trigger indexes them, a repeated backfill doubles an+ * index's postings, and a table rebuild (synchronize, `addColumn`) renumbers rowids around deleted rows+ * and leaves every key naming another message. A rowid anti-join cannot see any of that; the rank=1+ * integrity check recomputes every document from the content table and fails on all of it.+ */+export async function repairSqliteMessageFts(database: { query(sql: string): Promise<unknown> }): Promise<void> {+ const tables = (await database.query(+ `SELECT name FROM sqlite_master WHERE type='table' AND name='messages_fts'`,+ )) as unknown[];+ if (tables.length === 0) return;++ const integrityCheck = `INSERT INTO "messages_fts"("messages_fts", "rank") VALUES ('integrity-check', 1)`;+ try {+ await database.query(integrityCheck);+ } catch (error) {+ const failure = error as { code?: string; driverError?: { code?: string } } | null;+ // Any other failure (busy, I/O, a missing content table) says nothing about the index: rebuilding on it+ // would only hide the cause.+ if ((failure?.driverError?.code ?? failure?.code) !== 'SQLITE_CORRUPT_VTAB') throw error;+ await database.query(`INSERT INTO "messages_fts"("messages_fts") VALUES ('rebuild')`);+ await database.query(integrityCheck);+ }+}