Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(message): convert legacy message types in a migration

Yudhi Armyndharis20faf6bb5ab6
diff --git a/docs/05-database-design.md b/docs/05-database-design.md
index 534531da9..831befba3 100644
--- a/docs/05-database-design.md
+++ b/docs/05-database-design.md
@@ -927,7 +927,8 @@ src/database/migrations/ # data connection (pluggable)
├── 1787100000000-AddSendIdempotencyKeys.ts # send_idempotency_keys (opt-in send retry guard)
├── 1787200000000-DeduplicateTerminalWebhookFailures.ts # one terminal record per webhook and idempotency key
├── 1790812800000-AddMessageWindowIndexes.ts # compound timestamp indexes for message-time pagination
-└── 1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts # (pluginId, instanceId, deliveryId) index for the dead-letter guards
+├── 1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts # (pluginId, instanceId, deliveryId) index for the dead-letter guards
+└── 1791300000000-BackfillLegacyMessageTypes.ts # converts raw wwebjs message types to the neutral vocabulary, once
```
> [!NOTE]
diff --git a/src/database/migrations/1791300000000-BackfillLegacyMessageTypes.ts b/src/database/migrations/1791300000000-BackfillLegacyMessageTypes.ts
new file mode 100644
index 000000000..73656a9ba
--- /dev/null
+++ b/src/database/migrations/1791300000000-BackfillLegacyMessageTypes.ts
@@ -0,0 +1,46 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+/**
+ * One-time backfill of `messages.type` to the engine-neutral vocabulary (#265).
+ *
+ * Older incoming rows were persisted with raw whatsapp-web.js tokens (`chat`/`ptt`/`vcard`); the engine
+ * now emits neutral types (`text`/`voice`/`contact`), and the dashboard chat view and the message stats read
+ * these rows. A boot-time service used to run the conversion on every start. `type` has no index, so each
+ * start read the whole table (four times, before the HTTP port opened), and on PostgreSQL the runtime
+ * pool's statement timeout cancelled it once the table was large. A migration runs once, on a pool without
+ * that timeout. The service now runs only where migrations never run: SQLite under DATABASE_SYNCHRONIZE=true.
+ *
+ * call_log and poll_creation are left out on purpose: a row stored before #265 carries no call or poll
+ * metadata, so relabelling it would render an empty call or poll bubble. The mapping is forward-only and
+ * collision-free (the neutral targets were never valid raw tokens), so it is idempotent and `down` is a
+ * no-op. One statement, so the table is read once. `backfill` is shared with the boot service and with
+ * import-data, which restores rows from archives taken before the neutral vocabulary.
+ */
+export class BackfillLegacyMessageTypes1791300000000 implements MigrationInterface {
+ name = 'BackfillLegacyMessageTypes1791300000000';
+
+ static async backfill(queryRunner: QueryRunner): Promise<void> {
+ if (!(await queryRunner.hasTable('messages'))) return;
+ await queryRunner.query(
+ `UPDATE "messages" SET "type" = CASE "type" ` +
+ `WHEN 'chat' THEN 'text' WHEN 'buttons_response' THEN 'text' WHEN 'list_response' THEN 'text' ` +
+ `WHEN 'template_button_reply' THEN 'text' WHEN 'ptt' THEN 'voice' ` +
+ `WHEN 'vcard' THEN 'contact' WHEN 'multi_vcard' THEN 'contact' END ` +
+ `WHERE "type" IN ('chat', 'buttons_response', 'list_response', 'template_button_reply', 'ptt', 'vcard', 'multi_vcard')`,
+ );
+ }
+
+ public async up(queryRunner: QueryRunner): Promise<void> {
+ // Boot and CLI migrations run on a pool without the runtime statement_timeout (pg-boot-migrations.ts),
+ // and MigrationExecutor wraps a lone pending migration in its own transaction, so lift any role- or
+ // database-level default for this transaction only, exactly like AddWebhookDeliveryFailureLookupIndex.
+ if (queryRunner.dataSource.options.type === 'postgres') {
+ await queryRunner.query('SET LOCAL statement_timeout = 0');
+ }
+ await BackfillLegacyMessageTypes1791300000000.backfill(queryRunner);
+ }
+
+ public async down(): Promise<void> {
+ // Forward-only: the raw tokens are not recoverable once converted.
+ }
+}
diff --git a/src/database/migrations/__tests__/1791300000000-BackfillLegacyMessageTypes.spec.ts b/src/database/migrations/__tests__/1791300000000-BackfillLegacyMessageTypes.spec.ts
new file mode 100644
index 000000000..b7b08eed1
--- /dev/null
+++ b/src/database/migrations/__tests__/1791300000000-BackfillLegacyMessageTypes.spec.ts
@@ -0,0 +1,74 @@
+import { DataSource } from 'typeorm';
+import { BackfillLegacyMessageTypes1791300000000 } from '../1791300000000-BackfillLegacyMessageTypes';
+
+describe('BackfillLegacyMessageTypes migration', () => {
+ let ds: DataSource;
+
+ beforeEach(async () => {
+ ds = new DataSource({ type: 'better-sqlite3', database: ':memory:' });
+ await ds.initialize();
+ await ds.query(`CREATE TABLE "messages" ("id" varchar PRIMARY KEY NOT NULL, "type" varchar NOT NULL)`);
+ const types = [
+ 'chat',
+ 'buttons_response',
+ 'list_response',
+ 'template_button_reply',
+ 'ptt',
+ 'vcard',
+ 'multi_vcard',
+ // Already neutral, or deliberately left alone.
+ 'text',
+ 'voice',
+ 'contact',
+ 'image',
+ 'call_log',
+ 'poll_creation',
+ ];
+ await ds.query(`INSERT INTO "messages" ("id","type") VALUES ${types.map((t, i) => `('m${i}', '${t}')`).join(',')}`);
+ });
+
+ afterEach(async () => {
+ await ds.destroy();
+ });
+
+ const typeOf = async (): Promise<Record<string, string>> => {
+ const rows = await ds.query<{ id: string; type: string }[]>(`SELECT "id","type" FROM "messages" ORDER BY "id"`);
+ return Object.fromEntries(rows.map(r => [r.id, r.type]));
+ };
+
+ it('rewrites every legacy token to the neutral type and touches nothing else', async () => {
+ await new BackfillLegacyMessageTypes1791300000000().up(ds.createQueryRunner());
+
+ const after = await typeOf();
+ expect([after.m0, after.m1, after.m2, after.m3]).toEqual(['text', 'text', 'text', 'text']);
+ expect(after.m4).toBe('voice');
+ expect([after.m5, after.m6]).toEqual(['contact', 'contact']);
+ // text, voice, contact stay; image, call_log and poll_creation are left as stored.
+ expect([after.m7, after.m8, after.m9, after.m10, after.m11, after.m12]).toEqual([
+ 'text',
+ 'voice',
+ 'contact',
+ 'image',
+ 'call_log',
+ 'poll_creation',
+ ]);
+ });
+
+ it('is idempotent and its down() changes nothing', async () => {
+ const migration = new BackfillLegacyMessageTypes1791300000000();
+ const runner = ds.createQueryRunner();
+
+ await migration.up(runner);
+ const once = await typeOf();
+ await migration.up(runner);
+ await migration.down();
+
+ expect(await typeOf()).toEqual(once);
+ });
+
+ it('does nothing on a database that has no messages table yet', async () => {
+ await ds.query(`DROP TABLE "messages"`);
+
+ await expect(BackfillLegacyMessageTypes1791300000000.backfill(ds.createQueryRunner())).resolves.toBeUndefined();
+ });
+});
diff --git a/src/modules/infra/infra-data.controller.spec.ts b/src/modules/infra/infra-data.controller.spec.ts
index 095947564..f530249c9 100644
--- a/src/modules/infra/infra-data.controller.spec.ts
+++ b/src/modules/infra/infra-data.controller.spec.ts
@@ -2426,6 +2426,42 @@ describe('InfraDataController.importData status_updates + runtime reconciliation
expect(await ds.query('SELECT lid, phone FROM lid_mappings')).toEqual([{ lid: '3', phone: '628123' }]);
});
+ // Boot no longer converts the raw whatsapp-web.js tokens on a migration-managed database, so an archive taken
+ // before the neutral vocabulary has to be converted by the import that restores it.
+ it('converts the raw message types of a restored archive and leaves the neutral ones alone', async () => {
+ await seedSession('s1');
+ const dump = await build().exportData();
+ const at = dump.tables.sessions[0].updatedAt;
+ const message = (id: string, type: string) => ({
+ id,
+ sessionId: 's1',
+ waMessageId: `W-${id}`,
+ chatId: '[email protected]',
+ chatName: null,
+ author: null,
+ body: 'hello',
+ type,
+ direction: MessageDirection.INCOMING,
+ timestamp: 1,
+ metadata: null,
+ status: MessageStatus.DELIVERED,
+ createdAt: at,
+ mediaPath: null,
+ mediaMimetype: null,
+ });
+ const types = ['chat', 'ptt', 'vcard', 'multi_vcard', 'image', 'call_log', 'text'];
+
+ const res = await build().importData({
+ tables: { ...dump.tables, messages: types.map((type, i) => message(`m${i}`, type)) },
+ });
+
+ expect(res.imported).toBe(true);
+ const stored = await ds.query<Array<{ id: string; type: string }>>('SELECT id, type FROM messages ORDER BY id');
+ expect(stored.map(row => row.type)).toEqual(['text', 'voice', 'contact', 'contact', 'image', 'call_log', 'text']);
+ });
+
it('exports and restores automation_rules, which the session wipe would otherwise cascade away', async () => {
await seedSession('s1');
const ruleRepo = ds.getRepository(AutomationRule);
diff --git a/src/modules/infra/infra-data.service.ts b/src/modules/infra/infra-data.service.ts
index 47f6a70bc..1453ebd74 100644
--- a/src/modules/infra/infra-data.service.ts
+++ b/src/modules/infra/infra-data.service.ts
@@ -15,6 +15,7 @@ import { ScopeBindingService } from '../integration/scope-binding.service';
import { ReKeyChatStatesBySessionId1786500000000 } from '../../database/migrations/1786500000000-ReKeyChatStatesBySessionId';
import { ScrubRevokedMessageContent1786900000000 } from '../../database/migrations/1786900000000-ScrubRevokedMessageContent';
import { ScrubNonPhoneLidMappings1786950000000 } from '../../database/migrations/1786950000000-ScrubNonPhoneLidMappings';
+import { BackfillLegacyMessageTypes1791300000000 } from '../../database/migrations/1791300000000-BackfillLegacyMessageTypes';
import { Session as SessionEntity, SessionStatus } from '../session/entities/session.entity';
import { In } from 'typeorm';
import { DateUtils } from 'typeorm/util/DateUtils';
@@ -930,6 +931,9 @@ export class InfraDataService {
try {
await ScrubRevokedMessageContent1786900000000.scrub(queryRunner);
await ScrubNonPhoneLidMappings1786950000000.scrub(queryRunner);
+ // An archive taken before the neutral message vocabulary restores the raw wwebjs tokens, which
+ // the boot-time backfill no longer converts on a migration-managed database.
+ await BackfillLegacyMessageTypes1791300000000.backfill(queryRunner);
} catch (error) {
warnings.push(`Failed to clean restored rows: ${error instanceof Error ? error.message : String(error)}`);
}
diff --git a/src/modules/message/message-type-backfill.service.spec.ts b/src/modules/message/message-type-backfill.service.spec.ts
index f94a97de1..11be902de 100644
--- a/src/modules/message/message-type-backfill.service.spec.ts
+++ b/src/modules/message/message-type-backfill.service.spec.ts
@@ -1,37 +1,65 @@
-import { In, Repository } from 'typeorm';
+import { ConfigService } from '@nestjs/config';
+import { Repository } from 'typeorm';
import { MessageTypeBackfillService } from './message-type-backfill.service';
import { Message } from './entities/message.entity';
describe('MessageTypeBackfillService', () => {
- const makeService = (update: jest.Mock) => {
- const repo = { update } as unknown as Repository<Message>;
- return new MessageTypeBackfillService(repo);
+ const makeService = (
+ opts: { synchronize?: boolean; query?: jest.Mock<Promise<void>, [string]>; hasTable?: boolean } = {},
+ ) => {
+ const query = opts.query ?? jest.fn<Promise<void>, [string]>().mockResolvedValue(undefined);
+ const runner = {
+ hasTable: jest.fn().mockResolvedValue(opts.hasTable ?? true),
+ query,
+ release: jest.fn().mockResolvedValue(undefined),
+ };
+ const repo = { manager: { connection: { createQueryRunner: () => runner } } } as unknown as Repository<Message>;
+ const config = {
+ get: (key: string, fallback: unknown) => (key === 'dataDatabase.synchronize' ? opts.synchronize : fallback),
+ } as unknown as ConfigService;
+ return { service: new MessageTypeBackfillService(repo, config), runner, query };
};
- it('rewrites each legacy wwebjs token to the neutral MessageType on bootstrap', async () => {
- const update = jest.fn().mockResolvedValue({ affected: 1 });
- await makeService(update).onApplicationBootstrap();
-
- expect(update).toHaveBeenCalledTimes(4);
- expect(update).toHaveBeenCalledWith({ type: 'chat' }, { type: 'text' });
- // A tapped button, list row or template button: text, as mapWwebjsMessageType now reports them.
- expect(update).toHaveBeenCalledWith(
- { type: In(['buttons_response', 'list_response', 'template_button_reply']) },
- { type: 'text' },
- );
- expect(update).toHaveBeenCalledWith({ type: 'ptt' }, { type: 'voice' });
- // vcard + multi_vcard collapse to contact via an IN clause
- expect(update).toHaveBeenCalledWith({ type: In(['vcard', 'multi_vcard']) }, { type: 'contact' });
+ it('converts the legacy wwebjs tokens with one statement under DATABASE_SYNCHRONIZE=true', async () => {
+ const { service, runner, query } = makeService({ synchronize: true });
+ await service.onApplicationBootstrap();
+
+ expect(query).toHaveBeenCalledTimes(1);
+ const [sql] = query.mock.calls[0];
+ expect(sql).toContain(`WHEN 'chat' THEN 'text'`);
+ expect(sql).toContain(`WHEN 'ptt' THEN 'voice'`);
+ expect(sql).toContain(`WHEN 'multi_vcard' THEN 'contact'`);
+ expect(runner.release).toHaveBeenCalledTimes(1);
});
- it('is a no-op-safe write (idempotent): affected:0 on already-neutral rows does not throw', async () => {
- const update = jest.fn().mockResolvedValue({ affected: 0 });
- await expect(makeService(update).onApplicationBootstrap()).resolves.toBeUndefined();
- expect(update).toHaveBeenCalledTimes(4);
+ it.each([false, undefined])(
+ 'leaves a migration-managed database to the migration (synchronize: %s)',
+ async synchronize => {
+ const { service, runner, query } = makeService({ synchronize });
+ await service.onApplicationBootstrap();
+
+ expect(query).not.toHaveBeenCalled();
+ expect(runner.release).not.toHaveBeenCalled();
+ },
+ );
+
+ it('does nothing when the config is not injected', async () => {
+ const query = jest.fn();
+ const repo = {
+ manager: { connection: { createQueryRunner: () => ({ query }) } },
+ } as unknown as Repository<Message>;
+
+ await expect(new MessageTypeBackfillService(repo).onApplicationBootstrap()).resolves.toBeUndefined();
+ expect(query).not.toHaveBeenCalled();
});
- it('does not crash boot if the backfill query fails', async () => {
- const update = jest.fn().mockRejectedValue(new Error('db down'));
- await expect(makeService(update).onApplicationBootstrap()).resolves.toBeUndefined();
+ it('does not crash boot if the backfill query fails, and still releases the runner', async () => {
+ const { service, runner } = makeService({
+ synchronize: true,
+ query: jest.fn().mockRejectedValue(new Error('db down')),
+ });
+
+ await expect(service.onApplicationBootstrap()).resolves.toBeUndefined();
+ expect(runner.release).toHaveBeenCalledTimes(1);
});
});
diff --git a/src/modules/message/message-type-backfill.service.ts b/src/modules/message/message-type-backfill.service.ts
index ec0e2e28f..9867febb1 100644
--- a/src/modules/message/message-type-backfill.service.ts
+++ b/src/modules/message/message-type-backfill.service.ts
@@ -1,23 +1,20 @@
-import { Injectable, OnApplicationBootstrap } from '@nestjs/common';
+import { Injectable, OnApplicationBootstrap, Optional } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
-import { In, Repository } from 'typeorm';
+import { Repository } from 'typeorm';
import { Message } from './entities/message.entity';
import { createLogger } from '../../common/services/logger.service';
+import { BackfillLegacyMessageTypes1791300000000 } from '../../database/migrations/1791300000000-BackfillLegacyMessageTypes';
/**
- * One-time, idempotent backfill of `messages.type` to the engine-neutral vocabulary (#265).
+ * Backfill of `messages.type` to the engine-neutral vocabulary (#265) for the one setup where no migration
+ * ever runs: a SQLite data DB under DATABASE_SYNCHRONIZE=true (the dev and e2e setup, where `migrationsRun` is
+ * false). Every other database got the same conversion once, from BackfillLegacyMessageTypes, which runs
+ * on a pool without the runtime statement timeout. Running it at every boot there meant a full scan of the
+ * unindexed `type` column on each start, ahead of the HTTP listener.
*
- * Older incoming rows were persisted with raw whatsapp-web.js tokens (`chat`/`ptt`/`vcard`); the
- * engine now emits neutral types (`text`/`voice`/`contact`) at the adapter boundary, and the
- * dashboard chat view + message stats read these persisted rows. Without the backfill, old rows
- * render wrong (text as a `[chat]` media bubble, voice notes as document links) and stats split the
- * same kind across old/new tokens.
- *
- * This runs on startup in EVERY DB mode. A TypeORM data migration would NOT suffice: a SQLite data DB
- * under the opt-in DATABASE_SYNCHRONIZE=true (e.g. docker-compose.dev.yml) runs with `migrationsRun`
- * false, so migrations never run there. The mapping is forward-only and collision-free (the neutral targets were never valid
- * raw tokens, and passthrough kinds already match), so re-running on already-converted rows is a
- * no-op — safe to execute on every boot.
+ * The conversion is idempotent (the neutral targets were never valid raw tokens), so running it on every
+ * boot of a synchronized database is safe, and it stays a single statement in the migration, which owns it.
*/
@Injectable()
export class MessageTypeBackfillService implements OnApplicationBootstrap {
@@ -26,33 +23,16 @@ export class MessageTypeBackfillService implements OnApplicationBootstrap {
constructor(
@InjectRepository(Message, 'data')
private readonly messageRepository: Repository<Message>,
+ @Optional()
+ private readonly configService?: ConfigService,
) {}
async onApplicationBootstrap(): Promise<void> {
- // [legacy wwebjs token(s)] -> neutral MessageType, for the tokens mapWwebjsMessageType renames.
- // call_log and poll_creation are left out on purpose: a pre-#265 row carries no call or poll
- // metadata, so relabelling it would render an empty call or poll bubble.
- const conversions: Array<{ from: string[]; to: string }> = [
- { from: ['chat'], to: 'text' },
- { from: ['buttons_response', 'list_response', 'template_button_reply'], to: 'text' },
- { from: ['ptt'], to: 'voice' },
- { from: ['vcard', 'multi_vcard'], to: 'contact' },
- ];
+ if (this.configService?.get<boolean>('dataDatabase.synchronize', false) !== true) return;
- let total = 0;
+ const runner = this.messageRepository.manager.connection.createQueryRunner();
try {
- for (const { from, to } of conversions) {
- const result = await this.messageRepository.update(
- { type: from.length === 1 ? from[0] : In(from) },
- { type: to },
- );
- total += result.affected ?? 0;
- }
- if (total > 0) {
- this.logger.log(`Backfilled ${total} legacy message type(s) to the neutral vocabulary`, {
- action: 'message_type_backfill',
- });
- }
+ await BackfillLegacyMessageTypes1791300000000.backfill(runner);
} catch (error) {
// Non-fatal: new rows are already neutral; only historical rows are affected. Surface it so it
// can be re-run (e.g. via a restart) rather than crashing boot.
@@ -61,6 +41,8 @@ export class MessageTypeBackfillService implements OnApplicationBootstrap {
error instanceof Error ? error.stack : String(error),
{ action: 'message_type_backfill_failed' },
);
+ } finally {
+ await runner.release();
}
}
}