Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(message): convert legacy message types in a migration
Yudhi Armyndharis20faf6bb5ab6
diff --git a/docs/05-database-design.md b/docs/05-database-design.mdindex 534531da9..831befba3 100644--- a/docs/05-database-design.md+++ b/docs/05-database-design.md@@ -927,7 +927,8 @@ src/database/migrations/ # data connection (pluggable)├── 1787100000000-AddSendIdempotencyKeys.ts # send_idempotency_keys (opt-in send retry guard)├── 1787200000000-DeduplicateTerminalWebhookFailures.ts # one terminal record per webhook and idempotency key├── 1790812800000-AddMessageWindowIndexes.ts # compound timestamp indexes for message-time pagination-└── 1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts # (pluginId, instanceId, deliveryId) index for the dead-letter guards+├── 1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts # (pluginId, instanceId, deliveryId) index for the dead-letter guards+└── 1791300000000-BackfillLegacyMessageTypes.ts # converts raw wwebjs message types to the neutral vocabulary, once```> [!NOTE]diff --git a/src/database/migrations/1791300000000-BackfillLegacyMessageTypes.ts b/src/database/migrations/1791300000000-BackfillLegacyMessageTypes.tsnew file mode 100644index 000000000..73656a9ba--- /dev/null+++ b/src/database/migrations/1791300000000-BackfillLegacyMessageTypes.ts@@ -0,0 +1,46 @@+import { MigrationInterface, QueryRunner } from 'typeorm';++/**+ * One-time backfill of `messages.type` to the engine-neutral vocabulary (#265).+ *+ * Older incoming rows were persisted with raw whatsapp-web.js tokens (`chat`/`ptt`/`vcard`); the engine+ * now emits neutral types (`text`/`voice`/`contact`), and the dashboard chat view and the message stats read+ * these rows. A boot-time service used to run the conversion on every start. `type` has no index, so each+ * start read the whole table (four times, before the HTTP port opened), and on PostgreSQL the runtime+ * pool's statement timeout cancelled it once the table was large. A migration runs once, on a pool without+ * that timeout. The service now runs only where migrations never run: SQLite under DATABASE_SYNCHRONIZE=true.+ *+ * call_log and poll_creation are left out on purpose: a row stored before #265 carries no call or poll+ * metadata, so relabelling it would render an empty call or poll bubble. The mapping is forward-only and+ * collision-free (the neutral targets were never valid raw tokens), so it is idempotent and `down` is a+ * no-op. One statement, so the table is read once. `backfill` is shared with the boot service and with+ * import-data, which restores rows from archives taken before the neutral vocabulary.+ */+export class BackfillLegacyMessageTypes1791300000000 implements MigrationInterface {+ name = 'BackfillLegacyMessageTypes1791300000000';++ static async backfill(queryRunner: QueryRunner): Promise<void> {+ if (!(await queryRunner.hasTable('messages'))) return;+ await queryRunner.query(+ `UPDATE "messages" SET "type" = CASE "type" ` ++ `WHEN 'chat' THEN 'text' WHEN 'buttons_response' THEN 'text' WHEN 'list_response' THEN 'text' ` ++ `WHEN 'template_button_reply' THEN 'text' WHEN 'ptt' THEN 'voice' ` ++ `WHEN 'vcard' THEN 'contact' WHEN 'multi_vcard' THEN 'contact' END ` ++ `WHERE "type" IN ('chat', 'buttons_response', 'list_response', 'template_button_reply', 'ptt', 'vcard', 'multi_vcard')`,+ );+ }++ public async up(queryRunner: QueryRunner): Promise<void> {+ // Boot and CLI migrations run on a pool without the runtime statement_timeout (pg-boot-migrations.ts),+ // and MigrationExecutor wraps a lone pending migration in its own transaction, so lift any role- or+ // database-level default for this transaction only, exactly like AddWebhookDeliveryFailureLookupIndex.+ if (queryRunner.dataSource.options.type === 'postgres') {+ await queryRunner.query('SET LOCAL statement_timeout = 0');+ }+ await BackfillLegacyMessageTypes1791300000000.backfill(queryRunner);+ }++ public async down(): Promise<void> {+ // Forward-only: the raw tokens are not recoverable once converted.+ }+}diff --git a/src/database/migrations/__tests__/1791300000000-BackfillLegacyMessageTypes.spec.ts b/src/database/migrations/__tests__/1791300000000-BackfillLegacyMessageTypes.spec.tsnew file mode 100644index 000000000..b7b08eed1--- /dev/null+++ b/src/database/migrations/__tests__/1791300000000-BackfillLegacyMessageTypes.spec.ts@@ -0,0 +1,74 @@+import { DataSource } from 'typeorm';+import { BackfillLegacyMessageTypes1791300000000 } from '../1791300000000-BackfillLegacyMessageTypes';++describe('BackfillLegacyMessageTypes migration', () => {+ let ds: DataSource;++ beforeEach(async () => {+ ds = new DataSource({ type: 'better-sqlite3', database: ':memory:' });+ await ds.initialize();+ await ds.query(`CREATE TABLE "messages" ("id" varchar PRIMARY KEY NOT NULL, "type" varchar NOT NULL)`);+ const types = [+ 'chat',+ 'buttons_response',+ 'list_response',+ 'template_button_reply',+ 'ptt',+ 'vcard',+ 'multi_vcard',+ // Already neutral, or deliberately left alone.+ 'text',+ 'voice',+ 'contact',+ 'image',+ 'call_log',+ 'poll_creation',+ ];+ await ds.query(`INSERT INTO "messages" ("id","type") VALUES ${types.map((t, i) => `('m${i}', '${t}')`).join(',')}`);+ });++ afterEach(async () => {+ await ds.destroy();+ });++ const typeOf = async (): Promise<Record<string, string>> => {+ const rows = await ds.query<{ id: string; type: string }[]>(`SELECT "id","type" FROM "messages" ORDER BY "id"`);+ return Object.fromEntries(rows.map(r => [r.id, r.type]));+ };++ it('rewrites every legacy token to the neutral type and touches nothing else', async () => {+ await new BackfillLegacyMessageTypes1791300000000().up(ds.createQueryRunner());++ const after = await typeOf();+ expect([after.m0, after.m1, after.m2, after.m3]).toEqual(['text', 'text', 'text', 'text']);+ expect(after.m4).toBe('voice');+ expect([after.m5, after.m6]).toEqual(['contact', 'contact']);+ // text, voice, contact stay; image, call_log and poll_creation are left as stored.+ expect([after.m7, after.m8, after.m9, after.m10, after.m11, after.m12]).toEqual([+ 'text',+ 'voice',+ 'contact',+ 'image',+ 'call_log',+ 'poll_creation',+ ]);+ });++ it('is idempotent and its down() changes nothing', async () => {+ const migration = new BackfillLegacyMessageTypes1791300000000();+ const runner = ds.createQueryRunner();++ await migration.up(runner);+ const once = await typeOf();+ await migration.up(runner);+ await migration.down();++ expect(await typeOf()).toEqual(once);+ });++ it('does nothing on a database that has no messages table yet', async () => {+ await ds.query(`DROP TABLE "messages"`);++ await expect(BackfillLegacyMessageTypes1791300000000.backfill(ds.createQueryRunner())).resolves.toBeUndefined();+ });+});diff --git a/src/modules/infra/infra-data.controller.spec.ts b/src/modules/infra/infra-data.controller.spec.tsindex 095947564..f530249c9 100644--- a/src/modules/infra/infra-data.controller.spec.ts+++ b/src/modules/infra/infra-data.controller.spec.ts@@ -2426,6 +2426,42 @@ describe('InfraDataController.importData status_updates + runtime reconciliationexpect(await ds.query('SELECT lid, phone FROM lid_mappings')).toEqual([{ lid: '3', phone: '628123' }]);});+ // Boot no longer converts the raw whatsapp-web.js tokens on a migration-managed database, so an archive taken+ // before the neutral vocabulary has to be converted by the import that restores it.+ it('converts the raw message types of a restored archive and leaves the neutral ones alone', async () => {+ await seedSession('s1');+ const dump = await build().exportData();+ const at = dump.tables.sessions[0].updatedAt;+ const message = (id: string, type: string) => ({+ id,+ sessionId: 's1',+ waMessageId: `W-${id}`,+ chatId: '[email protected]',+ chatName: null,+ author: null,+ from: '[email protected]',+ to: '[email protected]',+ body: 'hello',+ type,+ direction: MessageDirection.INCOMING,+ timestamp: 1,+ metadata: null,+ status: MessageStatus.DELIVERED,+ createdAt: at,+ mediaPath: null,+ mediaMimetype: null,+ });+ const types = ['chat', 'ptt', 'vcard', 'multi_vcard', 'image', 'call_log', 'text'];++ const res = await build().importData({+ tables: { ...dump.tables, messages: types.map((type, i) => message(`m${i}`, type)) },+ });++ expect(res.imported).toBe(true);+ const stored = await ds.query<Array<{ id: string; type: string }>>('SELECT id, type FROM messages ORDER BY id');+ expect(stored.map(row => row.type)).toEqual(['text', 'voice', 'contact', 'contact', 'image', 'call_log', 'text']);+ });+it('exports and restores automation_rules, which the session wipe would otherwise cascade away', async () => {await seedSession('s1');const ruleRepo = ds.getRepository(AutomationRule);diff --git a/src/modules/infra/infra-data.service.ts b/src/modules/infra/infra-data.service.tsindex 47f6a70bc..1453ebd74 100644--- a/src/modules/infra/infra-data.service.ts+++ b/src/modules/infra/infra-data.service.ts@@ -15,6 +15,7 @@ import { ScopeBindingService } from '../integration/scope-binding.service';import { ReKeyChatStatesBySessionId1786500000000 } from '../../database/migrations/1786500000000-ReKeyChatStatesBySessionId';import { ScrubRevokedMessageContent1786900000000 } from '../../database/migrations/1786900000000-ScrubRevokedMessageContent';import { ScrubNonPhoneLidMappings1786950000000 } from '../../database/migrations/1786950000000-ScrubNonPhoneLidMappings';+import { BackfillLegacyMessageTypes1791300000000 } from '../../database/migrations/1791300000000-BackfillLegacyMessageTypes';import { Session as SessionEntity, SessionStatus } from '../session/entities/session.entity';import { In } from 'typeorm';import { DateUtils } from 'typeorm/util/DateUtils';@@ -930,6 +931,9 @@ export class InfraDataService {try {await ScrubRevokedMessageContent1786900000000.scrub(queryRunner);await ScrubNonPhoneLidMappings1786950000000.scrub(queryRunner);+ // An archive taken before the neutral message vocabulary restores the raw wwebjs tokens, which+ // the boot-time backfill no longer converts on a migration-managed database.+ await BackfillLegacyMessageTypes1791300000000.backfill(queryRunner);} catch (error) {warnings.push(`Failed to clean restored rows: ${error instanceof Error ? error.message : String(error)}`);}diff --git a/src/modules/message/message-type-backfill.service.spec.ts b/src/modules/message/message-type-backfill.service.spec.tsindex f94a97de1..11be902de 100644--- a/src/modules/message/message-type-backfill.service.spec.ts+++ b/src/modules/message/message-type-backfill.service.spec.ts@@ -1,37 +1,65 @@-import { In, Repository } from 'typeorm';+import { ConfigService } from '@nestjs/config';+import { Repository } from 'typeorm';import { MessageTypeBackfillService } from './message-type-backfill.service';import { Message } from './entities/message.entity';describe('MessageTypeBackfillService', () => {- const makeService = (update: jest.Mock) => {- const repo = { update } as unknown as Repository<Message>;- return new MessageTypeBackfillService(repo);+ const makeService = (+ opts: { synchronize?: boolean; query?: jest.Mock<Promise<void>, [string]>; hasTable?: boolean } = {},+ ) => {+ const query = opts.query ?? jest.fn<Promise<void>, [string]>().mockResolvedValue(undefined);+ const runner = {+ hasTable: jest.fn().mockResolvedValue(opts.hasTable ?? true),+ query,+ release: jest.fn().mockResolvedValue(undefined),+ };+ const repo = { manager: { connection: { createQueryRunner: () => runner } } } as unknown as Repository<Message>;+ const config = {+ get: (key: string, fallback: unknown) => (key === 'dataDatabase.synchronize' ? opts.synchronize : fallback),+ } as unknown as ConfigService;+ return { service: new MessageTypeBackfillService(repo, config), runner, query };};- it('rewrites each legacy wwebjs token to the neutral MessageType on bootstrap', async () => {- const update = jest.fn().mockResolvedValue({ affected: 1 });- await makeService(update).onApplicationBootstrap();-- expect(update).toHaveBeenCalledTimes(4);- expect(update).toHaveBeenCalledWith({ type: 'chat' }, { type: 'text' });- // A tapped button, list row or template button: text, as mapWwebjsMessageType now reports them.- expect(update).toHaveBeenCalledWith(- { type: In(['buttons_response', 'list_response', 'template_button_reply']) },- { type: 'text' },- );- expect(update).toHaveBeenCalledWith({ type: 'ptt' }, { type: 'voice' });- // vcard + multi_vcard collapse to contact via an IN clause- expect(update).toHaveBeenCalledWith({ type: In(['vcard', 'multi_vcard']) }, { type: 'contact' });+ it('converts the legacy wwebjs tokens with one statement under DATABASE_SYNCHRONIZE=true', async () => {+ const { service, runner, query } = makeService({ synchronize: true });+ await service.onApplicationBootstrap();++ expect(query).toHaveBeenCalledTimes(1);+ const [sql] = query.mock.calls[0];+ expect(sql).toContain(`WHEN 'chat' THEN 'text'`);+ expect(sql).toContain(`WHEN 'ptt' THEN 'voice'`);+ expect(sql).toContain(`WHEN 'multi_vcard' THEN 'contact'`);+ expect(runner.release).toHaveBeenCalledTimes(1);});- it('is a no-op-safe write (idempotent): affected:0 on already-neutral rows does not throw', async () => {- const update = jest.fn().mockResolvedValue({ affected: 0 });- await expect(makeService(update).onApplicationBootstrap()).resolves.toBeUndefined();- expect(update).toHaveBeenCalledTimes(4);+ it.each([false, undefined])(+ 'leaves a migration-managed database to the migration (synchronize: %s)',+ async synchronize => {+ const { service, runner, query } = makeService({ synchronize });+ await service.onApplicationBootstrap();++ expect(query).not.toHaveBeenCalled();+ expect(runner.release).not.toHaveBeenCalled();+ },+ );++ it('does nothing when the config is not injected', async () => {+ const query = jest.fn();+ const repo = {+ manager: { connection: { createQueryRunner: () => ({ query }) } },+ } as unknown as Repository<Message>;++ await expect(new MessageTypeBackfillService(repo).onApplicationBootstrap()).resolves.toBeUndefined();+ expect(query).not.toHaveBeenCalled();});- it('does not crash boot if the backfill query fails', async () => {- const update = jest.fn().mockRejectedValue(new Error('db down'));- await expect(makeService(update).onApplicationBootstrap()).resolves.toBeUndefined();+ it('does not crash boot if the backfill query fails, and still releases the runner', async () => {+ const { service, runner } = makeService({+ synchronize: true,+ query: jest.fn().mockRejectedValue(new Error('db down')),+ });++ await expect(service.onApplicationBootstrap()).resolves.toBeUndefined();+ expect(runner.release).toHaveBeenCalledTimes(1);});});diff --git a/src/modules/message/message-type-backfill.service.ts b/src/modules/message/message-type-backfill.service.tsindex ec0e2e28f..9867febb1 100644--- a/src/modules/message/message-type-backfill.service.ts+++ b/src/modules/message/message-type-backfill.service.ts@@ -1,23 +1,20 @@-import { Injectable, OnApplicationBootstrap } from '@nestjs/common';+import { Injectable, OnApplicationBootstrap, Optional } from '@nestjs/common';+import { ConfigService } from '@nestjs/config';import { InjectRepository } from '@nestjs/typeorm';-import { In, Repository } from 'typeorm';+import { Repository } from 'typeorm';import { Message } from './entities/message.entity';import { createLogger } from '../../common/services/logger.service';+import { BackfillLegacyMessageTypes1791300000000 } from '../../database/migrations/1791300000000-BackfillLegacyMessageTypes';/**- * One-time, idempotent backfill of `messages.type` to the engine-neutral vocabulary (#265).+ * Backfill of `messages.type` to the engine-neutral vocabulary (#265) for the one setup where no migration+ * ever runs: a SQLite data DB under DATABASE_SYNCHRONIZE=true (the dev and e2e setup, where `migrationsRun` is+ * false). Every other database got the same conversion once, from BackfillLegacyMessageTypes, which runs+ * on a pool without the runtime statement timeout. Running it at every boot there meant a full scan of the+ * unindexed `type` column on each start, ahead of the HTTP listener.*- * Older incoming rows were persisted with raw whatsapp-web.js tokens (`chat`/`ptt`/`vcard`); the- * engine now emits neutral types (`text`/`voice`/`contact`) at the adapter boundary, and the- * dashboard chat view + message stats read these persisted rows. Without the backfill, old rows- * render wrong (text as a `[chat]` media bubble, voice notes as document links) and stats split the- * same kind across old/new tokens.- *- * This runs on startup in EVERY DB mode. A TypeORM data migration would NOT suffice: a SQLite data DB- * under the opt-in DATABASE_SYNCHRONIZE=true (e.g. docker-compose.dev.yml) runs with `migrationsRun`- * false, so migrations never run there. The mapping is forward-only and collision-free (the neutral targets were never valid- * raw tokens, and passthrough kinds already match), so re-running on already-converted rows is a- * no-op — safe to execute on every boot.+ * The conversion is idempotent (the neutral targets were never valid raw tokens), so running it on every+ * boot of a synchronized database is safe, and it stays a single statement in the migration, which owns it.*/@Injectable()export class MessageTypeBackfillService implements OnApplicationBootstrap {@@ -26,33 +23,16 @@ export class MessageTypeBackfillService implements OnApplicationBootstrap {constructor(@InjectRepository(Message, 'data')private readonly messageRepository: Repository<Message>,+ @Optional()+ private readonly configService?: ConfigService,) {}async onApplicationBootstrap(): Promise<void> {- // [legacy wwebjs token(s)] -> neutral MessageType, for the tokens mapWwebjsMessageType renames.- // call_log and poll_creation are left out on purpose: a pre-#265 row carries no call or poll- // metadata, so relabelling it would render an empty call or poll bubble.- const conversions: Array<{ from: string[]; to: string }> = [- { from: ['chat'], to: 'text' },- { from: ['buttons_response', 'list_response', 'template_button_reply'], to: 'text' },- { from: ['ptt'], to: 'voice' },- { from: ['vcard', 'multi_vcard'], to: 'contact' },- ];+ if (this.configService?.get<boolean>('dataDatabase.synchronize', false) !== true) return;- let total = 0;+ const runner = this.messageRepository.manager.connection.createQueryRunner();try {- for (const { from, to } of conversions) {- const result = await this.messageRepository.update(- { type: from.length === 1 ? from[0] : In(from) },- { type: to },- );- total += result.affected ?? 0;- }- if (total > 0) {- this.logger.log(`Backfilled ${total} legacy message type(s) to the neutral vocabulary`, {- action: 'message_type_backfill',- });- }+ await BackfillLegacyMessageTypes1791300000000.backfill(runner);} catch (error) {// Non-fatal: new rows are already neutral; only historical rows are affected. Surface it so it// can be re-run (e.g. via a restart) rather than crashing boot.@@ -61,6 +41,8 @@ export class MessageTypeBackfillService implements OnApplicationBootstrap {error instanceof Error ? error.stack : String(error),{ action: 'message_type_backfill_failed' },);+ } finally {+ await runner.release();}}}