Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(db): drop NUL from free-text columns on write
Yudhi Armyndharis0557440433da
diff --git a/src/common/transformers/nul-free.transformer.spec.ts b/src/common/transformers/nul-free.transformer.spec.tsnew file mode 100644index 000000000..e47e3a2cc--- /dev/null+++ b/src/common/transformers/nul-free.transformer.spec.ts@@ -0,0 +1,91 @@+import { DataSource } from 'typeorm';+import { Message } from '../../modules/message/entities/message.entity';+import { StatusUpdate } from '../../modules/status-store/entities/status-update.entity';+import { IntegrationDeliveryFailure } from '../../modules/integration/entities/integration-delivery-failure.entity';+import { WebhookDeliveryFailure } from '../../modules/webhook/entities/webhook-delivery-failure.entity';++// PostgreSQL rejects U+0000 in text and varchar, so free text from WhatsApp, a plugin error or a webhook+// receiver that carried one failed its write there and the row was lost. SQLite stores it, which makes+// it the dialect that shows whether the character still reaches the column.+describe('free-text columns drop NUL characters on write', () => {+ let ds: DataSource;+ const N = '\u0000';++ beforeAll(async () => {+ ds = new DataSource({+ type: 'better-sqlite3',+ database: ':memory:',+ entities: [Message, StatusUpdate, IntegrationDeliveryFailure, WebhookDeliveryFailure],+ synchronize: true,+ });+ await ds.initialize();+ });++ afterAll(() => ds.destroy());++ const stored = (table: string, columns: string[]): Promise<Record<string, string>[]> =>+ ds.query(`SELECT ${columns.map(c => `"${c}"`).join(', ')} FROM "${table}"`);++ it('messages: body, chatName and mediaMimetype, on insert, multi-row insert and update', async () => {+ const repo = ds.getRepository(Message);+ const row = { sessionId: 's', chatId: '[email protected]', from: '[email protected]', to: 'me', type: 'text' };+ await repo.insert({ ...row, waMessageId: 'w1', body: `a${N}b`, chatName: `n${N}` });+ await repo+ .createQueryBuilder()+ .insert()+ .values([{ ...row, waMessageId: 'w2', body: `h${N}` }])+ .orIgnore()+ .execute();+ await repo.update({ waMessageId: 'w2' }, { body: `e${N}`, mediaMimetype: `image/jpeg${N}` });+ expect(await stored('messages', ['body', 'chatName', 'mediaMimetype'])).toEqual(+ expect.arrayContaining([+ { body: 'ab', chatName: 'n', mediaMimetype: null },+ { body: 'e', chatName: null, mediaMimetype: 'image/jpeg' },+ ]),+ );+ });++ it('status_updates: caption, contact names and mediaMimetype, across a save and a re-save', async () => {+ const repo = ds.getRepository(StatusUpdate);+ const row = repo.create({+ sessionId: 's',+ contactJid: '[email protected]',+ contactName: `c${N}`,+ contactPushName: `p${N}`,+ waStatusId: 'st1',+ type: 'image',+ caption: `x${N}`,+ mediaOmitted: true,+ postedAt: 1,+ expiresAt: 2,+ });+ await repo.save(row);+ row.mediaMimetype = `image/png${N}`;+ await repo.save(row);+ expect(await stored('status_updates', ['caption', 'contactName', 'contactPushName', 'mediaMimetype'])).toEqual([+ { caption: 'x', contactName: 'c', contactPushName: 'p', mediaMimetype: 'image/png' },+ ]);+ });++ it('the lastError of both dead-letter tables', async () => {+ await ds.getRepository(IntegrationDeliveryFailure).save({+ direction: 'inbound',+ pluginId: 'p',+ instanceId: 'i',+ attempts: 1,+ lastError: `Unexpected token 'a', "a${N}" is not valid JSON`,+ });+ await ds.getRepository(WebhookDeliveryFailure).insert({+ webhookId: 'w',+ sessionId: 's',+ event: 'message.received',+ url: 'https://example.com/hook',+ attempts: 3,+ lastError: `HTTP 500: Bad${N}`,+ });+ expect(await stored('integration_delivery_failures', ['lastError'])).toEqual([+ { lastError: `Unexpected token 'a', "a" is not valid JSON` },+ ]);+ expect(await stored('webhook_delivery_failures', ['lastError'])).toEqual([{ lastError: 'HTTP 500: Bad' }]);+ });+});diff --git a/src/common/transformers/nul-free.transformer.ts b/src/common/transformers/nul-free.transformer.tsnew file mode 100644index 000000000..71c297e8b--- /dev/null+++ b/src/common/transformers/nul-free.transformer.ts@@ -0,0 +1,12 @@+import { ValueTransformer } from 'typeorm';++/**+ * Drops U+0000 from free text on its way into a column. PostgreSQL rejects the character in text and+ * varchar, so a WhatsApp message, push name, caption or mimetype carrying one, or an error message+ * quoting one, failed its INSERT or UPDATE. Only for text that is never a lookup key: an id stored+ * without its NUL would stop matching the id the engine reports.+ */+export const NulFreeTransformer: ValueTransformer = {+ to: (value: unknown) => (typeof value === 'string' ? value.replaceAll('\u0000', '') : value),+ from: (value: unknown) => value,+};diff --git a/src/modules/integration/entities/integration-delivery-failure.entity.ts b/src/modules/integration/entities/integration-delivery-failure.entity.tsindex fdb08014c..5354535d9 100644--- a/src/modules/integration/entities/integration-delivery-failure.entity.ts+++ b/src/modules/integration/entities/integration-delivery-failure.entity.ts@@ -1,5 +1,6 @@import { Column, CreateDateColumn, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';import { jsonColumnType } from '../../../common/utils/column-types';+import { NulFreeTransformer } from '../../../common/transformers/nul-free.transformer';// DLQ-of-record for both inbound (ingress) and outbound (provider egress) delivery failures.// Generalizes webhook_delivery_failures. sessionId is provenance (no FK).@@ -27,7 +28,7 @@ export class IntegrationDeliveryFailure {@Column({ type: 'int' })attempts!: number;- @Column({ type: 'text' })+ @Column({ type: 'text', transformer: NulFreeTransformer })lastError!: string;@Column({ type: jsonColumnType(), nullable: true })diff --git a/src/modules/message/entities/message.entity.ts b/src/modules/message/entities/message.entity.tsindex 68ca21bed..307c95f52 100644--- a/src/modules/message/entities/message.entity.ts+++ b/src/modules/message/entities/message.entity.ts@@ -1,5 +1,6 @@import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, Index, ValueTransformer } from 'typeorm';import { jsonColumnType } from '../../../common/utils/column-types';+import { NulFreeTransformer } from '../../../common/transformers/nul-free.transformer';/*** A `bigint` column reads back as a string on PostgreSQL (pg avoids >2^53 precision loss) but as a@@ -60,7 +61,7 @@ export class Message {/** The sender's contact name (pushName) as the engine reported it: in a group that is the member, not* the group. Null on legacy rows and on rows that carried no contact. */- @Column({ nullable: true })+ @Column({ nullable: true, transformer: NulFreeTransformer })chatName?: string;/**@@ -78,7 +79,7 @@ export class Message {@Column()to!: string;- @Column({ type: 'text', nullable: true })+ @Column({ type: 'text', nullable: true, transformer: NulFreeTransformer })body!: string;@Column({ default: 'text' })@@ -115,7 +116,7 @@ export class Message {/** Mimetype of the archived media, so the read endpoint can serve a Content-Type without* depending on the inline copy. Null whenever `mediaPath` is. */- @Column({ nullable: true })+ @Column({ nullable: true, transformer: NulFreeTransformer })mediaMimetype?: string;@Column({diff --git a/src/modules/status-store/entities/status-update.entity.ts b/src/modules/status-store/entities/status-update.entity.tsindex 674b875d2..8f7711bc9 100644--- a/src/modules/status-store/entities/status-update.entity.ts+++ b/src/modules/status-store/entities/status-update.entity.ts@@ -1,5 +1,6 @@import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';import { bigintToNumberTransformer } from '../../message/entities/message.entity';+import { NulFreeTransformer } from '../../../common/transformers/nul-free.transformer';@Entity('status_updates')@Index(['sessionId', 'contactJid'])@@ -15,10 +16,10 @@ export class StatusUpdate {@Column()contactJid!: string;- @Column({ nullable: true })+ @Column({ nullable: true, transformer: NulFreeTransformer })contactName?: string;- @Column({ nullable: true })+ @Column({ nullable: true, transformer: NulFreeTransformer })contactPushName?: string;/** Engine status id, e.g. false_status@broadcast_<hash>. */@@ -28,14 +29,14 @@ export class StatusUpdate {@Column()type!: 'text' | 'image' | 'video' | 'voice';- @Column({ type: 'text', nullable: true })+ @Column({ type: 'text', nullable: true, transformer: NulFreeTransformer })caption?: string;/** Relative path under the media store; null for text or omitted media. */@Column({ nullable: true })mediaPath?: string;- @Column({ nullable: true })+ @Column({ nullable: true, transformer: NulFreeTransformer })mediaMimetype?: string;@Column({ default: false })diff --git a/src/modules/webhook/entities/webhook-delivery-failure.entity.ts b/src/modules/webhook/entities/webhook-delivery-failure.entity.tsindex 2839c4e06..f0c8b8d37 100644--- a/src/modules/webhook/entities/webhook-delivery-failure.entity.ts+++ b/src/modules/webhook/entities/webhook-delivery-failure.entity.ts@@ -1,4 +1,5 @@import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, Index } from 'typeorm';+import { NulFreeTransformer } from '../../../common/transformers/nul-free.transformer';/*** A durable record of a webhook delivery that exhausted all of its retries. The queued path (BullMQ)@@ -46,7 +47,7 @@ export class WebhookDeliveryFailure {@Column({ type: 'int', nullable: true })lastStatusCode!: number | null;- @Column({ type: 'text' })+ @Column({ type: 'text', transformer: NulFreeTransformer })lastError!: string;/** When the delivery was finally abandoned. */