Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(db): drop NUL from free-text columns on write

Yudhi Armyndharis0557440433da
diff --git a/src/common/transformers/nul-free.transformer.spec.ts b/src/common/transformers/nul-free.transformer.spec.ts
new file mode 100644
index 000000000..e47e3a2cc
--- /dev/null
+++ b/src/common/transformers/nul-free.transformer.spec.ts
@@ -0,0 +1,91 @@
+import { DataSource } from 'typeorm';
+import { Message } from '../../modules/message/entities/message.entity';
+import { StatusUpdate } from '../../modules/status-store/entities/status-update.entity';
+import { IntegrationDeliveryFailure } from '../../modules/integration/entities/integration-delivery-failure.entity';
+import { WebhookDeliveryFailure } from '../../modules/webhook/entities/webhook-delivery-failure.entity';
+
+// PostgreSQL rejects U+0000 in text and varchar, so free text from WhatsApp, a plugin error or a webhook
+// receiver that carried one failed its write there and the row was lost. SQLite stores it, which makes
+// it the dialect that shows whether the character still reaches the column.
+describe('free-text columns drop NUL characters on write', () => {
+ let ds: DataSource;
+ const N = '\u0000';
+
+ beforeAll(async () => {
+ ds = new DataSource({
+ type: 'better-sqlite3',
+ database: ':memory:',
+ entities: [Message, StatusUpdate, IntegrationDeliveryFailure, WebhookDeliveryFailure],
+ synchronize: true,
+ });
+ await ds.initialize();
+ });
+
+ afterAll(() => ds.destroy());
+
+ const stored = (table: string, columns: string[]): Promise<Record<string, string>[]> =>
+ ds.query(`SELECT ${columns.map(c => `"${c}"`).join(', ')} FROM "${table}"`);
+
+ it('messages: body, chatName and mediaMimetype, on insert, multi-row insert and update', async () => {
+ const repo = ds.getRepository(Message);
+ const row = { sessionId: 's', chatId: '[email protected]', from: '[email protected]', to: 'me', type: 'text' };
+ await repo.insert({ ...row, waMessageId: 'w1', body: `a${N}b`, chatName: `n${N}` });
+ await repo
+ .createQueryBuilder()
+ .insert()
+ .values([{ ...row, waMessageId: 'w2', body: `h${N}` }])
+ .orIgnore()
+ .execute();
+ await repo.update({ waMessageId: 'w2' }, { body: `e${N}`, mediaMimetype: `image/jpeg${N}` });
+ expect(await stored('messages', ['body', 'chatName', 'mediaMimetype'])).toEqual(
+ expect.arrayContaining([
+ { body: 'ab', chatName: 'n', mediaMimetype: null },
+ { body: 'e', chatName: null, mediaMimetype: 'image/jpeg' },
+ ]),
+ );
+ });
+
+ it('status_updates: caption, contact names and mediaMimetype, across a save and a re-save', async () => {
+ const repo = ds.getRepository(StatusUpdate);
+ const row = repo.create({
+ sessionId: 's',
+ contactJid: '[email protected]',
+ contactName: `c${N}`,
+ contactPushName: `p${N}`,
+ waStatusId: 'st1',
+ type: 'image',
+ caption: `x${N}`,
+ mediaOmitted: true,
+ postedAt: 1,
+ expiresAt: 2,
+ });
+ await repo.save(row);
+ row.mediaMimetype = `image/png${N}`;
+ await repo.save(row);
+ expect(await stored('status_updates', ['caption', 'contactName', 'contactPushName', 'mediaMimetype'])).toEqual([
+ { caption: 'x', contactName: 'c', contactPushName: 'p', mediaMimetype: 'image/png' },
+ ]);
+ });
+
+ it('the lastError of both dead-letter tables', async () => {
+ await ds.getRepository(IntegrationDeliveryFailure).save({
+ direction: 'inbound',
+ pluginId: 'p',
+ instanceId: 'i',
+ attempts: 1,
+ lastError: `Unexpected token 'a', "a${N}" is not valid JSON`,
+ });
+ await ds.getRepository(WebhookDeliveryFailure).insert({
+ webhookId: 'w',
+ sessionId: 's',
+ event: 'message.received',
+ url: 'https://example.com/hook',
+ attempts: 3,
+ lastError: `HTTP 500: Bad${N}`,
+ });
+ expect(await stored('integration_delivery_failures', ['lastError'])).toEqual([
+ { lastError: `Unexpected token 'a', "a" is not valid JSON` },
+ ]);
+ expect(await stored('webhook_delivery_failures', ['lastError'])).toEqual([{ lastError: 'HTTP 500: Bad' }]);
+ });
+});
diff --git a/src/common/transformers/nul-free.transformer.ts b/src/common/transformers/nul-free.transformer.ts
new file mode 100644
index 000000000..71c297e8b
--- /dev/null
+++ b/src/common/transformers/nul-free.transformer.ts
@@ -0,0 +1,12 @@
+import { ValueTransformer } from 'typeorm';
+
+/**
+ * Drops U+0000 from free text on its way into a column. PostgreSQL rejects the character in text and
+ * varchar, so a WhatsApp message, push name, caption or mimetype carrying one, or an error message
+ * quoting one, failed its INSERT or UPDATE. Only for text that is never a lookup key: an id stored
+ * without its NUL would stop matching the id the engine reports.
+ */
+export const NulFreeTransformer: ValueTransformer = {
+ to: (value: unknown) => (typeof value === 'string' ? value.replaceAll('\u0000', '') : value),
+ from: (value: unknown) => value,
+};
diff --git a/src/modules/integration/entities/integration-delivery-failure.entity.ts b/src/modules/integration/entities/integration-delivery-failure.entity.ts
index fdb08014c..5354535d9 100644
--- a/src/modules/integration/entities/integration-delivery-failure.entity.ts
+++ b/src/modules/integration/entities/integration-delivery-failure.entity.ts
@@ -1,5 +1,6 @@
import { Column, CreateDateColumn, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';
import { jsonColumnType } from '../../../common/utils/column-types';
+import { NulFreeTransformer } from '../../../common/transformers/nul-free.transformer';
// DLQ-of-record for both inbound (ingress) and outbound (provider egress) delivery failures.
// Generalizes webhook_delivery_failures. sessionId is provenance (no FK).
@@ -27,7 +28,7 @@ export class IntegrationDeliveryFailure {
@Column({ type: 'int' })
attempts!: number;
- @Column({ type: 'text' })
+ @Column({ type: 'text', transformer: NulFreeTransformer })
lastError!: string;
@Column({ type: jsonColumnType(), nullable: true })
diff --git a/src/modules/message/entities/message.entity.ts b/src/modules/message/entities/message.entity.ts
index 68ca21bed..307c95f52 100644
--- a/src/modules/message/entities/message.entity.ts
+++ b/src/modules/message/entities/message.entity.ts
@@ -1,5 +1,6 @@
import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, Index, ValueTransformer } from 'typeorm';
import { jsonColumnType } from '../../../common/utils/column-types';
+import { NulFreeTransformer } from '../../../common/transformers/nul-free.transformer';
/**
* A `bigint` column reads back as a string on PostgreSQL (pg avoids >2^53 precision loss) but as a
@@ -60,7 +61,7 @@ export class Message {
/** The sender's contact name (pushName) as the engine reported it: in a group that is the member, not
* the group. Null on legacy rows and on rows that carried no contact. */
- @Column({ nullable: true })
+ @Column({ nullable: true, transformer: NulFreeTransformer })
chatName?: string;
/**
@@ -78,7 +79,7 @@ export class Message {
@Column()
to!: string;
- @Column({ type: 'text', nullable: true })
+ @Column({ type: 'text', nullable: true, transformer: NulFreeTransformer })
body!: string;
@Column({ default: 'text' })
@@ -115,7 +116,7 @@ export class Message {
/** Mimetype of the archived media, so the read endpoint can serve a Content-Type without
* depending on the inline copy. Null whenever `mediaPath` is. */
- @Column({ nullable: true })
+ @Column({ nullable: true, transformer: NulFreeTransformer })
mediaMimetype?: string;
@Column({
diff --git a/src/modules/status-store/entities/status-update.entity.ts b/src/modules/status-store/entities/status-update.entity.ts
index 674b875d2..8f7711bc9 100644
--- a/src/modules/status-store/entities/status-update.entity.ts
+++ b/src/modules/status-store/entities/status-update.entity.ts
@@ -1,5 +1,6 @@
import { Column, Entity, Index, PrimaryGeneratedColumn } from 'typeorm';
import { bigintToNumberTransformer } from '../../message/entities/message.entity';
+import { NulFreeTransformer } from '../../../common/transformers/nul-free.transformer';
@Entity('status_updates')
@Index(['sessionId', 'contactJid'])
@@ -15,10 +16,10 @@ export class StatusUpdate {
@Column()
contactJid!: string;
- @Column({ nullable: true })
+ @Column({ nullable: true, transformer: NulFreeTransformer })
contactName?: string;
- @Column({ nullable: true })
+ @Column({ nullable: true, transformer: NulFreeTransformer })
contactPushName?: string;
/** Engine status id, e.g. false_status@broadcast_<hash>. */
@@ -28,14 +29,14 @@ export class StatusUpdate {
@Column()
type!: 'text' | 'image' | 'video' | 'voice';
- @Column({ type: 'text', nullable: true })
+ @Column({ type: 'text', nullable: true, transformer: NulFreeTransformer })
caption?: string;
/** Relative path under the media store; null for text or omitted media. */
@Column({ nullable: true })
mediaPath?: string;
- @Column({ nullable: true })
+ @Column({ nullable: true, transformer: NulFreeTransformer })
mediaMimetype?: string;
@Column({ default: false })
diff --git a/src/modules/webhook/entities/webhook-delivery-failure.entity.ts b/src/modules/webhook/entities/webhook-delivery-failure.entity.ts
index 2839c4e06..f0c8b8d37 100644
--- a/src/modules/webhook/entities/webhook-delivery-failure.entity.ts
+++ b/src/modules/webhook/entities/webhook-delivery-failure.entity.ts
@@ -1,4 +1,5 @@
import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, Index } from 'typeorm';
+import { NulFreeTransformer } from '../../../common/transformers/nul-free.transformer';
/**
* A durable record of a webhook delivery that exhausted all of its retries. The queued path (BullMQ)
@@ -46,7 +47,7 @@ export class WebhookDeliveryFailure {
@Column({ type: 'int', nullable: true })
lastStatusCode!: number | null;
- @Column({ type: 'text' })
+ @Column({ type: 'text', transformer: NulFreeTransformer })
lastError!: string;
/** When the delivery was finally abandoned. */