Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(infra): count template variables in the export media budget

Yudhi Armyndharis5bfa4af7ea3a
diff --git a/src/modules/infra/export-tables.media-variables.spec.ts b/src/modules/infra/export-tables.media-variables.spec.ts
new file mode 100644
index 000000000..a1081fcff
--- /dev/null
+++ b/src/modules/infra/export-tables.media-variables.spec.ts
@@ -0,0 +1,345 @@
+import 'reflect-metadata';
+import { randomUUID } from 'crypto';
+import { DataSource } from 'typeorm';
+import { InfraDataService } from './infra-data.service';
+import { EXPORT_TABLES } from './export-tables';
+import { Session, SessionStatus } from '../session/entities/session.entity';
+import { Webhook } from '../webhook/entities/webhook.entity';
+import { Message } from '../message/entities/message.entity';
+import { MessageBatch, BatchStatus } from '../message/entities/message-batch.entity';
+import { Template } from '../template/entities/template.entity';
+import { BaileysStoredMessage } from '../../engine/adapters/baileys-stored-message.entity';
+import { LidMapping } from '../../engine/identity/lid-mapping.entity';
+import { ChatState } from '../../engine/adapters/baileys-chat-state.entity';
+import { PluginInstance } from '../integration/entities/plugin-instance.entity';
+import { ConversationMapping } from '../integration/entities/conversation-mapping.entity';
+import { IngressEvent } from '../integration/entities/ingress-event.entity';
+import { WebhookDeliveryFailure } from '../webhook/entities/webhook-delivery-failure.entity';
+import { WebhookOutboxEvent } from '../webhook/entities/webhook-outbox-event.entity';
+import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';
+import { StatusUpdate } from '../status-store/entities/status-update.entity';
+import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';
+import { AutomationRule } from '../automation/entities/automation-rule.entity';
+import { renderTemplate } from '../../common/utils/template-render';
+
+const entities = [
+ Session,
+ Webhook,
+ Message,
+ MessageBatch,
+ Template,
+ BaileysStoredMessage,
+ LidMapping,
+ ChatState,
+ PluginInstance,
+ ConversationMapping,
+ IngressEvent,
+ WebhookDeliveryFailure,
+ WebhookOutboxEvent,
+ IntegrationDeliveryFailure,
+ StatusUpdate,
+ AutomationRule,
+ SendIdempotencyKey,
+];
+
+async function seedSession(ds: DataSource): Promise<string> {
+ const id = randomUUID();
+ await ds.getRepository(Session).save(
+ ds.getRepository(Session).create({
+ id,
+ name: `session-${id}`,
+ status: SessionStatus.DISCONNECTED,
+ config: {},
+ phone: null,
+ pushName: null,
+ proxyUrl: null,
+ proxyType: null,
+ connectedAt: null,
+ lastActiveAt: null,
+ }),
+ );
+ return id;
+}
+
+const batchExport = EXPORT_TABLES.find(entry => entry.key === 'messageBatches')!;
+const strip = batchExport.inlineMedia!.strip;
+
+function entry(base64 = '{{photo}}', variables: Record<string, string> = { photo: 'QUJD' }) {
+ return {
+ chatId: '[email protected]',
+ type: 'image',
+ content: { image: { base64, mimetype: 'image/jpeg', caption: '{{label}}' } },
+ variables: { ...variables, label: 'description only' },
+ };
+}
+
+interface MediaItem {
+ content: Record<string, Record<string, string>>;
+ variables: Record<string, string>;
+}
+
+function stripped(items: unknown[], bytes: number, status = BatchStatus.PROCESSING) {
+ let remaining = bytes;
+ const admitted: number[] = [];
+ const omitted: number[] = [];
+ const original = JSON.stringify(items);
+ const row = { messages: original, status };
+ strip(row as never, encoded => {
+ if (encoded > remaining) {
+ omitted.push(encoded);
+ return true;
+ }
+ remaining -= encoded;
+ admitted.push(encoded);
+ return false;
+ });
+ expect(JSON.stringify(items)).toBe(original);
+ return { items: JSON.parse(row.messages) as MediaItem[], row, remaining, admitted, omitted };
+}
+
+describe('bulk export budgets payload variables', () => {
+ it('uses the actual registered callback', () => {
+ expect(typeof strip).toBe('function');
+ expect(batchExport.table).toBe('message_batches');
+ });
+ it.each([0, 3, 4])('keeps literal media exactly when it fits budget %i', bytes => {
+ const result = stripped([entry('QUJD', {})], bytes);
+ expect(result.items[0].content.image.base64).toBe(bytes >= 4 ? 'QUJD' : undefined);
+ expect(result.items[0].variables.label).toBe('description only');
+ });
+ it('omits the referenced payload variable at a zero budget', () => {
+ const result = stripped([entry()], 0).items[0];
+ expect(result.content.image.base64).toBeUndefined();
+ expect(result.variables.photo).toBeUndefined();
+ expect(result.variables.label).toBe('description only');
+ expect(result.content.image).toEqual({ mimetype: 'image/jpeg', caption: '{{label}}' });
+ });
+ it.each(['{{photo}}', '{photo}', '{{ photo }}'])('keeps a payload at its exact encoded size: %s', token => {
+ const result = stripped([entry(token)], 4).items[0];
+ expect(result.content.image.base64).toBe(token);
+ expect(renderTemplate(result.content.image.base64, result.variables)).toBe('QUJD');
+ });
+ it.each(['{{image.data}}', '{{ image-data }}'])('uses the renderer grammar for key %s', token => {
+ const name = token.includes('.') ? 'image.data' : 'image-data';
+ const result = stripped([entry(token, { [name]: 'QUJD' })], 0).items[0];
+ expect(result.content.image.base64).toBeUndefined();
+ expect(result.variables[name]).toBeUndefined();
+ expect(result.variables.label).toBe('description only');
+ });
+ it('omits an over-budget value even when its placeholder is smaller', () => {
+ const result = stripped([entry('{{p}}', { p: 'QUJDREVGSElKS0w=' })], 8).items[0];
+ expect(result.content.image.base64).toBeUndefined();
+ expect(result.variables.p).toBeUndefined();
+ });
+ it('preserves unrelated descriptive variables without charging them', () => {
+ const result = stripped([entry('{{p}}', { p: 'QUJD', unused: 'a descriptive value longer than the budget' })], 4);
+ expect(result.items[0].content.image.base64).toBe('{{p}}');
+ expect(result.items[0].variables.unused).toBe('a descriptive value longer than the budget');
+ expect(result.items[0].variables.label).toBe('description only');
+ });
+ it('keeps multiple referenced values at their total stored payload size', () => {
+ const result = stripped([entry('{{first}}{{second}}', { first: 'QUJD', second: 'REVG' })], 8).items[0];
+ expect(renderTemplate(result.content.image.base64, result.variables)).toBe('QUJDREVG');
+ expect(result.content.image.base64).toBe('{{first}}{{second}}');
+ });
+ it('omits every referenced payload value when their total does not fit', () => {
+ const result = stripped([entry('{{first}}{{second}}', { first: 'QUJD', second: 'REVG' })], 7).items[0];
+ expect(result.content.image.base64).toBeUndefined();
+ expect(result.variables).toEqual({ label: 'description only' });
+ });
+ it('charges one stored variable once when two media fields share it', () => {
+ const message = entry();
+ (message.content as Record<string, unknown>).document = { base64: '{{photo}}', filename: 'tiny.jpg' };
+ const result = stripped([message], 4).items[0];
+ expect(result.content.image.base64).toBe('{{photo}}');
+ expect(result.content.document.base64).toBe('{{photo}}');
+ expect(result.variables.photo).toBe('QUJD');
+ });
+ it('does not leave shared references behind when their variable is omitted', () => {
+ const message = entry();
+ (message.content as Record<string, unknown>).document = { base64: '{{photo}}', filename: 'tiny.jpg' };
+ const result = stripped([message], 0).items[0];
+ expect(result.content.image.base64).toBeUndefined();
+ expect(result.content.document.base64).toBeUndefined();
+ expect(result.variables.photo).toBeUndefined();
+ expect(result.content.document.filename).toBe('tiny.jpg');
+ });
+ it('charges copies in separate message objects separately', () => {
+ const result = stripped([entry(), entry()], 4).items;
+ expect(result[0].variables.photo).toBe('QUJD');
+ expect(result[0].content.image.base64).toBe('{{photo}}');
+ expect(result[1].content.image.base64).toBeUndefined();
+ expect(result[1].variables.photo).toBeUndefined();
+ });
+ it('shares the budget between literal and templated media', () => {
+ const result = stripped([entry('QUJD', {}), entry()], 8).items;
+ expect(result[0].content.image.base64).toBe('QUJD');
+ expect(result[1].content.image.base64).toBe('{{photo}}');
+ expect(result[1].variables.photo).toBe('QUJD');
+ });
+ it('keeps an accepted shared variable when a later field does not fit', () => {
+ const message = entry('{{photo}}', { photo: 'QUJD', extra: 'REVGSElK' });
+ (message.content as Record<string, unknown>).document = { base64: '{{photo}}{{extra}}' };
+ const result = stripped([message], 4).items[0];
+ expect(result.content.image.base64).toBe('{{photo}}');
+ expect(result.content.document.base64).toBeUndefined();
+ expect(result.variables).toEqual({ photo: 'QUJD', label: 'description only' });
+ });
+ it('preserves a shared variable accepted after an earlier field did not fit', () => {
+ const message = entry('{{photo}}{{extra}}', { photo: 'QUJD', extra: 'REVGSElK' });
+ (message.content as Record<string, unknown>).document = { base64: '{{photo}}' };
+ const result = stripped([message], 4).items[0];
+ expect(result.content.image.base64).toBeUndefined();
+ expect(result.content.document.base64).toBe('{{photo}}');
+ expect(result.variables).toEqual({ photo: 'QUJD', label: 'description only' });
+ });
+ it('charges a repeated variable once in its serialized item', () => {
+ const result = stripped([entry('{{photo}}{{photo}}')], 4).items[0];
+ expect(result.content.image.base64).toBe('{{photo}}{{photo}}');
+ expect(result.variables.photo).toBe('QUJD');
+ });
+ it('charges literal payload segments alongside referenced variables', () => {
+ const kept = stripped([entry('QUJD{{photo}}')], 8).items[0];
+ expect(kept.content.image.base64).toBe('QUJD{{photo}}');
+ expect(kept.variables.photo).toBe('QUJD');
+ const omitted = stripped([entry('QUJD{{photo}}')], 7).items[0];
+ expect(omitted.content.image.base64).toBeUndefined();
+ expect(omitted.variables.photo).toBeUndefined();
+ });
+ it.each(Object.values(BatchStatus))('enforces the same export budget for a %s batch', status => {
+ const result = stripped([entry()], 0, status);
+ expect(result.row.status).toBe(status);
+ expect(result.items[0].variables.photo).toBeUndefined();
+ expect(result.items[0].content.image.base64).toBeUndefined();
+ });
+ it.each(['not-json', '{}', 'null'])('does not rewrite malformed historical content %s', raw => {
+ const row = { messages: raw };
+ const spend = jest.fn(() => true);
+ strip(row as never, spend);
+ expect(row.messages).toBe(raw);
+ expect(spend).not.toHaveBeenCalled();
+ });
+ it('tolerates partial historical list entries and preserves URL references', () => {
+ const items = [
+ null,
+ {},
+ { content: null },
+ { content: { image: { url: 'https://example.invalid/image' } } },
+ entry(),
+ ];
+ const result = stripped(items, 0).items;
+ expect(result.slice(0, 4)).toEqual(items.slice(0, 4));
+ expect(result[4].variables.photo).toBeUndefined();
+ });
+});
+describe('real SQLite export reads templated batch payloads', () => {
+ let ds: DataSource;
+ let service: InfraDataService;
+ let sessionId: string;
+ const previousBudget = process.env.EXPORT_INLINE_MEDIA_BUDGET_BYTES;
+ beforeEach(async () => {
+ delete process.env.DATABASE_TYPE;
+ ds = await new DataSource({
+ type: 'better-sqlite3',
+ database: ':memory:',
+ entities,
+ synchronize: true,
+ }).initialize();
+ service = new InfraDataService(
+ { get: (key: string, fallback?: unknown) => (key === 'dataDatabase.type' ? 'sqlite' : fallback) } as never,
+ ds,
+ );
+ sessionId = await seedSession(ds);
+ });
+ afterEach(async () => {
+ await ds?.destroy();
+ if (previousBudget === undefined) delete process.env.EXPORT_INLINE_MEDIA_BUDGET_BYTES;
+ else process.env.EXPORT_INLINE_MEDIA_BUDGET_BYTES = previousBudget;
+ });
+ async function seedBatch(index: number, status = BatchStatus.PROCESSING): Promise<string> {
+ const id = randomUUID();
+ await ds.getRepository(MessageBatch).save(
+ ds.getRepository(MessageBatch).create({
+ id,
+ batchId: `batch-${id}`,
+ sessionId,
+ status,
+ messages: [entry()],
+ currentIndex: 0,
+ startedAt: null,
+ completedAt: null,
+ createdAt: new Date(1577836800000 + index * 1000),
+ }),
+ );
+ return id;
+ }
+ it('exports omitted payloads without changing stored or caller-owned values', async () => {
+ process.env.EXPORT_INLINE_MEDIA_BUDGET_BYTES = '0';
+ const id = await seedBatch(1);
+ const before = await ds.getRepository(MessageBatch).findOneByOrFail({ id });
+ const original = JSON.stringify(before.messages);
+ const dump = await service.exportData();
+ const messages = JSON.parse(dump.tables.messageBatches[0].messages as string) as MediaItem[];
+ expect(messages[0].variables.photo).toBeUndefined();
+ expect(messages[0].content.image.base64).toBeUndefined();
+ expect(messages[0].variables.label).toBe('description only');
+ expect(dump.omittedInlineMedia.messageBatches).toBeGreaterThan(0);
+ expect(JSON.stringify(before.messages)).toBe(original);
+ expect(JSON.stringify((await ds.getRepository(MessageBatch).findOneByOrFail({ id })).messages)).toBe(original);
+ });
+ it('spends one budget newest-first across the 200-row read boundary and keeps key order', async () => {
+ process.env.EXPORT_INLINE_MEDIA_BUDGET_BYTES = '4';
+ let newest = '';
+ for (let index = 0; index < 201; index++) newest = await seedBatch(index);
+ let keyOrder: string[] = [];
+ const query = ds.query.bind(ds);
+ const spy = jest.spyOn(ds, 'query').mockImplementation((async (...args: Parameters<typeof query>) => {
+ const sql = args[0];
+ const rows: unknown = await query(...args);
+ if (sql.startsWith('SELECT id, "created_at"') && sql.endsWith('FROM message_batches')) {
+ keyOrder = (rows as Array<{ id: string }>).map(row => row.id);
+ }
+ return rows;
+ }) as typeof ds.query);
+ try {
+ const dump = await service.exportData();
+ const rows = dump.tables.messageBatches;
+ expect(keyOrder).toHaveLength(201);
+ expect(rows.map(row => row.id)).toEqual(keyOrder);
+ const retained = rows.filter(
+ row => (JSON.parse(row.messages as string) as MediaItem[])[0].variables.photo !== undefined,
+ );
+ expect(retained.map(row => row.id)).toEqual([newest]);
+ expect(
+ rows.filter(row => (JSON.parse(row.messages as string) as MediaItem[])[0].content.image.base64 !== undefined),
+ ).toHaveLength(1);
+ expect(dump.omittedInlineMedia.messageBatches).toBe(200);
+ } finally {
+ spy.mockRestore();
+ }
+ });
+ it('excludes rows inserted after the key read and drops rows deleted before their chunk', async () => {
+ process.env.EXPORT_INLINE_MEDIA_BUDGET_BYTES = '100';
+ const kept = await seedBatch(1);
+ const deleted = await seedBatch(2);
+ let added = '';
+ const query = ds.query.bind(ds);
+ const spy = jest.spyOn(ds, 'query').mockImplementation((async (...args: Parameters<typeof query>) => {
+ const sql = args[0];
+ const rows: unknown = await query(...args);
+ if (sql.startsWith('SELECT id, "created_at"')) {
+ await ds.getRepository(MessageBatch).delete({ id: deleted });
+ added = await seedBatch(3);
+ }
+ return rows;
+ }) as typeof ds.query);
+ try {
+ const dump = await service.exportData();
+ expect(dump.tables.messageBatches.map(row => row.id)).toEqual([kept]);
+ expect(added).not.toBe('');
+ expect(await ds.getRepository(MessageBatch).findOneBy({ id: added })).not.toBeNull();
+ } finally {
+ spy.mockRestore();
+ }
+ });
+});
diff --git a/src/modules/infra/export-tables.ts b/src/modules/infra/export-tables.ts
index 0bb1452b8..361e88580 100644
--- a/src/modules/infra/export-tables.ts
+++ b/src/modules/infra/export-tables.ts
@@ -6,6 +6,7 @@ import type {
MessageBatchRow,
WebhookDeliveryFailureRow,
} from './migration-tables.types';
+import { renderTemplate } from '../../common/utils/template-render';
/**
* A `data` value that is a POINTER rather than bytes. `metadata.media.data` holds `base64 || dto.url!`
@@ -146,13 +147,46 @@ function stripBatchInlineMedia(row: MessageBatchRow, exceedsBudget: (encodedByte
for (const entry of parsed) {
const content = (entry as { content?: unknown } | null)?.content;
if (typeof content !== 'object' || content === null) continue;
+ // A `base64` may be a placeholder whose payload sits in the item's variables, so those bytes are the field's
+ // too. A variable two fields share is stored once: it is charged to the first and dropped only if every
+ // field that names it was refused.
+ const rawVariables = (entry as { variables?: unknown }).variables;
+ const variables =
+ rawVariables && typeof rawVariables === 'object' && !Array.isArray(rawVariables)
+ ? (rawVariables as Record<string, unknown>)
+ : {};
+ const kept = new Set<string>();
+ const refused = new Set<string>();
for (const key of ['image', 'video', 'audio', 'document']) {
const media = (content as Record<string, unknown>)[key] as { base64?: unknown } | null | undefined;
if (!media || typeof media !== 'object' || typeof media.base64 !== 'string') continue;
- if (!exceedsBudget(Buffer.byteLength(media.base64, 'utf8'))) continue;
+ const template = media.base64;
+ // Asked of the renderer itself, so the placeholder grammar is the one the send path substitutes.
+ const named = template.includes('{')
+ ? Object.keys(variables).filter(
+ name => variables[name] != null && renderTemplate(template, { [name]: '' }) !== template,
+ )
+ : [];
+ const scaffolding = renderTemplate(template, Object.fromEntries(named.map(name => [name, ''])));
+ const encodedBytes = named.reduce(
+ (bytes, name) => {
+ if (kept.has(name)) return bytes;
+ const value = variables[name];
+ return bytes + Buffer.byteLength(typeof value === 'string' ? value : JSON.stringify(value), 'utf8');
+ },
+ Buffer.byteLength(scaffolding, 'utf8'),
+ );
+ if (!exceedsBudget(encodedBytes)) {
+ for (const name of named) kept.add(name);
+ continue;
+ }
+ for (const name of named) refused.add(name);
delete media.base64;
stripped = true;
}
+ for (const name of refused) {
+ if (!kept.has(name)) delete variables[name];
+ }
}
if (stripped) row.messages = JSON.stringify(parsed);
}