Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(infra): read export media in chunks within the inline budget
Yudhi Armyndharis8f287884603f
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.mdindex 4f266f8a8..c8c9671cb 100644--- a/docs/06-api-specification.md+++ b/docs/06-api-specification.md@@ -5624,7 +5624,7 @@ Request a graceful server restart, optionally orchestrating Docker profiles (add#### GET /api/infra/export-data-Export every row of the 16 migration tables from the Data DB as JSON. Read-only, but runs raw `SELECT *` on the `data` DataSource.+Export every row of the 16 migration tables from the Data DB as JSON. Read-only, but runs raw SQL on the `data` DataSource: `SELECT *` for most tables, while `messages` and `message_batches` are read as ids first and then as full rows in chunks, newest-first, of at most 200 rows and max(`EXPORT_INLINE_MEDIA_BUDGET_BYTES`, 1 MiB) of stored payload each (a larger single row is read on its own). Each chunk's payloads are dropped against the budget below before the next is read, so the export holds the rows it returns, one chunk and the budget's worth of media, never every inline payload in the database at once. Rows keep the order the database returns them in.**Auth:** API key (ADMIN)diff --git a/src/modules/infra/export-tables.ts b/src/modules/infra/export-tables.tsindex 9c86628aa..c81567603 100644--- a/src/modules/infra/export-tables.ts+++ b/src/modules/infra/export-tables.ts@@ -161,11 +161,23 @@ export interface ExportTable<K extends keyof MigrationTables = keyof MigrationTaoptional?: boolean;/** In-place row mutation applied right after the read (redaction, artifact stripping). */afterRead?: (rows: MigrationTables[K]) => void;- /** Spends the export's shared inline-media budget on this table's payloads. */+ /**+ * Spends the export's shared inline-media budget on this table's payloads. Such a table is never+ * read whole: the export reads `id`, `recencyColumn` and the stored size of `payloadColumn` for+ * every row, then the full rows in chunks, newest-first, stripping each chunk before reading the next.+ */inlineMedia?: {/** Which arm of `omittedInlineMedia` reports this table's dropped payloads. */bucket: 'messages' | 'messageBatches';- /** Recency key: the budget is spent newest-first, so the most recent media survives. */+ /** Column `newestFirst` reads. */+ recencyColumn: keyof MigrationTables[K][number] & string;+ /** Column holding the inline payloads; its stored size bounds each chunk of full rows. */+ payloadColumn: keyof MigrationTables[K][number] & string;+ /**+ * Recency key: the budget is spent newest-first, so the most recent media survives. It is called+ * on the key rows, which hold only `id`, `recencyColumn` and the payload size, so it must read+ * nothing but `recencyColumn`.+ */newestFirst: (row: MigrationTables[K][number]) => number;/** Drops the row's inline payload in place when it does not fit the budget. */strip: (row: MigrationTables[K][number], exceedsBudget: (encodedBytes: number) => boolean) => void;@@ -182,6 +194,8 @@ export type AnyExportTable = Omit<ExportTable, 'afterRead' | 'inlineMedia'> & {afterRead?: (rows: never[]) => void;inlineMedia?: {bucket: 'messages' | 'messageBatches';+ recencyColumn: string;+ payloadColumn: string;newestFirst: (row: never) => number;strip: (row: never, exceedsBudget: (encodedBytes: number) => boolean) => void;};@@ -214,6 +228,8 @@ export const EXPORT_TABLES: AnyExportTable[] = [afterRead: stripBodyTs,inlineMedia: {bucket: 'messages',+ recencyColumn: 'timestamp',+ payloadColumn: 'metadata',newestFirst: (row: MessageRow) => Number(row.timestamp),strip: stripInlineMediaPayload,},@@ -224,6 +240,8 @@ export const EXPORT_TABLES: AnyExportTable[] = [optional: true,inlineMedia: {bucket: 'messageBatches',+ recencyColumn: 'created_at',+ payloadColumn: 'messages',newestFirst: (row: MessageBatchRow) => Date.parse(row.created_at),strip: stripBatchInlineMedia,},diff --git a/src/modules/infra/infra-data.controller.spec.ts b/src/modules/infra/infra-data.controller.spec.tsindex 3c3929ab2..4ec269dcb 100644--- a/src/modules/infra/infra-data.controller.spec.ts+++ b/src/modules/infra/infra-data.controller.spec.ts@@ -907,6 +907,102 @@ describe('InfraDataController.importData round-trips export-data (no silent mess});});+ it('reads the media tables in chunks yet exports what a whole-table read did', async () => {+ // Mixed sizes, tied timestamps, recency keys that are not finite (text in a bigint column, a+ // created_at no parser accepts), URL pointers, rows without media, one payload bigger than a+ // whole chunk, and ids that do not follow insertion order: everything the newest-first order and+ // the first-fit spending of the budget depend on.+ await seedSession('s1');+ const photo = (bytes: number): string => Buffer.alloc(bytes, 7).toString('base64');+ await ds.transaction(async manager => {+ for (let i = 0; i < 450; i++) {+ const timestamp = i % 37 === 0 ? null : i % 41 === 0 ? 'not-a-number' : 1_700_000_000 + (i % 50);+ const data =+ i === 200+ ? photo(1_200_000)+ : i % 61 === 0+ ? 'https://cdn.example.com/p.png'+ : photo(((i * 7919) % 6000) + 1);+ const metadata = i % 53 === 0 ? null : JSON.stringify({ media: { mimetype: 'image/jpeg', data } });+ await manager.query(+ `INSERT INTO messages (id, "sessionId", "waMessageId", "chatId", "from", "to", type, direction, timestamp, metadata, status, "createdAt")+ VALUES (?, 's1', ?, 'c1', 'a', 'b', 'image', 'incoming', ?, ?, 'delivered', '2026-01-01T00:00:00.000Z')`,+ [`m-${(i * 7) % 450}`, `WA-${i}`, timestamp, metadata],+ );+ }+ for (let i = 0; i < 12; i++) {+ const createdAt = i % 5 === 0 ? 'garbage' : `2026-0${1 + (i % 3)}-01 00:00:00`;+ const messages = JSON.stringify([+ { chatId: 'c1', type: 'image', content: { image: { base64: photo(500 + i * 700), mimetype: 'image/png' } } },+ ]);+ await manager.query(+ `INSERT INTO message_batches (id, batch_id, session_id, status, messages, current_index, created_at, updated_at)+ VALUES (?, ?, 's1', 'processing', ?, 0, ?, ?)`,+ [`b-${(i * 5) % 12}`, `BATCH-${i}`, messages, createdAt, createdAt],+ );+ }+ });++ // The export as it was: each media table read whole, then the shared budget spent over a+ // newest-first copy with a non-finite key read as 0, messages before batches.+ const wholeTableExport = async (budgetBytes: number) => {+ let spent = 0;+ const tables: Record<string, unknown[]> = {};+ const omitted = { messages: 0, messageBatches: 0 };+ for (const entry of EXPORT_TABLES) {+ const media = entry.inlineMedia;+ if (!media) continue;+ const rows = await ds.query<never[]>(`SELECT * FROM ${entry.table}`);+ entry.afterRead?.(rows);+ const key = (row: never): number => {+ const value = media.newestFirst(row);+ return Number.isFinite(value) ? value : 0;+ };+ for (const row of [...rows].sort((a, b) => key(b) - key(a))) {+ media.strip(row, bytes => {+ if (spent + bytes > budgetBytes) {+ omitted[media.bucket] += 1;+ return true;+ }+ spent += bytes;+ return false;+ });+ }+ tables[entry.key] = rows;+ }+ return { tables, omitted };+ };++ const omittedByBudget: unknown[] = [];+ for (const budget of [0, 150_000, 1_700_000, 1_760_000, 10_000_000]) {+ await withBudget(budget, async () => {+ const expected = await wholeTableExport(budget);+ const query = jest.spyOn(ds, 'query');+ const dump = await controller.exportData();+ const reads = query.mock.calls.map(([sql]) => sql);+ query.mockRestore();++ expect(dump.tables.messages).toEqual(expected.tables.messages);+ expect(dump.tables.messageBatches).toEqual(expected.tables.messageBatches);+ expect(dump.omittedInlineMedia).toEqual(expected.omitted);+ expect(dump.counts).toMatchObject({ messages: 450, messageBatches: 12 });+ expect(dump.skippedTables).toEqual([]);+ expect(reads).not.toContain('SELECT * FROM messages');+ expect(reads).not.toContain('SELECT * FROM message_batches');+ expect(reads.filter(sql => sql.startsWith('SELECT * FROM messages WHERE id IN')).length).toBeGreaterThan(2);+ omittedByBudget.push(expected.omitted);+ });+ }+ // The budgets above make both tables keep some payloads and drop others, not just all or nothing.+ expect(omittedByBudget).toEqual([+ { messages: 434, messageBatches: 12 },+ { messages: 392, messageBatches: 12 },+ { messages: 7, messageBatches: 12 },+ { messages: 1, messageBatches: 5 },+ { messages: 0, messageBatches: 0 },+ ]);+ });+it('round-trips plugin instances + integration delivery failures (Integration Fabric + DLQ)', async () => {await seedSession('s1');await ds.getRepository(PluginInstance).save(@@ -1692,35 +1788,108 @@ describe('InfraDataController.exportData optional-table strictness', () => {// exportData validates its table registry against the DataSource's entity metadata before reading;// the fake stands in for a fully-migrated connection so the registry check passes and the tests// below pin the QUERY-time strictness instead.- const fakeDataSource = (query: jest.Mock): unknown => ({+ const fakeDataSource = (query: jest.Mock, type: string): unknown => ({query,+ options: { type },entityMetadatas: EXPORT_TABLES.map(entry => ({ tableName: entry.table })),});- const build = (query: jest.Mock) =>- new InfraDataController(new InfraDataService(cfg as never, fakeDataSource(query) as never));+ const build = (query: jest.Mock, type = 'better-sqlite3') =>+ new InfraDataController(new InfraDataService(cfg as never, fakeDataSource(query, type) as never));+ // Any read of the messages table, whichever of the export's statements it is.+ const readsMessages = (sql: string): boolean => / FROM messages\b/.test(sql);++ type Row = Record<string, unknown>;+ const mediaData = (row: Row): string | undefined =>+ typeof row.metadata === 'string'+ ? (JSON.parse(row.metadata) as { media?: { data?: string } } | null)?.media?.data+ : undefined;++ /**+ * A messages table answering each statement the way a database would: the key read, a full-row+ * read by id (in table order, not the order the ids were asked in) and a bare `SELECT *`. Every+ * read hands out fresh copies, and `heldBeforeRead` records the payload bytes still carried by rows+ * handed out earlier at each full-row read. Every other table reads empty.+ */+ const messagesTable = (rows: Row[]) => {+ const handedOut: Row[] = [];+ const heldBeforeRead: number[] = [];+ const handOut = (subset: Row[]): Row[] => {+ heldBeforeRead.push(handedOut.reduce((sum, row) => sum + Buffer.byteLength(mediaData(row) ?? ''), 0));+ const copies = subset.map(row => ({ ...row }));+ handedOut.push(...copies);+ return copies;+ };+ const query = jest.fn((sql: string, params: unknown[] = []) => {+ if (!readsMessages(sql)) return Promise.resolve([]);+ if (sql.startsWith('SELECT id, "timestamp", OCTET_LENGTH("metadata") AS "payloadBytes" FROM')) {+ return Promise.resolve(+ rows.map(row => ({+ id: row.id,+ timestamp: row.timestamp,+ payloadBytes: typeof row.metadata === 'string' ? Buffer.byteLength(row.metadata) : null,+ })),+ );+ }+ if (sql.startsWith('SELECT * FROM messages WHERE id IN (')) {+ return Promise.resolve(handOut(rows.filter(row => params.includes(row.id))));+ }+ if (sql === 'SELECT * FROM messages') return Promise.resolve(handOut(rows));+ return Promise.reject(new Error(`unexpected read: ${sql}`));+ });+ return Object.assign(query, { heldBeforeRead });+ };+ const chunkReads = (query: jest.Mock): Array<[string, unknown[]]> =>+ (query.mock.calls as Array<[string, unknown[]?]>)+ .filter(([sql]) => readsMessages(sql) && !sql.startsWith('SELECT id,'))+ .map(([sql, params]) => [sql, params ?? []]);+ const photoRow = (id: string, timestamp: unknown, data: string | null): Row => ({+ id,+ sessionId: 's1',+ waMessageId: `WA-${id}`,+ chatId: 'c1',+ chatName: null,+ author: null,+ from: 'a',+ to: 'b',+ body: null,+ type: data === null ? 'text' : 'image',+ direction: 'incoming',+ timestamp,+ metadata: data === null ? null : JSON.stringify({ media: { mimetype: 'image/jpeg', data } }),+ status: 'delivered',+ createdAt: '2026-01-01T00:00:00.000Z',+ });+ const withExportBudget = async <T>(bytes: number, run: () => Promise<T>): Promise<T> => {+ const prev = process.env.EXPORT_INLINE_MEDIA_BUDGET_BYTES;+ process.env.EXPORT_INLINE_MEDIA_BUDGET_BYTES = String(bytes);+ try {+ return await run();+ } finally {+ if (prev === undefined) delete process.env.EXPORT_INLINE_MEDIA_BUDGET_BYTES;+ else process.env.EXPORT_INLINE_MEDIA_BUDGET_BYTES = prev;+ }+ };it('rethrows a non-missing-table error (lock/IO/timeout) instead of reporting a partial export as complete', async () => {// A lock on ONE optional table must fail the whole export: silently exporting without that table// produces a backup the import then treats as authoritative, DELETing the table's existing rows.const query = jest.fn((sql: string) =>- sql === 'SELECT * FROM messages'- ? Promise.reject(new Error('SQLITE_BUSY: database is locked'))- : Promise.resolve([]),+ readsMessages(sql) ? Promise.reject(new Error('SQLITE_BUSY: database is locked')) : Promise.resolve([]),);await expect(build(query).exportData()).rejects.toThrow(/database is locked/);});it('tolerates a genuinely absent optional table — and marks it in skippedTables', async () => {const missing = Object.assign(new Error('no such table: messages'), { name: 'SqliteError' });- const query = jest.fn((sql: string) =>- sql === 'SELECT * FROM messages' ? Promise.reject(missing) : Promise.resolve([]),- );+ const query = jest.fn((sql: string) => (readsMessages(sql) ? Promise.reject(missing) : Promise.resolve([])));const dump = await build(query).exportData();expect(dump.counts.messages).toBe(0);expect(dump.tables.messages).toEqual([]);expect(dump.skippedTables).toContain('messages');// Every other table still exported (all empty here, none skipped).expect(dump.skippedTables).toEqual(['messages']);+ // The key read found the table missing; nothing read it again.+ expect(query.mock.calls.filter(([sql]) => readsMessages(sql))).toHaveLength(1);});it('strips the Postgres generated body_ts tsvector from exported message rows', async () => {@@ -1744,12 +1913,83 @@ describe('InfraDataController.exportData optional-table strictness', () => {createdAt: '2026-01-01T00:00:00.000Z',body_ts: "'hello'",};- const query = jest.fn((sql: string) => Promise.resolve(sql === 'SELECT * FROM messages' ? [messageRow] : []));- const dump = await build(query).exportData();+ const dump = await build(messagesTable([messageRow])).exportData();expect(dump.tables.messages).toHaveLength(1);expect(dump.tables.messages[0]).not.toHaveProperty('body_ts');expect(dump.tables.messages[0].body).toBe('hello');});++ it('reads messages in bounded chunks and holds no more than the budget of payload between them', async () => {+ // 450 photos of 40 KB of base64 each, 18 MB in all, of which a 100 KB budget keeps the newest two.+ // A bare `SELECT *` put every one of them on the heap before the budget dropped any.+ const rows = Array.from({ length: 450 }, (_, i) =>+ photoRow(`m${i}`, 1_700_000_000 + i, Buffer.alloc(30_000, i).toString('base64')),+ );+ const query = messagesTable(rows);++ const dump = await withExportBudget(100_000, () => build(query).exportData());++ const reads = chunkReads(query);+ expect(reads.map(([sql]) => sql)).not.toContain('SELECT * FROM messages');+ expect(reads.length).toBeGreaterThan(1);+ for (const [, ids] of reads) {+ const bytes = rows.filter(row => ids.includes(row.id)).reduce((sum, row) => sum + String(row.metadata).length, 0);+ expect(ids.length).toBeLessThanOrEqual(200);+ if (ids.length > 1) expect(bytes).toBeLessThanOrEqual(1024 * 1024);+ }+ for (const held of query.heldBeforeRead) expect(held).toBeLessThanOrEqual(100_000);++ // Same rows in table order, the same two newest payloads kept, every other one counted.+ expect(dump.tables.messages.map(row => row.id)).toEqual(rows.map(row => row.id));+ const kept = dump.tables.messages.filter(row => mediaData(row as never) !== undefined).map(row => row.id);+ expect(kept).toEqual(['m448', 'm449']);+ expect(dump.omittedInlineMedia).toEqual({ messages: 448, messageBatches: 0 });+ expect(dump.counts.messages).toBe(450);+ });++ it.each([+ ['better-sqlite3', '?'],+ ['postgres', '$'],+ ])('binds the chunk ids in the %s placeholder style, at most 200 per read', async (type, style) => {+ // No media at all, so only the row cap splits the reads.+ const rows = Array.from({ length: 201 }, (_, i) => photoRow(`m${i}`, 1_700_000_000 + i, null));+ const query = messagesTable(rows);++ const dump = await build(query, type).exportData();++ const placeholders = (n: number): string =>+ Array.from({ length: n }, (_, i) => (style === '$' ? `$${i + 1}` : '?')).join(', ');+ const newestFirstIds = rows.map(row => row.id).reverse();+ expect(chunkReads(query)).toEqual([+ [`SELECT * FROM messages WHERE id IN (${placeholders(200)})`, newestFirstIds.slice(0, 200)],+ [`SELECT * FROM messages WHERE id IN (${placeholders(1)})`, newestFirstIds.slice(200)],+ ]);+ expect(dump.counts.messages).toBe(201);+ });++ it('fails the export when the media table disappears between its key read and a chunk read', async () => {+ // Only the key read may report the table as skipped. Past it, a missing table means rows were+ // already read, and exporting them as "skipped" or as a partial table would both be wrong.+ const missing = Object.assign(new Error('no such table: messages'), { name: 'SqliteError' });+ const table = messagesTable([photoRow('m1', 1_700_000_000, 'QUJD')]);+ const query = jest.fn((sql: string, params?: unknown[]) =>+ sql.startsWith('SELECT * FROM messages') ? Promise.reject(missing) : table(sql, params),+ );++ await expect(build(query).exportData()).rejects.toThrow(/no such table: messages/);+ });++ it('leaves out a row deleted between the key read and its chunk read', async () => {+ const table = messagesTable([photoRow('m1', 1_700_000_000, null), photoRow('m2', 1_700_000_001, null)]);+ const query = jest.fn((sql: string, params: unknown[] = []) =>+ table(sql, sql.includes(' WHERE id IN (') ? params.filter(id => id !== 'm1') : params),+ );++ const dump = await build(query).exportData();++ expect(dump.tables.messages.map(row => row.id)).toEqual(['m2']);+ expect(dump.counts.messages).toBe(1);+ });});describe('InfraDataController.importData status_updates + runtime reconciliation', () => {diff --git a/src/modules/infra/infra-data.service.ts b/src/modules/infra/infra-data.service.tsindex 5a4fd8c3c..ffeefdccf 100644--- a/src/modules/infra/infra-data.service.ts+++ b/src/modules/infra/infra-data.service.ts@@ -17,7 +17,7 @@ import { Session as SessionEntity, SessionStatus } from '../session/entities/sesimport { In } from 'typeorm';import { DateUtils } from 'typeorm/util/DateUtils';import type { MigrationTables, TableCounts } from './migration-tables.types';-import { EXPORT_TABLES, EXPORT_TABLE_EXCLUSIONS } from './export-tables';+import { EXPORT_TABLES, EXPORT_TABLE_EXCLUSIONS, type AnyExportTable } from './export-tables';import { TABLE_IMPORTERS } from './table-importers';/**@@ -188,6 +188,14 @@ function newestFirst<T>(rows: readonly T[], at: (row: T) => number): T[] {return [...rows].sort((a, b) => key(b) - key(a));}+/**+ * A chunk of full-row reads from an inline-media table holds at most max(budget, 1 MiB) of stored+ * payload (and always at least one row), as the message list does. The row cap keeps a chunk of+ * small rows well inside every driver's bound-parameter limit.+ */+const EXPORT_READ_CHUNK_MIN_BYTES = 1024 * 1024;+const EXPORT_READ_CHUNK_MAX_ROWS = 200;+/*** Spends the shared budget, and remembers what it refused.*@@ -196,11 +204,19 @@ function newestFirst<T>(rows: readonly T[], at: (row: T) => number): T[] {* is deliberately the same shape a payload skipped on the way in gets — so without a count, a* truncated backup is indistinguishable from a complete one, both on inspection and on restore.*/-function createInlineMediaBudget(): { exceeds: (encodedBytes: number) => boolean; droppedPayloads: () => number } {+interface InlineMediaBudget {+ /** The configured budget, which also sizes the export's chunked reads. */+ bytes: number;+ exceeds: (encodedBytes: number) => boolean;+ droppedPayloads: () => number;+}++function createInlineMediaBudget(): InlineMediaBudget {const budget = exportInlineMediaBudgetBytes();let spent = 0;let dropped = 0;return {+ bytes: budget,exceeds: (encodedBytes: number): boolean => {if (spent + encodedBytes > budget) {dropped += 1;@@ -346,13 +362,13 @@ export class InfraDataService {// A skipped table is surfaced in `skippedTables` (and logged as a warning) so an operator can tell// "not migrated yet" apart from "exported empty".const skippedTables: string[] = [];- const queryOptionalTable = async (table: string): Promise<unknown[]> => {+ const readTable = async (entry: AnyExportTable, sql: string): Promise<unknown[]> => {try {- return await this.dataDataSource.query(`SELECT * FROM ${table}`);+ return await this.dataDataSource.query(sql);} catch (error) {- if (!isMissingTableError(error)) throw error;- skippedTables.push(table);- this.logger.warn('Optional table does not exist in this DB; exporting without it', { table });+ if (!entry.optional || !isMissingTableError(error)) throw error;+ skippedTables.push(entry.table);+ this.logger.warn('Optional table does not exist in this DB; exporting without it', { table: entry.table });return [];}};@@ -360,7 +376,7 @@ export class InfraDataService {// One budget shared by the tables that carry a full inline payload (messages and// message_batches today), so the total is what is bounded rather than each table separately.const inlineMediaBudget = createInlineMediaBudget();- // The per-bucket drop counts are measured as the delta around each budgeted table's loop, which+ // The per-bucket drop counts are measured as the delta around each budgeted table's read, which// attributes every refusal to the table that spent it and keeps the snapshot semantics the// response documents: messages are served first, batches spend what is left.const droppedByBucket: Record<'messages' | 'messageBatches', number> = { messages: 0, messageBatches: 0 };@@ -368,18 +384,15 @@ export class InfraDataService {const counts = {} as TableCounts;for (const entry of EXPORT_TABLES) {- const rows: unknown[] = entry.optional- ? await queryOptionalTable(entry.table)- : await this.dataDataSource.query(`SELECT * FROM ${entry.table}`);+ const droppedBefore = inlineMediaBudget.droppedPayloads();+ const rows: unknown[] = entry.inlineMedia+ ? await this.readInlineMediaTable(entry, entry.inlineMedia, sql => readTable(entry, sql), inlineMediaBudget)+ : await readTable(entry, `SELECT * FROM ${entry.table}`);// `rows` was read for exactly this entry's table, so it holds the row type the entry's hooks// declare. That correlation is what the erased entry type cannot carry, and this loop is the// one place it is known — so the casts live here rather than at each hook.entry.afterRead?.(rows as never[]);if (entry.inlineMedia) {- const droppedBefore = inlineMediaBudget.droppedPayloads();- for (const row of newestFirst(rows, entry.inlineMedia.newestFirst as (row: unknown) => number)) {- entry.inlineMedia.strip(row as never, inlineMediaBudget.exceeds);- }droppedByBucket[entry.inlineMedia.bucket] = inlineMediaBudget.droppedPayloads() - droppedBefore;}tables[entry.key] = rows as never;@@ -401,6 +414,64 @@ export class InfraDataService {};}+ /**+ * Read a table whose rows carry inline media without ever holding all of its payloads at once.+ *+ * A `SELECT *` put every payload on the heap before the budget could drop one, so a media-heavy+ * database could exhaust memory for a response that only ever carries the budget's worth. Instead+ * every row is read as `id`, its recency key and its payload's stored size; the full rows follow+ * newest-first in chunks, and each chunk is stripped against the budget before the next is read.+ * Peak memory is the stripped rows, one chunk and the budget.+ *+ * The budget sees the rows in exactly the order `newestFirst` gave it over a `SELECT *`, and the+ * result keeps the key read's row order, which is the order `SELECT *` returned; the caller applies+ * `afterRead` to it as to any other table. A row deleted between the key read and its chunk is left+ * out; one inserted after the key read is not exported.+ */+ private async readInlineMediaTable(+ entry: AnyExportTable,+ media: NonNullable<AnyExportTable['inlineMedia']>,+ readKeys: (sql: string) => Promise<unknown[]>,+ budget: InlineMediaBudget,+ ): Promise<unknown[]> {+ const keys = (await readKeys(+ `SELECT id, "${media.recencyColumn}", OCTET_LENGTH("${media.payloadColumn}") AS "payloadBytes" FROM ${entry.table}`,+ )) as Array<{ id: unknown; payloadBytes: number | string | null }>;+ const chunkLimit = Math.max(budget.bytes, EXPORT_READ_CHUNK_MIN_BYTES);+ const stripped = new Map<unknown, unknown>();+ let chunk: unknown[] = [];+ let chunkBytes = 0;++ const flush = async (): Promise<void> => {+ const isPostgres = this.dataDataSource.options.type === 'postgres';+ const placeholders = chunk.map((_, i) => (isPostgres ? `$${i + 1}` : '?')).join(', ');+ const rows = await this.dataDataSource.query<Array<{ id: unknown }>>(+ `SELECT * FROM ${entry.table} WHERE id IN (${placeholders})`,+ chunk,+ );+ // IN returns the rows in no particular order; the budget is spent in the chunk's own order.+ const byId = new Map(rows.map(row => [row.id, row]));+ for (const id of chunk) {+ const row = byId.get(id);+ if (!row) continue;+ media.strip(row as never, budget.exceeds);+ stripped.set(id, row);+ }+ chunk = [];+ chunkBytes = 0;+ };++ for (const key of newestFirst(keys, media.newestFirst as (row: unknown) => number)) {+ const bytes = Number(key.payloadBytes) || 0;+ const full = chunk.length === EXPORT_READ_CHUNK_MAX_ROWS || chunkBytes + bytes > chunkLimit;+ if (chunk.length > 0 && full) await flush();+ chunk.push(key.id);+ chunkBytes += bytes;+ }+ if (chunk.length > 0) await flush();+ return keys.filter(key => stripped.has(key.id)).map(key => stripped.get(key.id));+ }+async importData(data: {tables: Partial<MigrationTables>;force?: boolean;