Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(message): recheck the selection when loading list rows

Yudhi Armyndharisfa4374a01e75
diff --git a/src/modules/message/message-list-media-budget.spec.ts b/src/modules/message/message-list-media-budget.spec.ts
index 3b58a79ed..726a8e239 100644
--- a/src/modules/message/message-list-media-budget.spec.ts
+++ b/src/modules/message/message-list-media-budget.spec.ts
@@ -1,5 +1,5 @@
import 'reflect-metadata';
-import { DataSource, FindManyOptions, Repository } from 'typeorm';
+import { DataSource, Repository, SelectQueryBuilder } from 'typeorm';
import { BadRequestException } from '@nestjs/common';
import { MessageService, spendInlineMediaBudget } from './message.service';
import { Message, MessageDirection } from './entities/message.entity';
@@ -127,7 +127,7 @@ describe('message list reads inline media within the budget', () => {
const budget = 4 * MiB;
process.env.MESSAGE_LIST_INLINE_MEDIA_BUDGET_BYTES = String(budget);
await seed(Array.from({ length: 10 }, (_, i) => ({ id: `m${i}`, metadata: media(payload(3 * MiB)) })));
- const find = jest.spyOn(repository, 'find');
+ const find = jest.spyOn(SelectQueryBuilder.prototype, 'getMany');
const { messages } = await service.getMessages(SESSION, { limit: 100 });
@@ -143,7 +143,7 @@ describe('message list reads inline media within the budget', () => {
it('reads a page of plain text in one query', async () => {
await seed(Array.from({ length: 20 }, (_, i) => ({ id: `t${i}`, metadata: null })));
- const find = jest.spyOn(repository, 'find');
+ const find = jest.spyOn(SelectQueryBuilder.prototype, 'getMany');
const { messages, total } = await service.getMessages(SESSION, { limit: 5, offset: 5 });
@@ -180,14 +180,78 @@ describe('message list reads inline media within the budget', () => {
{ id: 'keep', metadata: null },
{ id: 'gone', metadata: null },
]);
- const find = repository.find.bind(repository);
- jest.spyOn(repository, 'find').mockImplementation(async (options?: FindManyOptions<Message>) => {
+ const getMany = Object.getOwnPropertyDescriptor(SelectQueryBuilder.prototype, 'getMany')!
+ .value as SelectQueryBuilder<Message>['getMany'];
+ jest.spyOn(SelectQueryBuilder.prototype, 'getMany').mockImplementationOnce(async function (
+ this: SelectQueryBuilder<Message>,
+ ) {
await repository.delete({ id: 'gone' });
- return find(options);
+ return getMany.call(this);
});
const { messages } = await service.getMessages(SESSION);
expect(messages.map(m => m.id)).toEqual(['keep']);
});
+ it.each([
+ ['chat', { chatId: '[email protected]' }],
+ ['sender', { from: '[email protected]' }],
+ ['direction', { direction: MessageDirection.OUTGOING }],
+ ['type', { type: 'document' }],
+ ['time', { timestamp: 2000 }],
+ ['session', { sessionId: 'other' }],
+ ] as const)('omits a row whose %s changed after page selection', async (_field, patch) => {
+ await seed(['old', 'middle', 'new'].map(id => ({ id, metadata: null, chatId: '[email protected]' })));
+ await repository.update({ sessionId: SESSION }, { type: 'image', timestamp: 1000 });
+ const getMany = Object.getOwnPropertyDescriptor(SelectQueryBuilder.prototype, 'getMany')!
+ .value as SelectQueryBuilder<Message>['getMany'];
+ jest.spyOn(SelectQueryBuilder.prototype, 'getMany').mockImplementationOnce(async function (
+ this: SelectQueryBuilder<Message>,
+ ) {
+ await repository.update({ id: 'middle' }, patch);
+ return getMany.call(this);
+ });
+ const { messages, total } = await service.getMessages(SESSION, {
+ chatId: '[email protected]',
+ direction: MessageDirection.INCOMING,
+ type: 'image',
+ since: 1000000,
+ until: 1001000,
+ });
+ expect(total).toBe(3);
+ expect(messages.map(message => message.id)).toEqual(['new', 'old']);
+ });
+
+ it('rechecks the sender author alternative when loading selected rows', async () => {
+ await seed([{ id: 'authored', metadata: null, chatId: '[email protected]' }]);
+ await repository.update({ id: 'authored' }, { from: '[email protected]', author: '[email protected]' });
+ const getMany = Object.getOwnPropertyDescriptor(SelectQueryBuilder.prototype, 'getMany')!
+ .value as SelectQueryBuilder<Message>['getMany'];
+ jest.spyOn(SelectQueryBuilder.prototype, 'getMany').mockImplementationOnce(async function (
+ this: SelectQueryBuilder<Message>,
+ ) {
+ await repository.update({ id: 'authored' }, { author: '[email protected]' });
+ return getMany.call(this);
+ });
+ const page = await service.getMessages(SESSION, { from: '[email protected]' });
+ expect(page.total).toBe(1);
+ expect(page.messages).toEqual([]);
+ });
+
+ it('rechecks an exact message reference when loading selected rows', async () => {
+ await seed([{ id: 'referenced', metadata: null }]);
+ await repository.update({ id: 'referenced' }, { waMessageId: 'selected-reference' });
+ const getMany = Object.getOwnPropertyDescriptor(SelectQueryBuilder.prototype, 'getMany')!
+ .value as SelectQueryBuilder<Message>['getMany'];
+ jest.spyOn(SelectQueryBuilder.prototype, 'getMany').mockImplementationOnce(async function (
+ this: SelectQueryBuilder<Message>,
+ ) {
+ await repository.update({ id: 'referenced' }, { waMessageId: 'replacement-reference' });
+ return getMany.call(this);
+ });
+ const page = await service.getMessages(SESSION, { messageId: 'selected-reference' });
+ expect(page.total).toBe(1);
+ expect(page.messages).toEqual([]);
+ });
});
diff --git a/src/modules/message/message-selection-restore.pg.spec.ts b/src/modules/message/message-selection-restore.pg.spec.ts
new file mode 100644
index 000000000..b6075bb4d
--- /dev/null
+++ b/src/modules/message/message-selection-restore.pg.spec.ts
@@ -0,0 +1,259 @@
+/* istanbul ignore file -- PG-gated: only runs under DATABASE_TYPE=postgres (test-postgres CI job);
+ skipped in the default test job, so its lines would be unread and skew the global coverage gate. */
+import 'reflect-metadata';
+import { randomBytes } from 'crypto';
+import { DataSource, Repository } from 'typeorm';
+import { InfraDataService } from '../infra/infra-data.service';
+import { Session, SessionStatus } from '../session/entities/session.entity';
+import { Webhook } from '../webhook/entities/webhook.entity';
+import { Message, MessageDirection } from './entities/message.entity';
+import { MessageService, GetMessagesOptions } from './message.service';
+import { MessageBatch } from './entities/message-batch.entity';
+import { Template } from '../template/entities/template.entity';
+import { BaileysStoredMessage } from '../../engine/adapters/baileys-stored-message.entity';
+import { LidMapping } from '../../engine/identity/lid-mapping.entity';
+import { ChatState } from '../../engine/adapters/baileys-chat-state.entity';
+import { PluginInstance } from '../integration/entities/plugin-instance.entity';
+import { ConversationMapping } from '../integration/entities/conversation-mapping.entity';
+import { IngressEvent } from '../integration/entities/ingress-event.entity';
+import { WebhookDeliveryFailure } from '../webhook/entities/webhook-delivery-failure.entity';
+import { WebhookOutboxEvent } from '../webhook/entities/webhook-outbox-event.entity';
+import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';
+import { StatusUpdate } from '../status-store/entities/status-update.entity';
+import { SendIdempotencyKey } from './entities/send-idempotency-key.entity';
+import { AutomationRule } from '../automation/entities/automation-rule.entity';
+import { ApiKey, ApiKeyRole } from '../auth/entities/api-key.entity';
+import { ChatScopeService } from '../auth/chat-scope.service';
+import { postgresUtcExtra } from '../../database/postgres-utc';
+
+const enabled = process.env.DATABASE_TYPE === 'postgres';
+const deferred = () => {
+ let resolve!: () => void;
+ const promise = new Promise<void>(r => {
+ resolve = r;
+ });
+ return { promise, resolve };
+};
+(enabled ? describe : describe.skip)('stored message selection during data restore', () => {
+ const schema = `message_selection_${randomBytes(4).toString('hex')}`;
+ const SID = '00000000-0000-4000-8000-000000000001';
+ const OLD = '00000000-0000-4000-8000-000000000011';
+ const MID = '00000000-0000-4000-8000-000000000012';
+ const NEW = '00000000-0000-4000-8000-000000000013';
+ const A = '[email protected]';
+ const B = '[email protected]';
+ let admin: DataSource;
+ let ds: DataSource;
+ let main: DataSource;
+ let repo: Repository<Message>;
+ let service: MessageService;
+ let infra: InfraDataService;
+ let key: ApiKey;
+ let releaseGate: (() => void) | undefined;
+ let previousBudget: string | undefined;
+ const chatScope = new ChatScopeService();
+
+ beforeAll(async () => {
+ const options = {
+ type: 'postgres' as const,
+ host: process.env.DATABASE_HOST || '127.0.0.1',
+ port: Number(process.env.DATABASE_PORT || 5432),
+ username: process.env.DATABASE_USERNAME || 'fixture',
+ password: process.env.DATABASE_PASSWORD || 'fixture',
+ database: process.env.DATABASE_NAME || 'fixture',
+ };
+ admin = await new DataSource(options).initialize();
+ await admin.query(`CREATE SCHEMA "${schema}"`);
+ ds = await new DataSource({
+ ...options,
+ schema,
+ extra: {
+ ...postgresUtcExtra(),
+ options: `-c search_path=${schema},public`,
+ max: 3,
+ statement_timeout: 5000,
+ connectionTimeoutMillis: 2000,
+ },
+ entities: [
+ Session,
+ Webhook,
+ Message,
+ MessageBatch,
+ Template,
+ BaileysStoredMessage,
+ LidMapping,
+ ChatState,
+ PluginInstance,
+ ConversationMapping,
+ IngressEvent,
+ WebhookDeliveryFailure,
+ WebhookOutboxEvent,
+ IntegrationDeliveryFailure,
+ StatusUpdate,
+ AutomationRule,
+ SendIdempotencyKey,
+ ],
+ synchronize: true,
+ }).initialize();
+ main = await new DataSource({
+ type: 'better-sqlite3',
+ database: ':memory:',
+ entities: [ApiKey],
+ synchronize: true,
+ }).initialize();
+ key = await main.getRepository(ApiKey).save(
+ main.getRepository(ApiKey).create({
+ name: 'scoped-viewer',
+ keyHash: '0'.repeat(64),
+ keyPrefix: 'dummy-prefix',
+ role: ApiKeyRole.VIEWER,
+ allowedSessions: [SID],
+ allowedChats: [A],
+ allowedIps: null,
+ }),
+ );
+ repo = ds.getRepository(Message);
+ service = new MessageService(
+ repo,
+ {} as never,
+ {} as never,
+ {} as never,
+ {
+ findLidsForPhone: jest.fn().mockResolvedValue([]),
+ findPhoneForLid: jest.fn().mockResolvedValue(null),
+ } as never,
+ {} as never,
+ {} as never,
+ );
+ infra = new InfraDataService(
+ { get: (name: string, fallback: unknown) => (name === 'dataDatabase.type' ? 'postgres' : fallback) } as never,
+ ds,
+ );
+ });
+ afterAll(async () => {
+ releaseGate?.();
+ if (ds?.isInitialized) await ds.destroy();
+ if (main?.isInitialized) await main.destroy();
+ if (admin?.isInitialized) {
+ await admin.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`);
+ await admin.destroy();
+ }
+ });
+ beforeEach(async () => {
+ previousBudget = process.env.MESSAGE_LIST_INLINE_MEDIA_BUDGET_BYTES;
+ await ds.createQueryBuilder().delete().from(Message).execute();
+ await ds.createQueryBuilder().delete().from(Session).execute();
+ await ds.getRepository(Session).save(
+ ds.getRepository(Session).create({
+ id: SID,
+ name: 'kept-session',
+ status: SessionStatus.DISCONNECTED,
+ phone: null,
+ pushName: null,
+ config: {},
+ proxyUrl: null,
+ proxyType: null,
+ connectedAt: null,
+ lastActiveAt: null,
+ }),
+ );
+ for (const [id, second] of [
+ [OLD, 0],
+ [MID, 1],
+ [NEW, 2],
+ ] as const) {
+ await repo.insert({
+ id,
+ sessionId: SID,
+ waMessageId: id,
+ chatId: A,
+ from: A,
+ to: 'me',
+ body: `original-${second}`,
+ type: 'image',
+ timestamp: 1000 + second,
+ direction: MessageDirection.INCOMING,
+ createdAt: new Date(Date.UTC(2026, 0, 1, 0, 0, second)),
+ metadata: { media: { mimetype: 'image/png', data: 'A'.repeat(1000) } },
+ });
+ }
+ });
+ afterEach(() => {
+ releaseGate?.();
+ releaseGate = undefined;
+ jest.restoreAllMocks();
+ if (previousBudget === undefined) delete process.env.MESSAGE_LIST_INLINE_MEDIA_BUDGET_BYTES;
+ else process.env.MESSAGE_LIST_INLINE_MEDIA_BUDGET_BYTES = previousBudget;
+ });
+
+ it.each(['chat', 'time', 'type'] as const)('preserves the original %s selection after data restore', async drift => {
+ const archive = await infra.exportData();
+ const row = archive.tables.messages.find(message => message.id === MID)!;
+ expect(row).toBeDefined();
+ row.body = 'replacement-outside-selection';
+ if (drift === 'chat') row.chatId = B;
+ if (drift === 'time') row.timestamp = 2000;
+ if (drift === 'type') row.type = 'document';
+ const selection: GetMessagesOptions = {
+ chatId: A,
+ inlineMedia: false,
+ ...(drift === 'time' ? { since: 1000000, until: 1003000, orderBy: 'timestamp' as const } : {}),
+ ...(drift === 'type' ? { type: 'image' } : {}),
+ };
+ expect(await chatScope.allows(key, A)).toBe(true);
+ expect(await chatScope.allows(key, B)).toBe(false);
+ chatScope.requireChat(key, A);
+ const entered = deferred();
+ const proceed = deferred();
+ releaseGate = proceed.resolve;
+ const createRunner = ds.createQueryRunner.bind(ds);
+ let paused = false;
+ jest.spyOn(ds, 'createQueryRunner').mockImplementation((...args) => {
+ const runner = createRunner(...args);
+ const query = runner.query.bind(runner);
+ jest.spyOn(runner, 'query').mockImplementation(async (sql, ...parameters) => {
+ if (!paused && /^\s*SELECT/i.test(sql) && sql.includes('"metadata"') && !sql.includes('OCTET_LENGTH')) {
+ paused = true;
+ entered.resolve();
+ await proceed.promise;
+ }
+ return query(sql, ...parameters) as Promise<unknown>;
+ });
+ return runner;
+ });
+ const reading = service.getMessages(SID, selection);
+ await entered.promise;
+ try {
+ expect(await infra.importData(archive as never)).toMatchObject({ imported: true, warnings: [] });
+ // Data restore does not replace the separate main DB's key or its A-only allowlist.
+ const currentKey = await main.getRepository(ApiKey).findOneByOrFail({ id: key.id });
+ expect(currentKey.allowedChats).toEqual([A]);
+ expect(await chatScope.allows(currentKey, B)).toBe(false);
+ } finally {
+ proceed.resolve();
+ }
+ const page = await reading;
+ expect(page.messages.filter(message => message.id !== MID).map(message => message.id)).toEqual([NEW, OLD]);
+ expect(new Set(page.messages.map(message => message.id)).size).toBe(page.messages.length);
+ for (const message of page.messages) {
+ expect(message.chatId).toBe(A);
+ expect(message.body).not.toBe('replacement-outside-selection');
+ if (drift === 'time') {
+ expect(message.timestamp).toBeGreaterThanOrEqual(1000);
+ expect(message.timestamp).toBeLessThan(1003);
+ }
+ if (drift === 'type') expect(message.type).toBe('image');
+ }
+ });
+ it('keeps newest-first ordering and encoded inline budget without restore', async () => {
+ process.env.MESSAGE_LIST_INLINE_MEDIA_BUDGET_BYTES = '1500';
+ const page = await service.getMessages(SID, { chatId: A });
+ expect(page.messages.map(message => message.id)).toEqual([NEW, MID, OLD]);
+ const inline = page.messages.map(message => (message.metadata.media as { data?: string }).data);
+ expect(inline.map(data => typeof data === 'string')).toEqual([true, false, false]);
+ expect(inline.reduce((bytes, data) => bytes + (data?.length ?? 0), 0)).toBeLessThanOrEqual(1500);
+ const metadataOnly = await service.getMessages(SID, { chatId: A, inlineMedia: false });
+ expect(metadataOnly.messages.map(message => message.id)).toEqual([NEW, MID, OLD]);
+ for (const message of metadataOnly.messages) expect(message.metadata.media).toMatchObject({ omitted: true });
+ });
+});
diff --git a/src/modules/message/message.service.spec.ts b/src/modules/message/message.service.spec.ts
index 581517798..919489d57 100644
--- a/src/modules/message/message.service.spec.ts
+++ b/src/modules/message/message.service.spec.ts
@@ -34,6 +34,7 @@ function withListRead<T extends object>(qb: T, repository: Partial<Repository<Me
select: jest.fn().mockReturnValue(qb),
addSelect: jest.fn().mockReturnValue(qb),
clone: jest.fn().mockReturnValue(qb),
+ getMany: jest.fn().mockImplementation(() => Promise.resolve(rows())),
getCount: jest.fn().mockImplementation(() => Promise.resolve(rows().length)),
getRawMany: jest
.fn()
@@ -255,7 +256,7 @@ describe('MessageService', () => {
const result = await service.getMessages('sess-1', { after: 'm-1', offset: 500 });
- expect(qb.skip).not.toHaveBeenCalled();
+ expect(qb.skip.mock.calls.every(([value]) => value === undefined)).toBe(true);
const [clause, params] = qb.andWhere.mock.calls[0] as [string, Record<string, unknown>];
// rowid, not id: the stub repository carries no manager, which reads as "not postgres".
expect(clause).toContain('(message.createdAt, message.rowid) <');
diff --git a/src/modules/message/message.service.ts b/src/modules/message/message.service.ts
index 30d8250ef..73f03f064 100644
--- a/src/modules/message/message.service.ts
+++ b/src/modules/message/message.service.ts
@@ -1,6 +1,6 @@
import { Injectable, BadRequestException, NotFoundException, Optional } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
-import { In, Repository } from 'typeorm';
+import { In, Repository, SelectQueryBuilder } from 'typeorm';
import { EngineRegistry } from '../../engine/engine-registry.service';
import { MessageProjector } from '../session/message-projector.service';
import { SendTextMessageDto, SendMediaMessageDto, SendAudioMessageDto, MessageResponseDto } from './dto';
@@ -455,7 +455,7 @@ export class MessageService implements PluginMessagePort {
// base64 in metadata.media.data. Spent newest-first in the selected order, so the
// most recently viewed media still arrives inline and the rest keeps its omitted marker.
return {
- messages: await this.readPageSpendingBudget(sessionId, page, inlineMediaBudget),
+ messages: await this.readPageSpendingBudget(query, page, inlineMediaBudget),
total,
...(unknownTimestampTotal !== undefined ? { unknownTimestampTotal } : {}),
};
@@ -486,10 +486,10 @@ export class MessageService implements PluginMessagePort {
* Load the rows of a page in its order, in chunks of at most `max(budget, 1 MiB)` stored metadata
* bytes (at least one row per chunk), spending the inline media budget on each chunk before the
* next is read. Loading the whole page at once put every payload on the heap before the budget
- * dropped most of them. A row deleted since the page was picked is left out.
+ * dropped most of them. Recheck the selection so deleted or replaced rows outside it are left out.
*/
private async readPageSpendingBudget(
- sessionId: string,
+ selection: SelectQueryBuilder<Message>,
page: Array<{ id: string; metadataLength: number | string | null }>,
budgetBytes: number,
): Promise<Message[]> {
@@ -500,7 +500,13 @@ export class MessageService implements PluginMessagePort {
let chunkBytes = 0;
const flush = async (): Promise<void> => {
- const rows = await this.messageRepository.find({ where: { id: In(chunk), sessionId } });
+ const rows = await selection
+ .clone()
+ .select('message')
+ .take(undefined)
+ .skip(undefined)
+ .andWhere('message.id IN (:...chunkIds)', { chunkIds: chunk })
+ .getMany();
const byId = new Map(rows.map(row => [row.id, row]));
for (const id of chunk) {
const row = byId.get(id);