Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
import { ConfigService } from '@nestjs/config';
import { randomUUID } from 'node:crypto';
import { DataSource, Repository, SelectQueryBuilder } from 'typeorm';
import { Session, SessionStatus } from '../session/entities/session.entity';
import { Webhook } from './entities/webhook.entity';
import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';
import { WebhookDeliveryService } from './webhook-delivery.service';
import { MAX_WEBHOOK_REDRIVE_LIMIT, WebhookRedriveService } from './webhook-redrive.service';
import { MAX_TIMER_MS } from '../../common/services/shutdown-budget';
import { recordWebhookDeliveryFailure } from './utils/record-delivery-failure';
describe('WebhookRedriveService', () => {
let ds: DataSource;
let failures: Repository<WebhookDeliveryFailure>;
let webhooks: Repository<Webhook>;
let primary: Webhook;
let otherSession: Webhook;
let retentionHours: number;
let drainMs: number;
let delivery: { redeliver: jest.MockedFunction<WebhookDeliveryService['redeliver']> };
let service: WebhookRedriveService;
const addFailure = (overrides: Partial<WebhookDeliveryFailure> = {}): Promise<WebhookDeliveryFailure> =>
failures.save(
failures.create({
webhookId: primary.id,
sessionId: primary.sessionId,
event: 'message.received',
url: primary.url,
idempotencyKey: randomUUID(),
deliveryId: randomUUID(),
attempts: 3,
lastStatusCode: 503,
lastError: 'HTTP 503: receiver unavailable',
payload: { id: 'msg-1', body: 'hi' },
createdAt: new Date(Date.now() - 60_000),
...overrides,
}),
);
beforeEach(async () => {
ds = new DataSource({
type: 'better-sqlite3',
database: ':memory:',
entities: [Session, Webhook, WebhookDeliveryFailure],
synchronize: true,
});
await ds.initialize();
failures = ds.getRepository(WebhookDeliveryFailure);
webhooks = ds.getRepository(Webhook);
for (const id of ['sess-1', 'sess-2']) {
await ds.getRepository(Session).save({ id, name: id, status: SessionStatus.READY, config: {} });
}
[primary, otherSession] = await webhooks.save(
['sess-1', 'sess-2'].map(sessionId =>
webhooks.create({
sessionId,
url: `https://${sessionId}.example/h`,
events: ['message.received'],
active: true,
retryCount: 3,
}),
),
);
retentionHours = 24;
drainMs = 5000;
delivery = {
redeliver: jest.fn<
ReturnType<WebhookDeliveryService['redeliver']>,
Parameters<WebhookDeliveryService['redeliver']>
>(async (webhook, sessionId, event, key, data) => {
// A preflight refusal (a body over the payload cap after the hooks): reported as failed, with
// nothing recorded, so the row keeps the attempt count it had.
if (data.refuse === true) return 'failed';
if (data.fail === true) {
const recorded = await recordWebhookDeliveryFailure(
failures,
{ error: jest.fn() },
{
webhookId: webhook.id,
sessionId,
event,
idempotencyKey: key,
url: webhook.url,
attempts: 1,
lastStatusCode: 503,
lastError: 'HTTP 503: receiver unavailable',
payload: data,
},
);
return recorded === null ? 'unrecorded' : 'failed';
}
await failures.delete({ webhookId: webhook.id, idempotencyKey: key });
return 'delivered';
}),
};
service = new WebhookRedriveService(
webhooks,
failures,
delivery as unknown as WebhookDeliveryService,
{
get: (key: string) => (key === 'webhook.shutdownDrainMs' ? drainMs : retentionHours),
} as unknown as ConfigService,
);
});
afterEach(async () => {
jest.restoreAllMocks();
await ds.destroy();
});
it('replays the stored payload and key with one attempt, and removes the delivered row', async () => {
const row = await addFailure();
expect(await service.redrive({})).toEqual({
redriven: 1,
delivered: 1,
enqueued: 0,
failed: 0,
skipped: 0,
remaining: 0,
});
expect(delivery.redeliver).toHaveBeenCalledWith(
expect.objectContaining({ id: primary.id }),
row.sessionId,
row.event,
row.idempotencyKey,
row.payload,
{ singleAttempt: true },
);
expect(await failures.count()).toBe(0);
expect(await service.redrive({})).toMatchObject({ delivered: 0, remaining: 0 });
});
it('counts a replay the delivery path refused before sending, so that row does not head every batch', async () => {
const refused = await addFailure({ attempts: 1, payload: { id: 'big', refuse: true } });
const other = await addFailure({ attempts: 1, createdAt: new Date(Date.now() - 30_000) });
expect(await service.redrive({ limit: 1 })).toMatchObject({ failed: 1, remaining: 2 });
// The recorder raised nothing for a refusal, so the redrive counts the attempt itself, once.
expect((await failures.findOneByOrFail({ id: refused.id })).attempts).toBe(2);
await service.redrive({ limit: 1 });
expect(delivery.redeliver.mock.calls.map(call => call[3])).toEqual([refused.idempotencyKey, other.idempotencyKey]);
});
it('raises the attempts of a replay the recorder already counted by one, not two', async () => {
const row = await addFailure({ attempts: 3, payload: { id: 'm', fail: true } });
await service.redrive({});
expect((await failures.findOneByOrFail({ id: row.id })).attempts).toBe(4);
});
it('moves a cancelled replay behind other rows without discarding its retained payload', async () => {
const cancelled = await addFailure({ attempts: 1, payload: { id: 'cancelled' } });
const next = await addFailure({ attempts: 1, createdAt: new Date(Date.now() - 30_000) });
const deliver = delivery.redeliver.getMockImplementation()!;
delivery.redeliver.mockImplementation((...args) =>
args[3] === cancelled.idempotencyKey ? Promise.resolve('cancelled') : deliver(...args),
);
expect(await service.redrive({ limit: 1 })).toMatchObject({ skipped: 1, failed: 0, remaining: 2 });
expect(await service.redrive({ limit: 1 })).toMatchObject({ delivered: 1, skipped: 0, remaining: 1 });
expect(delivery.redeliver.mock.calls.map(call => call[3])).toEqual([cancelled.idempotencyKey, next.idempotencyKey]);
const retained = await failures
.createQueryBuilder('failure')
.addSelect('failure.payload')
.where('failure.id = :id', { id: cancelled.id })
.getOneOrFail();
expect(retained).toMatchObject({
attempts: 2,
payload: cancelled.payload,
idempotencyKey: cancelled.idempotencyKey,
});
});
it('reads each retained payload only as its replay starts, never the whole batch at once', async () => {
for (let i = 0; i < 6; i++) await addFailure({ payload: { id: `m${i}`, body: 'x'.repeat(200) } });
const statements: string[] = [];
jest.spyOn(ds.logger, 'logQuery').mockImplementation(query => void statements.push(query));
await service.redrive({});
// A batch holds up to 500 rows but runs four replays at a time; loading the bodies with the batch
// held every one of them in memory for the whole call.
const selectList = (statement: string) => /^SELECT\s+(.*?)\s+FROM\s/is.exec(statement)?.[1] ?? '';
const payloadReads = statements.filter(
statement => /webhook_delivery_failures/i.test(statement) && /payload/i.test(selectList(statement)),
);
expect(payloadReads).toHaveLength(6);
expect(delivery.redeliver).toHaveBeenCalledTimes(6);
});
it('skips a row that lost its copy between the batch and its replay, without sending an empty event', async () => {
const rows: WebhookDeliveryFailure[] = [];
for (let i = 0; i < 5; i++) {
rows.push(await addFailure({ createdAt: new Date(Date.now() - 120_000 + i * 1000), payload: { id: `m${i}` } }));
}
// Four replays run at once, so the fifth row's body is read only after one of them ends. Remove it from
// inside the first replay, the way a second node or the payload retention sweep would.
delivery.redeliver.mockImplementationOnce(async (webhook, _sessionId, _event, key) => {
await failures.delete({ id: rows[4].id });
await failures.delete({ webhookId: webhook.id, idempotencyKey: key });
return 'delivered';
});
expect(await service.redrive({})).toEqual({
redriven: 4,
delivered: 4,
enqueued: 0,
failed: 0,
skipped: 1,
remaining: 0,
});
expect(delivery.redeliver).toHaveBeenCalledTimes(4);
expect(delivery.redeliver.mock.calls.map(call => call[3])).not.toContain(rows[4].idempotencyKey);
});
it('selects eligible subscriptions before the batch limit, past more than 100 old unavailable rows', async () => {
const disabled = await webhooks.save(webhooks.create({ ...primary, id: randomUUID(), active: false }));
const unsubscribed = await webhooks.save(
webhooks.create({ ...primary, id: randomUUID(), events: ['message.ack'] }),
);
const unavailable = [
{ webhookId: randomUUID() },
{ webhookId: disabled.id },
{ webhookId: unsubscribed.id },
{ webhookId: otherSession.id, sessionId: primary.sessionId },
];
for (let i = 0; i < 104; i++) {
await addFailure({ ...unavailable[i % unavailable.length], createdAt: new Date(Date.now() - 120_000) });
}
const healthy = await addFailure();
expect(await service.redrive({})).toMatchObject({ delivered: 1, skipped: 0, remaining: 0 });
expect(delivery.redeliver).toHaveBeenCalledTimes(1);
expect(delivery.redeliver.mock.calls[0][3]).toBe(healthy.idempotencyKey);
expect(await failures.count()).toBe(104);
});
it('moves a failed batch behind never-replayed rows', async () => {
for (let i = 0; i < 100; i++) {
await addFailure({ payload: { fail: true }, createdAt: new Date(Date.now() - 120_000) });
}
await addFailure({ payload: { healthy: 1 } });
await addFailure({ payload: { healthy: 2 } });
expect(await service.redrive({})).toMatchObject({ failed: 100, delivered: 0, remaining: 102 });
expect(await service.redrive({ limit: 2 })).toMatchObject({ failed: 0, delivered: 2, remaining: 100 });
expect(await failures.countBy({ attempts: 4 })).toBe(100);
});
it('requires a terminal keyed payload within the current retention window', async () => {
await addFailure({ createdAt: new Date(Date.now() - 25 * 60 * 60 * 1000) });
await addFailure({ attempts: 0 });
await addFailure({ payload: null });
await addFailure({ idempotencyKey: null as never });
const fresh = await addFailure();
expect(await service.redrive({})).toMatchObject({ delivered: 1, remaining: 0 });
expect(delivery.redeliver.mock.calls[0][3]).toBe(fresh.idempotencyKey);
expect(await failures.count()).toBe(4);
});
it('returns no replayable rows when retention is off, even before payload cleanup', async () => {
await addFailure();
retentionHours = 0;
expect(await service.redrive({})).toMatchObject({ redriven: 0, remaining: 0 });
expect(delivery.redeliver).not.toHaveBeenCalled();
expect(await failures.count()).toBe(1);
});
it('narrows both delivery and remaining to the same ids, webhook and allowed sessions', async () => {
const chosen = await addFailure();
await addFailure();
const outside = await addFailure({ webhookId: otherSession.id, sessionId: otherSession.sessionId });
expect(
await service.redrive({ ids: [chosen.id, outside.id], webhookId: primary.id }, [primary.sessionId]),
).toMatchObject({ delivered: 1, remaining: 0 });
expect(delivery.redeliver).toHaveBeenCalledTimes(1);
expect(delivery.redeliver.mock.calls[0][3]).toBe(chosen.idempotencyKey);
expect(await failures.count()).toBe(2);
expect(await service.redrive({ sessionId: otherSession.sessionId }, [primary.sessionId])).toMatchObject({
redriven: 0,
remaining: 0,
});
expect(await service.redrive({ webhookId: otherSession.id }, [primary.sessionId])).toMatchObject({
redriven: 0,
remaining: 0,
});
expect(await service.redrive({ ids: [] })).toMatchObject({ redriven: 0, remaining: 0 });
expect(delivery.redeliver).toHaveBeenCalledTimes(1);
expect(await service.redrive({ sessionId: otherSession.sessionId }, null)).toMatchObject({
delivered: 1,
remaining: 0,
});
});
it('allows wildcard subscriptions to replay another event', async () => {
await webhooks.update(primary.id, { events: ['*'] });
await addFailure({ event: 'session.disconnected' });
expect(await service.redrive({})).toMatchObject({ delivered: 1, remaining: 0 });
expect(delivery.redeliver.mock.calls[0][2]).toBe('session.disconnected');
});
it('keeps opt-in events out of a wildcard subscription, so they cannot pin the head of a batch', async () => {
// message.receipt must be named; '*' no longer covers it. Rows left over from a time the webhook did
// name it would otherwise be selected, skipped without a counter change, and selected again forever.
await webhooks.update(primary.id, { events: ['*'] });
for (let i = 0; i < 3; i++) {
await addFailure({ event: 'message.receipt', createdAt: new Date(Date.now() - 120_000) });
}
const wanted = await addFailure({ event: 'message.received' });
expect(await service.redrive({ limit: 2 })).toMatchObject({ delivered: 1, skipped: 0, remaining: 0 });
expect(delivery.redeliver).toHaveBeenCalledTimes(1);
expect(delivery.redeliver.mock.calls[0][3]).toBe(wanted.idempotencyKey);
});
it('replays an opt-in event once the webhook names it', async () => {
await webhooks.update(primary.id, { events: ['*', 'message.receipt'] });
await addFailure({ event: 'message.receipt' });
expect(await service.redrive({})).toMatchObject({ delivered: 1, remaining: 0 });
expect(delivery.redeliver.mock.calls[0][2]).toBe('message.receipt');
});
it('matches exact event members and ignores non-array subscriptions', async () => {
await webhooks.update(primary.id, { events: ['message.received.extra'] });
await addFailure();
const malformed = await webhooks.save(
webhooks.create({ ...primary, id: randomUUID(), events: { event: 'message.received' } as never }),
);
await addFailure({ webhookId: malformed.id });
expect(await service.redrive({})).toMatchObject({ redriven: 0, remaining: 0 });
expect(delivery.redeliver).not.toHaveBeenCalled();
const exact = await addFailure({ event: 'message.received.extra' });
expect(await service.redrive({})).toMatchObject({ delivered: 1, remaining: 0 });
expect(delivery.redeliver.mock.calls[0][3]).toBe(exact.idempotencyKey);
});
it('supports unrestricted redrive with more than 1000 active webhooks', async () => {
const ids = Array.from({ length: 1001 }, () => randomUUID());
await webhooks
.createQueryBuilder()
.insert()
.values(ids.map(id => ({ id, sessionId: primary.sessionId, url: primary.url, events: ['message.received'] })))
.updateEntity(false)
.execute();
const row = await addFailure({ webhookId: ids.at(-1) });
expect(await service.redrive({})).toMatchObject({ delivered: 1, remaining: 0 });
expect(delivery.redeliver.mock.calls[0][3]).toBe(row.idempotencyKey);
});
it('orders equal-attempt rows by creation time and id', async () => {
const older = new Date(Date.now() - 120_000);
const first = await addFailure({
id: '00000000-0000-4000-8000-000000000001',
createdAt: older,
payload: { fail: true },
});
const second = await addFailure({
id: '00000000-0000-4000-8000-000000000002',
createdAt: older,
payload: { fail: true },
});
await addFailure();
expect(await service.redrive({ limit: 2 })).toMatchObject({ failed: 2, remaining: 3 });
expect(delivery.redeliver.mock.calls.map(call => call[3])).toEqual([first.idempotencyKey, second.idempotencyKey]);
});
it('clamps an internal request to the maximum batch', async () => {
for (let i = 0; i <= MAX_WEBHOOK_REDRIVE_LIMIT; i++) await addFailure({ payload: { fail: true } });
expect(await service.redrive({ limit: 10_000 })).toMatchObject({
failed: MAX_WEBHOOK_REDRIVE_LIMIT,
remaining: MAX_WEBHOOK_REDRIVE_LIMIT + 1,
});
expect(delivery.redeliver).toHaveBeenCalledTimes(MAX_WEBHOOK_REDRIVE_LIMIT);
});
it('counts legacy enqueued and cancelled outcomes, and increments a rejected replay', async () => {
await addFailure();
await addFailure();
await addFailure();
delivery.redeliver
.mockResolvedValueOnce('enqueued')
.mockResolvedValueOnce('cancelled')
.mockRejectedValueOnce(new Error('receiver failed'));
expect(await service.redrive({})).toMatchObject({ redriven: 1, enqueued: 1, failed: 1, skipped: 1, remaining: 3 });
expect(await failures.countBy({ attempts: 4 })).toBe(2);
});
it('continues the batch when recording a failed replay fails', async () => {
await addFailure({ payload: { fail: true }, createdAt: new Date(Date.now() - 120_000) });
await addFailure();
jest.spyOn(failures, 'update').mockRejectedValueOnce(new Error('database unavailable'));
jest.spyOn(failures, 'increment').mockRejectedValueOnce(new Error('database unavailable'));
expect(await service.redrive({})).toMatchObject({ failed: 1, delivered: 1, remaining: 1 });
});
it('serializes overlapping calls and keeps the chain usable after a failed batch', async () => {
await addFailure();
jest.spyOn(failures, 'createQueryBuilder').mockImplementationOnce(() => {
throw new Error('database unavailable');
});
await expect(service.redrive({})).rejects.toThrow('database unavailable');
let release!: () => void;
const gate = new Promise<void>(resolve => (release = resolve));
const deliver = delivery.redeliver.getMockImplementation()!;
delivery.redeliver.mockImplementationOnce(async (...args) => {
await gate;
return deliver(...args);
});
const first = service.redrive({});
const second = service.redrive({});
await new Promise(resolve => setImmediate(resolve));
expect(delivery.redeliver).toHaveBeenCalledTimes(1);
release();
expect(await first).toMatchObject({ delivered: 1, remaining: 0 });
expect(await second).toMatchObject({ delivered: 0, remaining: 0 });
expect(delivery.redeliver).toHaveBeenCalledTimes(1);
});
it('bounds parallel direct replays to four', async () => {
for (let i = 0; i < 10; i++) await addFailure();
const deliver = delivery.redeliver.getMockImplementation()!;
let active = 0;
let peak = 0;
delivery.redeliver.mockImplementation(async (...args) => {
active++;
peak = Math.max(peak, active);
await new Promise(resolve => setImmediate(resolve));
const outcome = await deliver(...args);
active--;
return outcome;
});
expect(await service.redrive({})).toMatchObject({ delivered: 10, remaining: 0 });
expect(peak).toBe(4);
});
it('stops a batch at shutdown and waits for the replays in hand', async () => {
for (let i = 0; i < 6; i++) await addFailure();
const deliver = delivery.redeliver.getMockImplementation()!;
let release!: () => void;
const gate = new Promise<void>(resolve => (release = resolve));
let inHand!: () => void;
const fourInHand = new Promise<void>(resolve => (inHand = resolve));
delivery.redeliver.mockImplementation(async (...args) => {
if (delivery.redeliver.mock.calls.length === 4) inHand();
await gate;
return deliver(...args);
});
const batch = service.redrive({});
await fourInHand;
// The batch has read its rows; a call queued behind it must read none.
const select = jest.spyOn(SelectQueryBuilder.prototype, 'getMany');
const queued = service.redrive({});
let destroyed = false;
const destroy = service.onModuleDestroy().then(() => (destroyed = true));
for (let i = 0; i < 5; i++) await new Promise(resolve => setImmediate(resolve));
expect(destroyed).toBe(false);
release();
await destroy;
expect(await batch).toMatchObject({ delivered: 4, remaining: 2 });
expect(await queued).toMatchObject({ delivered: 0, remaining: 2 });
expect(select).not.toHaveBeenCalled();
expect(delivery.redeliver).toHaveBeenCalledTimes(4);
});
describe('shutdown wait bound', () => {
const holdReplay = async (): Promise<() => void> => {
await addFailure();
let release!: () => void;
const gate = new Promise<void>(resolve => (release = resolve));
const deliver = delivery.redeliver.getMockImplementation()!;
delivery.redeliver.mockImplementation(async (...args) => {
await gate;
return deliver(...args);
});
void service.redrive({});
await new Promise(resolve => setImmediate(resolve));
expect(delivery.redeliver).toHaveBeenCalledTimes(1);
return release;
};
it('stops waiting for a replay that never settles after WEBHOOK_SHUTDOWN_DRAIN_MS and logs it', async () => {
drainMs = 30;
const release = await holdReplay();
const warn = jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined);
const startedAt = Date.now();
await service.onModuleDestroy();
expect(Date.now() - startedAt).toBeGreaterThanOrEqual(25);
expect(warn).toHaveBeenCalledWith(
expect.stringContaining('redrive still running'),
expect.objectContaining({ action: 'webhook_redrive_shutdown_timeout', waitMs: 30 }),
);
release();
});
it('caps the wait at the largest timer delay when the drain exceeds it', async () => {
drainMs = 2 ** 31 + 1000;
const release = await holdReplay();
const warn = jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined);
const setTimeoutSpy = jest.spyOn(global, 'setTimeout');
const destroy = service.onModuleDestroy();
release();
await destroy;
expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_MS, false);
expect(warn).not.toHaveBeenCalled();
});
it('returns without waiting when the drain is 0', async () => {
drainMs = 0;
const release = await holdReplay();
const warn = jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined);
await service.onModuleDestroy();
expect(warn).toHaveBeenCalledTimes(1);
release();
});
it('resolves as soon as the replay settles and logs nothing', async () => {
drainMs = 60_000;
const release = await holdReplay();
const warn = jest.spyOn(service['logger'], 'warn').mockImplementation(() => undefined);
const destroy = service.onModuleDestroy();
release();
await destroy;
expect(warn).not.toHaveBeenCalled();
});
});
});