Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
import { DataSource, Repository } from 'typeorm';
import { IngressEvent } from './entities/ingress-event.entity';
import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';
import { IntegrationRetentionService } from './integration-retention.service';
import { IngressReconcilerService } from './ingress-reconciler.service';
import { IngressEnqueueService } from './ingress-enqueue.service';
import { INGRESS_DISPATCH_TIMEOUT_MS } from './integration.constants';
import { PluginInstanceService } from './plugin-instance.service';
import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';
import { Session } from '../session/entities/session.entity';
import { LoggerService } from '../../common/services/logger.service';
const daysAgo = (d: number): Date => {
const dt = new Date();
dt.setDate(dt.getDate() - d);
return dt;
};
describe('IntegrationRetentionService.pruneOlderThan', () => {
let ds: DataSource;
let service: IntegrationRetentionService;
beforeEach(async () => {
ds = new DataSource({
type: 'better-sqlite3',
database: ':memory:',
entities: [IngressEvent, IntegrationDeliveryFailure],
synchronize: true,
});
await ds.initialize();
service = new IntegrationRetentionService(
ds.getRepository(IngressEvent),
ds.getRepository(IntegrationDeliveryFailure),
{ deadLetterAgedPending: jest.fn().mockResolvedValue(0) } as unknown as IngressReconcilerService,
);
const events = ds.getRepository(IngressEvent);
const failures = ds.getRepository(IntegrationDeliveryFailure);
// Two old + two recent rows in each table. createdAt is set explicitly so the test controls the
// window boundary rather than the @CreateDateColumn "now" default.
await events
.createQueryBuilder()
.insert()
.values([
{
id: 'ev-old-1',
instanceId: 'i',
pluginId: 'p',
providerDeliveryId: 'd1',
route: 'r',
payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
createdAt: daysAgo(30),
},
{
id: 'ev-old-2',
instanceId: 'i',
pluginId: 'p',
providerDeliveryId: 'd2',
route: 'r',
payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
createdAt: daysAgo(15),
},
{
id: 'ev-new-1',
instanceId: 'i',
pluginId: 'p',
providerDeliveryId: 'd3',
route: 'r',
payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
createdAt: daysAgo(3),
},
{
id: 'ev-new-2',
instanceId: 'i',
pluginId: 'p',
providerDeliveryId: 'd4',
route: 'r',
payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
createdAt: daysAgo(1),
},
])
.execute();
await failures
.createQueryBuilder()
.insert()
.values([
{
direction: 'inbound',
pluginId: 'p',
instanceId: 'i',
attempts: 1,
lastError: 'x',
redriven: false,
createdAt: daysAgo(40),
},
{
direction: 'outbound',
pluginId: 'p',
instanceId: 'i',
attempts: 2,
lastError: 'y',
redriven: false,
createdAt: daysAgo(20),
},
{
direction: 'inbound',
pluginId: 'p',
instanceId: 'i',
attempts: 1,
lastError: 'z',
redriven: false,
createdAt: daysAgo(5),
},
{
direction: 'inbound',
pluginId: 'p',
instanceId: 'i',
attempts: 1,
lastError: 'w',
redriven: false,
createdAt: daysAgo(2),
},
])
.execute();
});
afterEach(async () => {
if (ds.isInitialized) await ds.destroy();
});
it('deletes only rows older than the window and keeps recent ones', async () => {
const result = await service.pruneOlderThan(10);
expect(result).toEqual({ events: 2, failures: 2 });
const remainingEvents = await ds.getRepository(IngressEvent).find({ order: { id: 'ASC' } });
const remainingFailures = await ds.getRepository(IntegrationDeliveryFailure).find({ order: { createdAt: 'ASC' } });
expect(remainingEvents.map(e => e.id)).toEqual(['ev-new-1', 'ev-new-2']);
expect(remainingFailures).toHaveLength(2);
expect(remainingFailures.every(f => f.createdAt > daysAgo(10))).toBe(true);
});
it('applies independent windows per table (dedup short, failures long)', async () => {
// Events older than 2d: 30d/15d/3d (3 rows). Failures older than 10d: 40d/20d (2 rows).
const result = await service.pruneOlderThan(2, 10);
expect(result).toEqual({ events: 3, failures: 2 });
expect((await ds.getRepository(IngressEvent).find({ order: { id: 'ASC' } })).map(e => e.id)).toEqual(['ev-new-2']);
expect(await ds.getRepository(IntegrationDeliveryFailure).count()).toBe(2);
});
it('skips the failures prune entirely when the failures window is null', async () => {
const result = await service.pruneOlderThan(2, null);
expect(result).toEqual({ events: 3, failures: 0 });
expect(await ds.getRepository(IntegrationDeliveryFailure).count()).toBe(4);
});
it('deletes nothing when the window exceeds every row age', async () => {
const result = await service.pruneOlderThan(365);
expect(result).toEqual({ events: 0, failures: 0 });
expect(await ds.getRepository(IngressEvent).count()).toBe(4);
expect(await ds.getRepository(IntegrationDeliveryFailure).count()).toBe(4);
});
it('deletes everything when the window is 0 days', async () => {
// A 0-day cutoff = everything strictly older than "now" is pruned. (onModuleInit never passes a
// 0-day events window — INGRESS_DEDUP_RETENTION_DAYS <= 0 clamps to the default; here we confirm
// the method itself is bounded by age.)
const result = await service.pruneOlderThan(0);
expect(result.events + result.failures).toBe(8);
});
// Note: the "deletes only rows older than the window" case above is itself the bounded-delete proof —
// an unbounded DELETE would have removed all rows (counts of 4/4), not just the aged subset.
});
describe('IntegrationRetentionService.onModuleInit (retention scheduling)', () => {
const original = process.env.INGRESS_RETENTION_DAYS;
const originalDedup = process.env.INGRESS_DEDUP_RETENTION_DAYS;
const mockRepos = () => {
const eventsDelete = jest.fn().mockResolvedValue({ affected: 0 });
const failuresDelete = jest.fn().mockResolvedValue({ affected: 0 });
return {
eventsDelete,
failuresDelete,
events: { delete: eventsDelete } as unknown as Repository<IngressEvent>,
failures: { delete: failuresDelete } as unknown as Repository<IntegrationDeliveryFailure>,
reconciler: { deadLetterAgedPending: jest.fn().mockResolvedValue(0) } as unknown as IngressReconcilerService,
};
};
afterEach(() => {
if (original === undefined) delete process.env.INGRESS_RETENTION_DAYS;
else process.env.INGRESS_RETENTION_DAYS = original;
if (originalDedup === undefined) delete process.env.INGRESS_DEDUP_RETENTION_DAYS;
else process.env.INGRESS_DEDUP_RETENTION_DAYS = originalDedup;
});
it('disables only the failures prune when INGRESS_RETENTION_DAYS <= 0 — dedup pruning still applies', async () => {
process.env.INGRESS_RETENTION_DAYS = '0';
delete process.env.INGRESS_DEDUP_RETENTION_DAYS;
const repos = mockRepos();
const svc = new IntegrationRetentionService(repos.events, repos.failures, repos.reconciler);
svc.onModuleInit();
// Let the startup prune promise settle (the method runs before the log .then()).
await new Promise(resolve => setImmediate(resolve));
// Dedup rows are never an audit log — their prune cannot be disabled to infinity.
expect(repos.eventsDelete).toHaveBeenCalled();
expect(repos.failuresDelete).not.toHaveBeenCalled();
svc.onModuleDestroy();
});
it('defaults to 7-day dedup + 90-day failures windows, prunes once at startup, and schedules daily', () => {
delete process.env.INGRESS_RETENTION_DAYS;
delete process.env.INGRESS_DEDUP_RETENTION_DAYS;
const repos = mockRepos();
const svc = new IntegrationRetentionService(repos.events, repos.failures, repos.reconciler);
jest.useFakeTimers();
try {
const pruneSpy = jest.spyOn(svc, 'pruneOlderThan');
svc.onModuleInit();
// Startup prune fires immediately (7-day dedup + 90-day failures defaults).
expect(pruneSpy).toHaveBeenCalledWith(7, 90);
pruneSpy.mockClear();
// The recurring timer fires daily thereafter.
jest.advanceTimersByTime(24 * 60 * 60 * 1000);
expect(pruneSpy).toHaveBeenCalledWith(7, 90);
svc.onModuleDestroy();
// After destroy, advancing the timer must NOT trigger further prunes.
pruneSpy.mockClear();
jest.advanceTimersByTime(24 * 60 * 60 * 1000);
expect(pruneSpy).not.toHaveBeenCalled();
} finally {
jest.useRealTimers();
}
});
it('honors an explicit INGRESS_DEDUP_RETENTION_DAYS window', () => {
delete process.env.INGRESS_RETENTION_DAYS;
process.env.INGRESS_DEDUP_RETENTION_DAYS = '21';
const repos = mockRepos();
const svc = new IntegrationRetentionService(repos.events, repos.failures, repos.reconciler);
const pruneSpy = jest.spyOn(svc, 'pruneOlderThan');
svc.onModuleInit();
expect(pruneSpy).toHaveBeenCalledWith(21, 90);
svc.onModuleDestroy();
});
it('clamps INGRESS_DEDUP_RETENTION_DAYS <= 0 to the default instead of disabling dedup pruning', () => {
delete process.env.INGRESS_RETENTION_DAYS;
process.env.INGRESS_DEDUP_RETENTION_DAYS = '0';
const repos = mockRepos();
const svc = new IntegrationRetentionService(repos.events, repos.failures, repos.reconciler);
const pruneSpy = jest.spyOn(svc, 'pruneOlderThan');
svc.onModuleInit();
// <=0 would mean an unbounded dedup table — the trap this knob exists to avoid.
expect(pruneSpy).toHaveBeenCalledWith(7, 90);
svc.onModuleDestroy();
});
});
describe('IntegrationRetentionService.pruneOlderThan with undispatched events', () => {
let ds: DataSource;
let events: Repository<IngressEvent>;
let failures: Repository<IntegrationDeliveryFailure>;
let reconciler: IngressReconcilerService;
let service: IntegrationRetentionService;
let resolveInstance: jest.Mock;
let existingJobState: jest.Mock;
let logError: jest.SpyInstance;
let logWarn: jest.SpyInstance;
const SESSION_ID = '11111111-1111-4111-8111-111111111111';
beforeEach(async () => {
ds = new DataSource({
type: 'better-sqlite3',
database: ':memory:',
entities: [IngressEvent, IntegrationDeliveryFailure, Session],
synchronize: true,
});
await ds.initialize();
events = ds.getRepository(IngressEvent);
failures = ds.getRepository(IntegrationDeliveryFailure);
await ds.getRepository(Session).save({ id: SESSION_ID, name: 'sess-1' });
// A disabled instance, no loaded plugin and the queue off: the hand-off must not depend on the
// instance being enabled or its plugin being loaded.
resolveInstance = jest.fn().mockResolvedValue({ enabled: false });
existingJobState = jest.fn().mockResolvedValue(undefined);
reconciler = new IngressReconcilerService(
events,
failures,
{ existingJobState } as unknown as IngressEnqueueService,
{ getPlugin: () => undefined } as unknown as PluginLoaderService,
{ resolve: resolveInstance } as unknown as PluginInstanceService,
);
service = new IntegrationRetentionService(events, failures, reconciler);
logWarn = jest.spyOn(LoggerService.prototype, 'warn').mockImplementation(() => undefined);
logError = jest.spyOn(LoggerService.prototype, 'error').mockImplementation(() => undefined);
});
afterEach(async () => {
jest.restoreAllMocks();
if (ds.isInitialized) await ds.destroy();
});
const insertEvent = (id: string, over: Partial<IngressEvent>) =>
events
.createQueryBuilder()
.insert()
.values({
id,
instanceId: 'inst',
pluginId: 'plug',
providerDeliveryId: `d-${id}`,
route: 'chatwoot',
payload: { headers: {}, query: {}, body: '{"msg":"hello"}', rawBody: '{"msg":"hello"}', method: 'POST' },
sessionId: SESSION_ID,
dispatchState: 'pending',
createdAt: daysAgo(8),
...over,
})
.execute();
it('dead-letters an aged pending event with its payload instead of deleting it unrecoverably', async () => {
await insertEvent('stranded', {});
const first = await service.pruneOlderThan(7, 90);
// The payload is not lost: it now lives in a redrivable dead-letter row, written before the marker
// was settled, and the settled marker goes with the next run.
expect(first).toEqual({ events: 0, failures: 0 });
expect(await events.findOneByOrFail({ id: 'stranded' })).toMatchObject({ dispatchState: 'failed', payload: null });
expect(await service.pruneOlderThan(7, 90)).toEqual({ events: 1, failures: 0 });
expect(await events.count()).toBe(0);
const [dlq] = await failures.find();
expect(dlq).toMatchObject({
direction: 'inbound',
pluginId: 'plug',
instanceId: 'inst',
sessionId: SESSION_ID,
deliveryId: 'd-stranded',
lastError: 'not dispatched within the ingress dedup retention window',
redriven: false,
});
expect(dlq.payload).toMatchObject({
route: 'chatwoot',
method: 'POST',
ingress: { body: '{"msg":"hello"}', rawBody: '{"msg":"hello"}' },
});
});
it('hands off the healthy rows behind one whose stored payload is not JSON', async () => {
// Only an imported or hand-edited row can hold such text. Read with the batch it failed the whole
// read, so nothing behind it was ever dead-lettered and every run failed the same way.
await insertEvent('unreadable', { createdAt: daysAgo(10) });
await insertEvent('healthy-1', {});
await insertEvent('healthy-2', {});
await ds.query(`UPDATE ingress_events SET payload = '{oops' WHERE id = 'unreadable'`);
expect(await service.pruneOlderThan(7, 90)).toEqual({ events: 0, failures: 0 });
expect(await events.findOneByOrFail({ id: 'healthy-1' })).toMatchObject({ dispatchState: 'failed', payload: null });
expect(await events.findOneByOrFail({ id: 'healthy-2' })).toMatchObject({ dispatchState: 'failed', payload: null });
expect((await failures.find()).map(row => row.deliveryId).sort()).toEqual(['d-healthy-1', 'd-healthy-2']);
// Kept for an operator to remove: neither settled nor deleted, and the next run still moves on.
expect(await ds.query(`SELECT "dispatchState", payload FROM ingress_events WHERE id = 'unreadable'`)).toEqual([
{ dispatchState: 'pending', payload: '{oops' },
]);
expect(await service.pruneOlderThan(7, 90)).toEqual({ events: 2, failures: 0 });
expect(await events.count()).toBe(1);
});
it('retires the payload without a second dead letter when the live path already wrote one', async () => {
await insertEvent('inline-failed', { dispatchAttempts: 1 });
await failures.save({
direction: 'inbound',
pluginId: 'plug',
instanceId: 'inst',
deliveryId: 'd-inline-failed',
attempts: 1,
lastError: 'sandbox 5xx',
payload: { route: 'chatwoot', ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' } },
redriven: false,
});
const update = jest.spyOn(events, 'update');
await service.pruneOlderThan(7, 90);
// Settled by the hand-off: the payload is retired against the existing dead letter.
expect(update).toHaveBeenCalledWith(
{ id: 'inline-failed', dispatchState: 'pending' },
{ dispatchState: 'failed', payload: null },
);
const [dlq] = await failures.find();
expect(dlq).toMatchObject({ deliveryId: 'd-inline-failed', lastError: 'sandbox 5xx', redriven: false });
expect(await failures.count()).toBe(1);
});
it('writes a fresh dead letter when the only one for the delivery id was already redriven', async () => {
await insertEvent('resent', {});
const old = await failures.save({
direction: 'inbound',
pluginId: 'plug',
instanceId: 'inst',
deliveryId: 'd-resent',
attempts: 1,
lastError: 'sandbox 5xx',
payload: null,
redriven: true,
});
await service.pruneOlderThan(7, 90);
const open = await failures.find({ where: { deliveryId: 'd-resent', redriven: false } });
expect(open).toHaveLength(1);
expect(open[0].id).not.toBe(old.id);
expect(open[0].payload).toMatchObject({ ingress: { rawBody: '{"msg":"hello"}' } });
});
it('keeps an aged pending event with its payload when the dead-letter write fails', async () => {
await insertEvent('stranded', {});
jest.spyOn(failures, 'save').mockRejectedValueOnce(new Error('disk full'));
const result = await service.pruneOlderThan(7, 90);
expect(result.events).toBe(0);
const kept = await events.findOneByOrFail({ id: 'stranded' });
expect(kept.dispatchState).toBe('pending');
expect(kept.payload).toMatchObject({ rawBody: '{"msg":"hello"}' });
expect(await failures.count()).toBe(0);
});
it('retires its own dead letter when a concurrent dispatch settles the event first', async () => {
await insertEvent('raced', {});
jest.spyOn(events, 'update').mockImplementationOnce(async () => {
await events.query(`UPDATE ingress_events SET "dispatchState" = 'dispatched', payload = NULL WHERE id = 'raced'`);
return { affected: 0, raw: [], generatedMaps: [] };
});
await service.pruneOlderThan(7, 90);
const [dlq] = await failures.find();
expect(dlq).toMatchObject({ deliveryId: 'd-raced', redriven: true });
});
it('keeps its own dead letter redrivable when a concurrent failure mark settles the event first', async () => {
await insertEvent('raced', {});
jest.spyOn(events, 'update').mockImplementationOnce(async () => {
await events.query(`UPDATE ingress_events SET "dispatchState" = 'failed', payload = NULL WHERE id = 'raced'`);
return { affected: 0, raw: [], generatedMaps: [] };
});
await service.pruneOlderThan(7, 90);
const [dlq] = await failures.find();
expect(dlq).toMatchObject({ deliveryId: 'd-raced', redriven: false });
});
describe('when another node dead-letters the same delivery concurrently', () => {
const LOW_ID = '00000000-0000-4000-8000-000000000000';
const HIGH_ID = 'ffffffff-ffff-4fff-bfff-ffffffffffff';
// The other node's dead letter lands after this node's open-row check, so both write one.
const otherNodeDeadLetter = (id: string) =>
failures.insert({
id,
direction: 'inbound',
pluginId: 'plug',
instanceId: 'inst',
deliveryId: 'd-raced',
attempts: 0,
lastError: 'not dispatched within the ingress dedup retention window',
payload: { route: 'chatwoot', ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' } },
redriven: false,
});
const openIds = async () =>
(await failures.find({ where: { deliveryId: 'd-raced', redriven: false } })).map(f => f.id);
it('retires its own dead letter when it loses the failure mark to a node with a lower-id row', async () => {
await insertEvent('raced', {});
jest.spyOn(events, 'update').mockImplementationOnce(async () => {
await otherNodeDeadLetter(LOW_ID);
await events.query(`UPDATE ingress_events SET "dispatchState" = 'failed', payload = NULL WHERE id = 'raced'`);
return { affected: 0, raw: [], generatedMaps: [] };
});
await service.pruneOlderThan(7, 90);
expect(await openIds()).toEqual([LOW_ID]);
expect(await failures.count()).toBe(2);
});
it('retires its own dead letter when it wins the failure mark but a lower-id row exists', async () => {
await insertEvent('raced', {});
const update = events.update.bind(events);
jest.spyOn(events, 'update').mockImplementationOnce(async (criteria, partial) => {
await otherNodeDeadLetter(LOW_ID);
return update(criteria, partial);
});
await service.pruneOlderThan(7, 90);
expect(await openIds()).toEqual([LOW_ID]);
expect(await events.findOneByOrFail({ id: 'raced' })).toMatchObject({ dispatchState: 'failed', payload: null });
});
it('keeps its own dead letter when the other node holds the higher id', async () => {
await insertEvent('raced', {});
jest.spyOn(events, 'update').mockImplementationOnce(async () => {
await otherNodeDeadLetter(HIGH_ID);
await events.query(`UPDATE ingress_events SET "dispatchState" = 'failed', payload = NULL WHERE id = 'raced'`);
return { affected: 0, raw: [], generatedMaps: [] };
});
await service.pruneOlderThan(7, 90);
// The other writer, a hand-off or a sweep, retires its own row once its check sees this one.
const open = await openIds();
expect(open).toHaveLength(2);
expect(open).toContain(HIGH_ID);
});
});
it('still prunes aged settled markers, legacy rows, and pending rows without a payload', async () => {
await insertEvent('dispatched', { dispatchState: 'dispatched', payload: null });
await insertEvent('failed', { dispatchState: 'failed', payload: null });
await insertEvent('legacy', { dispatchState: null });
await insertEvent('empty', { payload: null });
await insertEvent('recent', { createdAt: daysAgo(1) });
const result = await service.pruneOlderThan(7, 90);
expect(result.events).toBe(4);
expect((await events.find()).map(e => e.id)).toEqual(['recent']);
expect(await failures.count()).toBe(0);
});
it('writes no dead letter for a deleted instance and prunes the row', async () => {
await insertEvent('orphan', {});
resolveInstance.mockResolvedValue(null);
await service.pruneOlderThan(7, 90);
const result = await service.pruneOlderThan(7, 90);
expect(result.events).toBe(1);
expect(await events.count()).toBe(0);
expect(await failures.count()).toBe(0);
});
it('writes no dead letter for a deleted session, but keeps one for a wildcard scope', async () => {
await insertEvent('gone', { sessionId: '22222222-2222-4222-8222-222222222222' });
await insertEvent('wildcard', { sessionId: '*' });
await service.pruneOlderThan(7, 90);
expect(await events.find({ where: { dispatchState: 'failed' } })).toHaveLength(2);
expect((await failures.find()).map(f => f.deliveryId)).toEqual(['d-wildcard']);
});
it('writes no dead letter while a queue job still owns the delivery', async () => {
await insertEvent('queued', {});
existingJobState.mockResolvedValue('waiting');
const update = jest.spyOn(events, 'update');
await service.pruneOlderThan(7, 90);
expect(update).toHaveBeenCalledWith(
{ id: 'queued', dispatchState: 'pending' },
{ dispatchState: 'dispatched', payload: null },
);
expect(await failures.count()).toBe(0);
});
it('closes an inline-failure dead letter once a queue job owns the delivery', async () => {
await insertEvent('queued', { dispatchAttempts: 1 });
await failures.save({
direction: 'inbound',
pluginId: 'plug',
instanceId: 'inst',
deliveryId: 'd-queued',
attempts: 1,
lastError: 'sandbox 5xx',
payload: { route: 'chatwoot', ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' } },
redriven: false,
});
existingJobState.mockResolvedValue('waiting');
await service.pruneOlderThan(7, 90);
// The job delivers it, so a manual redrive of the old row would deliver it twice.
expect(await events.findOneByOrFail({ id: 'queued' })).toMatchObject({ dispatchState: 'dispatched' });
expect(await failures.find()).toEqual([expect.objectContaining({ deliveryId: 'd-queued', redriven: true })]);
});
it('dead-letters an event whose queue job failed', async () => {
await insertEvent('job-failed', {});
existingJobState.mockResolvedValue('failed');
await service.pruneOlderThan(7, 90);
// A failed job delivers nothing, so the payload must survive in a redrivable dead letter.
expect(await events.findOneByOrFail({ id: 'job-failed' })).toMatchObject({
dispatchState: 'failed',
payload: null,
});
const open = await failures.find({ where: { direction: 'inbound', redriven: false } });
expect(open).toHaveLength(1);
expect(open[0]).toMatchObject({ deliveryId: 'd-job-failed' });
expect(open[0].payload).toMatchObject({ ingress: { rawBody: '{"msg":"hello"}' } });
});
it('still prunes aged events when the dead-letter prune fails', async () => {
await insertEvent('dispatched', { dispatchState: 'dispatched', payload: null });
jest.spyOn(failures, 'delete').mockRejectedValueOnce(new Error('locked'));
const result = await service.pruneOlderThan(7, 90);
expect(result).toEqual({ events: 1, failures: 0 });
expect(await events.count()).toBe(0);
});
// A dead letter older than the 90-day window, so the first step of a run has something to remove.
const insertAgedFailure = () =>
failures
.createQueryBuilder()
.insert()
.values({
direction: 'inbound',
pluginId: 'plug',
instanceId: 'inst',
attempts: 1,
lastError: 'old',
redriven: false,
createdAt: daysAgo(100),
})
.execute();
it('still hands off an aged pending event when the dedup delete fails', async () => {
await insertEvent('stranded', {});
await insertAgedFailure();
jest.spyOn(events, 'delete').mockRejectedValueOnce(new Error('locked'));
const result = await service.pruneOlderThan(7, 90);
// The failed delete counts as nothing removed, the dead-letter prune before it still counts, and the
// hand-off after it still ran: a locked table must not hold the day's hand-off back another 24 hours.
expect(result).toEqual({ events: 0, failures: 1 });
expect(await events.findOneByOrFail({ id: 'stranded' })).toMatchObject({ dispatchState: 'failed', payload: null });
expect(await failures.count()).toBe(1);
expect(logError).toHaveBeenCalledWith(
'Integration ingress retention: dedup prune failed',
expect.stringContaining('locked'),
);
});
it('reports what the prunes removed when the hand-off fails', async () => {
await insertEvent('dispatched', { dispatchState: 'dispatched', payload: null });
await insertAgedFailure();
jest.spyOn(reconciler, 'deadLetterAgedPending').mockRejectedValueOnce(new Error('queue unreachable'));
const result = await service.pruneOlderThan(7, 90);
expect(result).toEqual({ events: 1, failures: 1 });
expect(await events.count()).toBe(0);
expect(logError).toHaveBeenCalledWith(
'Integration ingress retention: undispatched event hand-off failed',
expect.stringContaining('queue unreachable'),
);
});
it('hands off a backlog larger than one batch', async () => {
for (let i = 0; i < 250; i++) await insertEvent(`b${i}`, {});
await service.pruneOlderThan(7, 90);
expect(await events.count({ where: { dispatchState: 'failed' } })).toBe(250);
expect(await failures.count()).toBe(250);
});
it('lets the event loop run between rows, which a synchronous driver would otherwise starve for the whole pass', async () => {
for (let i = 0; i < 120; i++) await insertEvent(`y${i}`, {});
// better-sqlite3 answers every statement at once, so without a yield the pass is one unbroken chain of
// microtasks: no timer, request or keep-alive callback runs until it ends, however many rows it holds.
let ticks = 0;
let probing = true;
const probe = (): void => {
if (!probing) return;
ticks += 1;
setImmediate(probe);
};
setImmediate(probe);
await service.pruneOlderThan(7, 90);
probing = false;
expect(await events.count({ where: { dispatchState: 'failed' } })).toBe(120);
expect(ticks).toBeGreaterThan(60);
});
it('prunes aged settled markers while the hand-off is still waiting', async () => {
await insertEvent('dispatched', { dispatchState: 'dispatched', payload: null });
await insertEvent('stranded', {});
// A job-state lookup against a stalled Redis that has not returned yet.
const lookedUp = new Promise<void>(resolve =>
existingJobState.mockImplementation(() => {
resolve();
return new Promise(() => {});
}),
);
void service.pruneOlderThan(7, 90);
await lookedUp;
expect((await events.find()).map(e => e.id)).toEqual(['stranded']);
});
it('keeps every row when no dead-letter write succeeds', async () => {
for (let i = 0; i < 100; i++) await insertEvent(`f${i}`, {});
jest.spyOn(failures, 'save').mockRejectedValue(new Error('disk full'));
const result = await service.pruneOlderThan(7, 90);
expect(result.events).toBe(0);
expect(await events.count({ where: { dispatchState: 'pending' } })).toBe(100);
expect(logError).toHaveBeenCalledTimes(100);
expect(logError).toHaveBeenCalledWith(
'Failed to dead-letter an undispatched ingress event; kept for the next run',
'disk full',
expect.objectContaining({ action: 'ingress_event_retention_dead_letter_failed' }),
);
});
it('tries a row whose dead-letter write keeps failing once per run and hands off the rows behind it', async () => {
// A full batch of older rows whose write always fails, ahead of rows that hand off normally.
for (let i = 0; i < 100; i++) await insertEvent(`bad${i}`, { createdAt: daysAgo(9) });
for (let i = 0; i < 20; i++) await insertEvent(`ok${i}`, {});
const realSave = failures.save.bind(failures) as (
row: IntegrationDeliveryFailure,
) => Promise<IntegrationDeliveryFailure>;
jest.spyOn(failures, 'save').mockImplementation(async row => {
const dlq = row as IntegrationDeliveryFailure;
if (dlq.deliveryId?.startsWith('d-bad')) throw new Error('row rejected');
return realSave(dlq);
});
await service.pruneOlderThan(7, 90);
expect(logError).toHaveBeenCalledTimes(100);
expect(await events.count({ where: { dispatchState: 'pending' } })).toBe(100);
expect(await events.count({ where: { dispatchState: 'failed' } })).toBe(20);
expect(await failures.count()).toBe(20);
});
it('stops a run once ten batches of rows could not be handed off', async () => {
for (let i = 0; i < 25; i++) await insertEvent(`f${i}`, {});
jest.spyOn(failures, 'save').mockRejectedValue(new Error('disk full'));
expect(await reconciler.deadLetterAgedPending(daysAgo(7), 2)).toBe(0);
expect(logError).toHaveBeenCalledTimes(20);
expect(await events.count({ where: { dispatchState: 'pending' } })).toBe(25);
expect(logWarn).toHaveBeenCalledWith(
'Ingress retention hand-off stopped early; the remaining events wait for the next run',
expect.objectContaining({ kept: 20, action: 'ingress_event_retention_handoff_stopped' }),
);
});
it('passes over pending rows whose payload is unreadable and hands off the rows behind them', async () => {
await insertEvent('unreadable1', { createdAt: daysAgo(9) });
await insertEvent('unreadable2', { createdAt: daysAgo(9) });
await insertEvent('stranded', {});
// A JSON null passes the payload IS NOT NULL filter but reads back as no payload.
await events.query(`UPDATE ingress_events SET payload = 'null' WHERE id LIKE 'unreadable%'`);
const find = events.find.bind(events);
let fetches = 0;
jest.spyOn(events, 'find').mockImplementation(opts => {
if (++fetches > 5) throw new Error('the hand-off keeps fetching the same rows');
return find(opts);
});
expect(await reconciler.deadLetterAgedPending(daysAgo(7), 2)).toBe(1);
expect(await events.findOneByOrFail({ id: 'stranded' })).toMatchObject({ dispatchState: 'failed', payload: null });
expect(await events.count({ where: { dispatchState: 'pending' } })).toBe(2);
});
// Stands in for a reconcile sweep holding the guard the hand-off shares with it.
const holdSweep = () => {
const guarded = reconciler as unknown as { inFlight?: Promise<void> };
let settle!: () => void;
guarded.inFlight = new Promise(resolve => (settle = resolve));
return () => {
guarded.inFlight = undefined;
settle();
};
};
// Resolves once the retention run has called the hand-off, with the hand-off's own promise.
const handOffCalled = () =>
new Promise<{ running: Promise<number> }>(resolve => {
const handOff = reconciler.deadLetterAgedPending.bind(reconciler);
jest.spyOn(reconciler, 'deadLetterAgedPending').mockImplementation((...args) => {
const running = handOff(...args);
resolve({ running });
return running;
});
});
it('waits for a running reconcile sweep, then hands off the aged pending rows', async () => {
await insertEvent('stranded', {});
const finishSweep = holdSweep();
const called = handOffCalled();
const pruned = service.pruneOlderThan(7, 90);
const { running } = await called;
const stateDuringSweep = await Promise.race([
running.then(() => 'returned'),
new Promise(resolve => setImmediate(resolve, 'waiting')),
]);
const duringSweep = await events.findOneByOrFail({ id: 'stranded' });
finishSweep();
await pruned;
expect(stateDuringSweep).toBe('waiting');
expect(duringSweep.payload).not.toBeNull();
expect(await events.findOneByOrFail({ id: 'stranded' })).toMatchObject({ dispatchState: 'failed', payload: null });
expect(await failures.count()).toBe(1);
});
it('stops waiting for a running sweep once shutdown begins, and keeps the rows', async () => {
await insertEvent('stranded', {});
const finishSweep = holdSweep();
const called = handOffCalled();
const pruned = service.pruneOlderThan(7, 90);
await called;
const destroyed = reconciler.onModuleDestroy();
finishSweep();
await Promise.all([pruned, destroyed]);
expect((await events.findOneByOrFail({ id: 'stranded' })).payload).not.toBeNull();
expect(await failures.count()).toBe(0);
});
it('stops the hand-off at the next row once shutdown begins, and destroy waits for the row in hand', async () => {
await insertEvent('first', { createdAt: daysAgo(9) });
await insertEvent('second', {});
const realSave = failures.save.bind(failures) as (
row: IntegrationDeliveryFailure,
) => Promise<IntegrationDeliveryFailure>;
let saving!: () => void;
const entered = new Promise<void>(resolve => (saving = resolve));
let release!: () => void;
const held = new Promise<void>(resolve => (release = resolve));
jest.spyOn(failures, 'save').mockImplementationOnce(async row => {
saving();
await held;
return realSave(row as IntegrationDeliveryFailure);
});
const pruned = service.pruneOlderThan(7, 90);
await entered;
let destroyed = false;
const destroy = reconciler.onModuleDestroy().then(() => (destroyed = true));
await new Promise(setImmediate);
const destroyedWhileHeld = destroyed;
release();
await destroy;
const firstAtDestroy = (await events.findOneByOrFail({ id: 'first' })).dispatchState;
await pruned;
expect(destroyedWhileHeld).toBe(false);
expect(firstAtDestroy).toBe('failed');
expect(await failures.count()).toBe(1);
const rows = await events.find({ order: { createdAt: 'ASC' } });
expect(rows.map(r => r.dispatchState)).toEqual(['failed', 'pending']);
expect(rows[1].payload).not.toBeNull();
});
it('stops waiting on destroy for a hand-off lookup that does not return, and keeps its row', async () => {
await insertEvent('stranded', {});
let answer!: (state: undefined) => void;
const lookedUp = new Promise<void>(resolve =>
existingJobState.mockImplementation(() => {
resolve();
return new Promise(settle => (answer = settle));
}),
);
const pruned = service.pruneOlderThan(7, 90);
await lookedUp;
let destroyed = false;
let destroyedBeforeDeadline: boolean;
let destroyedAtDeadline: boolean;
jest.useFakeTimers({ doNotFake: ['nextTick', 'setImmediate', 'queueMicrotask'] });
try {
void reconciler.onModuleDestroy().then(() => (destroyed = true));
await jest.advanceTimersByTimeAsync(INGRESS_DISPATCH_TIMEOUT_MS - 1);
await new Promise(setImmediate);
destroyedBeforeDeadline = destroyed;
await jest.advanceTimersByTimeAsync(1);
await new Promise(setImmediate);
destroyedAtDeadline = destroyed;
} finally {
jest.useRealTimers();
}
// The lookup returns once the queue connection closes later in shutdown; the row is left as it was.
answer(undefined);
await pruned;
expect(destroyedBeforeDeadline).toBe(false);
expect(destroyedAtDeadline).toBe(true);
expect(await failures.count()).toBe(0);
const row = await events.findOneByOrFail({ id: 'stranded' });
expect(row.dispatchState).toBe('pending');
expect(row.payload).not.toBeNull();
});
});