Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
import {
IngressEnqueueService,
resolveIngressJobOptions,
buildIngressDeadLetterRow,
sanitizeIngressJobId,
} from './ingress-enqueue.service';
import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';
import { ConfigService } from '@nestjs/config';
import { producerConnectWaitMs } from '../queue/redis-connection';
describe('IngressEnqueueService', () => {
const data = {
pluginId: 'chatwoot',
instanceId: 'acct1',
route: 'chatwoot',
deliveryId: 'd1',
sessionId: 'sess-1',
payload: { headers: {}, query: {}, body: '{}', rawBody: '{}' },
};
let loader: jest.Mocked<Partial<PluginLoaderService>>;
let config: jest.Mocked<Partial<ConfigService>>;
let queue: { add: jest.Mock; waitUntilReady: jest.Mock };
beforeEach(() => {
loader = { dispatchWebhookForInstance: jest.fn().mockResolvedValue(undefined) };
config = { get: jest.fn() };
queue = { add: jest.fn().mockResolvedValue(undefined), waitUntilReady: jest.fn().mockResolvedValue(undefined) };
});
it('adds a job to the ingress queue keyed by the namespaced jobId when queueing is enabled and a queue is present', async () => {
(config.get as jest.Mock).mockReturnValue(true);
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
expect(await svc.enqueue(data, 'd1')).toEqual({ outcome: 'queued' });
expect(queue.add).toHaveBeenCalledWith('ingress', data, {
jobId: sanitizeIngressJobId('d1', 'chatwoot\u0000acct1'),
attempts: 3,
backoff: { type: 'exponential', delay: 5000 },
});
expect(loader.dispatchWebhookForInstance).not.toHaveBeenCalled();
});
describe('existingJobState', () => {
it('reads the state of the job under the same namespaced jobId enqueue() would use', async () => {
(config.get as jest.Mock).mockReturnValue(true);
const getJobState = jest.fn().mockResolvedValue('failed');
const svc = new IngressEnqueueService(
loader as PluginLoaderService,
config as ConfigService,
{
...queue,
getJobState,
} as never,
);
expect(await svc.existingJobState(data, 'd1')).toBe('failed');
expect(getJobState).toHaveBeenCalledWith(sanitizeIngressJobId('d1', 'chatwoot\u0000acct1'));
});
it('reports no job when the queue holds none, is off, or cannot answer', async () => {
(config.get as jest.Mock).mockReturnValue(true);
const getJobState = jest.fn().mockResolvedValueOnce('unknown').mockRejectedValueOnce(new Error('ECONNREFUSED'));
const svc = new IngressEnqueueService(
loader as PluginLoaderService,
config as ConfigService,
{
...queue,
getJobState,
} as never,
);
expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
getJobState.mockClear();
(config.get as jest.Mock).mockReturnValue(false);
expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
expect(getJobState).not.toHaveBeenCalled();
});
// IngressProcessor re-queues a delivery whose dead-letter write failed under a copy id: that copy
// owns the delivery while the original stays failed, or after removeOnFail pruned it.
it('reports a live or completed re-queued copy of a failed or pruned job', async () => {
(config.get as jest.Mock).mockReturnValue(true);
const base = sanitizeIngressJobId('d1', 'chatwoot\u0000acct1');
const states: Record<string, string> = { [base]: 'failed', [`${base}-requeued-2`]: 'delayed' };
const getJobState = jest.fn((id: string) => Promise.resolve(states[id] ?? 'unknown'));
const svc = new IngressEnqueueService(
loader as PluginLoaderService,
config as ConfigService,
{ ...queue, getJobState } as never,
);
expect(await svc.existingJobState(data, 'd1')).toBe('delayed');
delete states[base];
expect(await svc.existingJobState(data, 'd1')).toBe('delayed');
states[`${base}-requeued-2`] = 'completed';
expect(await svc.existingJobState(data, 'd1')).toBe('completed');
states[`${base}-requeued-2`] = 'failed';
states[base] = 'failed';
expect(await svc.existingJobState(data, 'd1')).toBe('failed');
delete states[base];
expect(await svc.existingJobState(data, 'd1')).toBeUndefined();
});
});
// BullMQ refuses integer jobIds and colon ids that do not split into exactly 3 parts; before the
// sanitizer those throws read as "Redis unreachable" in the catch-all and silently degraded the
// delivery to inline dispatch (no retry, no backoff, blocked redrive loop).
describe('sanitizeIngressJobId', () => {
it('hashes ids BullMQ would accept too, so the namespace always applies', () => {
for (const id of ['d1', 'evt_abc-123', 'a:b:c', '0abc']) {
expect(sanitizeIngressJobId(id)).toMatch(/^ing-[0-9a-f]{40}$/);
}
});
it('namespaces a non-numeric id two instances share (one Svix message fanned out to both)', () => {
const a = sanitizeIngressJobId('msg_2LJx9', 'chatwoot\u0000acct-1');
const b = sanitizeIngressJobId('msg_2LJx9', 'chatwoot\u0000acct-2');
expect(a).not.toBe(b);
expect(sanitizeIngressJobId('msg_2LJx9', 'chatwoot\u0000acct-1')).toBe(a);
});
it('hashes the shapes BullMQ refuses, deterministically', () => {
const integer = sanitizeIngressJobId('12345');
const redrive = sanitizeIngressJobId('redrive:3f8e2a1b-9c4d-4e6f-8a2b-7d1c0e5f9a3b');
const twoPart = sanitizeIngressJobId('a:b');
const zeroLedThreePart = sanitizeIngressJobId('0:tenant:123'); // Queue.addJob: "cannot start with '0:'"
for (const hashed of [integer, redrive, twoPart, zeroLedThreePart]) {
expect(hashed).toMatch(/^ing-[0-9a-f]{40}$/);
expect(hashed).not.toContain(':');
}
expect(sanitizeIngressJobId('12345')).toBe(integer);
expect(sanitizeIngressJobId('redrive:3f8e2a1b-9c4d-4e6f-8a2b-7d1c0e5f9a3b')).toBe(redrive);
expect(sanitizeIngressJobId('0')).toMatch(/^ing-/); // Queue.addJob rejects the literal '0' too
});
it('coerces a non-string id (a duplicated header can surface as string[]) instead of throwing', () => {
const coerced = sanitizeIngressJobId(['0:tenant', '123'] as unknown as string);
expect(coerced).toMatch(/^ing-[0-9a-f]{40}$/);
});
it('namespaces the hash by plugin/instance so two instances sharing a numeric id do not collide', () => {
// BullMQ dedups jobIds across the whole shared queue while the DB dedup is
// (pluginId, instanceId, providerDeliveryId); numeric provider ids are per-account sequences.
const a = sanitizeIngressJobId('12345', 'chatwoot\u0000acct-1');
const b = sanitizeIngressJobId('12345', 'chatwoot\u0000acct-2');
expect(a).toMatch(/^ing-[0-9a-f]{40}$/);
expect(a).not.toBe(b);
expect(sanitizeIngressJobId('12345', 'chatwoot\u0000acct-1')).toBe(a); // stable for replays
});
it.each([
['a numeric provider dedup header', '12345'],
['the redrive replay id', 'redrive:3f8e2a1b-9c4d-4e6f-8a2b-7d1c0e5f9a3b'],
['a two-part colon id', 'a:b'],
['a zero-led three-part colon id', '0:tenant:123'],
])('still queues (%s) instead of tripping the inline fallback', async (_label, jobId) => {
(config.get as jest.Mock).mockReturnValue(true);
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
expect(await svc.enqueue(data, jobId)).toEqual({ outcome: 'queued' });
expect(queue.add).toHaveBeenCalledWith('ingress', data, {
jobId: sanitizeIngressJobId(jobId, `${data.pluginId}\u0000${data.instanceId}`),
attempts: 3,
backoff: { type: 'exponential', delay: 5000 },
});
expect(loader.dispatchWebhookForInstance).not.toHaveBeenCalled();
});
});
it('dispatches inline when queueing is disabled, even if a queue instance is present', async () => {
(config.get as jest.Mock).mockReturnValue(false);
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
expect(await svc.enqueue(data, 'd1')).toEqual({ outcome: 'dispatched' });
expect(queue.add).not.toHaveBeenCalled();
expect(loader.dispatchWebhookForInstance).toHaveBeenCalledWith(data);
});
it('dispatches inline when no queue instance is injected (QUEUE_ENABLED unset)', async () => {
(config.get as jest.Mock).mockReturnValue(true);
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, undefined);
expect(await svc.enqueue(data, 'd1')).toEqual({ outcome: 'dispatched' });
expect(loader.dispatchWebhookForInstance).toHaveBeenCalledWith(data);
});
// A plugin the boot restore has not reached has no worker. With the queue on, the job waits in the queue and
// IngressProcessor holds it; dispatching inline is the one path that would have to run the plugin right away.
describe('defersInlineDispatch', () => {
const restoring = (ids: string[]) => {
loader.isRestorePending = jest.fn((id: string) => ids.includes(id));
};
it('defers a plugin the restore has not reached while dispatch is inline', () => {
restoring(['chatwoot']);
(config.get as jest.Mock).mockReturnValue(false);
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
expect(svc.defersInlineDispatch('chatwoot')).toBe(true);
expect(svc.defersInlineDispatch('other')).toBe(false);
});
it('also defers when the queue is switched on but no queue instance was injected', () => {
restoring(['chatwoot']);
(config.get as jest.Mock).mockReturnValue(true);
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, undefined);
expect(svc.defersInlineDispatch('chatwoot')).toBe(true);
});
it('does not defer when a queue takes the job', () => {
restoring(['chatwoot']);
(config.get as jest.Mock).mockReturnValue(true);
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
expect(svc.defersInlineDispatch('chatwoot')).toBe(false);
});
it('does not defer once the restore is done with the plugin', () => {
restoring([]);
(config.get as jest.Mock).mockReturnValue(false);
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
expect(svc.defersInlineDispatch('chatwoot')).toBe(false);
});
});
describe('onApplicationBootstrap (queue-wiring tripwire)', () => {
const ORIGINAL = process.env.QUEUE_ENABLED;
afterEach(() => {
if (ORIGINAL === undefined) delete process.env.QUEUE_ENABLED;
else process.env.QUEUE_ENABLED = ORIGINAL;
});
it('throws when QUEUE_ENABLED=true but no queue resolved (broken wiring must fail the boot)', () => {
process.env.QUEUE_ENABLED = 'true';
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, undefined);
expect(() => svc.onApplicationBootstrap()).toThrow(
/QUEUE_ENABLED=true but the 'ingress-queue' BullMQ queue did not resolve/,
);
});
it('does not throw when QUEUE_ENABLED=true and the queue resolved', () => {
process.env.QUEUE_ENABLED = 'true';
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
expect(() => svc.onApplicationBootstrap()).not.toThrow();
});
it.each(['false', undefined])(
'does not throw when QUEUE_ENABLED=%s and no queue resolved (inline is the contract)',
value => {
if (value === undefined) delete process.env.QUEUE_ENABLED;
else process.env.QUEUE_ENABLED = value;
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, undefined);
expect(() => svc.onApplicationBootstrap()).not.toThrow();
},
);
});
it('swallows an inline dispatch error and returns outcome "failed" rather than throwing (row stays redrivable)', async () => {
(loader.dispatchWebhookForInstance as jest.Mock).mockRejectedValue(new Error('boom'));
(config.get as jest.Mock).mockReturnValue(false);
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, undefined);
expect(await svc.enqueue(data, 'd1')).toEqual({ outcome: 'failed', error: 'boom' });
});
it('falls back to inline dispatch (never throws) when queue.add() fails, e.g. Redis unreachable', async () => {
// Without this, the throw would 500 the ingress request; the provider retries, dedup returns
// "duplicate", and the already-persisted event is lost forever (no job, no DLQ row).
(config.get as jest.Mock).mockReturnValue(true);
queue.add.mockRejectedValue(new Error('Redis connection is closed'));
const svc = new IngressEnqueueService(loader as PluginLoaderService, config as ConfigService, queue as never);
expect(await svc.enqueue(data, 'd1')).toEqual({ outcome: 'dispatched' });
expect(queue.add).toHaveBeenCalledWith('ingress', data, {
jobId: sanitizeIngressJobId('d1', 'chatwoot\u0000acct1'),
attempts: 3,
backoff: { type: 'exponential', delay: 5000 },
});
expect(loader.dispatchWebhookForInstance).toHaveBeenCalledWith(data);
});
// Until the first connect, BullMQ holds every queue call without sending a command and retries the
// connection for ever, so enableOfflineQueue:false never rejects: with Redis unreachable since boot,
// the ingress request, a redrive and the reconciler's row in hand would wait indefinitely.
it('falls back to inline dispatch and reports no job when Redis has not connected within the connect timeout', async () => {
(config.get as jest.Mock).mockReturnValue(true);
const pending = () => new Promise<never>(() => {});
queue.waitUntilReady.mockImplementation(pending);
queue.add.mockImplementation(pending);
const getJobState = jest.fn(pending);
const svc = new IngressEnqueueService(
loader as PluginLoaderService,
config as ConfigService,
{ ...queue, getJobState } as never,
);
let results: unknown;
jest.useFakeTimers();
try {
void Promise.all([svc.existingJobState(data, 'd1'), svc.enqueue(data, 'd1')]).then(r => (results = r));
await jest.advanceTimersByTimeAsync(producerConnectWaitMs() - 1);
expect(results).toBeUndefined();
await jest.advanceTimersByTimeAsync(1);
} finally {
jest.useRealTimers();
}
expect(results).toEqual([undefined, { outcome: 'dispatched' }]);
expect(getJobState).not.toHaveBeenCalled();
expect(queue.add).not.toHaveBeenCalled();
expect(loader.dispatchWebhookForInstance).toHaveBeenCalledWith(data);
});
describe('resolveIngressJobOptions', () => {
it('defaults to 3 attempts with exponential backoff', () => {
expect(resolveIngressJobOptions()).toEqual({ attempts: 3, backoff: { type: 'exponential', delay: 5000 } });
});
it('honors INGRESS_MAX_ATTEMPTS / INGRESS_RETRY_DELAY_MS overrides', () => {
const prevA = process.env.INGRESS_MAX_ATTEMPTS;
const prevD = process.env.INGRESS_RETRY_DELAY_MS;
try {
process.env.INGRESS_MAX_ATTEMPTS = '5';
process.env.INGRESS_RETRY_DELAY_MS = '1000';
expect(resolveIngressJobOptions()).toEqual({ attempts: 5, backoff: { type: 'exponential', delay: 1000 } });
} finally {
if (prevA === undefined) delete process.env.INGRESS_MAX_ATTEMPTS;
else process.env.INGRESS_MAX_ATTEMPTS = prevA;
if (prevD === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;
else process.env.INGRESS_RETRY_DELAY_MS = prevD;
}
});
it.each(['1e1', '0x10', '0'])('falls back to 3 attempts for INGRESS_MAX_ATTEMPTS=%p', raw => {
const prevA = process.env.INGRESS_MAX_ATTEMPTS;
try {
process.env.INGRESS_MAX_ATTEMPTS = raw;
expect(resolveIngressJobOptions().attempts).toBe(3);
} finally {
if (prevA === undefined) delete process.env.INGRESS_MAX_ATTEMPTS;
else process.env.INGRESS_MAX_ATTEMPTS = prevA;
}
});
it('treats a blank INGRESS_RETRY_DELAY_MS as unset, not as 0', () => {
const prevD = process.env.INGRESS_RETRY_DELAY_MS;
try {
process.env.INGRESS_RETRY_DELAY_MS = '';
expect(resolveIngressJobOptions()).toEqual({ attempts: 3, backoff: { type: 'exponential', delay: 5000 } });
} finally {
if (prevD === undefined) delete process.env.INGRESS_RETRY_DELAY_MS;
else process.env.INGRESS_RETRY_DELAY_MS = prevD;
}
});
});
describe('buildIngressDeadLetterRow', () => {
it('mirrors the ingress-processor dead-letter shape so a redrive reads it back (attempts=1, redriven=false)', () => {
expect(buildIngressDeadLetterRow(data, 'boom')).toEqual({
direction: 'inbound',
pluginId: 'chatwoot',
instanceId: 'acct1',
sessionId: 'sess-1',
deliveryId: 'd1',
attempts: 1,
lastError: 'boom',
payload: { route: 'chatwoot', providerConversationId: undefined, ingress: data.payload },
redriven: false,
});
});
it('defaults sessionId to null and supplies a fallback lastError when the error is absent', () => {
const row = buildIngressDeadLetterRow({ ...data, sessionId: undefined }, undefined);
expect(row.sessionId).toBeNull();
expect(row.lastError).toBe('inline ingress dispatch failed');
});
});
});