Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
import { ConflictException } from '@nestjs/common';
import { SessionTakeoverService } from './session-takeover.service';
import { ServerShuttingDownException, SessionStoppedException } from '../session/session-engine-controls';
import { Session, SessionStatus } from '../session/entities/session.entity';
import type { SessionService } from '../session/session.service';
import type { SessionOwnershipService } from '../session/session-ownership.service';
import type { ConfigService } from '@nestjs/config';
import type { ShutdownService } from '../../common/services/shutdown.service';
/**
* The sweep is the retry that boot auto-start never had: both live incidents (a container recreate
* landing inside the old identity's lease, and a crashed peer) left sessions sitting disconnected
* until a manual POST /start. What has to hold: only lapsed-lease sessions worth resuming are
* started, and a lost claim race is a non-event. The claim on the start path fails the previous
* holder's stuck batches (see the ownership and bulk message specs).
*/
describe('SessionTakeoverService', () => {
const lapsed = (over: Partial<Session> = {}): Session =>
({
id: `id-${over.name ?? 'x'}`,
name: 'x',
status: SessionStatus.READY,
phone: '628123',
nodeId: 'dead-node',
...over,
}) as Session;
const build = (
rows: Session[],
opts: { autoStart?: boolean; startImpl?: jest.Mock; maxConcurrent?: number; slotHolders?: number } = {},
): {
svc: SessionTakeoverService;
start: jest.Mock;
markLapsedDisconnected: jest.Mock;
} => {
const start = opts.startImpl ?? jest.fn().mockResolvedValue({});
const markLapsedDisconnected = jest.fn().mockResolvedValue([]);
const config = {
get: (key: string, def?: unknown) =>
({
features: { autoStartSessions: opts.autoStart ?? true },
'session.takeoverSweepMs': 30_000,
'sessions.maxConcurrent': opts.maxConcurrent,
})[key as 'features'] ?? def,
} as unknown as ConfigService;
const hasStartCapacity = jest.fn((max: number) => (opts.slotHolders ?? 0) < max);
const svc = new SessionTakeoverService(
{ start, markLapsedDisconnected, hasStartCapacity } as unknown as SessionService,
{
lapsedHeldByOthers: jest.fn().mockResolvedValue(rows),
leaseTtlMs: 60_000,
} as unknown as SessionOwnershipService,
config,
);
return { svc, start, markLapsedDisconnected };
};
afterEach(() => {
jest.useRealTimers();
});
// onModuleDestroy only cleared the interval. A sweep already in flight was neither aborted nor
// awaited, and neither sweep() nor the start path consulted any shutting-down signal — so on a
// rolling restart an engine could be constructed and registered AFTER the shutdown path had torn
// the registry down, and ownership.claim() could pin a session to a process that was exiting,
// leaving it unclaimable by any peer until the lease lapsed.
describe('a sweep racing shutdown', () => {
it('adopts nothing once the module is being destroyed', async () => {
const { svc, start } = build([lapsed({ name: 'a' }), lapsed({ name: 'b' })]);
svc.onModuleDestroy();
await svc.sweep();
expect(start).not.toHaveBeenCalled();
});
it('stops adopting the rest of the batch when shutdown begins mid-sweep', async () => {
const svcRef: { current?: { onModuleDestroy: () => void } } = {};
const startImpl = jest.fn().mockImplementation(() => {
svcRef.current?.onModuleDestroy(); // shutdown starts while the first adoption is in flight
return Promise.resolve({});
});
const { svc, start } = build([lapsed({ name: 'a' }), lapsed({ name: 'b' }), lapsed({ name: 'c' })], {
startImpl,
});
svcRef.current = svc;
await svc.sweep();
expect(start).toHaveBeenCalledTimes(1);
});
// Negative twin: an ordinary sweep must still adopt everything eligible.
it('still adopts the whole batch when no shutdown is in progress', async () => {
const { svc, start } = build([lapsed({ name: 'a' }), lapsed({ name: 'b' })]);
await svc.sweep();
expect(start).toHaveBeenCalledTimes(2);
});
});
it('adopts a lapsed authenticated session through the ordinary start path', async () => {
const { svc, start } = build([lapsed({ name: 'a' })]);
await svc.sweep();
expect(start).toHaveBeenCalledWith('id-a');
});
it('skips sessions not worth resuming: unauthenticated, mid-pairing, or operator-flagged failed', async () => {
const { svc, start } = build([
lapsed({ name: 'no-phone', phone: undefined as unknown as string }),
lapsed({ name: 'pairing', status: SessionStatus.QR_READY }),
lapsed({ name: 'failed', status: SessionStatus.FAILED }),
]);
await svc.sweep();
expect(start).not.toHaveBeenCalled();
});
it('leaves a session an operator stopped down, and adopts one nobody stopped', async () => {
const { svc, start } = build([
lapsed({ name: 'stopped', status: SessionStatus.DISCONNECTED, desiredState: 'stopped' }),
lapsed({ name: 'dropped', status: SessionStatus.DISCONNECTED, desiredState: null }),
]);
await svc.sweep();
expect(start.mock.calls).toEqual([['id-dropped']]);
});
it('a lost claim race is a non-event: the next candidate still starts', async () => {
jest.useFakeTimers();
const start = jest.fn().mockRejectedValueOnce(new ConflictException('held elsewhere')).mockResolvedValueOnce({});
const { svc } = build([lapsed({ name: 'raced' }), lapsed({ name: 'ours' })], { startImpl: start });
const sweep = svc.sweep();
// The second launch waits out the inter-launch stagger, to the millisecond.
await jest.advanceTimersByTimeAsync(1999);
expect(start).toHaveBeenCalledTimes(1);
await jest.advanceTimersByTimeAsync(1);
expect(start).toHaveBeenCalledTimes(2);
await sweep;
expect(start.mock.calls).toEqual([['id-raced'], ['id-ours']]);
});
// A start the cap refuses still claims the lapsed lease first, and the refusal then releases it to
// nobody: no peer adopts a row without a holder, so the session would stay down. A node at its cap
// leaves the lease alone for a peer with room.
it('adopts nothing while this node is at MAX_CONCURRENT_SESSIONS', async () => {
const { svc, start } = build([lapsed({ name: 'a' })], { maxConcurrent: 1, slotHolders: 1 });
await svc.sweep();
expect(start).not.toHaveBeenCalled();
});
it('adopts while this node is still below MAX_CONCURRENT_SESSIONS', async () => {
const { svc, start } = build([lapsed({ name: 'a' })], { maxConcurrent: 2, slotHolders: 1 });
await svc.sweep();
expect(start).toHaveBeenCalledWith('id-a');
});
it('a session stopped after the sweep read it is logged as skipped, not as a lost claim race', async () => {
const start = jest.fn().mockRejectedValueOnce(new SessionStoppedException('Session id-halted was stopped'));
const { svc } = build([lapsed({ name: 'halted' })], { startImpl: start });
const debug = jest.spyOn((svc as unknown as { logger: { debug: jest.Mock } }).logger, 'debug');
await svc.sweep();
expect(debug.mock.calls.map(([message]) => message as string)).toEqual([
'Session halted skipped: stopped by an operator',
]);
});
it('a start refused by shutdown ends the sweep quietly instead of logging a failed takeover', async () => {
const start = jest.fn().mockRejectedValueOnce(new ServerShuttingDownException()).mockResolvedValue({});
const { svc } = build([lapsed({ name: 'late' }), lapsed({ name: 'next' })], { startImpl: start });
const logger = (svc as unknown as { logger: { warn: jest.Mock; debug: jest.Mock } }).logger;
const warn = jest.spyOn(logger, 'warn');
const debug = jest.spyOn(logger, 'debug');
await svc.sweep();
expect(start).toHaveBeenCalledTimes(1);
expect(warn).not.toHaveBeenCalled();
expect(debug.mock.calls.map(([message]) => message as string)).toEqual([
'Takeover of session late abandoned: shutting down',
]);
});
it('a non-conflict start failure is logged and does not abort the rest of the sweep', async () => {
jest.useFakeTimers();
const start = jest.fn().mockRejectedValueOnce(new Error('chromium died')).mockResolvedValueOnce({});
const { svc } = build([lapsed({ name: 'boom' }), lapsed({ name: 'fine' })], { startImpl: start });
const logger = (svc as unknown as { logger: { warn: jest.Mock; debug: jest.Mock } }).logger;
const warn = jest.spyOn(logger, 'warn');
const debug = jest.spyOn(logger, 'debug');
const sweep = svc.sweep();
await jest.advanceTimersByTimeAsync(2000); // the inter-launch stagger
await expect(sweep).resolves.toBeUndefined();
expect(start).toHaveBeenCalledTimes(2);
expect(warn).toHaveBeenCalledTimes(1);
expect(warn).toHaveBeenCalledWith(
'Takeover start failed for session boom',
expect.objectContaining({ sessionId: 'id-boom', error: 'chromium died' }),
);
expect(debug).not.toHaveBeenCalled();
});
it('the AUTO_START_SESSIONS opt-out arms the sweep but starts nothing', async () => {
// The opt-out means "no spontaneous engine starts", not "leave a dead node's sessions reporting
// READY forever". The sweep is the only thing that revisits those rows, so it has to keep
// running; only the adopting half is gated.
jest.useFakeTimers();
const { svc, start, markLapsedDisconnected } = build([lapsed({ name: 'a' })], { autoStart: false });
svc.onApplicationBootstrap();
expect(jest.getTimerCount()).toBe(1);
jest.useRealTimers();
await svc.sweep();
expect(markLapsedDisconnected).toHaveBeenCalledTimes(1);
expect(start).not.toHaveBeenCalled();
});
it('corrects stale statuses BEFORE adopting, so the write cannot land on a live engine', async () => {
const order: string[] = [];
const { svc, start, markLapsedDisconnected } = build([lapsed({ name: 'a' })]);
markLapsedDisconnected.mockImplementation(() => {
order.push('mark');
return Promise.resolve([]);
});
start.mockImplementation(() => {
order.push('start');
return Promise.resolve({});
});
await svc.sweep();
expect(order).toEqual(['mark', 'start']);
});
it('a failed status correction does not cost the pass its adoptions', async () => {
// A failed correction leaves no write behind, so nothing it guards against can land on an engine
// this pass starts. Letting it abort the sweep would turn a database blip into a tick with no
// failover at all.
const { svc, start, markLapsedDisconnected } = build([lapsed({ name: 'a' })]);
markLapsedDisconnected.mockRejectedValue(new Error('SQLITE_BUSY: database is locked'));
await expect(svc.sweep()).resolves.toBeUndefined();
expect(start).toHaveBeenCalledWith('id-a');
});
it('gives the reset a cutoff of two lease TTLs, so a healthy peer that lapsed once is left alone', async () => {
const { svc, markLapsedDisconnected } = build([lapsed({ name: 'a' })]);
const before = Date.now();
await svc.sweep();
const after = Date.now();
const [, goneBefore] = markLapsedDisconnected.mock.calls[0] as [Session[], Date];
// 60s TTL x 2: anything whose lease expired inside the last two minutes is still presumed alive.
// The service reads the clock somewhere inside sweep(), so its cutoff is bounded by the readings either side.
expect(goneBefore.getTime()).toBeGreaterThanOrEqual(before - 120_000);
expect(goneBefore.getTime()).toBeLessThanOrEqual(after - 120_000);
});
it('arms the timer when auto-start is on, and tears it down on destroy', () => {
jest.useFakeTimers();
const { svc } = build([]);
svc.onApplicationBootstrap();
expect(jest.getTimerCount()).toBe(1);
svc.onModuleDestroy();
expect(jest.getTimerCount()).toBe(0);
});
it('never stacks sweeps: a tick that fires while one is in flight is skipped', async () => {
jest.useFakeTimers();
let resolveFirst: () => void = () => undefined;
const start = jest.fn().mockImplementation(
() =>
new Promise<void>(resolve => {
resolveFirst = resolve;
}),
);
const ownershipCalls = jest.fn().mockResolvedValue([lapsed({ name: 'slow' })]);
const config = {
get: (key: string, def?: unknown) =>
({ features: { autoStartSessions: true }, 'session.takeoverSweepMs': 1000 })[key as 'features'] ?? def,
} as unknown as ConfigService;
const svc = new SessionTakeoverService(
{ start, markLapsedDisconnected: jest.fn().mockResolvedValue([]) } as unknown as SessionService,
{ lapsedHeldByOthers: ownershipCalls, leaseTtlMs: 60_000 } as unknown as SessionOwnershipService,
config,
);
svc.onApplicationBootstrap();
await jest.advanceTimersByTimeAsync(1000); // tick 1: sweep starts, hangs in start()
await jest.advanceTimersByTimeAsync(2000); // ticks 2-3 fire while it is in flight
expect(ownershipCalls).toHaveBeenCalledTimes(1);
resolveFirst();
svc.onModuleDestroy();
});
/**
* There are TWO shutdown guards: one at sweep entry and one per adoption. Every existing case has
* at least one eligible session, so the per-adoption guard stops the work either way and the entry
* guard binds nothing — deleting it left the whole suite green.
*
* The ownership QUERY is what separates them: it runs before the per-adoption guard, so a sweep
* that never queries proves the entry guard fired. That also matters on its own — a sweep entered
* during shutdown otherwise still hits the database on every remaining tick.
*/
/**
* `onModuleDestroy` runs at `app.close()` — AFTER the bounded drain. On SIGTERM main.ts flips
* readiness to 503 and waits SHUTDOWN_DELAY_MS first, and throughout that window the sweep timer is
* still armed while the local flag is still false: a tick can launch Chromium and claim an
* ownership lease for a process that is about to exit, which is the rolling-restart case this
* guard exists for. Two siblings in this repo already consult ShutdownService for exactly that —
* session-engine-lifecycle and the liveness watchdog.
*/
it('does not adopt during the drain window, before onModuleDestroy runs', async () => {
const ownershipCalls = jest.fn().mockResolvedValue([lapsed({ name: 'any' })]);
const start = jest.fn().mockResolvedValue(undefined);
const config = {
get: (key: string, def?: unknown) =>
({ features: { autoStartSessions: true }, 'session.takeoverSweepMs': 1000 })[key as 'features'] ?? def,
} as unknown as ConfigService;
const svc = new SessionTakeoverService(
{ start } as unknown as SessionService,
{ lapsedHeldByOthers: ownershipCalls } as unknown as SessionOwnershipService,
config,
{ isShuttingDown: () => true } as unknown as ShutdownService,
);
// Deliberately NOT calling onModuleDestroy: this is the drain window, where the signal has
// arrived but Nest has not torn the module down yet.
await svc.sweep();
expect(ownershipCalls).not.toHaveBeenCalled();
expect(start).not.toHaveBeenCalled();
});
it('a sweep entered during shutdown does not even query ownership', async () => {
const ownershipCalls = jest.fn().mockResolvedValue([lapsed({ name: 'any' })]);
const config = {
get: (key: string, def?: unknown) =>
({ features: { autoStartSessions: true }, 'session.takeoverSweepMs': 1000 })[key as 'features'] ?? def,
} as unknown as ConfigService;
const svc = new SessionTakeoverService(
{ start: jest.fn().mockResolvedValue(undefined) } as unknown as SessionService,
{ lapsedHeldByOthers: ownershipCalls } as unknown as SessionOwnershipService,
config,
);
svc.onModuleDestroy(); // flips the shutting-down signal
await svc.sweep();
expect(ownershipCalls).not.toHaveBeenCalled();
});
});