Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(plugins): keep a late hook handler inside the re-entrancy guard
Yudhi Armyndharisc440ab07220e
diff --git a/src/core/plugins/sandbox/plugin-worker-host.spec.ts b/src/core/plugins/sandbox/plugin-worker-host.spec.tsindex c84291be0..c9336e56a 100644--- a/src/core/plugins/sandbox/plugin-worker-host.spec.ts+++ b/src/core/plugins/sandbox/plugin-worker-host.spec.ts@@ -884,6 +884,85 @@ describe('PluginWorkerHost', () => {expect(ch.sent).toContainEqual({ kind: 'cap-result', id: 4, ok: true, result: true });ch.crash(); // drains the pending dispatch and its timer});++ // A handler slower than the host's hook timeout keeps running in the worker, and what it sends from+ // there names the dispatch that timed out. Honouring that claim only until the timeout let the send+ // run outside the guard, fire message:sending again and dispatch the same slow handler once more.+ describe('a handler that outlives the host timeout', () => {+ const fakeTimers = (): void => {+ jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick', 'queueMicrotask'] });+ };+ const guardedResult = async (ch: FakeChannel, id: number, inFlight: string[]): Promise<unknown> => {+ ch.reply({ kind: 'cap', id, verb: 'messages.sendText', args: [], inFlight });+ await flush();+ await flush();+ const reply = ch.sent.find(m => m.kind === 'cap-result' && m.id === id);+ return reply && reply.kind === 'cap-result' && reply.ok ? reply.result : reply;+ };+ const build = () => {+ const hm = new HookManager();+ const ch = new FakeChannel();+ hm.register('moderation', SENDING, () => Promise.resolve({ continue: false }));+ const capDispatcher = async (): Promise<unknown> => (await hm.execute(SENDING, {}, { source: 'cap' })).continue;+ const host = new PluginWorkerHost(ch, capDispatcher, undefined, undefined, undefined, (events, run) =>+ hm.runInFlight(events as HookEvent[], run),+ );+ const dispatch = (event: string) => host.dispatchHook({ event, data: {}, source: 'test', timeoutMs: 5000 });+ return { ch, dispatch };+ };++ afterEach(() => jest.useRealTimers());++ it('keeps guarding its capability calls after the timeout, until the worker answers', async () => {+ fakeTimers();+ const { ch, dispatch } = build();+ const timedOut = dispatch('message:sending');+ await jest.advanceTimersByTimeAsync(5000);+ await expect(timedOut).resolves.toEqual({ continue: true });++ // The late send still short-circuits (true), rather than running the veto chain (false).+ await expect(guardedResult(ch, 1, ['message:sending'])).resolves.toBe(true);++ // The handler finally returns: the claim is given back, and the same call is no longer guarded.+ const hook = ch.sent.find(m => m.kind === 'hook');+ ch.reply({ kind: 'hook-result', id: hook?.kind === 'hook' ? hook.id : -1, continue: true });+ await expect(guardedResult(ch, 2, ['message:sending'])).resolves.toBe(false);+ ch.crash();+ });++ it('gives the claim back when the worker exits', async () => {+ fakeTimers();+ const { ch, dispatch } = build();+ const timedOut = dispatch('message:sending');+ await jest.advanceTimersByTimeAsync(5000);+ await timedOut;+ await expect(guardedResult(ch, 1, ['message:sending'])).resolves.toBe(true);++ ch.crash();++ // Nothing honours a claim on a dead worker's behalf: a call that still arrives runs the chain.+ await expect(guardedResult(ch, 2, ['message:sending'])).resolves.toBe(false);+ });++ it('holds a bounded number of late claims, giving the oldest back first', async () => {+ fakeTimers();+ const { ch, dispatch } = build();+ const first = dispatch('message:sending');+ await jest.advanceTimersByTimeAsync(5000);+ await first;+ await expect(guardedResult(ch, 1, ['message:sending'])).resolves.toBe(true);++ // 32 more timed-out dispatches of another event push the first one past the bound.+ for (let i = 0; i < 32; i++) {+ const next = dispatch('message:sent');+ await jest.advanceTimersByTimeAsync(5000);+ await next;+ }++ await expect(guardedResult(ch, 2, ['message:sending'])).resolves.toBe(false);+ ch.crash();+ });+ });});describe('post-crash dead-check (fail-fast, no stall)', () => {diff --git a/src/core/plugins/sandbox/plugin-worker-host.ts b/src/core/plugins/sandbox/plugin-worker-host.tsindex 802506054..20426474c 100644--- a/src/core/plugins/sandbox/plugin-worker-host.ts+++ b/src/core/plugins/sandbox/plugin-worker-host.ts@@ -27,6 +27,13 @@ const SEND_CAP_TIMEOUT_FACTOR = 4;/** Longest worker-reported error kept for an exit message and the plugin's status. */const MAX_WORKER_ERROR_CHARS = 500;+/**+ * Timed-out hook dispatches whose in-flight claim is still honoured, per worker. A handler that never+ * returns would otherwise keep one entry per dispatch for as long as the worker lives; past this many,+ * the oldest claim is given back.+ */+const MAX_LATE_HOOK_CLAIMS = 32;+/** Worker replies that finish a host request; one arriving during a liveness probe restarts its window. */const PROGRESS_KINDS: ReadonlySet<WorkerToHostMessage['kind']> = new Set(['lifecycle-result',@@ -92,6 +99,9 @@ export class PluginWorkerHost {// this set bounds that claim, so a worker can never mark an event in flight that the host did not// actually dispatch to it (HookManager's AsyncLocalStorage guard does not span the IPC boundary).private readonly inFlightHookEvents = new Map<string, number>();+ // Dispatches that timed out host-side while their handler may still be running in the worker, oldest+ // first, each with the release of the in-flight claim it still holds. See holdLateClaim.+ private readonly lateHookClaims = new Map<number, () => void>();// Worker-initiated capability calls currently running host-side, bounded by maxInFlightCaps.private inFlightCaps = 0;@@ -191,17 +201,29 @@ export class PluginWorkerHost {const inFlight = [...new Set([...(options.inFlight ?? []), options.event])];for (const event of inFlight) this.incInFlightHook(event);return new Promise(resolve => {- // settle decrements the in-flight counters on every exit path (worker result, timeout, or crash- // drain) since it is what the hookPending entry's resolve runs.- const settle = (result: { continue: boolean; data?: unknown; error?: string }): void => {+ let released = false;+ const release = (): void => {+ if (released) return;+ released = true;+ this.lateHookClaims.delete(id);for (const event of inFlight) this.decInFlightHook(event);+ };+ // settle gives the in-flight claim back on every exit path that ends the dispatch (the worker's+ // result, a late one included, or the crash drain) since it is what the hookPending entry's resolve+ // runs. Resolving a second time is a no-op, so a late result only releases the claim.+ const settle = (result: { continue: boolean; data?: unknown; error?: string }): void => {+ release();resolve(result);};const timer = setTimeout(() => {- this.hookPending.delete(id);options.onTimeout?.();this.probeLiveness();- settle({ continue: true });+ // The chain moves on without this plugin, but its handler is still running in the worker, and+ // a capability call it makes from here on still names this dispatch. Dropping the claim now+ // would let that call run outside the re-entrancy guard, re-fire the event and dispatch the+ // same slow handler again, one more send every timeout for as long as the plugin stays enabled.+ resolve({ continue: true });+ this.holdLateClaim(id, release);}, options.timeoutMs);this.hookPending.set(id, { resolve: settle, timer });this.channel.postMessage({@@ -217,6 +239,20 @@ export class PluginWorkerHost {});}+ /**+ * Keep the claim of a timed-out dispatch until the worker answers it or exits, bounded by+ * {@link MAX_LATE_HOOK_CLAIMS}. The hookPending entry stays too, so the late result finds it.+ */+ private holdLateClaim(id: number, release: () => void): void {+ this.lateHookClaims.set(id, release);+ if (this.lateHookClaims.size <= MAX_LATE_HOOK_CLAIMS) return;+ const [oldest, giveBack] = this.lateHookClaims.entries().next().value as [number, () => void];+ const waiter = this.hookPending.get(oldest);+ if (waiter) clearTimeout(waiter.timer);+ this.hookPending.delete(oldest); // an answer this late is ignored+ giveBack();+ }+/*** Dispatch a verified inbound webhook to the worker and await its handler result. Cloned from* dispatchHook and bounded by `timeoutMs`: a slow or wedged worker resolves ok:false with 504, and a