Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(plugins): keep a late hook handler inside the re-entrancy guard

Yudhi Armyndharisc440ab07220e
diff --git a/src/core/plugins/sandbox/plugin-worker-host.spec.ts b/src/core/plugins/sandbox/plugin-worker-host.spec.ts
index c84291be0..c9336e56a 100644
--- a/src/core/plugins/sandbox/plugin-worker-host.spec.ts
+++ b/src/core/plugins/sandbox/plugin-worker-host.spec.ts
@@ -884,6 +884,85 @@ describe('PluginWorkerHost', () => {
expect(ch.sent).toContainEqual({ kind: 'cap-result', id: 4, ok: true, result: true });
ch.crash(); // drains the pending dispatch and its timer
});
+
+ // A handler slower than the host's hook timeout keeps running in the worker, and what it sends from
+ // there names the dispatch that timed out. Honouring that claim only until the timeout let the send
+ // run outside the guard, fire message:sending again and dispatch the same slow handler once more.
+ describe('a handler that outlives the host timeout', () => {
+ const fakeTimers = (): void => {
+ jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick', 'queueMicrotask'] });
+ };
+ const guardedResult = async (ch: FakeChannel, id: number, inFlight: string[]): Promise<unknown> => {
+ ch.reply({ kind: 'cap', id, verb: 'messages.sendText', args: [], inFlight });
+ await flush();
+ await flush();
+ const reply = ch.sent.find(m => m.kind === 'cap-result' && m.id === id);
+ return reply && reply.kind === 'cap-result' && reply.ok ? reply.result : reply;
+ };
+ const build = () => {
+ const hm = new HookManager();
+ const ch = new FakeChannel();
+ hm.register('moderation', SENDING, () => Promise.resolve({ continue: false }));
+ const capDispatcher = async (): Promise<unknown> => (await hm.execute(SENDING, {}, { source: 'cap' })).continue;
+ const host = new PluginWorkerHost(ch, capDispatcher, undefined, undefined, undefined, (events, run) =>
+ hm.runInFlight(events as HookEvent[], run),
+ );
+ const dispatch = (event: string) => host.dispatchHook({ event, data: {}, source: 'test', timeoutMs: 5000 });
+ return { ch, dispatch };
+ };
+
+ afterEach(() => jest.useRealTimers());
+
+ it('keeps guarding its capability calls after the timeout, until the worker answers', async () => {
+ fakeTimers();
+ const { ch, dispatch } = build();
+ const timedOut = dispatch('message:sending');
+ await jest.advanceTimersByTimeAsync(5000);
+ await expect(timedOut).resolves.toEqual({ continue: true });
+
+ // The late send still short-circuits (true), rather than running the veto chain (false).
+ await expect(guardedResult(ch, 1, ['message:sending'])).resolves.toBe(true);
+
+ // The handler finally returns: the claim is given back, and the same call is no longer guarded.
+ const hook = ch.sent.find(m => m.kind === 'hook');
+ ch.reply({ kind: 'hook-result', id: hook?.kind === 'hook' ? hook.id : -1, continue: true });
+ await expect(guardedResult(ch, 2, ['message:sending'])).resolves.toBe(false);
+ ch.crash();
+ });
+
+ it('gives the claim back when the worker exits', async () => {
+ fakeTimers();
+ const { ch, dispatch } = build();
+ const timedOut = dispatch('message:sending');
+ await jest.advanceTimersByTimeAsync(5000);
+ await timedOut;
+ await expect(guardedResult(ch, 1, ['message:sending'])).resolves.toBe(true);
+
+ ch.crash();
+
+ // Nothing honours a claim on a dead worker's behalf: a call that still arrives runs the chain.
+ await expect(guardedResult(ch, 2, ['message:sending'])).resolves.toBe(false);
+ });
+
+ it('holds a bounded number of late claims, giving the oldest back first', async () => {
+ fakeTimers();
+ const { ch, dispatch } = build();
+ const first = dispatch('message:sending');
+ await jest.advanceTimersByTimeAsync(5000);
+ await first;
+ await expect(guardedResult(ch, 1, ['message:sending'])).resolves.toBe(true);
+
+ // 32 more timed-out dispatches of another event push the first one past the bound.
+ for (let i = 0; i < 32; i++) {
+ const next = dispatch('message:sent');
+ await jest.advanceTimersByTimeAsync(5000);
+ await next;
+ }
+
+ await expect(guardedResult(ch, 2, ['message:sending'])).resolves.toBe(false);
+ ch.crash();
+ });
+ });
});
describe('post-crash dead-check (fail-fast, no stall)', () => {
diff --git a/src/core/plugins/sandbox/plugin-worker-host.ts b/src/core/plugins/sandbox/plugin-worker-host.ts
index 802506054..20426474c 100644
--- a/src/core/plugins/sandbox/plugin-worker-host.ts
+++ b/src/core/plugins/sandbox/plugin-worker-host.ts
@@ -27,6 +27,13 @@ const SEND_CAP_TIMEOUT_FACTOR = 4;
/** Longest worker-reported error kept for an exit message and the plugin's status. */
const MAX_WORKER_ERROR_CHARS = 500;
+/**
+ * Timed-out hook dispatches whose in-flight claim is still honoured, per worker. A handler that never
+ * returns would otherwise keep one entry per dispatch for as long as the worker lives; past this many,
+ * the oldest claim is given back.
+ */
+const MAX_LATE_HOOK_CLAIMS = 32;
+
/** Worker replies that finish a host request; one arriving during a liveness probe restarts its window. */
const PROGRESS_KINDS: ReadonlySet<WorkerToHostMessage['kind']> = new Set([
'lifecycle-result',
@@ -92,6 +99,9 @@ export class PluginWorkerHost {
// this set bounds that claim, so a worker can never mark an event in flight that the host did not
// actually dispatch to it (HookManager's AsyncLocalStorage guard does not span the IPC boundary).
private readonly inFlightHookEvents = new Map<string, number>();
+ // Dispatches that timed out host-side while their handler may still be running in the worker, oldest
+ // first, each with the release of the in-flight claim it still holds. See holdLateClaim.
+ private readonly lateHookClaims = new Map<number, () => void>();
// Worker-initiated capability calls currently running host-side, bounded by maxInFlightCaps.
private inFlightCaps = 0;
@@ -191,17 +201,29 @@ export class PluginWorkerHost {
const inFlight = [...new Set([...(options.inFlight ?? []), options.event])];
for (const event of inFlight) this.incInFlightHook(event);
return new Promise(resolve => {
- // settle decrements the in-flight counters on every exit path (worker result, timeout, or crash
- // drain) since it is what the hookPending entry's resolve runs.
- const settle = (result: { continue: boolean; data?: unknown; error?: string }): void => {
+ let released = false;
+ const release = (): void => {
+ if (released) return;
+ released = true;
+ this.lateHookClaims.delete(id);
for (const event of inFlight) this.decInFlightHook(event);
+ };
+ // settle gives the in-flight claim back on every exit path that ends the dispatch (the worker's
+ // result, a late one included, or the crash drain) since it is what the hookPending entry's resolve
+ // runs. Resolving a second time is a no-op, so a late result only releases the claim.
+ const settle = (result: { continue: boolean; data?: unknown; error?: string }): void => {
+ release();
resolve(result);
};
const timer = setTimeout(() => {
- this.hookPending.delete(id);
options.onTimeout?.();
this.probeLiveness();
- settle({ continue: true });
+ // The chain moves on without this plugin, but its handler is still running in the worker, and
+ // a capability call it makes from here on still names this dispatch. Dropping the claim now
+ // would let that call run outside the re-entrancy guard, re-fire the event and dispatch the
+ // same slow handler again, one more send every timeout for as long as the plugin stays enabled.
+ resolve({ continue: true });
+ this.holdLateClaim(id, release);
}, options.timeoutMs);
this.hookPending.set(id, { resolve: settle, timer });
this.channel.postMessage({
@@ -217,6 +239,20 @@ export class PluginWorkerHost {
});
}
+ /**
+ * Keep the claim of a timed-out dispatch until the worker answers it or exits, bounded by
+ * {@link MAX_LATE_HOOK_CLAIMS}. The hookPending entry stays too, so the late result finds it.
+ */
+ private holdLateClaim(id: number, release: () => void): void {
+ this.lateHookClaims.set(id, release);
+ if (this.lateHookClaims.size <= MAX_LATE_HOOK_CLAIMS) return;
+ const [oldest, giveBack] = this.lateHookClaims.entries().next().value as [number, () => void];
+ const waiter = this.hookPending.get(oldest);
+ if (waiter) clearTimeout(waiter.timer);
+ this.hookPending.delete(oldest); // an answer this late is ignored
+ giveBack();
+ }
+
/**
* Dispatch a verified inbound webhook to the worker and await its handler result. Cloned from
* dispatchHook and bounded by `timeoutMs`: a slow or wedged worker resolves ok:false with 504, and a