Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: recycle WebGPU worker on device death so retries recover without reload
Emre Sokullu659616f165fc
diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.jsindex 220d26d9b..173aff0e3 100644--- a/src/chrome/src/offscreen/inference-worker.js+++ b/src/chrome/src/offscreen/inference-worker.js@@ -385,7 +385,7 @@ async function enrichWebGpuExecutionError(error) {const suffix = [details.length ? `GPU detail: ${details.join(' ')}` : '',adapter ? `Adapter: ${adapter}.` : '',- 'Close other GPU-heavy tabs/apps and retry with a short prompt; a retry re-creates the on-device session from the cached download, it does not re-download. If it persists, this GPU/driver cannot execute this model with the current WebGPU runtime.',+ 'Close other GPU-heavy tabs/apps and retry with a short prompt; a retry restarts the on-device worker and re-creates the session from the cached download, it does not re-download. If it persists, this GPU/driver cannot execute this model with the current WebGPU runtime.',].filter(Boolean).join(' ');return new Error(`${error?.message || String(error)} ${suffix}`);}@@ -411,9 +411,44 @@ async function handleWebGpuExecutionFailure(error) {const enriched = await enrichWebGpuExecutionError(error);markWebgpuRuntimeDirty();await disposeAllRuntimes();+ noteWebgpuExecutionFailure(error);return enriched;}+// Consecutive WebGPU execution failures mean the device itself is dead, not+// just one session: onnxruntime-web initializes its Dawn device exactly once+// per worker lifetime, so no in-worker rebuild can recover. Signal the host to+// terminate and recreate this worker; the next request then starts from a+// fresh device while weights reload from the browser cache.+let webgpuExecutionFailureStreak = 0;+const WEBGPU_WORKER_RECYCLE_FAILURES = 2;+const WEBGPU_CERTAIN_DEVICE_DEATH_PATTERN = /external Instance reference|device lost/i;++function requestWebgpuWorkerRecycle(reason) {+ webgpuExecutionFailureStreak = 0;+ // Defer to a later macrotask so the failing request's `{ id, ok: false }`+ // response (posted in the current microtask chain) reaches the host first.+ // The caller then sees the enriched error instead of the recycle notice.+ setTimeout(() => {+ try {+ self.postMessage({ type: 'webgpu-device-dead', reason });+ } catch {}+ }, 0);+}++function noteWebgpuExecutionFailure(error) {+ webgpuExecutionFailureStreak += 1;+ const message = error?.message || String(error);+ const certainDeath = WEBGPU_CERTAIN_DEVICE_DEATH_PATTERN.test(message);+ if (certainDeath || webgpuExecutionFailureStreak >= WEBGPU_WORKER_RECYCLE_FAILURES) {+ requestWebgpuWorkerRecycle(certainDeath ? 'certain-device-death' : 'repeated-execution-failures');+ }+}++function noteWebgpuExecutionSuccess() {+ webgpuExecutionFailureStreak = 0;+}+function postProgress(modelId, event) {if (modelId === activeTextDownloadModelId && !textDownloadCancelMode) {const file = String(event?.file || event?.name || '');@@ -600,6 +635,7 @@ async function getVisionRuntime(modelId, dtype, device, {if (isWebGpuExecutionFailure(failure)) {markWebgpuRuntimeDirty();await disposeAllRuntimes();+ noteWebgpuExecutionFailure(failure);throw await enrichWebGpuExecutionError(failure);}throw failure;@@ -753,6 +789,7 @@ async function getTextRuntime(modelId, dtype, device, { localFilesOnly = false }if (isWebGpuExecutionFailure(error)) {markWebgpuRuntimeDirty();await disposeAllRuntimes();+ noteWebgpuExecutionFailure(error);throw await enrichWebGpuExecutionError(error);}throw error;@@ -1159,10 +1196,11 @@ async function runVision(payload, requestId) {error.name = 'AbortError';throw error;}- const inputLength = inputs.input_ids.dims.at(-1);- const generated = outputs.slice(null, [inputLength, null]);- const decoded = runtime.processor.batch_decode(generated, { skip_special_tokens: true });- return String(decoded?.[0] || '').trim();+ const inputLength = inputs.input_ids.dims.at(-1);+ const generated = outputs.slice(null, [inputLength, null]);+ const decoded = runtime.processor.batch_decode(generated, { skip_special_tokens: true });+ noteWebgpuExecutionSuccess();+ return String(decoded?.[0] || '').trim();} finally {activeVisionGenerations.delete(requestId);cancelledVisionGenerations.delete(requestId);@@ -1317,6 +1355,7 @@ async function runMultimodalText(payload) {const generated = outputs.slice(null, [inputLength, null]);const decoded = runtime.processor.batch_decode(generated, { skip_special_tokens: true });const result = splitThinking(String(decoded?.[0] || '').trim());+ noteWebgpuExecutionSuccess();return { content: result.content, reasoningContent: result.reasoningContent };}@@ -1371,6 +1410,7 @@ async function runText(payload) {if (result.incompleteReasoning) {throw new Error(`${modelId} used its generation budget before finishing reasoning. Retry with a shorter prompt.`);}+ noteWebgpuExecutionSuccess();return { content: result.content, reasoningContent: result.reasoningContent };}diff --git a/src/chrome/src/offscreen/vision-inference-host.js b/src/chrome/src/offscreen/vision-inference-host.jsindex e7a6afd96..2f3c170d6 100644--- a/src/chrome/src/offscreen/vision-inference-host.js+++ b/src/chrome/src/offscreen/vision-inference-host.js@@ -22,6 +22,9 @@ let visionPreloadKey = '';let visionPreloadLifecycle = null;const timedOutVisionRequests = new Map();const WORKER_INITIALIZATION_TIMEOUT_MS = 15_000;+const VISION_WORKER_RECYCLE_COOLDOWN_MS = 30_000;+// Start one full window in the past so the first signal always recycles.+let lastVisionWorkerRecycleAt = -VISION_WORKER_RECYCLE_COOLDOWN_MS;const VISION_INFERENCE_TIMEOUT_MS = 90_000;const CANCELLATION_GRACE_PERIOD_MS = 5_000;const DOWNLOAD_STALL_TIMEOUT_MS = 2 * 60_000;@@ -480,6 +483,23 @@ async function ensureVisionWorker() {visionWorker = worker;worker.addEventListener('message', event => {if (visionWorker !== worker) return;+ // The worker asks for a fresh context when its WebGPU device dies:+ // onnxruntime-web initializes its Dawn device once per worker lifetime,+ // so no in-worker session rebuild can recover. Terminate and recreate;+ // the next request re-initializes and reloads weights from the cache.+ // Stale workers (already replaced) are ignored by the guard above.+ if (event.data?.type === 'webgpu-device-dead') {+ // A persistently poisoned GPU process would otherwise recycle (and+ // reload ~2 GB into) every attempt; back off instead and surface errors+ // until the window passes, then recycle on the next signal.+ if (Date.now() - lastVisionWorkerRecycleAt < VISION_WORKER_RECYCLE_COOLDOWN_MS) return;+ lastVisionWorkerRecycleAt = Date.now();+ resetVisionWorker(deadlineError(+ 'webgpu_device_recreated',+ `The WebGPU device died (${event.data?.reason || 'execution failure'}); the worker was recreated.`,+ ));+ return;+ }settleVisionRequest(event.data);});worker.addEventListener('error', event => {diff --git a/test/run.js b/test/run.jsindex 34c2e6228..8a47dc91b 100644--- a/test/run.js+++ b/test/run.js@@ -63136,6 +63136,10 @@ test('WebGPU worker follows local text-generation and WebBrain VL vision contracassert.match(worker, /load_image\(imageUrls\[0\]\)/);assert.match(worker, /type === 'multimodal-text-chat'[\s\S]*?runMultimodalText\(payload\)/);assert.match(host, /message\.runtime === 'onnx-vl'[\s\S]*?'multimodal-text-chat'/);+ assert.match(worker, /type: 'webgpu-device-dead'/,+ 'the worker must signal the host when its WebGPU device dies');+ assert.match(host, /type === 'webgpu-device-dead'[\s\S]*?resetVisionWorker/,+ 'the host must recycle the worker on a device-dead signal');assert.match(worker, /session_file_names:[\s\S]*?vision_encoder: 'embed_images'[\s\S]*?decoder_model_merged: 'decoder'/,'the legacy LFM2.5-VL-1.6B ONNX filenames must be mapped into the Transformers.js runtime');assert.match(worker, /image_processor_config_file: 'processor_config\.json'[\s\S]*?chat_template_file: 'chat_template\.jinja'/,@@ -63824,6 +63828,31 @@ test('vision inference host enforces deadlines and recreates poisoned workers',assert.equal(inferenceRetry.content, 'recovered vision','a timed-out inference poisoned the next serialized request');+ const deviceDeath = createHarness({});+ await deviceDeath.dispatch({ type: 'webgpu-vision-probe', model: WEBGPU_VISION_MODEL_ID });+ await deviceDeath.drain();+ assert.equal(deviceDeath.workers.length, 1);+ deviceDeath.workers[0].emit({ type: 'webgpu-device-dead', reason: 'repeated-execution-failures' });+ await deviceDeath.drain();+ assert.equal(deviceDeath.workers[0].terminated, true,+ 'a device-dead signal did not recycle the worker');+ const afterRecycle = await deviceDeath.dispatch({ type: 'webgpu-vision-chat', model: WEBGPU_VISION_MODEL_ID });+ assert.equal(deviceDeath.workers.length, 2, 'the next request did not boot a fresh worker');+ assert.equal(afterRecycle.content, 'recovered vision',+ 'a request after device-death recovery did not succeed');+ // A second signal within the cooldown must not recycle again (a+ // persistently poisoned GPU would otherwise reload ~2 GB every attempt).+ deviceDeath.workers[1].emit({ type: 'webgpu-device-dead', reason: 'certain-device-death' });+ await deviceDeath.drain();+ assert.equal(deviceDeath.workers[1].terminated, false,+ 'a device-dead signal inside the cooldown recycled the worker again');+ assert.equal(deviceDeath.workers.length, 2, 'the cooldown signal booted an extra worker');+ await deviceDeath.advance(30_000);+ deviceDeath.workers[1].emit({ type: 'webgpu-device-dead', reason: 'certain-device-death' });+ await deviceDeath.drain();+ assert.equal(deviceDeath.workers[1].terminated, true,+ 'a device-dead signal after the cooldown did not recycle the worker');+const queuedInference = createHarness({ hangChatCount: 1, cancelQueued: true });const hungQueuedInference = queuedInference.dispatch({ type: 'webgpu-vision-chat', model: WEBGPU_VISION_MODEL_ID });await queuedInference.drain();@@ -64460,7 +64489,7 @@ test('WebGPU worker replays text tool history and applies model-specific generatassert.equal(poisonedText.ok, false);assert.match(poisonedText.error, /OrtRun|mapAsync/,'a dead WebGPU device must still surface the enriched execution error');- assert.match(poisonedText.error, /re-creates the on-device session/,+ assert.match(poisonedText.error, /re-creates the session/,'the enriched error must explain that a retry rebuilds the session');assert.equal(globalThis.__webgpuRuntimeCounts.textDisposals, textDisposalsBeforePoison + 1,'an execution failure must dispose the poisoned text runtime');@@ -64506,6 +64535,43 @@ test('WebGPU worker replays text tool history and applies model-specific generatassert.equal(recoveredVision.content, 'vision answer');assert.equal(globalThis.__webgpuRuntimeCounts.visionModelLoads, visionLoadsBeforePoison + 1,'the next vision turn after a device failure must rebuild the session');++ const deviceDeadSignals = () => posted.filter(message => message?.type === 'webgpu-device-dead');+ // The recycle signal is deferred to a later macrotask so the enriched+ // failure response always delivers first; wait it out before counting.+ const settleSignals = () => new Promise(resolve => setTimeout(resolve, 25));+ const failTextChat = async () => {+ const id = requestId++;+ await workerListener({ data: { id, type: 'text-chat', payload: textPayload } });+ await settleSignals();+ return posted.find(message => message.id === id);+ };+ // The text poison above already carried certain device death, so it must+ // have signalled immediately; the streak is 0 again after the recoveries.+ await settleSignals();+ assert.equal(deviceDeadSignals().length, 1, 'certain device death must recycle on the first failure');+ assert.equal(deviceDeadSignals()[0].reason, 'certain-device-death');+ globalThis.__webgpuInstanceExecutionError = 'failed to call OrtRun(): transient failure';+ assert.equal((await failTextChat()).ok, false);+ assert.equal(deviceDeadSignals().length, 1, 'the first consecutive failure must stay in-worker');+ // The second consecutive failure proves the device itself is dead: the+ // worker must ask the host for a fresh context, but only after the+ // enriched failure response has already been delivered.+ const secondFailure = await failTextChat();+ assert.equal(secondFailure.ok, false);+ assert.match(secondFailure.error, /OrtRun|mapAsync/,+ 'the failing turn must surface the enriched error, not the recycle notice');+ assert.equal(deviceDeadSignals().length, 2, 'repeated execution failures must request a worker recycle');+ assert.equal(deviceDeadSignals()[1].reason, 'repeated-execution-failures');+ assert.ok(posted.indexOf(secondFailure) < posted.indexOf(deviceDeadSignals()[1]),+ 'the enriched error response must deliver before the recycle signal');+ // A success resets the streak: fail, recover, fail again must not signal.+ globalThis.__webgpuInstanceExecutionError = '';+ assert.equal((await dispatch('text-chat', textPayload)).content, 'text answer');+ globalThis.__webgpuInstanceExecutionError = 'failed to call OrtRun(): another transient failure';+ assert.equal((await failTextChat()).ok, false);+ assert.equal(deviceDeadSignals().length, 2, 'a success must reset the failure streak');+ globalThis.__webgpuInstanceExecutionError = '';} finally {if (previousSelf === undefined) delete globalThis.self;else globalThis.self = previousSelf;