Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Make the in-browser WebGPU models report their output limit and stop on cancel
Claudeaddd83f155f1
diff --git a/docs/architecture.md b/docs/architecture.mdindex 521f6a01d..7b54f3f27 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -527,7 +527,10 @@ answers with whatever text the provider returned) the responseis flagged `truncated`, and the card says so and offers **Continue in side panel** insteadof passing a cut-off translation off as complete. Each call carries a job id, and a cardthat is closed, retried or retargeted sends `WB_SELECTION_SHORTCUT_INLINE_CANCEL` so the-background aborts the paid request it no longer needs; the card also claims its request id+background aborts the paid request it no longer needs (for the in-browser WebGPU models the+provider also sends `webgpu-chat-cancel`, and the offscreen worker interrupts the running+generation or drops a queued one; those workers report `outputTruncated` when an answer used+its token budget, since they have no finish reason); the card also claims its request idbefore it waits for the language bundle, so a double click starts only one call. Selectionsover 4,000 characters skip the card and use the side-panel flow. The card draws the answer as DOMnodes (paragraphs, lists, bold, italic and code), never as HTML, and takes the sourcediff --git a/src/chrome/src/context-menu-storage.js b/src/chrome/src/context-menu-storage.jsindex 199722d7e..56375f671 100644--- a/src/chrome/src/context-menu-storage.js+++ b/src/chrome/src/context-menu-storage.js@@ -296,6 +296,8 @@ export function inlineErrorWasCut(error) {}export function inlineAnswerWasCut(result, maxTokens) {+ // The in-browser WebGPU models have no finish reason; their worker reports the limit directly.+ if (result?.outputTruncated === true) return true;const raw = result?.raw;const reason = [result?.finishReason,diff --git a/src/chrome/src/offscreen/bonsai-worker.js b/src/chrome/src/offscreen/bonsai-worker.jsindex 3c593764f..18aa00c18 100644--- a/src/chrome/src/offscreen/bonsai-worker.js+++ b/src/chrome/src/offscreen/bonsai-worker.js@@ -16,6 +16,22 @@ const CACHE_NAME = 'bitgpu-models-v1';const TEXT_DOWNLOAD_EVENT = 'text-download-state';const WEBGPU_BONSAI27_MAX_NEW_TOKENS = 2048;const WEBGPU_BONSAI27_THINK_BUDGET = 128;+// Text generations that carry a requestKey can be cancelled by their caller; the+// controller's signal is handed to the runtime, which stops decoding when it fires.+const textRequestControllers = new Map();+const TEXT_REQUEST_KEY_PATTERN = /^[a-z0-9-]{8,64}$/i;++function textRequestKey(payload) {+ const key = typeof payload?.requestKey === 'string' ? payload.requestKey : '';+ return TEXT_REQUEST_KEY_PATTERN.test(key) ? key : '';+}++function throwIfTextCancelled(signal) {+ if (!signal?.aborted) return;+ const error = new Error('The generation was cancelled.');+ error.name = 'AbortError';+ throw error;+}const WEBGPU_BONSAI27_MAX_SEQ_LEN = 4096;const OOM_HINT = 'This machine cannot hold Bonsai 27B in GPU memory. Use LFM2.5 2.6B instead.';@@ -828,12 +844,14 @@ function normalizeBitgpuToolCalls(toolCalls) {}).filter(Boolean);}-async function runText(payload) {+async function runText(payload, signal = null) {+ throwIfTextCancelled(signal);const modelId = assertBonsaiModel(payload?.modelId);if (!await isTextModelReady(modelId, payload?.dtype || WEBGPU_BONSAI27_DTYPE)) {throw new Error(`${modelId} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it before chatting.`);}const runtime = await getTextRuntime({ localFilesOnly: true });+ throwIfTextCancelled(signal);const requestedTokens = Number(payload?.options?.maxTokens);const maxTokens = Number.isFinite(requestedTokens)? Math.max(1, Math.min(WEBGPU_BONSAI27_MAX_NEW_TOKENS, Math.round(requestedTokens)))@@ -849,10 +867,13 @@ async function runText(payload) {topP: 0.85,topK: 20,tools: tools.length ? tools : undefined,+ ...(signal ? { signal } : {}),});} catch (error) {+ if (signal?.aborted) throwIfTextCancelled(signal);throw mapGpuError(error);}+ throwIfTextCancelled(signal);const content = String(result?.text || '').trim();const reasoningContent = String(result?.thinkText || '').trim() || null;const toolCalls = normalizeBitgpuToolCalls(result?.toolCalls);@@ -860,7 +881,14 @@ async function runText(payload) {throw new Error(`${modelId} used its generation budget before finishing reasoning. Retry with a shorter prompt.`);}if (!content && !toolCalls.length) throw new Error('The WebGPU model returned no generated text or tool calls.');- return { content, reasoningContent, toolCalls };+ // The runtime reports why decoding stopped, so a cut-off answer is not mistaken for a finished one.+ const finishReason = typeof result?.finishReason === 'string' ? result.finishReason : '';+ return {+ content,+ reasoningContent,+ toolCalls,+ ...(finishReason ? { outputTruncated: finishReason === 'length' } : {}),+ };}async function probeRuntime() {@@ -950,14 +978,28 @@ self.addEventListener('message', async (event) => {self.postMessage({ id, ok: true, disposed: true });return;}+ if (type === 'cancel-text') {+ const requestKey = typeof payload?.requestKey === 'string' ? payload.requestKey : '';+ const controller = textRequestControllers.get(requestKey);+ controller?.abort();+ self.postMessage({ id, ok: true, cancelled: !!controller });+ return;+ }if (type === 'text-chat') {- const result = await enqueueModelOperation(() => runText(payload));- self.postMessage({- id,- ok: true,- ...result,- raw: { model: payload?.modelId || WEBGPU_BONSAI27_MODEL_ID },- });+ const requestKey = textRequestKey(payload);+ const controller = requestKey ? new AbortController() : null;+ if (controller) textRequestControllers.set(requestKey, controller);+ try {+ const result = await enqueueModelOperation(() => runText(payload, controller?.signal || null));+ self.postMessage({+ id,+ ok: true,+ ...result,+ raw: { model: payload?.modelId || WEBGPU_BONSAI27_MODEL_ID },+ });+ } finally {+ if (controller && textRequestControllers.get(requestKey) === controller) textRequestControllers.delete(requestKey);+ }return;}throw new Error(`Unknown Bonsai WebGPU worker message: ${type || 'missing type'}`);diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.jsindex 008753f78..2db952cd0 100644--- a/src/chrome/src/offscreen/inference-worker.js+++ b/src/chrome/src/offscreen/inference-worker.js@@ -102,6 +102,57 @@ let visionDownloadAbortController = null;const activeVisionGenerations = new Map();const queuedVisionGenerations = new Set();const cancelledVisionGenerations = new Set();+// Text generations that carry a requestKey (the in-page answer cards do) can be+// cancelled: one still queued never starts, and a running one is interrupted at+// its next token instead of finishing a result nobody is waiting for.+const queuedTextGenerations = new Set();+const activeTextGenerations = new Map();+const cancelledTextGenerations = new Set();+const TEXT_REQUEST_KEY_PATTERN = /^[a-z0-9-]{8,64}$/i;+// A re-tokenized answer that used this much of its budget is reported as cut off.+const WEBGPU_OUTPUT_LIMIT_RATIO = 0.97;++function textRequestKey(payload) {+ const key = typeof payload?.requestKey === 'string' ? payload.requestKey : '';+ return TEXT_REQUEST_KEY_PATTERN.test(key) ? key : '';+}++function throwIfTextCancelled(requestKey) {+ if (!requestKey || !cancelledTextGenerations.has(requestKey)) return;+ const error = new Error('The generation was cancelled.');+ error.name = 'AbortError';+ throw error;+}++function beginTextGeneration(requestKey, library) {+ if (!requestKey || !library?.InterruptableStoppingCriteria) return null;+ const criteria = new library.InterruptableStoppingCriteria();+ activeTextGenerations.set(requestKey, criteria);+ return criteria;+}++function endTextGeneration(requestKey) {+ if (requestKey) activeTextGenerations.delete(requestKey);+}++// Tokens in a generated text, counted with the model's own tokenizer; null when it cannot.+function countGeneratedTokens(tokenizer, text) {+ try {+ if (typeof tokenizer !== 'function' || !text) return null;+ const count = Number(tokenizer(text, { add_special_tokens: false })?.input_ids?.dims?.at?.(-1));+ return Number.isFinite(count) ? count : null;+ } catch {+ return null;+ }+}++// The worker has no finish reason for these models, so a generation that used+// (nearly) all of its token budget is reported as cut off for the caller to act on.+function outputLimitField(generatedTokens, maxNewTokens, ratio = WEBGPU_OUTPUT_LIMIT_RATIO) {+ return Number.isFinite(generatedTokens) && maxNewTokens > 0+ ? { outputTruncated: generatedTokens >= Math.max(1, Math.floor(maxNewTokens * ratio)) }+ : {};+}let lastTextProgressPostAt = 0;let webGpuAdapterProbePromise = null;let webGpuAdapterSummary = '';@@ -1326,6 +1377,8 @@ function addLegacyVlTools(messages, tools) {}async function runMultimodalText(payload) {+ const requestKey = textRequestKey(payload);+ throwIfTextCancelled(requestKey);const modelId = assertOnnxTextModel(payload?.modelId);if (!isLfm25VlModel(modelId)) {throw new Error(`${modelId} is not a shipped WebGPU multimodal model.`);@@ -1344,6 +1397,7 @@ async function runMultimodalText(payload) {owner: 'text',readiness: 'text',});+ throwIfTextCancelled(requestKey);const tools = Array.isArray(payload?.options?.tools) ? payload.options.tools : [];let { messages, imageUrls } = prepareMultimodalMessages(payload?.messages);if (modelId === WEBGPU_LFM25_VL_16B_MODEL_ID) messages = addLegacyVlTools(messages, tools);@@ -1366,26 +1420,38 @@ async function runMultimodalText(payload) {: 800;lastWebGpuDeviceError = '';lastWebGpuDeviceLost = '';+ const criteria = beginTextGeneration(requestKey, runtime.library);let outputs;try {outputs = await runtime.model.generate({...inputs,do_sample: false,max_new_tokens: maxNewTokens,+ ...(criteria ? { stopping_criteria: [criteria] } : {}),});} catch (error) {if (isWebGpuExecutionFailure(error)) throw await handleWebGpuExecutionFailure(error);throw error;+ } finally {+ endTextGeneration(requestKey);}+ throwIfTextCancelled(requestKey);const inputLength = inputs.input_ids.dims.at(-1);const generated = outputs.slice(null, [inputLength, null]);const decoded = runtime.processor.batch_decode(generated, { skip_special_tokens: true });const result = splitThinking(String(decoded?.[0] || '').trim());noteWebgpuExecutionSuccess();- return { content: result.content, reasoningContent: result.reasoningContent };+ // The generated tensor is exact, so reaching the budget means no end-of-text token came first.+ return {+ content: result.content,+ reasoningContent: result.reasoningContent,+ ...outputLimitField(Number(generated?.dims?.at?.(-1)), maxNewTokens, 1),+ };}async function runText(payload) {+ const requestKey = textRequestKey(payload);+ throwIfTextCancelled(requestKey);const modelId = assertOnnxTextModel(payload?.modelId);const device = payload?.device || 'webgpu';const dtype = payload?.dtype || 'q4f16';@@ -1397,6 +1463,7 @@ async function runText(payload) {throw new Error(`${modelId} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it before chatting.`);}const runtime = await getTextRuntime(modelId, dtype, device, { localFilesOnly: true });+ throwIfTextCancelled(requestKey);if (payload?.requireTools === true) assertToolCapableTextRuntime(runtime, modelId);const requestedTokens = Number(payload?.options?.maxTokens);const maxTokenLimit = usesLongOutputBudget@@ -1408,12 +1475,14 @@ async function runText(payload) {const tools = Array.isArray(payload?.options?.tools) ? payload.options.tools : [];lastWebGpuDeviceError = '';lastWebGpuDeviceLost = '';+ const criteria = beginTextGeneration(requestKey, runtime.library);let output;try {output = await runtime.pipeline(prepareTextMessages(payload?.messages), {do_sample: Boolean(sampling),...(sampling || {}),max_new_tokens: maxNewTokens,+ ...(criteria ? { stopping_criteria: [criteria] } : {}),tools: tools.length ? tools : undefined,// Reasoning templates re-open `<think>` for the pending turn and collapse// completed thinking in history; the rest suppress thinking outright.@@ -1424,7 +1493,10 @@ async function runText(payload) {} catch (error) {if (isWebGpuExecutionFailure(error)) throw await handleWebGpuExecutionFailure(error);throw error;+ } finally {+ endTextGeneration(requestKey);}+ throwIfTextCancelled(requestKey);const generated = output?.[0]?.generated_text;const content = Array.isArray(generated)? generated.at(-1)?.content@@ -1437,7 +1509,11 @@ async function runText(payload) {throw new Error(`${modelId} used its generation budget before finishing reasoning. Retry with a shorter prompt.`);}noteWebgpuExecutionSuccess();- return { content: result.content, reasoningContent: result.reasoningContent };+ return {+ content: result.content,+ reasoningContent: result.reasoningContent,+ ...outputLimitField(countGeneratedTokens(runtime.tokenizer, content), maxNewTokens),+ };}async function probeRuntime() {@@ -1501,6 +1577,18 @@ self.addEventListener('message', async event => {});return;}+ if (type === 'cancel-text') {+ const requestKey = typeof payload?.requestKey === 'string' ? payload.requestKey : '';+ const queued = queuedTextGenerations.has(requestKey);+ const active = activeTextGenerations.has(requestKey);+ const cancellable = TEXT_REQUEST_KEY_PATTERN.test(requestKey) && (queued || active);+ if (cancellable) {+ cancelledTextGenerations.add(requestKey);+ activeTextGenerations.get(requestKey)?.interrupt?.();+ }+ self.postMessage({ id, ok: true, cancelled: cancellable, queued: queued && !active, active });+ return;+ }if (type === 'init') {workerConfig = payload;self.postMessage({ id, ok: true });@@ -1649,24 +1737,25 @@ self.addEventListener('message', async event => {}return;}- if (type === 'text-chat') {- const result = await enqueueModelOperation(() => runText(payload));- self.postMessage({- id,- ok: true,- ...result,- raw: { model: payload?.modelId || '' },- });- return;- }- if (type === 'multimodal-text-chat') {- const result = await enqueueModelOperation(() => runMultimodalText(payload));- self.postMessage({- id,- ok: true,- ...result,- raw: { model: payload?.modelId || '' },- });+ if (type === 'text-chat' || type === 'multimodal-text-chat') {+ const requestKey = textRequestKey(payload);+ if (requestKey) queuedTextGenerations.add(requestKey);+ try {+ const result = await enqueueModelOperation(+ () => (type === 'text-chat' ? runText(payload) : runMultimodalText(payload)),+ );+ self.postMessage({+ id,+ ok: true,+ ...result,+ raw: { model: payload?.modelId || '' },+ });+ } finally {+ if (requestKey) {+ queuedTextGenerations.delete(requestKey);+ cancelledTextGenerations.delete(requestKey);+ }+ }return;}throw new Error(`Unknown WebGPU worker message: ${type || 'missing type'}`);diff --git a/src/chrome/src/offscreen/vision-inference-host.js b/src/chrome/src/offscreen/vision-inference-host.jsindex c8f682946..b94e3ce79 100644--- a/src/chrome/src/offscreen/vision-inference-host.js+++ b/src/chrome/src/offscreen/vision-inference-host.js@@ -538,6 +538,7 @@ async function ensureVisionWorker() {const WEBGPU_MESSAGE_TYPES = new Set(['webgpu-chat',+ 'webgpu-chat-cancel','webgpu-download-start','webgpu-download-pause','webgpu-download-stop',@@ -716,6 +717,17 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {sendResponse(disposed[0] || { ok: true, disposed: true });return;}+ if (message.type === 'webgpu-chat-cancel') {+ // Stop a text generation its caller no longer wants. Both workers are asked+ // because the host does not know which model the request was routed to.+ const requestKey = typeof message.requestKey === 'string' ? message.requestKey : '';+ const cancels = [];+ if (visionWorker) cancels.push(sendVisionWorkerMessage('cancel-text', { requestKey }).catch(() => null));+ if (bonsaiWorker) cancels.push(sendBonsaiWorkerMessage('cancel-text', { requestKey }).catch(() => null));+ await Promise.all(cancels);+ sendResponse({ ok: true });+ return;+ }if (message.type === 'webgpu-chat') {const workerMessageType = message.runtime === 'onnx-vl'? 'multimodal-text-chat'@@ -725,6 +737,7 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {device: message.device,dtype: message.dtype,requireTools: message.requireTools === true,+ requestKey: typeof message.requestKey === 'string' ? message.requestKey : '',messages: message.messages || [],options: message.options || {},}, { exclusive: true, runtime: message.runtime }));diff --git a/src/chrome/src/providers/webgpu.js b/src/chrome/src/providers/webgpu.jsindex dcb988864..42c13d936 100644--- a/src/chrome/src/providers/webgpu.js+++ b/src/chrome/src/providers/webgpu.js@@ -2,6 +2,7 @@import { BaseLLMProvider } from './base.js';import { ensureOffscreen } from '../offscreen/ensure.js';+import { responseAbortError } from '../network/response-body.js';export const WEBGPU_VISION_MODEL_ID = 'webbrain-one/webbrain-vl-2-450M-onnx';export const WEBGPU_MODEL_ID = 'LiquidAI/LFM2.5-2.6B-ONNX';@@ -240,8 +241,18 @@ export function webgpuModelRequiresToolTemplate(modelId) {return !isShippedWebgpuPreset(normalized);}+function newWebgpuRequestKey() {+ return globalThis.crypto?.randomUUID?.()+ || `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 12)}`;+}+class WebGPUOffscreenProvider extends BaseLLMProvider {- async _dispatch(message, { timeoutMs = 0 } = {}) {+ /**+ * Send one request to the offscreen host. With a signal, an abort rejects at+ * once (the caller stops waiting) and `onAbort` tells the host to stop the work.+ */+ async _dispatch(message, { timeoutMs = 0, signal = null, onAbort = null } = {}) {+ if (signal?.aborted) throw responseAbortError(signal);await ensureOffscreen();return await new Promise((resolve, reject) => {let settled = false;@@ -249,8 +260,13 @@ class WebGPUOffscreenProvider extends BaseLLMProvider {if (settled) return;settled = true;if (timeoutId) clearTimeout(timeoutId);+ signal?.removeEventListener?.('abort', abortListener);callback(value);};+ const abortListener = () => {+ try { onAbort?.(); } catch { /* the request is abandoned either way */ }+ finish(reject, responseAbortError(signal));+ };const timeoutId = timeoutMs > 0? setTimeout(() => {const error = new Error(`In-browser WebGPU request timed out after ${timeoutMs}ms (${message?.type || 'unknown'}).`);@@ -258,6 +274,7 @@ class WebGPUOffscreenProvider extends BaseLLMProvider {finish(reject, error);}, timeoutMs): null;+ signal?.addEventListener?.('abort', abortListener, { once: true });try {chrome.runtime.sendMessage(message, (response) => {const lastError = chrome.runtime.lastError;@@ -270,6 +287,15 @@ class WebGPUOffscreenProvider extends BaseLLMProvider {});}+ // Ask the offscreen host to stop a text generation nobody is waiting for any more.+ _cancelChat(requestKey) {+ try {+ chrome.runtime.sendMessage({ type: 'webgpu-chat-cancel', requestKey }, () => {+ void chrome.runtime.lastError;+ });+ } catch { /* best effort: the generation then finishes on its own */ }+ }+async _testWebGPU() {try {const response = await this._dispatch(@@ -345,6 +371,7 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {}async chat(messages, options = {}) {+ if (options.signal?.aborted) throw responseAbortError(options.signal);if (this._messagesContainImage(messages) && !this.supportsVision) {throw new Error('The WebGPU chat model is text-only. Configure a separate model under Settings -> Multimodal for screenshots.');}@@ -352,6 +379,9 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {if (!download.ready) {throw new Error(`${webgpuModelDisplayName(this.model)} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it before chatting.`);}+ // A caller that can abort gets a key the worker can use to interrupt this generation.+ const signal = options.signal || null;+ const requestKey = signal ? newWebgpuRequestKey() : '';const response = await this._dispatch({type: 'webgpu-chat',model: this.model,@@ -359,12 +389,13 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {device: this.device,dtype: this.dtype,requireTools: this.requiresToolTemplate,+ ...(requestKey ? { requestKey } : {}),messages: this._chatMessages(messages, options),options: {maxTokens: options.maxTokens,tools: Array.isArray(options.tools) ? options.tools : [],},- });+ }, { signal, onAbort: requestKey ? () => this._cancelChat(requestKey) : null });if (!response || response.error) {const error = new Error(`In-browser WebGPU: ${response?.error || 'no response from the inference worker'}`);// A failed OrtRun leaves the WebGPU session/device in an unknown state,@@ -382,6 +413,9 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {toolCalls: Array.isArray(response.toolCalls) && response.toolCalls.length ? response.toolCalls : null,usage: null,raw: response.raw || null,+ // The worker has no finish reason for these models; it reports whether the+ // answer used up its token budget so callers can tell a cut-off from a finish.+ ...(typeof response.outputTruncated === 'boolean' ? { outputTruncated: response.outputTruncated } : {}),};}diff --git a/src/firefox/src/context-menu-storage.js b/src/firefox/src/context-menu-storage.jsindex e6f3ce14a..e8bfc94f2 100644--- a/src/firefox/src/context-menu-storage.js+++ b/src/firefox/src/context-menu-storage.js@@ -296,6 +296,8 @@ export function inlineErrorWasCut(error) {}export function inlineAnswerWasCut(result, maxTokens) {+ // The in-browser WebGPU models have no finish reason; their worker reports the limit directly.+ if (result?.outputTruncated === true) return true;const raw = result?.raw;const reason = [result?.finishReason,diff --git a/test/run.js b/test/run.jsindex 2198b3f3e..afb282827 100644--- a/test/run.js+++ b/test/run.js@@ -51805,6 +51805,8 @@ test('inline card detects an answer that stopped at the output cap on every provassert.equal(wasCut({ content: 'x', usage: { completion_tokens: 400 } }, 2000), false, `${label}: a short answer is complete`);assert.equal(wasCut({ content: 'x' }, 2000), false, `${label}: no signal at all is treated as complete`);assert.equal(wasCut(null, 2000), false, `${label}: a missing result is not a cut-off`);+ assert.equal(wasCut({ content: 'x', outputTruncated: true, usage: null }, 2000), true, `${label}: the in-browser WebGPU worker reports its own limit`);+ assert.equal(wasCut({ content: 'x', outputTruncated: false }, 2000), false, `${label}: a WebGPU answer that finished is complete`);}});@@ -51847,6 +51849,239 @@ test('inline card treats a Responses output-limit error as a cut-off with any pa}});+test('WebGPU worker interrupts a cancelled text generation, skips a queued one and reports a cut-off answer', async () => {+ const previousSelf = globalThis.self;+ const previousState = globalThis.__webgpuCancelTest;+ let workerListener = null;+ const posted = [];+ try {+ globalThis.self = {+ addEventListener(type, listener) {+ if (type === 'message') workerListener = listener;+ },+ postMessage(message) {+ posted.push(message);+ },+ };+ globalThis.__webgpuCancelTest = { calls: [], criteria: [], release: false, content: 'one two three four' };+ const workerUrl = `${pathToFileURL(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js')).href}?text-cancel-test`;+ await import(workerUrl);+ const runtimeModule = `+ export const env = { version: 'test', backends: { onnx: { wasm: {} } } };+ export class InterruptableStoppingCriteria {+ constructor() { this.interrupted = false; globalThis.__webgpuCancelTest.criteria.push(this); }+ interrupt() { this.interrupted = true; }+ reset() { this.interrupted = false; }+ }+ export async function pipeline() {+ const instance = async (input, generation) => {+ const state = globalThis.__webgpuCancelTest;+ state.calls.push(generation);+ // Generate until released, or until the stopping criteria is interrupted.+ while (!state.release && !generation.stopping_criteria?.[0]?.interrupted) {+ await new Promise(resolve => setTimeout(resolve, 1));+ }+ return [{ generated_text: [...input, { role: 'assistant', content: state.content }] }];+ };+ instance.model = {};+ // One token per word: enough to exercise the worker's cut-off check.+ instance.tokenizer = Object.assign(+ text => ({ input_ids: { dims: [1, String(text).split(/\\s+/).filter(Boolean).length] } }),+ { chat_template: '{% if tools %}{{ tools }}{% endif %}{{ messages }}' },+ );+ instance.dispose = async () => {};+ return instance;+ }+ `;+ let requestId = 1;+ const send = async (type, payload = {}) => {+ const id = requestId++;+ await workerListener({ data: { id, type, payload } });+ return posted.find(message => message.id === id);+ };+ const start = (type, payload) => {+ const id = requestId++;+ const done = workerListener({ data: { id, type, payload } }).then(() => posted.find(message => message.id === id));+ return { id, done };+ };+ const until = async (condition, what) => {+ for (let attempt = 0; attempt < 200 && !condition(); attempt++) {+ await new Promise(resolve => setTimeout(resolve, 2));+ }+ assert.ok(condition(), `timed out waiting for ${what}`);+ };+ const state = globalThis.__webgpuCancelTest;+ const initialized = await send('init', {+ transformersUrl: `data:text/javascript,${encodeURIComponent(runtimeModule)}`,+ wasmMjsUrl: 'test.mjs',+ wasmUrl: 'test.wasm',+ });+ assert.equal(initialized.ok, true);+ const payload = {+ modelId: 'text-model',+ device: 'webgpu',+ dtype: 'q4f16',+ messages: [{ role: 'user', content: 'Translate this.' }],+ options: { maxTokens: 4 },+ };+ assert.equal((await send('download-text', payload)).ok, true);++ // A generation that used its whole token budget is reported as cut off; a shorter one is not.+ state.release = true;+ const full = await send('text-chat', payload);+ assert.equal(full.ok, true, full.error);+ assert.equal(full.outputTruncated, true, 'an answer that used the whole budget must be flagged');+ assert.equal(state.calls.at(-1).stopping_criteria, undefined, 'a caller without a request key gets no stopping criteria');+ state.content = 'one two';+ const short = await send('text-chat', payload);+ assert.equal(short.outputTruncated, false, 'an answer well inside the budget is complete');+ state.content = 'one two three four';++ // Cancelling a running generation interrupts it and the caller gets a cancellation, not a result.+ state.release = false;+ const callsBefore = state.calls.length;+ const running = start('text-chat', { ...payload, requestKey: 'card-running-0001' });+ await until(() => state.calls.length === callsBefore + 1, 'the generation to start');+ assert.ok(state.calls.at(-1).stopping_criteria?.[0], 'a keyed generation must be interruptible');+ const cancelled = await send('cancel-text', { requestKey: 'card-running-0001' });+ assert.deepEqual([cancelled.cancelled, cancelled.active], [true, true]);+ const runningResult = await running.done;+ assert.equal(runningResult.ok, false);+ assert.match(runningResult.error, /cancelled/);+ assert.equal(state.criteria.at(-1).interrupted, true, 'the stopping criteria must have been interrupted');++ // A generation still queued behind another is skipped without ever reaching the model.+ const first = start('text-chat', { ...payload, requestKey: 'card-first-00001' });+ await until(() => state.calls.length === callsBefore + 2, 'the first generation to start');+ const second = start('text-chat', { ...payload, requestKey: 'card-second-0002' });+ await new Promise(resolve => setTimeout(resolve, 5));+ const queuedCancel = await send('cancel-text', { requestKey: 'card-second-0002' });+ assert.deepEqual([queuedCancel.cancelled, queuedCancel.queued, queuedCancel.active], [true, true, false]);+ state.release = true;+ assert.equal((await first.done).ok, true);+ const secondResult = await second.done;+ assert.equal(secondResult.ok, false);+ assert.match(secondResult.error, /cancelled/);+ assert.equal(state.calls.length, callsBefore + 2, 'a cancelled queued generation must never start');++ // Unknown, malformed and finished keys cancel nothing.+ for (const requestKey of ['not-a-job', 'x'.repeat(200), 'card-running-0001', '']) {+ const response = await send('cancel-text', { requestKey });+ assert.equal(response.cancelled, false, `a stray key (${requestKey.slice(0, 12)}) must not cancel anything`);+ }+ state.release = true;+ const afterwards = await send('text-chat', { ...payload, requestKey: 'card-running-0001' });+ assert.equal(afterwards.ok, true, 'a finished job must not leave a stale cancellation behind');+ } finally {+ if (previousSelf === undefined) delete globalThis.self;+ else globalThis.self = previousSelf;+ if (previousState === undefined) delete globalThis.__webgpuCancelTest;+ else globalThis.__webgpuCancelTest = previousState;+ }+});++test('WebGPU provider stops waiting and cancels the worker when its caller aborts, and passes the cut-off flag on', async () => {+ const previousChrome = globalThis.chrome;+ const sent = [];+ let respondToChat = null;+ try {+ globalThis.chrome = {+ offscreen: { hasDocument: async () => true },+ runtime: {+ lastError: undefined,+ sendMessage(message, callback) {+ sent.push(message);+ if (message.type === 'webgpu-download-status') callback?.({ ok: true, ready: true });+ else if (message.type === 'webgpu-chat') respondToChat = callback;+ else callback?.({ ok: true });+ },+ },+ };+ const until = async (condition, what) => {+ for (let attempt = 0; attempt < 200 && !condition(); attempt++) {+ await new Promise(resolve => setTimeout(resolve, 2));+ }+ assert.ok(condition(), `timed out waiting for ${what}`);+ };+ const provider = new WebGPUProvider({ model: WEBGPU_MODEL_ID });+ const messages = [{ role: 'user', content: 'Translate this.' }];+ const chatMessages = () => sent.filter(message => message.type === 'webgpu-chat');+ const cancelMessages = () => sent.filter(message => message.type === 'webgpu-chat-cancel');++ // Aborting rejects at once with the caller's reason and tells the worker which job to stop.+ const controller = new AbortController();+ const aborted = provider.chat(messages, { maxTokens: 50, signal: controller.signal });+ await until(() => chatMessages().length === 1, 'the chat request');+ const key = chatMessages()[0].requestKey;+ assert.match(key, /^[a-z0-9-]{8,64}$/i, 'a cancellable request carries a worker-safe key');+ assert.equal(cancelMessages().length, 0);+ controller.abort(new Error('The card was closed.'));+ await assert.rejects(aborted, /The card was closed\./);+ assert.deepEqual(cancelMessages().map(message => message.requestKey), [key], 'the worker is told which job to stop');+ respondToChat({ ok: true, content: 'late answer' });++ // A signal that is already aborted never sends anything to the worker.+ const before = sent.length;+ const preAborted = new AbortController();+ preAborted.abort();+ await assert.rejects(provider.chat(messages, { signal: preAborted.signal }), /abort/i);+ assert.equal(sent.length, before, 'an aborted request must not reach the worker');++ // Without a signal nothing changes: no key, no cancel, and the answer comes back as before.+ const plain = provider.chat(messages, { maxTokens: 50 });+ await until(() => chatMessages().length === 2, 'the unsigned chat request');+ assert.equal('requestKey' in chatMessages()[1], false);+ respondToChat({ ok: true, content: 'Hallo', outputTruncated: true });+ const plainResult = await plain;+ assert.equal(plainResult.content, 'Hallo');+ assert.equal(plainResult.outputTruncated, true, 'the worker\'s cut-off flag reaches the caller');+ assert.equal(cancelMessages().length, 1);++ // A signed request that completes keeps its answer and removes its abort listener.+ const finished = new AbortController();+ const done = provider.chat(messages, { maxTokens: 50, signal: finished.signal });+ await until(() => chatMessages().length === 3, 'the third chat request');+ respondToChat({ ok: true, content: 'Done', outputTruncated: false });+ const doneResult = await done;+ assert.equal(doneResult.outputTruncated, false);+ finished.abort();+ await new Promise(resolve => setTimeout(resolve, 5));+ assert.equal(cancelMessages().length, 1, 'aborting after the answer arrived must not cancel anything');++ // Results without the flag stay as they were for every other caller.+ const legacy = provider.chat(messages, { maxTokens: 50 });+ await until(() => chatMessages().length === 4, 'the fourth chat request');+ respondToChat({ ok: true, content: 'Old shape' });+ assert.equal('outputTruncated' in await legacy, false);+ } finally {+ if (previousChrome === undefined) delete globalThis.chrome;+ else globalThis.chrome = previousChrome;+ }+});++test('WebGPU host and workers carry the cancel request to the generation they stop', () => {+ const host = fs.readFileSync(path.join(ROOT, 'src/chrome/src/offscreen/vision-inference-host.js'), 'utf8');+ const worker = fs.readFileSync(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js'), 'utf8');+ const bonsai = fs.readFileSync(path.join(ROOT, 'src/chrome/src/offscreen/bonsai-worker.js'), 'utf8');++ assert.match(host, /'webgpu-chat',\s*'webgpu-chat-cancel',/, 'the host must accept the cancel message');+ assert.match(host, /requestKey: typeof message\.requestKey === 'string' \? message\.requestKey : '',\s*messages: message\.messages/, 'the host passes the key to the worker that runs the chat');+ const cancel = host.slice(host.indexOf("message.type === 'webgpu-chat-cancel'"), host.indexOf("message.type === 'webgpu-chat') {"));+ assert.match(cancel, /if \(visionWorker\) cancels\.push\(sendVisionWorkerMessage\('cancel-text'/, 'the ONNX worker is asked to cancel');+ assert.match(cancel, /if \(bonsaiWorker\) cancels\.push\(sendBonsaiWorkerMessage\('cancel-text'/, 'the BitGPU worker is asked to cancel');++ // The ONNX worker interrupts the running generation in both text paths.+ assert.equal((worker.match(/stopping_criteria: \[criteria\]/g) || []).length, 2, 'text and multimodal generation both take the stopping criteria');+ assert.equal((worker.match(/throwIfTextCancelled\(requestKey\)/g) || []).length >= 6, true, 'a cancelled request is dropped before and after generation in both paths');+ assert.match(worker, /\.\.\.outputLimitField\(Number\(generated\?\.dims\?\.at\?\.\(-1\)\), maxNewTokens, 1\)/, 'the multimodal path compares the exact generated length with its budget');+ assert.match(worker, /\.\.\.outputLimitField\(countGeneratedTokens\(runtime\.tokenizer, content\), maxNewTokens\)/, 'the text path compares the re-tokenized answer with its budget');++ // The BitGPU worker aborts the runtime's decode loop and reports its own finish reason.+ assert.match(bonsai, /\.\.\.\(signal \? \{ signal \} : \{\}\),/, 'the abort signal reaches the runtime');+ assert.match(bonsai, /controller\?\.abort\(\);/, 'a cancel message aborts the tracked request');+ assert.match(bonsai, /outputTruncated: finishReason === 'length'/, 'a length stop is reported as a cut-off');+});+test('inline card claims its request before waiting, cancels what it abandons and finishes cut-off answers in the panel', () => {for (const prefix of ['src/chrome', 'src/firefox']) {const content = fs.readFileSync(path.join(ROOT, prefix, 'src/content/selection-shortcut.js'), 'utf8');@@ -67054,7 +67289,7 @@ test('WebGPU worker follows local text-generation and WebBrain VL vision contracassert.match(bonsaiWorker, /opfsPartialName/);assert.match(bonsaiWorker, /tools: tools\.length \? tools : undefined/);assert.match(bonsaiWorker, /function normalizeBitgpuToolCalls/);- assert.match(bonsaiWorker, /return \{ content, reasoningContent, toolCalls \}/);+ assert.match(bonsaiWorker, /return \{\s*content,\s*reasoningContent,\s*toolCalls,/);assert.match(bonsaiWorker, /if \(queuedTextDownload === request\) queuedTextDownload = null/,'a completed download request must not clear a newer queued resume for the same model');assert.match(apocalypseScript, /if \(!preset \|\| preset\.id !== WEBGPU_COMPASS_TINY_V2_MODEL_ID\)[\s\S]*?setWebgpuDownloadState\(state, \{ syncActiveTransfer: true \}\)[\s\S]*?ensureFixedWebgpuProvider\(\{ force: true \}\)[\s\S]*?get_webgpu_download_status/,