Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Make the in-browser WebGPU models report their output limit and stop on cancel

Claudeaddd83f155f1
diff --git a/docs/architecture.md b/docs/architecture.md
index 521f6a01d..7b54f3f27 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -527,7 +527,10 @@ answers with whatever text the provider returned) the response
is flagged `truncated`, and the card says so and offers **Continue in side panel** instead
of passing a cut-off translation off as complete. Each call carries a job id, and a card
that is closed, retried or retargeted sends `WB_SELECTION_SHORTCUT_INLINE_CANCEL` so the
-background aborts the paid request it no longer needs; the card also claims its request id
+background aborts the paid request it no longer needs (for the in-browser WebGPU models the
+provider also sends `webgpu-chat-cancel`, and the offscreen worker interrupts the running
+generation or drops a queued one; those workers report `outputTruncated` when an answer used
+its token budget, since they have no finish reason); the card also claims its request id
before it waits for the language bundle, so a double click starts only one call. Selections
over 4,000 characters skip the card and use the side-panel flow. The card draws the answer as DOM
nodes (paragraphs, lists, bold, italic and code), never as HTML, and takes the source
diff --git a/src/chrome/src/context-menu-storage.js b/src/chrome/src/context-menu-storage.js
index 199722d7e..56375f671 100644
--- a/src/chrome/src/context-menu-storage.js
+++ b/src/chrome/src/context-menu-storage.js
@@ -296,6 +296,8 @@ export function inlineErrorWasCut(error) {
}
export function inlineAnswerWasCut(result, maxTokens) {
+ // The in-browser WebGPU models have no finish reason; their worker reports the limit directly.
+ if (result?.outputTruncated === true) return true;
const raw = result?.raw;
const reason = [
result?.finishReason,
diff --git a/src/chrome/src/offscreen/bonsai-worker.js b/src/chrome/src/offscreen/bonsai-worker.js
index 3c593764f..18aa00c18 100644
--- a/src/chrome/src/offscreen/bonsai-worker.js
+++ b/src/chrome/src/offscreen/bonsai-worker.js
@@ -16,6 +16,22 @@ const CACHE_NAME = 'bitgpu-models-v1';
const TEXT_DOWNLOAD_EVENT = 'text-download-state';
const WEBGPU_BONSAI27_MAX_NEW_TOKENS = 2048;
const WEBGPU_BONSAI27_THINK_BUDGET = 128;
+// Text generations that carry a requestKey can be cancelled by their caller; the
+// controller's signal is handed to the runtime, which stops decoding when it fires.
+const textRequestControllers = new Map();
+const TEXT_REQUEST_KEY_PATTERN = /^[a-z0-9-]{8,64}$/i;
+
+function textRequestKey(payload) {
+ const key = typeof payload?.requestKey === 'string' ? payload.requestKey : '';
+ return TEXT_REQUEST_KEY_PATTERN.test(key) ? key : '';
+}
+
+function throwIfTextCancelled(signal) {
+ if (!signal?.aborted) return;
+ const error = new Error('The generation was cancelled.');
+ error.name = 'AbortError';
+ throw error;
+}
const WEBGPU_BONSAI27_MAX_SEQ_LEN = 4096;
const OOM_HINT = 'This machine cannot hold Bonsai 27B in GPU memory. Use LFM2.5 2.6B instead.';
@@ -828,12 +844,14 @@ function normalizeBitgpuToolCalls(toolCalls) {
}).filter(Boolean);
}
-async function runText(payload) {
+async function runText(payload, signal = null) {
+ throwIfTextCancelled(signal);
const modelId = assertBonsaiModel(payload?.modelId);
if (!await isTextModelReady(modelId, payload?.dtype || WEBGPU_BONSAI27_DTYPE)) {
throw new Error(`${modelId} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it before chatting.`);
}
const runtime = await getTextRuntime({ localFilesOnly: true });
+ throwIfTextCancelled(signal);
const requestedTokens = Number(payload?.options?.maxTokens);
const maxTokens = Number.isFinite(requestedTokens)
? Math.max(1, Math.min(WEBGPU_BONSAI27_MAX_NEW_TOKENS, Math.round(requestedTokens)))
@@ -849,10 +867,13 @@ async function runText(payload) {
topP: 0.85,
topK: 20,
tools: tools.length ? tools : undefined,
+ ...(signal ? { signal } : {}),
});
} catch (error) {
+ if (signal?.aborted) throwIfTextCancelled(signal);
throw mapGpuError(error);
}
+ throwIfTextCancelled(signal);
const content = String(result?.text || '').trim();
const reasoningContent = String(result?.thinkText || '').trim() || null;
const toolCalls = normalizeBitgpuToolCalls(result?.toolCalls);
@@ -860,7 +881,14 @@ async function runText(payload) {
throw new Error(`${modelId} used its generation budget before finishing reasoning. Retry with a shorter prompt.`);
}
if (!content && !toolCalls.length) throw new Error('The WebGPU model returned no generated text or tool calls.');
- return { content, reasoningContent, toolCalls };
+ // The runtime reports why decoding stopped, so a cut-off answer is not mistaken for a finished one.
+ const finishReason = typeof result?.finishReason === 'string' ? result.finishReason : '';
+ return {
+ content,
+ reasoningContent,
+ toolCalls,
+ ...(finishReason ? { outputTruncated: finishReason === 'length' } : {}),
+ };
}
async function probeRuntime() {
@@ -950,14 +978,28 @@ self.addEventListener('message', async (event) => {
self.postMessage({ id, ok: true, disposed: true });
return;
}
+ if (type === 'cancel-text') {
+ const requestKey = typeof payload?.requestKey === 'string' ? payload.requestKey : '';
+ const controller = textRequestControllers.get(requestKey);
+ controller?.abort();
+ self.postMessage({ id, ok: true, cancelled: !!controller });
+ return;
+ }
if (type === 'text-chat') {
- const result = await enqueueModelOperation(() => runText(payload));
- self.postMessage({
- id,
- ok: true,
- ...result,
- raw: { model: payload?.modelId || WEBGPU_BONSAI27_MODEL_ID },
- });
+ const requestKey = textRequestKey(payload);
+ const controller = requestKey ? new AbortController() : null;
+ if (controller) textRequestControllers.set(requestKey, controller);
+ try {
+ const result = await enqueueModelOperation(() => runText(payload, controller?.signal || null));
+ self.postMessage({
+ id,
+ ok: true,
+ ...result,
+ raw: { model: payload?.modelId || WEBGPU_BONSAI27_MODEL_ID },
+ });
+ } finally {
+ if (controller && textRequestControllers.get(requestKey) === controller) textRequestControllers.delete(requestKey);
+ }
return;
}
throw new Error(`Unknown Bonsai WebGPU worker message: ${type || 'missing type'}`);
diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.js
index 008753f78..2db952cd0 100644
--- a/src/chrome/src/offscreen/inference-worker.js
+++ b/src/chrome/src/offscreen/inference-worker.js
@@ -102,6 +102,57 @@ let visionDownloadAbortController = null;
const activeVisionGenerations = new Map();
const queuedVisionGenerations = new Set();
const cancelledVisionGenerations = new Set();
+// Text generations that carry a requestKey (the in-page answer cards do) can be
+// cancelled: one still queued never starts, and a running one is interrupted at
+// its next token instead of finishing a result nobody is waiting for.
+const queuedTextGenerations = new Set();
+const activeTextGenerations = new Map();
+const cancelledTextGenerations = new Set();
+const TEXT_REQUEST_KEY_PATTERN = /^[a-z0-9-]{8,64}$/i;
+// A re-tokenized answer that used this much of its budget is reported as cut off.
+const WEBGPU_OUTPUT_LIMIT_RATIO = 0.97;
+
+function textRequestKey(payload) {
+ const key = typeof payload?.requestKey === 'string' ? payload.requestKey : '';
+ return TEXT_REQUEST_KEY_PATTERN.test(key) ? key : '';
+}
+
+function throwIfTextCancelled(requestKey) {
+ if (!requestKey || !cancelledTextGenerations.has(requestKey)) return;
+ const error = new Error('The generation was cancelled.');
+ error.name = 'AbortError';
+ throw error;
+}
+
+function beginTextGeneration(requestKey, library) {
+ if (!requestKey || !library?.InterruptableStoppingCriteria) return null;
+ const criteria = new library.InterruptableStoppingCriteria();
+ activeTextGenerations.set(requestKey, criteria);
+ return criteria;
+}
+
+function endTextGeneration(requestKey) {
+ if (requestKey) activeTextGenerations.delete(requestKey);
+}
+
+// Tokens in a generated text, counted with the model's own tokenizer; null when it cannot.
+function countGeneratedTokens(tokenizer, text) {
+ try {
+ if (typeof tokenizer !== 'function' || !text) return null;
+ const count = Number(tokenizer(text, { add_special_tokens: false })?.input_ids?.dims?.at?.(-1));
+ return Number.isFinite(count) ? count : null;
+ } catch {
+ return null;
+ }
+}
+
+// The worker has no finish reason for these models, so a generation that used
+// (nearly) all of its token budget is reported as cut off for the caller to act on.
+function outputLimitField(generatedTokens, maxNewTokens, ratio = WEBGPU_OUTPUT_LIMIT_RATIO) {
+ return Number.isFinite(generatedTokens) && maxNewTokens > 0
+ ? { outputTruncated: generatedTokens >= Math.max(1, Math.floor(maxNewTokens * ratio)) }
+ : {};
+}
let lastTextProgressPostAt = 0;
let webGpuAdapterProbePromise = null;
let webGpuAdapterSummary = '';
@@ -1326,6 +1377,8 @@ function addLegacyVlTools(messages, tools) {
}
async function runMultimodalText(payload) {
+ const requestKey = textRequestKey(payload);
+ throwIfTextCancelled(requestKey);
const modelId = assertOnnxTextModel(payload?.modelId);
if (!isLfm25VlModel(modelId)) {
throw new Error(`${modelId} is not a shipped WebGPU multimodal model.`);
@@ -1344,6 +1397,7 @@ async function runMultimodalText(payload) {
owner: 'text',
readiness: 'text',
});
+ throwIfTextCancelled(requestKey);
const tools = Array.isArray(payload?.options?.tools) ? payload.options.tools : [];
let { messages, imageUrls } = prepareMultimodalMessages(payload?.messages);
if (modelId === WEBGPU_LFM25_VL_16B_MODEL_ID) messages = addLegacyVlTools(messages, tools);
@@ -1366,26 +1420,38 @@ async function runMultimodalText(payload) {
: 800;
lastWebGpuDeviceError = '';
lastWebGpuDeviceLost = '';
+ const criteria = beginTextGeneration(requestKey, runtime.library);
let outputs;
try {
outputs = await runtime.model.generate({
...inputs,
do_sample: false,
max_new_tokens: maxNewTokens,
+ ...(criteria ? { stopping_criteria: [criteria] } : {}),
});
} catch (error) {
if (isWebGpuExecutionFailure(error)) throw await handleWebGpuExecutionFailure(error);
throw error;
+ } finally {
+ endTextGeneration(requestKey);
}
+ throwIfTextCancelled(requestKey);
const inputLength = inputs.input_ids.dims.at(-1);
const generated = outputs.slice(null, [inputLength, null]);
const decoded = runtime.processor.batch_decode(generated, { skip_special_tokens: true });
const result = splitThinking(String(decoded?.[0] || '').trim());
noteWebgpuExecutionSuccess();
- return { content: result.content, reasoningContent: result.reasoningContent };
+ // The generated tensor is exact, so reaching the budget means no end-of-text token came first.
+ return {
+ content: result.content,
+ reasoningContent: result.reasoningContent,
+ ...outputLimitField(Number(generated?.dims?.at?.(-1)), maxNewTokens, 1),
+ };
}
async function runText(payload) {
+ const requestKey = textRequestKey(payload);
+ throwIfTextCancelled(requestKey);
const modelId = assertOnnxTextModel(payload?.modelId);
const device = payload?.device || 'webgpu';
const dtype = payload?.dtype || 'q4f16';
@@ -1397,6 +1463,7 @@ async function runText(payload) {
throw new Error(`${modelId} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it before chatting.`);
}
const runtime = await getTextRuntime(modelId, dtype, device, { localFilesOnly: true });
+ throwIfTextCancelled(requestKey);
if (payload?.requireTools === true) assertToolCapableTextRuntime(runtime, modelId);
const requestedTokens = Number(payload?.options?.maxTokens);
const maxTokenLimit = usesLongOutputBudget
@@ -1408,12 +1475,14 @@ async function runText(payload) {
const tools = Array.isArray(payload?.options?.tools) ? payload.options.tools : [];
lastWebGpuDeviceError = '';
lastWebGpuDeviceLost = '';
+ const criteria = beginTextGeneration(requestKey, runtime.library);
let output;
try {
output = await runtime.pipeline(prepareTextMessages(payload?.messages), {
do_sample: Boolean(sampling),
...(sampling || {}),
max_new_tokens: maxNewTokens,
+ ...(criteria ? { stopping_criteria: [criteria] } : {}),
tools: tools.length ? tools : undefined,
// Reasoning templates re-open `<think>` for the pending turn and collapse
// completed thinking in history; the rest suppress thinking outright.
@@ -1424,7 +1493,10 @@ async function runText(payload) {
} catch (error) {
if (isWebGpuExecutionFailure(error)) throw await handleWebGpuExecutionFailure(error);
throw error;
+ } finally {
+ endTextGeneration(requestKey);
}
+ throwIfTextCancelled(requestKey);
const generated = output?.[0]?.generated_text;
const content = Array.isArray(generated)
? generated.at(-1)?.content
@@ -1437,7 +1509,11 @@ async function runText(payload) {
throw new Error(`${modelId} used its generation budget before finishing reasoning. Retry with a shorter prompt.`);
}
noteWebgpuExecutionSuccess();
- return { content: result.content, reasoningContent: result.reasoningContent };
+ return {
+ content: result.content,
+ reasoningContent: result.reasoningContent,
+ ...outputLimitField(countGeneratedTokens(runtime.tokenizer, content), maxNewTokens),
+ };
}
async function probeRuntime() {
@@ -1501,6 +1577,18 @@ self.addEventListener('message', async event => {
});
return;
}
+ if (type === 'cancel-text') {
+ const requestKey = typeof payload?.requestKey === 'string' ? payload.requestKey : '';
+ const queued = queuedTextGenerations.has(requestKey);
+ const active = activeTextGenerations.has(requestKey);
+ const cancellable = TEXT_REQUEST_KEY_PATTERN.test(requestKey) && (queued || active);
+ if (cancellable) {
+ cancelledTextGenerations.add(requestKey);
+ activeTextGenerations.get(requestKey)?.interrupt?.();
+ }
+ self.postMessage({ id, ok: true, cancelled: cancellable, queued: queued && !active, active });
+ return;
+ }
if (type === 'init') {
workerConfig = payload;
self.postMessage({ id, ok: true });
@@ -1649,24 +1737,25 @@ self.addEventListener('message', async event => {
}
return;
}
- if (type === 'text-chat') {
- const result = await enqueueModelOperation(() => runText(payload));
- self.postMessage({
- id,
- ok: true,
- ...result,
- raw: { model: payload?.modelId || '' },
- });
- return;
- }
- if (type === 'multimodal-text-chat') {
- const result = await enqueueModelOperation(() => runMultimodalText(payload));
- self.postMessage({
- id,
- ok: true,
- ...result,
- raw: { model: payload?.modelId || '' },
- });
+ if (type === 'text-chat' || type === 'multimodal-text-chat') {
+ const requestKey = textRequestKey(payload);
+ if (requestKey) queuedTextGenerations.add(requestKey);
+ try {
+ const result = await enqueueModelOperation(
+ () => (type === 'text-chat' ? runText(payload) : runMultimodalText(payload)),
+ );
+ self.postMessage({
+ id,
+ ok: true,
+ ...result,
+ raw: { model: payload?.modelId || '' },
+ });
+ } finally {
+ if (requestKey) {
+ queuedTextGenerations.delete(requestKey);
+ cancelledTextGenerations.delete(requestKey);
+ }
+ }
return;
}
throw new Error(`Unknown WebGPU worker message: ${type || 'missing type'}`);
diff --git a/src/chrome/src/offscreen/vision-inference-host.js b/src/chrome/src/offscreen/vision-inference-host.js
index c8f682946..b94e3ce79 100644
--- a/src/chrome/src/offscreen/vision-inference-host.js
+++ b/src/chrome/src/offscreen/vision-inference-host.js
@@ -538,6 +538,7 @@ async function ensureVisionWorker() {
const WEBGPU_MESSAGE_TYPES = new Set([
'webgpu-chat',
+ 'webgpu-chat-cancel',
'webgpu-download-start',
'webgpu-download-pause',
'webgpu-download-stop',
@@ -716,6 +717,17 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
sendResponse(disposed[0] || { ok: true, disposed: true });
return;
}
+ if (message.type === 'webgpu-chat-cancel') {
+ // Stop a text generation its caller no longer wants. Both workers are asked
+ // because the host does not know which model the request was routed to.
+ const requestKey = typeof message.requestKey === 'string' ? message.requestKey : '';
+ const cancels = [];
+ if (visionWorker) cancels.push(sendVisionWorkerMessage('cancel-text', { requestKey }).catch(() => null));
+ if (bonsaiWorker) cancels.push(sendBonsaiWorkerMessage('cancel-text', { requestKey }).catch(() => null));
+ await Promise.all(cancels);
+ sendResponse({ ok: true });
+ return;
+ }
if (message.type === 'webgpu-chat') {
const workerMessageType = message.runtime === 'onnx-vl'
? 'multimodal-text-chat'
@@ -725,6 +737,7 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
device: message.device,
dtype: message.dtype,
requireTools: message.requireTools === true,
+ requestKey: typeof message.requestKey === 'string' ? message.requestKey : '',
messages: message.messages || [],
options: message.options || {},
}, { exclusive: true, runtime: message.runtime }));
diff --git a/src/chrome/src/providers/webgpu.js b/src/chrome/src/providers/webgpu.js
index dcb988864..42c13d936 100644
--- a/src/chrome/src/providers/webgpu.js
+++ b/src/chrome/src/providers/webgpu.js
@@ -2,6 +2,7 @@
import { BaseLLMProvider } from './base.js';
import { ensureOffscreen } from '../offscreen/ensure.js';
+import { responseAbortError } from '../network/response-body.js';
export const WEBGPU_VISION_MODEL_ID = 'webbrain-one/webbrain-vl-2-450M-onnx';
export const WEBGPU_MODEL_ID = 'LiquidAI/LFM2.5-2.6B-ONNX';
@@ -240,8 +241,18 @@ export function webgpuModelRequiresToolTemplate(modelId) {
return !isShippedWebgpuPreset(normalized);
}
+function newWebgpuRequestKey() {
+ return globalThis.crypto?.randomUUID?.()
+ || `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 12)}`;
+}
+
class WebGPUOffscreenProvider extends BaseLLMProvider {
- async _dispatch(message, { timeoutMs = 0 } = {}) {
+ /**
+ * Send one request to the offscreen host. With a signal, an abort rejects at
+ * once (the caller stops waiting) and `onAbort` tells the host to stop the work.
+ */
+ async _dispatch(message, { timeoutMs = 0, signal = null, onAbort = null } = {}) {
+ if (signal?.aborted) throw responseAbortError(signal);
await ensureOffscreen();
return await new Promise((resolve, reject) => {
let settled = false;
@@ -249,8 +260,13 @@ class WebGPUOffscreenProvider extends BaseLLMProvider {
if (settled) return;
settled = true;
if (timeoutId) clearTimeout(timeoutId);
+ signal?.removeEventListener?.('abort', abortListener);
callback(value);
};
+ const abortListener = () => {
+ try { onAbort?.(); } catch { /* the request is abandoned either way */ }
+ finish(reject, responseAbortError(signal));
+ };
const timeoutId = timeoutMs > 0
? setTimeout(() => {
const error = new Error(`In-browser WebGPU request timed out after ${timeoutMs}ms (${message?.type || 'unknown'}).`);
@@ -258,6 +274,7 @@ class WebGPUOffscreenProvider extends BaseLLMProvider {
finish(reject, error);
}, timeoutMs)
: null;
+ signal?.addEventListener?.('abort', abortListener, { once: true });
try {
chrome.runtime.sendMessage(message, (response) => {
const lastError = chrome.runtime.lastError;
@@ -270,6 +287,15 @@ class WebGPUOffscreenProvider extends BaseLLMProvider {
});
}
+ // Ask the offscreen host to stop a text generation nobody is waiting for any more.
+ _cancelChat(requestKey) {
+ try {
+ chrome.runtime.sendMessage({ type: 'webgpu-chat-cancel', requestKey }, () => {
+ void chrome.runtime.lastError;
+ });
+ } catch { /* best effort: the generation then finishes on its own */ }
+ }
+
async _testWebGPU() {
try {
const response = await this._dispatch(
@@ -345,6 +371,7 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {
}
async chat(messages, options = {}) {
+ if (options.signal?.aborted) throw responseAbortError(options.signal);
if (this._messagesContainImage(messages) && !this.supportsVision) {
throw new Error('The WebGPU chat model is text-only. Configure a separate model under Settings -> Multimodal for screenshots.');
}
@@ -352,6 +379,9 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {
if (!download.ready) {
throw new Error(`${webgpuModelDisplayName(this.model)} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it before chatting.`);
}
+ // A caller that can abort gets a key the worker can use to interrupt this generation.
+ const signal = options.signal || null;
+ const requestKey = signal ? newWebgpuRequestKey() : '';
const response = await this._dispatch({
type: 'webgpu-chat',
model: this.model,
@@ -359,12 +389,13 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {
device: this.device,
dtype: this.dtype,
requireTools: this.requiresToolTemplate,
+ ...(requestKey ? { requestKey } : {}),
messages: this._chatMessages(messages, options),
options: {
maxTokens: options.maxTokens,
tools: Array.isArray(options.tools) ? options.tools : [],
},
- });
+ }, { signal, onAbort: requestKey ? () => this._cancelChat(requestKey) : null });
if (!response || response.error) {
const error = new Error(`In-browser WebGPU: ${response?.error || 'no response from the inference worker'}`);
// A failed OrtRun leaves the WebGPU session/device in an unknown state,
@@ -382,6 +413,9 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {
toolCalls: Array.isArray(response.toolCalls) && response.toolCalls.length ? response.toolCalls : null,
usage: null,
raw: response.raw || null,
+ // The worker has no finish reason for these models; it reports whether the
+ // answer used up its token budget so callers can tell a cut-off from a finish.
+ ...(typeof response.outputTruncated === 'boolean' ? { outputTruncated: response.outputTruncated } : {}),
};
}
diff --git a/src/firefox/src/context-menu-storage.js b/src/firefox/src/context-menu-storage.js
index e6f3ce14a..e8bfc94f2 100644
--- a/src/firefox/src/context-menu-storage.js
+++ b/src/firefox/src/context-menu-storage.js
@@ -296,6 +296,8 @@ export function inlineErrorWasCut(error) {
}
export function inlineAnswerWasCut(result, maxTokens) {
+ // The in-browser WebGPU models have no finish reason; their worker reports the limit directly.
+ if (result?.outputTruncated === true) return true;
const raw = result?.raw;
const reason = [
result?.finishReason,
diff --git a/test/run.js b/test/run.js
index 2198b3f3e..afb282827 100644
--- a/test/run.js
+++ b/test/run.js
@@ -51805,6 +51805,8 @@ test('inline card detects an answer that stopped at the output cap on every prov
assert.equal(wasCut({ content: 'x', usage: { completion_tokens: 400 } }, 2000), false, `${label}: a short answer is complete`);
assert.equal(wasCut({ content: 'x' }, 2000), false, `${label}: no signal at all is treated as complete`);
assert.equal(wasCut(null, 2000), false, `${label}: a missing result is not a cut-off`);
+ assert.equal(wasCut({ content: 'x', outputTruncated: true, usage: null }, 2000), true, `${label}: the in-browser WebGPU worker reports its own limit`);
+ assert.equal(wasCut({ content: 'x', outputTruncated: false }, 2000), false, `${label}: a WebGPU answer that finished is complete`);
}
});
@@ -51847,6 +51849,239 @@ test('inline card treats a Responses output-limit error as a cut-off with any pa
}
});
+test('WebGPU worker interrupts a cancelled text generation, skips a queued one and reports a cut-off answer', async () => {
+ const previousSelf = globalThis.self;
+ const previousState = globalThis.__webgpuCancelTest;
+ let workerListener = null;
+ const posted = [];
+ try {
+ globalThis.self = {
+ addEventListener(type, listener) {
+ if (type === 'message') workerListener = listener;
+ },
+ postMessage(message) {
+ posted.push(message);
+ },
+ };
+ globalThis.__webgpuCancelTest = { calls: [], criteria: [], release: false, content: 'one two three four' };
+ const workerUrl = `${pathToFileURL(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js')).href}?text-cancel-test`;
+ await import(workerUrl);
+ const runtimeModule = `
+ export const env = { version: 'test', backends: { onnx: { wasm: {} } } };
+ export class InterruptableStoppingCriteria {
+ constructor() { this.interrupted = false; globalThis.__webgpuCancelTest.criteria.push(this); }
+ interrupt() { this.interrupted = true; }
+ reset() { this.interrupted = false; }
+ }
+ export async function pipeline() {
+ const instance = async (input, generation) => {
+ const state = globalThis.__webgpuCancelTest;
+ state.calls.push(generation);
+ // Generate until released, or until the stopping criteria is interrupted.
+ while (!state.release && !generation.stopping_criteria?.[0]?.interrupted) {
+ await new Promise(resolve => setTimeout(resolve, 1));
+ }
+ return [{ generated_text: [...input, { role: 'assistant', content: state.content }] }];
+ };
+ instance.model = {};
+ // One token per word: enough to exercise the worker's cut-off check.
+ instance.tokenizer = Object.assign(
+ text => ({ input_ids: { dims: [1, String(text).split(/\\s+/).filter(Boolean).length] } }),
+ { chat_template: '{% if tools %}{{ tools }}{% endif %}{{ messages }}' },
+ );
+ instance.dispose = async () => {};
+ return instance;
+ }
+ `;
+ let requestId = 1;
+ const send = async (type, payload = {}) => {
+ const id = requestId++;
+ await workerListener({ data: { id, type, payload } });
+ return posted.find(message => message.id === id);
+ };
+ const start = (type, payload) => {
+ const id = requestId++;
+ const done = workerListener({ data: { id, type, payload } }).then(() => posted.find(message => message.id === id));
+ return { id, done };
+ };
+ const until = async (condition, what) => {
+ for (let attempt = 0; attempt < 200 && !condition(); attempt++) {
+ await new Promise(resolve => setTimeout(resolve, 2));
+ }
+ assert.ok(condition(), `timed out waiting for ${what}`);
+ };
+ const state = globalThis.__webgpuCancelTest;
+ const initialized = await send('init', {
+ transformersUrl: `data:text/javascript,${encodeURIComponent(runtimeModule)}`,
+ wasmMjsUrl: 'test.mjs',
+ wasmUrl: 'test.wasm',
+ });
+ assert.equal(initialized.ok, true);
+ const payload = {
+ modelId: 'text-model',
+ device: 'webgpu',
+ dtype: 'q4f16',
+ messages: [{ role: 'user', content: 'Translate this.' }],
+ options: { maxTokens: 4 },
+ };
+ assert.equal((await send('download-text', payload)).ok, true);
+
+ // A generation that used its whole token budget is reported as cut off; a shorter one is not.
+ state.release = true;
+ const full = await send('text-chat', payload);
+ assert.equal(full.ok, true, full.error);
+ assert.equal(full.outputTruncated, true, 'an answer that used the whole budget must be flagged');
+ assert.equal(state.calls.at(-1).stopping_criteria, undefined, 'a caller without a request key gets no stopping criteria');
+ state.content = 'one two';
+ const short = await send('text-chat', payload);
+ assert.equal(short.outputTruncated, false, 'an answer well inside the budget is complete');
+ state.content = 'one two three four';
+
+ // Cancelling a running generation interrupts it and the caller gets a cancellation, not a result.
+ state.release = false;
+ const callsBefore = state.calls.length;
+ const running = start('text-chat', { ...payload, requestKey: 'card-running-0001' });
+ await until(() => state.calls.length === callsBefore + 1, 'the generation to start');
+ assert.ok(state.calls.at(-1).stopping_criteria?.[0], 'a keyed generation must be interruptible');
+ const cancelled = await send('cancel-text', { requestKey: 'card-running-0001' });
+ assert.deepEqual([cancelled.cancelled, cancelled.active], [true, true]);
+ const runningResult = await running.done;
+ assert.equal(runningResult.ok, false);
+ assert.match(runningResult.error, /cancelled/);
+ assert.equal(state.criteria.at(-1).interrupted, true, 'the stopping criteria must have been interrupted');
+
+ // A generation still queued behind another is skipped without ever reaching the model.
+ const first = start('text-chat', { ...payload, requestKey: 'card-first-00001' });
+ await until(() => state.calls.length === callsBefore + 2, 'the first generation to start');
+ const second = start('text-chat', { ...payload, requestKey: 'card-second-0002' });
+ await new Promise(resolve => setTimeout(resolve, 5));
+ const queuedCancel = await send('cancel-text', { requestKey: 'card-second-0002' });
+ assert.deepEqual([queuedCancel.cancelled, queuedCancel.queued, queuedCancel.active], [true, true, false]);
+ state.release = true;
+ assert.equal((await first.done).ok, true);
+ const secondResult = await second.done;
+ assert.equal(secondResult.ok, false);
+ assert.match(secondResult.error, /cancelled/);
+ assert.equal(state.calls.length, callsBefore + 2, 'a cancelled queued generation must never start');
+
+ // Unknown, malformed and finished keys cancel nothing.
+ for (const requestKey of ['not-a-job', 'x'.repeat(200), 'card-running-0001', '']) {
+ const response = await send('cancel-text', { requestKey });
+ assert.equal(response.cancelled, false, `a stray key (${requestKey.slice(0, 12)}) must not cancel anything`);
+ }
+ state.release = true;
+ const afterwards = await send('text-chat', { ...payload, requestKey: 'card-running-0001' });
+ assert.equal(afterwards.ok, true, 'a finished job must not leave a stale cancellation behind');
+ } finally {
+ if (previousSelf === undefined) delete globalThis.self;
+ else globalThis.self = previousSelf;
+ if (previousState === undefined) delete globalThis.__webgpuCancelTest;
+ else globalThis.__webgpuCancelTest = previousState;
+ }
+});
+
+test('WebGPU provider stops waiting and cancels the worker when its caller aborts, and passes the cut-off flag on', async () => {
+ const previousChrome = globalThis.chrome;
+ const sent = [];
+ let respondToChat = null;
+ try {
+ globalThis.chrome = {
+ offscreen: { hasDocument: async () => true },
+ runtime: {
+ lastError: undefined,
+ sendMessage(message, callback) {
+ sent.push(message);
+ if (message.type === 'webgpu-download-status') callback?.({ ok: true, ready: true });
+ else if (message.type === 'webgpu-chat') respondToChat = callback;
+ else callback?.({ ok: true });
+ },
+ },
+ };
+ const until = async (condition, what) => {
+ for (let attempt = 0; attempt < 200 && !condition(); attempt++) {
+ await new Promise(resolve => setTimeout(resolve, 2));
+ }
+ assert.ok(condition(), `timed out waiting for ${what}`);
+ };
+ const provider = new WebGPUProvider({ model: WEBGPU_MODEL_ID });
+ const messages = [{ role: 'user', content: 'Translate this.' }];
+ const chatMessages = () => sent.filter(message => message.type === 'webgpu-chat');
+ const cancelMessages = () => sent.filter(message => message.type === 'webgpu-chat-cancel');
+
+ // Aborting rejects at once with the caller's reason and tells the worker which job to stop.
+ const controller = new AbortController();
+ const aborted = provider.chat(messages, { maxTokens: 50, signal: controller.signal });
+ await until(() => chatMessages().length === 1, 'the chat request');
+ const key = chatMessages()[0].requestKey;
+ assert.match(key, /^[a-z0-9-]{8,64}$/i, 'a cancellable request carries a worker-safe key');
+ assert.equal(cancelMessages().length, 0);
+ controller.abort(new Error('The card was closed.'));
+ await assert.rejects(aborted, /The card was closed\./);
+ assert.deepEqual(cancelMessages().map(message => message.requestKey), [key], 'the worker is told which job to stop');
+ respondToChat({ ok: true, content: 'late answer' });
+
+ // A signal that is already aborted never sends anything to the worker.
+ const before = sent.length;
+ const preAborted = new AbortController();
+ preAborted.abort();
+ await assert.rejects(provider.chat(messages, { signal: preAborted.signal }), /abort/i);
+ assert.equal(sent.length, before, 'an aborted request must not reach the worker');
+
+ // Without a signal nothing changes: no key, no cancel, and the answer comes back as before.
+ const plain = provider.chat(messages, { maxTokens: 50 });
+ await until(() => chatMessages().length === 2, 'the unsigned chat request');
+ assert.equal('requestKey' in chatMessages()[1], false);
+ respondToChat({ ok: true, content: 'Hallo', outputTruncated: true });
+ const plainResult = await plain;
+ assert.equal(plainResult.content, 'Hallo');
+ assert.equal(plainResult.outputTruncated, true, 'the worker\'s cut-off flag reaches the caller');
+ assert.equal(cancelMessages().length, 1);
+
+ // A signed request that completes keeps its answer and removes its abort listener.
+ const finished = new AbortController();
+ const done = provider.chat(messages, { maxTokens: 50, signal: finished.signal });
+ await until(() => chatMessages().length === 3, 'the third chat request');
+ respondToChat({ ok: true, content: 'Done', outputTruncated: false });
+ const doneResult = await done;
+ assert.equal(doneResult.outputTruncated, false);
+ finished.abort();
+ await new Promise(resolve => setTimeout(resolve, 5));
+ assert.equal(cancelMessages().length, 1, 'aborting after the answer arrived must not cancel anything');
+
+ // Results without the flag stay as they were for every other caller.
+ const legacy = provider.chat(messages, { maxTokens: 50 });
+ await until(() => chatMessages().length === 4, 'the fourth chat request');
+ respondToChat({ ok: true, content: 'Old shape' });
+ assert.equal('outputTruncated' in await legacy, false);
+ } finally {
+ if (previousChrome === undefined) delete globalThis.chrome;
+ else globalThis.chrome = previousChrome;
+ }
+});
+
+test('WebGPU host and workers carry the cancel request to the generation they stop', () => {
+ const host = fs.readFileSync(path.join(ROOT, 'src/chrome/src/offscreen/vision-inference-host.js'), 'utf8');
+ const worker = fs.readFileSync(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js'), 'utf8');
+ const bonsai = fs.readFileSync(path.join(ROOT, 'src/chrome/src/offscreen/bonsai-worker.js'), 'utf8');
+
+ assert.match(host, /'webgpu-chat',\s*'webgpu-chat-cancel',/, 'the host must accept the cancel message');
+ assert.match(host, /requestKey: typeof message\.requestKey === 'string' \? message\.requestKey : '',\s*messages: message\.messages/, 'the host passes the key to the worker that runs the chat');
+ const cancel = host.slice(host.indexOf("message.type === 'webgpu-chat-cancel'"), host.indexOf("message.type === 'webgpu-chat') {"));
+ assert.match(cancel, /if \(visionWorker\) cancels\.push\(sendVisionWorkerMessage\('cancel-text'/, 'the ONNX worker is asked to cancel');
+ assert.match(cancel, /if \(bonsaiWorker\) cancels\.push\(sendBonsaiWorkerMessage\('cancel-text'/, 'the BitGPU worker is asked to cancel');
+
+ // The ONNX worker interrupts the running generation in both text paths.
+ assert.equal((worker.match(/stopping_criteria: \[criteria\]/g) || []).length, 2, 'text and multimodal generation both take the stopping criteria');
+ assert.equal((worker.match(/throwIfTextCancelled\(requestKey\)/g) || []).length >= 6, true, 'a cancelled request is dropped before and after generation in both paths');
+ assert.match(worker, /\.\.\.outputLimitField\(Number\(generated\?\.dims\?\.at\?\.\(-1\)\), maxNewTokens, 1\)/, 'the multimodal path compares the exact generated length with its budget');
+ assert.match(worker, /\.\.\.outputLimitField\(countGeneratedTokens\(runtime\.tokenizer, content\), maxNewTokens\)/, 'the text path compares the re-tokenized answer with its budget');
+
+ // The BitGPU worker aborts the runtime's decode loop and reports its own finish reason.
+ assert.match(bonsai, /\.\.\.\(signal \? \{ signal \} : \{\}\),/, 'the abort signal reaches the runtime');
+ assert.match(bonsai, /controller\?\.abort\(\);/, 'a cancel message aborts the tracked request');
+ assert.match(bonsai, /outputTruncated: finishReason === 'length'/, 'a length stop is reported as a cut-off');
+});
+
test('inline card claims its request before waiting, cancels what it abandons and finishes cut-off answers in the panel', () => {
for (const prefix of ['src/chrome', 'src/firefox']) {
const content = fs.readFileSync(path.join(ROOT, prefix, 'src/content/selection-shortcut.js'), 'utf8');
@@ -67054,7 +67289,7 @@ test('WebGPU worker follows local text-generation and WebBrain VL vision contrac
assert.match(bonsaiWorker, /opfsPartialName/);
assert.match(bonsaiWorker, /tools: tools\.length \? tools : undefined/);
assert.match(bonsaiWorker, /function normalizeBitgpuToolCalls/);
- assert.match(bonsaiWorker, /return \{ content, reasoningContent, toolCalls \}/);
+ assert.match(bonsaiWorker, /return \{\s*content,\s*reasoningContent,\s*toolCalls,/);
assert.match(bonsaiWorker, /if \(queuedTextDownload === request\) queuedTextDownload = null/,
'a completed download request must not clear a newer queued resume for the same model');
assert.match(apocalypseScript, /if \(!preset \|\| preset\.id !== WEBGPU_COMPASS_TINY_V2_MODEL_ID\)[\s\S]*?setWebgpuDownloadState\(state, \{ syncActiveTransfer: true \}\)[\s\S]*?ensureFixedWebgpuProvider\(\{ force: true \}\)[\s\S]*?get_webgpu_download_status/,