Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Merge origin/main into feat/selection-bar-inline-cards

Claude35d91ca2b7b3
diff --cc src/chrome/src/offscreen/inference-worker.js
index 2db952cd0,8d3f01fdb..93746b4f1
--- a/src/chrome/src/offscreen/inference-worker.js
+++ b/src/chrome/src/offscreen/inference-worker.js
@@@ -1377,10 -1687,8 +1738,10 @@@ function addLegacyVlTools(messages, too
}
async function runMultimodalText(payload) {
+ const requestKey = textRequestKey(payload);
+ throwIfTextCancelled(requestKey);
const modelId = assertOnnxTextModel(payload?.modelId);
- if (!isLfm25VlModel(modelId)) {
+ if (!isMultimodalTextModel(modelId)) {
throw new Error(`${modelId} is not a shipped WebGPU multimodal model.`);
}
const device = payload?.device || 'webgpu';
@@@ -1397,7 -1705,7 +1758,8 @@@
owner: 'text',
readiness: 'text',
});
+ throwIfTextCancelled(requestKey);
+ if (payload?.requireTools === true) assertToolCapableTextRuntime(runtime, modelId);
const tools = Array.isArray(payload?.options?.tools) ? payload.options.tools : [];
let { messages, imageUrls } = prepareMultimodalMessages(payload?.messages);
if (modelId === WEBGPU_LFM25_VL_16B_MODEL_ID) messages = addLegacyVlTools(messages, tools);
@@@ -1420,22 -1727,30 +1781,34 @@@
: 800;
lastWebGpuDeviceError = '';
lastWebGpuDeviceLost = '';
+ const criteria = beginTextGeneration(requestKey, runtime.library);
let outputs;
+ let prefilled = null;
try {
+ if (WEBGPU_BONSAI2_VISION_MODEL_IDS.has(modelId)) {
+ prefilled = await prefillBonsai2InSlices(
+ runtime,
+ inputs,
+ bonsai2PrefillSliceTokens(payload?.options?.prefillSliceTokens),
+ );
+ }
outputs = await runtime.model.generate({
...inputs,
+ ...(prefilled ? { past_key_values: prefilled } : {}),
do_sample: false,
max_new_tokens: maxNewTokens,
+ ...(criteria ? { stopping_criteria: [criteria] } : {}),
});
} catch (error) {
if (isWebGpuExecutionFailure(error)) throw await handleWebGpuExecutionFailure(error);
throw error;
} finally {
+ endTextGeneration(requestKey);
+ // generate() keeps a caller-supplied cache alive; release its GPU buffers
+ // without masking a generation error.
+ if (prefilled) await prefilled.dispose().catch(() => {});
}
+ throwIfTextCancelled(requestKey);
const inputLength = inputs.input_ids.dims.at(-1);
const generated = outputs.slice(null, [inputLength, null]);
const decoded = runtime.processor.batch_decode(generated, { skip_special_tokens: true });
diff --cc src/chrome/src/providers/webgpu.js
index 2d2ae6389,81df4bf90..b1b70eb02
--- a/src/chrome/src/providers/webgpu.js
+++ b/src/chrome/src/providers/webgpu.js
@@@ -383,9 -435,7 +466,10 @@@ export class WebGPUProvider extends Web
if (!download.ready) {
throw new Error(`${webgpuModelDisplayName(this.model)} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it before chatting.`);
}
+ // A caller that can abort gets a key the worker can use to interrupt this generation.
+ const signal = options.signal || null;
+ const requestKey = signal ? newWebgpuRequestKey() : '';
+ const prefillSliceTokens = await this._prefillSliceTokens();
const response = await this._dispatch({
type: 'webgpu-chat',
model: this.model,
@@@ -398,8 -447,9 +482,9 @@@
options: {
maxTokens: options.maxTokens,
tools: Array.isArray(options.tools) ? options.tools : [],
+ ...(prefillSliceTokens == null ? {} : { prefillSliceTokens }),
},
- });
+ }, { signal, onAbort: requestKey ? () => this._cancelChat(requestKey) : null });
if (!response || response.error) {
const error = new Error(`In-browser WebGPU: ${response?.error || 'no response from the inference worker'}`);
// A failed OrtRun leaves the WebGPU session/device in an unknown state,
diff --cc test/run.js
index ed9284fca,297116189..5de381e65
--- a/test/run.js
+++ b/test/run.js
@@@ -67589,10 -67460,10 +68222,10 @@@ test('WebGPU worker follows local text-
assert.match(bonsaiWorker, /opfsPartialName/);
assert.match(bonsaiWorker, /tools: tools\.length \? tools : undefined/);
assert.match(bonsaiWorker, /function normalizeBitgpuToolCalls/);
- assert.match(bonsaiWorker, /return \{ content, reasoningContent, toolCalls \}/);
+ assert.match(bonsaiWorker, /return \{\s*content,\s*reasoningContent,\s*toolCalls,/);
assert.match(bonsaiWorker, /if \(queuedTextDownload === request\) queuedTextDownload = null/,
'a completed download request must not clear a newer queued resume for the same model');
- assert.match(apocalypseScript, /if \(!preset \|\| preset\.id !== WEBGPU_COMPASS_TINY_V2_MODEL_ID\)[\s\S]*?setWebgpuDownloadState\(state, \{ syncActiveTransfer: true \}\)[\s\S]*?ensureFixedWebgpuProvider\(\{ force: true \}\)[\s\S]*?get_webgpu_download_status/,
+ assert.match(apocalypseScript, /if \(!isSelectableWebgpuPreset\(preset\?\.id\)\)[\s\S]*?setWebgpuDownloadState\(state, \{ syncActiveTransfer: true \}\)[\s\S]*?ensureFixedWebgpuProvider\(\{ force: true \}\)[\s\S]*?get_webgpu_download_status/,
'Apocalypse Mode must replace a persisted custom WebGPU model with the checked shipped preset');
const resumeHelpersStart = bonsaiWorker.indexOf('function parseContentRange');
const resumeHelpersEnd = bonsaiWorker.indexOf('\n\nasync function fetchGgufForStorage', resumeHelpersStart);