Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

resolve merge conflicts in WebGPU offscreen worker

Emre Sokullu28e004fc8f62
diff --cc src/chrome/src/offscreen/inference-worker.js
index c2e522274,2db3fdb09..5c5c72228
--- a/src/chrome/src/offscreen/inference-worker.js
+++ b/src/chrome/src/offscreen/inference-worker.js
@@@ -368,21 -329,11 +380,12 @@@ function presentToPastName(name)
.replace('present', 'past_key_values');
}
- /**
- * Prefill all but the last prompt token in slices and return the cache, or
- * null when the prompt fits in one slice. generate() then continues from the
- * cache: the Qwen VL input preparation computes the full rotary positions and
- * feeds only the remaining token.
- *
- * `checkCancelled` throws when the request was cancelled. It runs before the embedding
- * passes and before every slice, so a card that was closed stops at the next slice
- * boundary instead of occupying the serialized GPU worker for the whole prefill.
- */
- export async function prefillBonsai2InSlices(runtime, inputs, sliceTokens, checkCancelled = () => {}) {
+ // Prefill the first `count` prompt tokens in slices of `sliceTokens` (one
+ // pass when 0) and return the recurrent/conv/KV cache. `beforeSlice` runs
+ // before each slice and may throw to stop (a cancelled decision).
+ async function prefillBonsai2Cache(runtime, inputs, sliceTokens, count, beforeSlice = null) {
const { model, library } = runtime;
- const length = inputs.input_ids.dims.at(-1);
- if (!sliceTokens || length - 1 <= sliceTokens) return null;
- checkCancelled();
++ beforeSlice?.();
let embeds = await model.encode_text({ input_ids: inputs.input_ids });
let attentionMask = inputs.attention_mask;
if (inputs.pixel_values) {
@@@ -429,6 -381,51 +433,51 @@@
}
}
+ /**
+ * Prefill all but the last prompt token in slices and return the cache, or
+ * null when the prompt fits in one slice. generate() then continues from the
+ * cache: the Qwen VL input preparation computes the full rotary positions and
+ * feeds only the remaining token.
+ */
-export async function prefillBonsai2InSlices(runtime, inputs, sliceTokens) {
++export async function prefillBonsai2InSlices(runtime, inputs, sliceTokens, checkCancelled = () => {}) {
+ const length = inputs.input_ids.dims.at(-1);
+ if (!sliceTokens || length - 1 <= sliceTokens) return null;
- return prefillBonsai2Cache(runtime, inputs, sliceTokens, length - 1);
++ return prefillBonsai2Cache(runtime, inputs, sliceTokens, length - 1, checkCancelled);
+ }
+
+ /**
+ * Next-token logits after appending `suffixIds` to a prefilled prompt. The
+ * prefix cache is only read: the continuation's cache outputs are released,
+ * so every question starts from the same page state.
+ */
+ export async function bonsai2ContinuationLogits(runtime, prefixInputs, cache, suffixIds, { withHidden = false } = {}) {
+ const { model, library } = runtime;
+ const prefixLength = prefixInputs.input_ids.dims.at(-1), suffixLength = suffixIds.dims.at(-1);
+ const total = prefixLength + suffixLength;
+ const fullIds = library.cat([prefixInputs.input_ids, suffixIds], 1);
+ const fullMask = new library.Tensor('int64', new BigInt64Array(total).fill(1n), [1, total]);
+ const [positions] = model.get_rope_index(fullIds, prefixInputs.image_grid_thw ?? null, null, fullMask);
+ const embeds = await model.encode_text({ input_ids: suffixIds });
+ const outputs = await model.forward({
+ inputs_embeds: embeds,
+ attention_mask: fullMask,
+ position_ids: positions.slice(null, null, [prefixLength, total]),
+ past_key_values: cache,
+ });
+ try {
+ const logits = Float32Array.from(outputs.logits.slice(null, -1, null).to('float32').data);
+ if (!withHidden) return logits;
+ // Present only in a decoder graph that exposes `decision_hidden`
+ // (scripts/models/bonsai/add_decision_hidden.py).
+ const hidden = outputs.decision_hidden ? Float32Array.from(outputs.decision_hidden.to('float32').data) : null;
+ return { logits, hidden };
+ } finally {
+ for (const tensor of Object.values(outputs)) {
+ if (tensor?.location === 'gpu-buffer') tensor.dispose();
+ }
+ }
+ }
+
export function normalizeBonsai2Config(config) {
return {
...config,
@@@ -1823,17 -1810,173 +1872,180 @@@ async function runMultimodalText(payloa
const decoded = runtime.processor.batch_decode(generated, { skip_special_tokens: true });
const result = splitThinking(String(decoded?.[0] || '').trim());
noteWebgpuExecutionSuccess();
- return { content: result.content, reasoningContent: result.reasoningContent };
+ // The generated tensor is exact, so reaching the budget means no end-of-text token came first.
+ return {
+ content: result.content,
+ reasoningContent: result.reasoningContent,
+ ...outputLimitField(Number(generated?.dims?.at?.(-1)), maxNewTokens, 1),
+ };
}
+ function encodeFloat16Base64(values) {
+ const half = new Uint16Array(values.length), view = new DataView(new ArrayBuffer(4));
+ for (let i = 0; i < values.length; i++) {
+ view.setFloat32(0, values[i]);
+ const bits = view.getUint32(0), sign = (bits >>> 16) & 0x8000, exponent = ((bits >>> 23) & 0xff) - 112, mantissa = (bits >>> 13) & 0x3ff;
+ half[i] = exponent <= 0 ? sign : exponent >= 31 ? sign | 0x7c00 : sign | (exponent << 10) | mantissa;
+ }
+ let binary = '';
+ const bytes = new Uint8Array(half.buffer);
+ for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
+ return btoa(binary);
+ }
+
+ // Option probabilities read from one forward pass per question (see
+ // webgpu-decisions.js). Only the Bonsai 2 Vision presets are supported: they
+ // use the chunked decoder whose cache the continuations share.
+ const DECISION_QUESTION_MARKER = '\u2063WEBBRAIN_DECISION_QUESTION\u2063';
+
+ function decisionCancelledError() {
+ const error = new Error('The in-browser decision was cancelled.');
+ error.name = 'AbortError';
+ error.code = 'webgpu_decision_cancelled';
+ return error;
+ }
+
+ // `isCancelled` reports a `cancel` for this request (a judge timeout or Stop).
+ // It is checked before loading, between prefill slices and before every
+ // continuation, so a timed-out decision frees the model queue promptly.
+ async function runMultimodalDecision(payload, { isCancelled = () => false } = {}) {
+ const stopIfCancelled = () => { if (isCancelled()) throw decisionCancelledError(); };
+ stopIfCancelled();
+ const modelId = assertOnnxTextModel(payload?.modelId);
+ if (!WEBGPU_BONSAI2_VISION_MODEL_IDS.has(modelId)) {
+ throw new Error(`${modelId} does not support in-browser decisions.`);
+ }
+ const request = normalizeDecisionRequest(payload);
+ const dtype = payload?.dtype || { embed_tokens: 'fp16', vision_encoder: 'fp16', decoder_model_merged: 'q2f16' };
+ if (!await isTextModelReady(modelId, dtype)) {
+ throw new Error(`${modelId} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it first.`);
+ }
+ const runtime = await getVisionRuntime(modelId, dtype, payload?.device || 'webgpu', {
+ localFilesOnly: true,
+ owner: 'text',
+ readiness: 'text',
+ });
+ stopIfCancelled();
+ const temperature = Number(payload?.temperature) > 0 ? Number(payload.temperature) : 1;
+ const messages = [
+ { role: 'system', content: [{ type: 'text', text: decisionSystemPrompt(payload?.promptVariant) }] },
+ { role: 'user', content: [
+ ...request.images.map(() => ({ type: 'image' })),
+ { type: 'text', text: `${decisionStateText(request.state)}\n\n${DECISION_QUESTION_MARKER}` },
+ ] },
+ ];
+ const rendered = runtime.processor.apply_chat_template(messages, { add_generation_prompt: true, enable_thinking: false });
+ const split = rendered.indexOf(DECISION_QUESTION_MARKER);
+ if (split < 0 || rendered.indexOf(DECISION_QUESTION_MARKER, split + 1) >= 0) {
+ throw new Error('The decision prompt did not render as expected.');
+ }
+ const prefixText = rendered.slice(0, split);
+ const tailText = rendered.slice(split + DECISION_QUESTION_MARKER.length);
+ const tokenizer = runtime.processor.tokenizer;
+ const codeIds = new Map();
+ const codeId = code => {
+ if (!codeIds.has(code)) {
+ const ids = tokenizer.encode(code, { add_special_tokens: false });
+ if (ids.length !== 1) throw new Error(`Decision answer code ${code} is not a single token.`);
+ codeIds.set(code, ids[0]);
+ }
+ return codeIds.get(code);
+ };
+ let prefix;
+ if (request.images.length) {
+ const images = await Promise.all(request.images.map(url => runtime.library.load_image(url)));
+ prefix = await processMultimodalInputs(runtime, modelId, images.length === 1 ? images[0] : images, prefixText);
+ } else {
+ prefix = tokenizer(prefixText, { add_special_tokens: false });
+ }
+ lastWebGpuDeviceError = '';
+ lastWebGpuDeviceLost = '';
+ const prefixLength = prefix.input_ids.dims.at(-1);
+ const started = performance.now();
+ let cache = null;
+ const answers = {};
+ const questionMs = [];
+ let suffixTokens = 0;
+ try {
+ cache = await prefillBonsai2Cache(runtime, prefix, bonsai2PrefillSliceTokens(payload?.prefillSliceTokens), prefixLength, stopIfCancelled);
+ const prefillMs = performance.now() - started;
+ // `probe` (the default) scores each option's verify continuation with the
+ // trained probe; `verify` returns the raw Yes margins; `readout` and
+ // `strict` read the option codes in one pass. All but `probe` are for the
+ // diagnostic fixture path.
+ const variant = ['readout', 'default', 'strict', 'verify', 'probe'].includes(payload?.promptVariant) ? payload.promptVariant : 'probe';
+ const verify = variant === 'verify' || variant === 'probe';
+ if (variant === 'probe' && modelId !== BONSAI_DECISION_PROBE.model) {
+ throw new Error(`No decision probe is trained for ${modelId}.`);
+ }
+ const returnHidden = payload?.returnHidden === true;
+ const yesId = verify ? codeId('Yes') : null, noId = verify ? codeId('No') : null;
+ for (const [id, question] of request.entries) {
+ const questionStarted = performance.now();
+ if (verify) {
+ // One continuation per option; the answer compares their Yes margins.
+ const margins = [], hidden = [];
+ const count = decisionOptions(question).names.length;
+ for (let index = 0; index < count; index++) {
+ const suffix = tokenizer(`${decisionVerifyText(question, index)}${tailText}`, { add_special_tokens: false });
+ suffixTokens += suffix.input_ids.dims.at(-1);
+ stopIfCancelled();
+ const output = await bonsai2ContinuationLogits(runtime, prefix, cache, suffix.input_ids, { withHidden: true });
+ margins.push(output.logits[yesId] - output.logits[noId]);
+ if (variant === 'probe' && !output.hidden) {
+ throw new Error('In-browser decisions need the updated Ternary Bonsai 2 2B Vision decoder. Download the model again in Settings > Providers > WebGPU.');
+ }
+ hidden.push(output.hidden);
+ }
+ const scores = variant === 'probe' ? bonsaiProbeScores(margins, hidden) : margins;
+ answers[id] = {
+ ...decisionAnswer(question, scores, { temperature }),
+ ...(returnHidden ? { margins, hidden: hidden.map(vector => (vector ? encodeFloat16Base64(vector) : null)) } : {}),
+ };
+ questionMs.push(Math.round(performance.now() - questionStarted));
+ continue;
+ }
+ const suffix = tokenizer(`${decisionQuestionText(question)}${tailText}`, { add_special_tokens: false });
+ suffixTokens += suffix.input_ids.dims.at(-1);
+ stopIfCancelled();
+ const logits = await bonsai2ContinuationLogits(runtime, prefix, cache, suffix.input_ids);
+ const ids = decisionOptions(question).codes.map(codeId);
+ let top = -Infinity;
+ for (const value of logits) if (value > top) top = value;
+ let total = 0;
+ for (const value of logits) total += Math.exp(value - top);
+ const mass = ids.reduce((sum, tokenId) => sum + Math.exp(logits[tokenId] - top), 0) / total;
+ answers[id] = decisionAnswer(question, ids.map(tokenId => logits[tokenId]), { temperature, mass });
+ questionMs.push(Math.round(performance.now() - questionStarted));
+ }
+ noteWebgpuExecutionSuccess();
+ return {
+ model: modelId,
+ provider: 'webgpu',
+ answers,
+ usage: { input_tokens: prefixLength + suffixTokens, output_tokens: 0 },
+ diagnostics: {
+ generated_tokens: 0,
+ prefix_tokens: prefixLength,
+ images: request.images.length,
+ prefill_ms: Math.round(prefillMs),
+ question_ms: questionMs,
+ total_ms: Math.round(performance.now() - started),
+ temperature,
+ prompt_variant: variant,
+ },
+ };
+ } catch (error) {
+ if (isWebGpuExecutionFailure(error)) throw await handleWebGpuExecutionFailure(error);
+ throw error;
+ } finally {
+ if (cache) await cache.dispose().catch(() => {});
+ }
+ }
+
async function runText(payload) {
+ const requestKey = textRequestKey(payload);
+ throwIfTextCancelled(requestKey);
const modelId = assertOnnxTextModel(payload?.modelId);
const device = payload?.device || 'webgpu';
const dtype = payload?.dtype || 'q4f16';
@@@ -2119,25 -2240,43 +2331,44 @@@ self.addEventListener('message', async
}
return;
}
- if (type === 'text-chat') {
- const result = await enqueueModelOperation(() => runText(payload));
- self.postMessage({
- id,
- ok: true,
- ...result,
- raw: { model: payload?.modelId || '' },
- });
- return;
- }
+ if (type === 'text-chat' || type === 'multimodal-text-chat') {
+ const requestKey = textRequestKey(payload);
+ if (requestKey) queuedTextGenerations.add(requestKey);
+ try {
+ const result = await enqueueModelOperation(
+ () => (type === 'text-chat' ? runText(payload) : runMultimodalText(payload)),
+ );
+ self.postMessage({
+ id,
+ ok: true,
+ ...result,
+ raw: { model: payload?.modelId || '' },
+ });
+ } finally {
+ if (requestKey) {
+ queuedTextGenerations.delete(requestKey);
+ cancelledTextGenerations.delete(requestKey);
+ }
+ }
+ if (type === 'multimodal-decide') {
+ // Cancellable through `cancel` like vision generations, while queued or running.
+ queuedVisionGenerations.add(id);
+ try {
+ const result = await enqueueModelOperation(async () => {
+ activeVisionGenerations.set(id, {});
+ try {
+ return await runMultimodalDecision(payload, { isCancelled: () => cancelledVisionGenerations.has(id) });
+ } finally {
+ activeVisionGenerations.delete(id);
+ }
+ });
+ self.postMessage({ id, ok: true, ...result });
+ } finally {
+ queuedVisionGenerations.delete(id);
+ cancelledVisionGenerations.delete(id);
+ }
+ return;
+ }
- if (type === 'multimodal-text-chat') {
- const result = await enqueueModelOperation(() => runMultimodalText(payload));
- self.postMessage({
- id,
- ok: true,
- ...result,
- raw: { model: payload?.modelId || '' },
- });
return;
}
throw new Error(`Unknown WebGPU worker message: ${type || 'missing type'}`);
diff --cc src/chrome/src/offscreen/vision-inference-host.js
index eb54b7ea4,55b4ab06a..3241e8e58
--- a/src/chrome/src/offscreen/vision-inference-host.js
+++ b/src/chrome/src/offscreen/vision-inference-host.js
@@@ -438,36 -438,15 +438,36 @@@ async function disposeOtherTextRuntime(
}
}
+// A chat is not in any worker's table until the worker receives it, and getting
+// it there can take a while (starting a worker, unloading the other runtime). A
+// cancel that arrives in that window is remembered here and checked right before
+// the request is handed over, so an abandoned card never starts a generation.
+const routingChatKeys = new Set();
+const cancelledChatKeys = new Set();
+const CHAT_REQUEST_KEY_PATTERN = /^[a-z0-9-]{8,64}$/i;
+
+function throwIfChatCancelled(requestKey) {
+ if (!requestKey || !cancelledChatKeys.has(requestKey)) return;
+ const error = new Error('The generation was cancelled.');
+ error.name = 'AbortError';
+ error.code = 'webgpu_chat_cancelled';
+ throw error;
+}
+
- async function sendTextWorkerMessage(modelId, type, payload = {}, { exclusive = false, runtime } = {}) {
+ async function sendTextWorkerMessage(modelId, type, payload = {}, { exclusive = false, runtime, onDispatch } = {}) {
+ const requestKey = typeof payload.requestKey === 'string' ? payload.requestKey : '';
if (isBitgpuTextModel(modelId, runtime) || isBitgpuTextModel(payload.modelId, payload.runtime)) {
if (exclusive) await disposeOtherTextRuntime('bitgpu');
+ throwIfChatCancelled(requestKey);
await ensureBonsaiWorker();
- return sendBonsaiWorkerMessage(type, payload);
+ throwIfChatCancelled(requestKey);
- return sendBonsaiWorkerMessage(type, payload);
++ return sendBonsaiWorkerMessage(type, payload, { onDispatch });
}
if (exclusive) await disposeOtherTextRuntime('onnx');
+ throwIfChatCancelled(requestKey);
await ensureVisionWorker();
+ throwIfChatCancelled(requestKey);
- return sendVisionWorkerMessage(type, payload);
+ return sendVisionWorkerMessage(type, payload, { onDispatch });
}
function defaultVisionWorkerTimeout(type) {
@@@ -557,9 -537,35 +558,36 @@@ async function ensureVisionWorker()
}
}
+ // In-browser decisions by the provider's decisionId. A judge timeout or Stop
+ // sends webgpu-decide-cancel, which cancels the worker operation instead of
+ // leaving it on the serialized model queue ahead of the next request.
+ const activeWebgpuDecisions = new Map();
+ // Cancels that arrived before their decision; dropped after a minute.
+ const cancelledWebgpuDecisions = new Set();
+
+ function cancelWebgpuDecisionWork(decision) {
+ if (!decision.worker || decision.worker !== visionWorker) return;
+ try {
+ decision.worker.postMessage({ id: nextVisionRequestId++, type: 'cancel', payload: { requestId: decision.requestId } });
+ } catch {}
+ }
+
+ function cancelWebgpuDecision(decisionId) {
+ const decision = activeWebgpuDecisions.get(decisionId);
+ if (decision) {
+ decision.cancelled = true;
+ cancelWebgpuDecisionWork(decision);
+ return;
+ }
+ cancelledWebgpuDecisions.add(decisionId);
+ setTimeout(() => cancelledWebgpuDecisions.delete(decisionId), 60_000);
+ }
+
const WEBGPU_MESSAGE_TYPES = new Set([
'webgpu-chat',
+ 'webgpu-chat-cancel',
+ 'webgpu-decide',
+ 'webgpu-decide-cancel',
'webgpu-download-start',
'webgpu-download-pause',
'webgpu-download-stop',
@@@ -577,17 -583,11 +605,22 @@@
chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
if (!WEBGPU_MESSAGE_TYPES.has(message?.type)) return false;
+ // Registered before anything is awaited, so a cancel that follows can always find the chat.
+ const chatKey = message.type === 'webgpu-chat' && CHAT_REQUEST_KEY_PATTERN.test(message.requestKey || '')
+ ? message.requestKey
+ : '';
+ if (chatKey) routingChatKeys.add(chatKey);
+ // A cancel is recorded synchronously for the same reason: it must not queue behind a
+ // worker start-up, and a chat still being routed has no worker-side request to stop yet.
+ const cancelKey = message.type === 'webgpu-chat-cancel' && typeof message.requestKey === 'string'
+ ? message.requestKey
+ : '';
+ if (cancelKey && routingChatKeys.has(cancelKey)) cancelledChatKeys.add(cancelKey);
+ if (message.type === 'webgpu-decide-cancel') {
+ if (typeof message.decisionId === 'string' && message.decisionId) cancelWebgpuDecision(message.decisionId);
+ sendResponse({ ok: true });
+ return false;
+ }
(async () => {
try {
// Bonsai is GGUF/bitgpu. Never boot Transformers.js for those messages —
@@@ -750,17 -749,40 +783,51 @@@
sendResponse(disposed[0] || { ok: true, disposed: true });
return;
}
+ if (message.type === 'webgpu-chat-cancel') {
+ // Stop a text generation its caller no longer wants. Both workers are asked
+ // because the host does not know which model the request was routed to.
+ const requestKey = typeof message.requestKey === 'string' ? message.requestKey : '';
+ const cancels = [];
+ if (visionWorker) cancels.push(sendVisionWorkerMessage('cancel-text', { requestKey }).catch(() => null));
+ if (bonsaiWorker) cancels.push(sendBonsaiWorkerMessage('cancel-text', { requestKey }).catch(() => null));
+ await Promise.all(cancels);
+ sendResponse({ ok: true });
+ return;
+ }
+ if (message.type === 'webgpu-decide') {
+ const decisionId = typeof message.decisionId === 'string' ? message.decisionId : '';
+ if (decisionId && cancelledWebgpuDecisions.delete(decisionId)) {
+ sendResponse({ ok: false, error: 'The in-browser decision was cancelled.', code: 'webgpu_decision_cancelled' });
+ return;
+ }
+ const decision = { requestId: null, worker: null, cancelled: false };
+ if (decisionId) activeWebgpuDecisions.set(decisionId, decision);
+ try {
+ sendResponse(await sendTextWorkerMessage(message.model, 'multimodal-decide', {
+ modelId: message.model,
+ device: message.device,
+ dtype: message.dtype,
+ state: message.state,
+ images: message.images || [],
+ questions: message.questions,
+ temperature: message.temperature,
+ promptVariant: message.promptVariant,
+ returnHidden: message.returnHidden === true,
+ prefillSliceTokens: message.prefillSliceTokens,
+ }, {
+ exclusive: true,
+ runtime: message.runtime,
+ onDispatch: (requestId, worker) => {
+ decision.requestId = requestId;
+ decision.worker = worker;
+ if (decision.cancelled) cancelWebgpuDecisionWork(decision);
+ },
+ }));
+ } finally {
+ if (decisionId) activeWebgpuDecisions.delete(decisionId);
+ }
+ return;
+ }
if (message.type === 'webgpu-chat') {
const workerMessageType = message.runtime === 'onnx-vl'
? 'multimodal-text-chat'