Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
resolve merge conflicts in WebGPU offscreen worker
Emre Sokullu28e004fc8f62
diff --cc src/chrome/src/offscreen/inference-worker.jsindex c2e522274,2db3fdb09..5c5c72228--- a/src/chrome/src/offscreen/inference-worker.js+++ b/src/chrome/src/offscreen/inference-worker.js@@@ -368,21 -329,11 +380,12 @@@ function presentToPastName(name).replace('present', 'past_key_values');}- /**- * Prefill all but the last prompt token in slices and return the cache, or- * null when the prompt fits in one slice. generate() then continues from the- * cache: the Qwen VL input preparation computes the full rotary positions and- * feeds only the remaining token.- *- * `checkCancelled` throws when the request was cancelled. It runs before the embedding- * passes and before every slice, so a card that was closed stops at the next slice- * boundary instead of occupying the serialized GPU worker for the whole prefill.- */- export async function prefillBonsai2InSlices(runtime, inputs, sliceTokens, checkCancelled = () => {}) {+ // Prefill the first `count` prompt tokens in slices of `sliceTokens` (one+ // pass when 0) and return the recurrent/conv/KV cache. `beforeSlice` runs+ // before each slice and may throw to stop (a cancelled decision).+ async function prefillBonsai2Cache(runtime, inputs, sliceTokens, count, beforeSlice = null) {const { model, library } = runtime;- const length = inputs.input_ids.dims.at(-1);- if (!sliceTokens || length - 1 <= sliceTokens) return null;- checkCancelled();++ beforeSlice?.();let embeds = await model.encode_text({ input_ids: inputs.input_ids });let attentionMask = inputs.attention_mask;if (inputs.pixel_values) {@@@ -429,6 -381,51 +433,51 @@@}}+ /**+ * Prefill all but the last prompt token in slices and return the cache, or+ * null when the prompt fits in one slice. generate() then continues from the+ * cache: the Qwen VL input preparation computes the full rotary positions and+ * feeds only the remaining token.+ */-export async function prefillBonsai2InSlices(runtime, inputs, sliceTokens) {++export async function prefillBonsai2InSlices(runtime, inputs, sliceTokens, checkCancelled = () => {}) {+ const length = inputs.input_ids.dims.at(-1);+ if (!sliceTokens || length - 1 <= sliceTokens) return null;- return prefillBonsai2Cache(runtime, inputs, sliceTokens, length - 1);++ return prefillBonsai2Cache(runtime, inputs, sliceTokens, length - 1, checkCancelled);+ }++ /**+ * Next-token logits after appending `suffixIds` to a prefilled prompt. The+ * prefix cache is only read: the continuation's cache outputs are released,+ * so every question starts from the same page state.+ */+ export async function bonsai2ContinuationLogits(runtime, prefixInputs, cache, suffixIds, { withHidden = false } = {}) {+ const { model, library } = runtime;+ const prefixLength = prefixInputs.input_ids.dims.at(-1), suffixLength = suffixIds.dims.at(-1);+ const total = prefixLength + suffixLength;+ const fullIds = library.cat([prefixInputs.input_ids, suffixIds], 1);+ const fullMask = new library.Tensor('int64', new BigInt64Array(total).fill(1n), [1, total]);+ const [positions] = model.get_rope_index(fullIds, prefixInputs.image_grid_thw ?? null, null, fullMask);+ const embeds = await model.encode_text({ input_ids: suffixIds });+ const outputs = await model.forward({+ inputs_embeds: embeds,+ attention_mask: fullMask,+ position_ids: positions.slice(null, null, [prefixLength, total]),+ past_key_values: cache,+ });+ try {+ const logits = Float32Array.from(outputs.logits.slice(null, -1, null).to('float32').data);+ if (!withHidden) return logits;+ // Present only in a decoder graph that exposes `decision_hidden`+ // (scripts/models/bonsai/add_decision_hidden.py).+ const hidden = outputs.decision_hidden ? Float32Array.from(outputs.decision_hidden.to('float32').data) : null;+ return { logits, hidden };+ } finally {+ for (const tensor of Object.values(outputs)) {+ if (tensor?.location === 'gpu-buffer') tensor.dispose();+ }+ }+ }+export function normalizeBonsai2Config(config) {return {...config,@@@ -1823,17 -1810,173 +1872,180 @@@ async function runMultimodalText(payloaconst decoded = runtime.processor.batch_decode(generated, { skip_special_tokens: true });const result = splitThinking(String(decoded?.[0] || '').trim());noteWebgpuExecutionSuccess();- return { content: result.content, reasoningContent: result.reasoningContent };+ // The generated tensor is exact, so reaching the budget means no end-of-text token came first.+ return {+ content: result.content,+ reasoningContent: result.reasoningContent,+ ...outputLimitField(Number(generated?.dims?.at?.(-1)), maxNewTokens, 1),+ };}+ function encodeFloat16Base64(values) {+ const half = new Uint16Array(values.length), view = new DataView(new ArrayBuffer(4));+ for (let i = 0; i < values.length; i++) {+ view.setFloat32(0, values[i]);+ const bits = view.getUint32(0), sign = (bits >>> 16) & 0x8000, exponent = ((bits >>> 23) & 0xff) - 112, mantissa = (bits >>> 13) & 0x3ff;+ half[i] = exponent <= 0 ? sign : exponent >= 31 ? sign | 0x7c00 : sign | (exponent << 10) | mantissa;+ }+ let binary = '';+ const bytes = new Uint8Array(half.buffer);+ for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);+ return btoa(binary);+ }++ // Option probabilities read from one forward pass per question (see+ // webgpu-decisions.js). Only the Bonsai 2 Vision presets are supported: they+ // use the chunked decoder whose cache the continuations share.+ const DECISION_QUESTION_MARKER = '\u2063WEBBRAIN_DECISION_QUESTION\u2063';++ function decisionCancelledError() {+ const error = new Error('The in-browser decision was cancelled.');+ error.name = 'AbortError';+ error.code = 'webgpu_decision_cancelled';+ return error;+ }++ // `isCancelled` reports a `cancel` for this request (a judge timeout or Stop).+ // It is checked before loading, between prefill slices and before every+ // continuation, so a timed-out decision frees the model queue promptly.+ async function runMultimodalDecision(payload, { isCancelled = () => false } = {}) {+ const stopIfCancelled = () => { if (isCancelled()) throw decisionCancelledError(); };+ stopIfCancelled();+ const modelId = assertOnnxTextModel(payload?.modelId);+ if (!WEBGPU_BONSAI2_VISION_MODEL_IDS.has(modelId)) {+ throw new Error(`${modelId} does not support in-browser decisions.`);+ }+ const request = normalizeDecisionRequest(payload);+ const dtype = payload?.dtype || { embed_tokens: 'fp16', vision_encoder: 'fp16', decoder_model_merged: 'q2f16' };+ if (!await isTextModelReady(modelId, dtype)) {+ throw new Error(`${modelId} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it first.`);+ }+ const runtime = await getVisionRuntime(modelId, dtype, payload?.device || 'webgpu', {+ localFilesOnly: true,+ owner: 'text',+ readiness: 'text',+ });+ stopIfCancelled();+ const temperature = Number(payload?.temperature) > 0 ? Number(payload.temperature) : 1;+ const messages = [+ { role: 'system', content: [{ type: 'text', text: decisionSystemPrompt(payload?.promptVariant) }] },+ { role: 'user', content: [+ ...request.images.map(() => ({ type: 'image' })),+ { type: 'text', text: `${decisionStateText(request.state)}\n\n${DECISION_QUESTION_MARKER}` },+ ] },+ ];+ const rendered = runtime.processor.apply_chat_template(messages, { add_generation_prompt: true, enable_thinking: false });+ const split = rendered.indexOf(DECISION_QUESTION_MARKER);+ if (split < 0 || rendered.indexOf(DECISION_QUESTION_MARKER, split + 1) >= 0) {+ throw new Error('The decision prompt did not render as expected.');+ }+ const prefixText = rendered.slice(0, split);+ const tailText = rendered.slice(split + DECISION_QUESTION_MARKER.length);+ const tokenizer = runtime.processor.tokenizer;+ const codeIds = new Map();+ const codeId = code => {+ if (!codeIds.has(code)) {+ const ids = tokenizer.encode(code, { add_special_tokens: false });+ if (ids.length !== 1) throw new Error(`Decision answer code ${code} is not a single token.`);+ codeIds.set(code, ids[0]);+ }+ return codeIds.get(code);+ };+ let prefix;+ if (request.images.length) {+ const images = await Promise.all(request.images.map(url => runtime.library.load_image(url)));+ prefix = await processMultimodalInputs(runtime, modelId, images.length === 1 ? images[0] : images, prefixText);+ } else {+ prefix = tokenizer(prefixText, { add_special_tokens: false });+ }+ lastWebGpuDeviceError = '';+ lastWebGpuDeviceLost = '';+ const prefixLength = prefix.input_ids.dims.at(-1);+ const started = performance.now();+ let cache = null;+ const answers = {};+ const questionMs = [];+ let suffixTokens = 0;+ try {+ cache = await prefillBonsai2Cache(runtime, prefix, bonsai2PrefillSliceTokens(payload?.prefillSliceTokens), prefixLength, stopIfCancelled);+ const prefillMs = performance.now() - started;+ // `probe` (the default) scores each option's verify continuation with the+ // trained probe; `verify` returns the raw Yes margins; `readout` and+ // `strict` read the option codes in one pass. All but `probe` are for the+ // diagnostic fixture path.+ const variant = ['readout', 'default', 'strict', 'verify', 'probe'].includes(payload?.promptVariant) ? payload.promptVariant : 'probe';+ const verify = variant === 'verify' || variant === 'probe';+ if (variant === 'probe' && modelId !== BONSAI_DECISION_PROBE.model) {+ throw new Error(`No decision probe is trained for ${modelId}.`);+ }+ const returnHidden = payload?.returnHidden === true;+ const yesId = verify ? codeId('Yes') : null, noId = verify ? codeId('No') : null;+ for (const [id, question] of request.entries) {+ const questionStarted = performance.now();+ if (verify) {+ // One continuation per option; the answer compares their Yes margins.+ const margins = [], hidden = [];+ const count = decisionOptions(question).names.length;+ for (let index = 0; index < count; index++) {+ const suffix = tokenizer(`${decisionVerifyText(question, index)}${tailText}`, { add_special_tokens: false });+ suffixTokens += suffix.input_ids.dims.at(-1);+ stopIfCancelled();+ const output = await bonsai2ContinuationLogits(runtime, prefix, cache, suffix.input_ids, { withHidden: true });+ margins.push(output.logits[yesId] - output.logits[noId]);+ if (variant === 'probe' && !output.hidden) {+ throw new Error('In-browser decisions need the updated Ternary Bonsai 2 2B Vision decoder. Download the model again in Settings > Providers > WebGPU.');+ }+ hidden.push(output.hidden);+ }+ const scores = variant === 'probe' ? bonsaiProbeScores(margins, hidden) : margins;+ answers[id] = {+ ...decisionAnswer(question, scores, { temperature }),+ ...(returnHidden ? { margins, hidden: hidden.map(vector => (vector ? encodeFloat16Base64(vector) : null)) } : {}),+ };+ questionMs.push(Math.round(performance.now() - questionStarted));+ continue;+ }+ const suffix = tokenizer(`${decisionQuestionText(question)}${tailText}`, { add_special_tokens: false });+ suffixTokens += suffix.input_ids.dims.at(-1);+ stopIfCancelled();+ const logits = await bonsai2ContinuationLogits(runtime, prefix, cache, suffix.input_ids);+ const ids = decisionOptions(question).codes.map(codeId);+ let top = -Infinity;+ for (const value of logits) if (value > top) top = value;+ let total = 0;+ for (const value of logits) total += Math.exp(value - top);+ const mass = ids.reduce((sum, tokenId) => sum + Math.exp(logits[tokenId] - top), 0) / total;+ answers[id] = decisionAnswer(question, ids.map(tokenId => logits[tokenId]), { temperature, mass });+ questionMs.push(Math.round(performance.now() - questionStarted));+ }+ noteWebgpuExecutionSuccess();+ return {+ model: modelId,+ provider: 'webgpu',+ answers,+ usage: { input_tokens: prefixLength + suffixTokens, output_tokens: 0 },+ diagnostics: {+ generated_tokens: 0,+ prefix_tokens: prefixLength,+ images: request.images.length,+ prefill_ms: Math.round(prefillMs),+ question_ms: questionMs,+ total_ms: Math.round(performance.now() - started),+ temperature,+ prompt_variant: variant,+ },+ };+ } catch (error) {+ if (isWebGpuExecutionFailure(error)) throw await handleWebGpuExecutionFailure(error);+ throw error;+ } finally {+ if (cache) await cache.dispose().catch(() => {});+ }+ }+async function runText(payload) {+ const requestKey = textRequestKey(payload);+ throwIfTextCancelled(requestKey);const modelId = assertOnnxTextModel(payload?.modelId);const device = payload?.device || 'webgpu';const dtype = payload?.dtype || 'q4f16';@@@ -2119,25 -2240,43 +2331,44 @@@ self.addEventListener('message', async}return;}- if (type === 'text-chat') {- const result = await enqueueModelOperation(() => runText(payload));- self.postMessage({- id,- ok: true,- ...result,- raw: { model: payload?.modelId || '' },- });- return;- }+ if (type === 'text-chat' || type === 'multimodal-text-chat') {+ const requestKey = textRequestKey(payload);+ if (requestKey) queuedTextGenerations.add(requestKey);+ try {+ const result = await enqueueModelOperation(+ () => (type === 'text-chat' ? runText(payload) : runMultimodalText(payload)),+ );+ self.postMessage({+ id,+ ok: true,+ ...result,+ raw: { model: payload?.modelId || '' },+ });+ } finally {+ if (requestKey) {+ queuedTextGenerations.delete(requestKey);+ cancelledTextGenerations.delete(requestKey);+ }+ }+ if (type === 'multimodal-decide') {+ // Cancellable through `cancel` like vision generations, while queued or running.+ queuedVisionGenerations.add(id);+ try {+ const result = await enqueueModelOperation(async () => {+ activeVisionGenerations.set(id, {});+ try {+ return await runMultimodalDecision(payload, { isCancelled: () => cancelledVisionGenerations.has(id) });+ } finally {+ activeVisionGenerations.delete(id);+ }+ });+ self.postMessage({ id, ok: true, ...result });+ } finally {+ queuedVisionGenerations.delete(id);+ cancelledVisionGenerations.delete(id);+ }+ return;+ }- if (type === 'multimodal-text-chat') {- const result = await enqueueModelOperation(() => runMultimodalText(payload));- self.postMessage({- id,- ok: true,- ...result,- raw: { model: payload?.modelId || '' },- });return;}throw new Error(`Unknown WebGPU worker message: ${type || 'missing type'}`);diff --cc src/chrome/src/offscreen/vision-inference-host.jsindex eb54b7ea4,55b4ab06a..3241e8e58--- a/src/chrome/src/offscreen/vision-inference-host.js+++ b/src/chrome/src/offscreen/vision-inference-host.js@@@ -438,36 -438,15 +438,36 @@@ async function disposeOtherTextRuntime(}}+// A chat is not in any worker's table until the worker receives it, and getting+// it there can take a while (starting a worker, unloading the other runtime). A+// cancel that arrives in that window is remembered here and checked right before+// the request is handed over, so an abandoned card never starts a generation.+const routingChatKeys = new Set();+const cancelledChatKeys = new Set();+const CHAT_REQUEST_KEY_PATTERN = /^[a-z0-9-]{8,64}$/i;++function throwIfChatCancelled(requestKey) {+ if (!requestKey || !cancelledChatKeys.has(requestKey)) return;+ const error = new Error('The generation was cancelled.');+ error.name = 'AbortError';+ error.code = 'webgpu_chat_cancelled';+ throw error;+}+- async function sendTextWorkerMessage(modelId, type, payload = {}, { exclusive = false, runtime } = {}) {+ async function sendTextWorkerMessage(modelId, type, payload = {}, { exclusive = false, runtime, onDispatch } = {}) {+ const requestKey = typeof payload.requestKey === 'string' ? payload.requestKey : '';if (isBitgpuTextModel(modelId, runtime) || isBitgpuTextModel(payload.modelId, payload.runtime)) {if (exclusive) await disposeOtherTextRuntime('bitgpu');+ throwIfChatCancelled(requestKey);await ensureBonsaiWorker();- return sendBonsaiWorkerMessage(type, payload);+ throwIfChatCancelled(requestKey);- return sendBonsaiWorkerMessage(type, payload);++ return sendBonsaiWorkerMessage(type, payload, { onDispatch });}if (exclusive) await disposeOtherTextRuntime('onnx');+ throwIfChatCancelled(requestKey);await ensureVisionWorker();+ throwIfChatCancelled(requestKey);- return sendVisionWorkerMessage(type, payload);+ return sendVisionWorkerMessage(type, payload, { onDispatch });}function defaultVisionWorkerTimeout(type) {@@@ -557,9 -537,35 +558,36 @@@ async function ensureVisionWorker()}}+ // In-browser decisions by the provider's decisionId. A judge timeout or Stop+ // sends webgpu-decide-cancel, which cancels the worker operation instead of+ // leaving it on the serialized model queue ahead of the next request.+ const activeWebgpuDecisions = new Map();+ // Cancels that arrived before their decision; dropped after a minute.+ const cancelledWebgpuDecisions = new Set();++ function cancelWebgpuDecisionWork(decision) {+ if (!decision.worker || decision.worker !== visionWorker) return;+ try {+ decision.worker.postMessage({ id: nextVisionRequestId++, type: 'cancel', payload: { requestId: decision.requestId } });+ } catch {}+ }++ function cancelWebgpuDecision(decisionId) {+ const decision = activeWebgpuDecisions.get(decisionId);+ if (decision) {+ decision.cancelled = true;+ cancelWebgpuDecisionWork(decision);+ return;+ }+ cancelledWebgpuDecisions.add(decisionId);+ setTimeout(() => cancelledWebgpuDecisions.delete(decisionId), 60_000);+ }+const WEBGPU_MESSAGE_TYPES = new Set(['webgpu-chat',+ 'webgpu-chat-cancel',+ 'webgpu-decide',+ 'webgpu-decide-cancel','webgpu-download-start','webgpu-download-pause','webgpu-download-stop',@@@ -577,17 -583,11 +605,22 @@@chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {if (!WEBGPU_MESSAGE_TYPES.has(message?.type)) return false;+ // Registered before anything is awaited, so a cancel that follows can always find the chat.+ const chatKey = message.type === 'webgpu-chat' && CHAT_REQUEST_KEY_PATTERN.test(message.requestKey || '')+ ? message.requestKey+ : '';+ if (chatKey) routingChatKeys.add(chatKey);+ // A cancel is recorded synchronously for the same reason: it must not queue behind a+ // worker start-up, and a chat still being routed has no worker-side request to stop yet.+ const cancelKey = message.type === 'webgpu-chat-cancel' && typeof message.requestKey === 'string'+ ? message.requestKey+ : '';+ if (cancelKey && routingChatKeys.has(cancelKey)) cancelledChatKeys.add(cancelKey);+ if (message.type === 'webgpu-decide-cancel') {+ if (typeof message.decisionId === 'string' && message.decisionId) cancelWebgpuDecision(message.decisionId);+ sendResponse({ ok: true });+ return false;+ }(async () => {try {// Bonsai is GGUF/bitgpu. Never boot Transformers.js for those messages —@@@ -750,17 -749,40 +783,51 @@@sendResponse(disposed[0] || { ok: true, disposed: true });return;}+ if (message.type === 'webgpu-chat-cancel') {+ // Stop a text generation its caller no longer wants. Both workers are asked+ // because the host does not know which model the request was routed to.+ const requestKey = typeof message.requestKey === 'string' ? message.requestKey : '';+ const cancels = [];+ if (visionWorker) cancels.push(sendVisionWorkerMessage('cancel-text', { requestKey }).catch(() => null));+ if (bonsaiWorker) cancels.push(sendBonsaiWorkerMessage('cancel-text', { requestKey }).catch(() => null));+ await Promise.all(cancels);+ sendResponse({ ok: true });+ return;+ }+ if (message.type === 'webgpu-decide') {+ const decisionId = typeof message.decisionId === 'string' ? message.decisionId : '';+ if (decisionId && cancelledWebgpuDecisions.delete(decisionId)) {+ sendResponse({ ok: false, error: 'The in-browser decision was cancelled.', code: 'webgpu_decision_cancelled' });+ return;+ }+ const decision = { requestId: null, worker: null, cancelled: false };+ if (decisionId) activeWebgpuDecisions.set(decisionId, decision);+ try {+ sendResponse(await sendTextWorkerMessage(message.model, 'multimodal-decide', {+ modelId: message.model,+ device: message.device,+ dtype: message.dtype,+ state: message.state,+ images: message.images || [],+ questions: message.questions,+ temperature: message.temperature,+ promptVariant: message.promptVariant,+ returnHidden: message.returnHidden === true,+ prefillSliceTokens: message.prefillSliceTokens,+ }, {+ exclusive: true,+ runtime: message.runtime,+ onDispatch: (requestId, worker) => {+ decision.requestId = requestId;+ decision.worker = worker;+ if (decision.cancelled) cancelWebgpuDecisionWork(decision);+ },+ }));+ } finally {+ if (decisionId) activeWebgpuDecisions.delete(decisionId);+ }+ return;+ }if (message.type === 'webgpu-chat') {const workerMessageType = message.runtime === 'onnx-vl'? 'multimodal-text-chat'