Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Prefill long Bonsai 2 Vision prompts in cache-carrying slices
Claude752942d4eb0b
diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.jsindex c1dc032b7..8d3f01fdb 100644--- a/src/chrome/src/offscreen/inference-worker.js+++ b/src/chrome/src/offscreen/inference-worker.js@@ -292,6 +292,86 @@ export async function refreshBonsai2DecoderGraph(modelId, {}}+// Long Bonsai 2 Vision prompts are prefilled in slices so no single GPU pass+// covers the whole prompt. On a GPU shared with the display (Apple silicon, a+// display-attached discrete card) one multi-second prefill starves window+// compositing and the desktop stutters; slices bound each pass. Each slice+// continues from the previous slice's recurrent/conv/KV cache, which the+// chunked decoder graph computes exactly like a single pass.+export const BONSAI2_DEFAULT_PREFILL_SLICE_TOKENS = 512;+const BONSAI2_MIN_PREFILL_SLICE_TOKENS = 64;+const BONSAI2_MAX_PREFILL_SLICE_TOKENS = 8192;++/** 0 disables slicing; invalid values use the default. */+export function bonsai2PrefillSliceTokens(value) {+ if (value === 0 || value === '0') return 0;+ const tokens = Math.round(Number(value));+ if (!Number.isFinite(tokens) || tokens <= 0) return BONSAI2_DEFAULT_PREFILL_SLICE_TOKENS;+ return Math.min(BONSAI2_MAX_PREFILL_SLICE_TOKENS, Math.max(BONSAI2_MIN_PREFILL_SLICE_TOKENS, tokens));+}++function presentToPastName(name) {+ return name+ .replace('present_conv', 'past_conv')+ .replace('present_recurrent', 'past_recurrent')+ .replace('present', 'past_key_values');+}++/**+ * Prefill all but the last prompt token in slices and return the cache, or+ * null when the prompt fits in one slice. generate() then continues from the+ * cache: the Qwen VL input preparation computes the full rotary positions and+ * feeds only the remaining token.+ */+export async function prefillBonsai2InSlices(runtime, inputs, sliceTokens) {+ const { model, library } = runtime;+ const length = inputs.input_ids.dims.at(-1);+ if (!sliceTokens || length - 1 <= sliceTokens) return null;+ let embeds = await model.encode_text({ input_ids: inputs.input_ids });+ let attentionMask = inputs.attention_mask;+ if (inputs.pixel_values) {+ const imageFeatures = await model.encode_image({+ pixel_values: inputs.pixel_values,+ image_grid_thw: inputs.image_grid_thw,+ });+ ({ inputs_embeds: embeds, attention_mask: attentionMask } = model._merge_input_ids_with_image_features({+ image_features: imageFeatures,+ inputs_embeds: embeds,+ input_ids: inputs.input_ids,+ attention_mask: attentionMask,+ }));+ }+ const [positionIds] = model.get_rope_index(+ inputs.input_ids,+ inputs.image_grid_thw ?? null,+ inputs.video_grid_thw ?? null,+ attentionMask,+ );+ let cache = null;+ try {+ for (let start = 0; start < length - 1; start += sliceTokens) {+ const end = Math.min(start + sliceTokens, length - 1);+ const outputs = await model.forward({+ inputs_embeds: embeds.slice(null, [start, end], null),+ attention_mask: attentionMask.slice(null, [0, end]),+ position_ids: positionIds.slice(null, null, [start, end]),+ past_key_values: cache,+ });+ const present = {};+ for (const [name, tensor] of Object.entries(outputs)) {+ if (name.startsWith('present')) present[presentToPastName(name)] = tensor;+ else if (tensor?.location === 'gpu-buffer') tensor.dispose();+ }+ if (cache) cache.update(present);+ else cache = new library.DynamicCache(present);+ }+ return cache;+ } catch (error) {+ await cache?.dispose?.();+ throw error;+ }+}+export function normalizeBonsai2Config(config) {return {...config,@@ -1648,15 +1728,28 @@ async function runMultimodalText(payload) {lastWebGpuDeviceError = '';lastWebGpuDeviceLost = '';let outputs;+ let prefilled = null;try {+ if (WEBGPU_BONSAI2_VISION_MODEL_IDS.has(modelId)) {+ prefilled = await prefillBonsai2InSlices(+ runtime,+ inputs,+ bonsai2PrefillSliceTokens(payload?.options?.prefillSliceTokens),+ );+ }outputs = await runtime.model.generate({...inputs,+ ...(prefilled ? { past_key_values: prefilled } : {}),do_sample: false,max_new_tokens: maxNewTokens,});} catch (error) {if (isWebGpuExecutionFailure(error)) throw await handleWebGpuExecutionFailure(error);throw error;+ } finally {+ // generate() keeps a caller-supplied cache alive; release its GPU buffers+ // without masking a generation error.+ if (prefilled) await prefilled.dispose().catch(() => {});}const inputLength = inputs.input_ids.dims.at(-1);const generated = outputs.slice(null, [inputLength, null]);diff --git a/src/chrome/src/providers/webgpu.js b/src/chrome/src/providers/webgpu.jsindex 2fecec279..1ee047eb5 100644--- a/src/chrome/src/providers/webgpu.js+++ b/src/chrome/src/providers/webgpu.js@@ -195,6 +195,10 @@ export const WEBGPU_MODEL_NOT_READY_ERROR = `${WEBGPU_COMPASS_TINY_V2_MODEL_ID}// `visionModel` endpoint so enabling the fallback never overwrites a user's// remote vision credentials or sends a Chromium-only provider type to Firefox.export const WEBGPU_VISION_ENABLED_KEY = 'webgpuVisionEnabled';+// Optional chrome.storage.local override for the Bonsai 2 Vision prefill slice+// (prompt tokens per GPU pass; 0 disables slicing). Smaller slices keep a GPU+// shared with the display responsive at a small throughput cost.+export const WEBGPU_PREFILL_SLICE_TOKENS_KEY = 'webgpuPrefillSliceTokens';// Legacy provenance key retained only so the one-time consent migration can// remove selections made automatically by older Apocalypse Mode builds.export const WEBGPU_VISION_AUTO_SELECTED_KEY = 'webgpuVisionAutoSelected';@@ -419,6 +423,7 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {if (!download.ready) {throw new Error(`${webgpuModelDisplayName(this.model)} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it before chatting.`);}+ const prefillSliceTokens = await this._prefillSliceTokens();const response = await this._dispatch({type: 'webgpu-chat',model: this.model,@@ -430,6 +435,7 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {options: {maxTokens: options.maxTokens,tools: Array.isArray(options.tools) ? options.tools : [],+ ...(prefillSliceTokens == null ? {} : { prefillSliceTokens }),},});if (!response || response.error) {@@ -452,6 +458,15 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {};}+ async _prefillSliceTokens() {+ try {+ const stored = await chrome.storage.local.get(WEBGPU_PREFILL_SLICE_TOKENS_KEY);+ return stored?.[WEBGPU_PREFILL_SLICE_TOKENS_KEY] ?? null;+ } catch {+ return null;+ }+ }+/** Probe the packaged runtime and adapter without downloading model weights. */async testConnection() {return this._testWebGPU();diff --git a/src/chrome/vendor/transformers/README.md b/src/chrome/vendor/transformers/README.mdindex 632e630c6..37f1ec2ab 100644--- a/src/chrome/vendor/transformers/README.md+++ b/src/chrome/vendor/transformers/README.md@@ -40,6 +40,15 @@ single causal graph with its own complete metadata. The Vision presets use`webbrain-one/Ternary-Bonsai-2-{2B,4B}-ONNX-WebGPU-Vision`, the Qwen processor'stext-first image API, and a bounded image resolution.+The Vision presets prefill long prompts in 512-token slices that carry the+recurrent, convolution and KV cache, then let `generate()` continue from that+cache. A single pass over a multi-thousand-token prompt can occupy a GPU shared+with the display (Apple silicon, a display-attached card) for seconds; slices+bound each pass at the cost of roughly one extra decoder run per slice. Greedy+answers match the single-pass path. `chrome.storage.local` key+`webgpuPrefillSliceTokens` overrides the slice size (64–8192 tokens; `0`+disables slicing).+For the split Bonsai text and Vision models, the worker corrects Transformers.js4.2's text-only rotary cache offset; image turns keep the native three-axispositions. Independent screenshots are processed separately before concatenatingdiff --git a/test/run.js b/test/run.jsindex c25afac4f..81405ebf4 100644--- a/test/run.js+++ b/test/run.js@@ -67653,6 +67653,102 @@ test('Bonsai 2 Vision worker replaces only a stale cached decoder graph', async}});+test('Bonsai 2 Vision prefill runs long prompts in cache-carrying slices', async () => {+ const previousSelf = globalThis.self;+ const ortSymbol = Symbol.for('onnxruntime');+ const previousOrt = globalThis[ortSymbol];+ try {+ globalThis.self = { addEventListener() {}, postMessage() {} };+ globalThis[ortSymbol] = await import(pathToFileURL(path.join(ROOT, 'src/chrome/vendor/transformers/onnxruntime-common/index.js')).href);+ const worker = await import(`${pathToFileURL(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js')).href}?bonsai2-prefill-slices`);+ const { Tensor, DynamicCache } = await import(pathToFileURL(path.join(ROOT, 'src/chrome/vendor/transformers/transformers.web.js')).href);+ assert.equal(worker.bonsai2PrefillSliceTokens(0), 0, '0 disables slicing');+ assert.equal(worker.bonsai2PrefillSliceTokens('0'), 0);+ assert.equal(worker.bonsai2PrefillSliceTokens(undefined), worker.BONSAI2_DEFAULT_PREFILL_SLICE_TOKENS);+ assert.equal(worker.bonsai2PrefillSliceTokens('nonsense'), worker.BONSAI2_DEFAULT_PREFILL_SLICE_TOKENS);+ assert.equal(worker.bonsai2PrefillSliceTokens(8), 64);+ assert.equal(worker.bonsai2PrefillSliceTokens(1e9), 8192);+ assert.equal(worker.bonsai2PrefillSliceTokens(300), 300);++ const L = 10, H = 3;+ const ids = new Tensor('int64', BigInt64Array.from({ length: L }, (_, i) => BigInt(100 + i)), [1, L]);+ const mask = new Tensor('int64', new BigInt64Array(L).fill(1n), [1, L]);+ const calls = [];+ const imageCalls = [];+ const model = {+ async encode_text({ input_ids }) {+ const n = input_ids.dims[1];+ return new Tensor('float32', Float32Array.from({ length: n * H }, (_, i) => i), [1, n, H]);+ },+ async encode_image(args) { imageCalls.push(['encode', args.image_grid_thw.dims]); return new Tensor('float32', new Float32Array(2 * H), [2, H]); },+ _merge_input_ids_with_image_features({ inputs_embeds, attention_mask, image_features }) {+ imageCalls.push(['merge', image_features.dims]);+ return { inputs_embeds, attention_mask };+ },+ get_rope_index(input_ids, grid, video, attention_mask) {+ const n = input_ids.dims[1];+ const data = new BigInt64Array(3 * n);+ for (let r = 0; r < 3; r++) for (let i = 0; i < n; i++) data[r * n + i] = BigInt(i);+ return [new Tensor('int64', data, [3, 1, n]), new Tensor('int64', [0n], [1, 1])];+ },+ async forward(inputs) {+ calls.push({+ embeds: inputs.inputs_embeds.dims.slice(),+ firstEmbed: inputs.inputs_embeds.data[0],+ mask: inputs.attention_mask.dims[1],+ positions: Array.from(inputs.position_ids.data.slice(0, inputs.position_ids.dims[2])).map(Number),+ past: inputs.past_key_values,+ });+ const n = calls.length;+ return {+ logits: new Tensor('float32', new Float32Array(1), [1, 1, 1]),+ 'present_conv.0': new Tensor('float32', [n], [1]),+ 'present_recurrent.0': new Tensor('float32', [n], [1]),+ 'present.3.key': new Tensor('float32', new Float32Array(inputs.attention_mask.dims[1]), [1, 1, inputs.attention_mask.dims[1], 1]),+ };+ },+ };+ const runtime = { model, library: { DynamicCache } };+ assert.equal(await worker.prefillBonsai2InSlices(runtime, { input_ids: ids, attention_mask: mask }, 0), null);+ assert.equal(await worker.prefillBonsai2InSlices(runtime, { input_ids: ids, attention_mask: mask }, 9), null,+ 'a prompt whose prefill fits in one slice keeps the single-pass path');+ assert.equal(calls.length, 0);++ const cache = await worker.prefillBonsai2InSlices(runtime, { input_ids: ids, attention_mask: mask }, 4);+ assert.deepEqual(calls.map(c => c.embeds), [[1, 4, H], [1, 4, H], [1, 1, H]],+ 'all but the last prompt token are prefilled in slices; generate() feeds the last one');+ assert.deepEqual(calls.map(c => c.firstEmbed), [0, 4 * H, 8 * H], 'slices cover consecutive prompt positions');+ assert.deepEqual(calls.map(c => c.mask), [4, 8, 9], 'each slice attends to every earlier prompt token');+ assert.deepEqual(calls.map(c => c.positions[0]), [0, 4, 8]);+ assert.equal(calls[0].past, null);+ assert.equal(calls[1].past, cache, 'later slices continue from the shared cache');+ assert.equal(calls[2].past, cache);+ assert.deepEqual(Object.keys(cache).sort(), ['past_conv.0', 'past_key_values.3.key', 'past_recurrent.0']);+ assert.equal(cache['past_conv.0'].data[0], 3, 'the cache holds the final slice state');+ assert.equal(cache.get_seq_length(), 9);++ calls.length = 0;+ const grid = new Tensor('int64', [1n, 2n, 2n], [1, 3]);+ await worker.prefillBonsai2InSlices(runtime, {+ input_ids: ids, attention_mask: mask,+ pixel_values: new Tensor('float32', new Float32Array(4), [4, 1]), image_grid_thw: grid,+ }, 4);+ assert.deepEqual(imageCalls, [['encode', [1, 3]], ['merge', [2, H]]],+ 'image features are merged once over the whole prompt before slicing');+ assert.equal(calls.length, 3);++ const workerSource = fs.readFileSync(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js'), 'utf8');+ assert.match(workerSource, /past_key_values: prefilled/);+ assert.match(workerSource, /if \(prefilled\) await prefilled\.dispose\(\)/,+ 'the caller-supplied cache must be released after generation');+ } finally {+ if (previousSelf === undefined) delete globalThis.self;+ else globalThis.self = previousSelf;+ if (previousOrt === undefined) delete globalThis[ortSymbol];+ else globalThis[ortSymbol] = previousOrt;+ }+});+test('Bonsai 2 worker keeps native rotary positions continuous for text and images', async () => {const previousSelf = globalThis.self;const ortSymbol = Symbol.for('onnxruntime');