Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Prefill long Bonsai 2 Vision prompts in cache-carrying slices

Claude752942d4eb0b
diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.js
index c1dc032b7..8d3f01fdb 100644
--- a/src/chrome/src/offscreen/inference-worker.js
+++ b/src/chrome/src/offscreen/inference-worker.js
@@ -292,6 +292,86 @@ export async function refreshBonsai2DecoderGraph(modelId, {
}
}
+// Long Bonsai 2 Vision prompts are prefilled in slices so no single GPU pass
+// covers the whole prompt. On a GPU shared with the display (Apple silicon, a
+// display-attached discrete card) one multi-second prefill starves window
+// compositing and the desktop stutters; slices bound each pass. Each slice
+// continues from the previous slice's recurrent/conv/KV cache, which the
+// chunked decoder graph computes exactly like a single pass.
+export const BONSAI2_DEFAULT_PREFILL_SLICE_TOKENS = 512;
+const BONSAI2_MIN_PREFILL_SLICE_TOKENS = 64;
+const BONSAI2_MAX_PREFILL_SLICE_TOKENS = 8192;
+
+/** 0 disables slicing; invalid values use the default. */
+export function bonsai2PrefillSliceTokens(value) {
+ if (value === 0 || value === '0') return 0;
+ const tokens = Math.round(Number(value));
+ if (!Number.isFinite(tokens) || tokens <= 0) return BONSAI2_DEFAULT_PREFILL_SLICE_TOKENS;
+ return Math.min(BONSAI2_MAX_PREFILL_SLICE_TOKENS, Math.max(BONSAI2_MIN_PREFILL_SLICE_TOKENS, tokens));
+}
+
+function presentToPastName(name) {
+ return name
+ .replace('present_conv', 'past_conv')
+ .replace('present_recurrent', 'past_recurrent')
+ .replace('present', 'past_key_values');
+}
+
+/**
+ * Prefill all but the last prompt token in slices and return the cache, or
+ * null when the prompt fits in one slice. generate() then continues from the
+ * cache: the Qwen VL input preparation computes the full rotary positions and
+ * feeds only the remaining token.
+ */
+export async function prefillBonsai2InSlices(runtime, inputs, sliceTokens) {
+ const { model, library } = runtime;
+ const length = inputs.input_ids.dims.at(-1);
+ if (!sliceTokens || length - 1 <= sliceTokens) return null;
+ let embeds = await model.encode_text({ input_ids: inputs.input_ids });
+ let attentionMask = inputs.attention_mask;
+ if (inputs.pixel_values) {
+ const imageFeatures = await model.encode_image({
+ pixel_values: inputs.pixel_values,
+ image_grid_thw: inputs.image_grid_thw,
+ });
+ ({ inputs_embeds: embeds, attention_mask: attentionMask } = model._merge_input_ids_with_image_features({
+ image_features: imageFeatures,
+ inputs_embeds: embeds,
+ input_ids: inputs.input_ids,
+ attention_mask: attentionMask,
+ }));
+ }
+ const [positionIds] = model.get_rope_index(
+ inputs.input_ids,
+ inputs.image_grid_thw ?? null,
+ inputs.video_grid_thw ?? null,
+ attentionMask,
+ );
+ let cache = null;
+ try {
+ for (let start = 0; start < length - 1; start += sliceTokens) {
+ const end = Math.min(start + sliceTokens, length - 1);
+ const outputs = await model.forward({
+ inputs_embeds: embeds.slice(null, [start, end], null),
+ attention_mask: attentionMask.slice(null, [0, end]),
+ position_ids: positionIds.slice(null, null, [start, end]),
+ past_key_values: cache,
+ });
+ const present = {};
+ for (const [name, tensor] of Object.entries(outputs)) {
+ if (name.startsWith('present')) present[presentToPastName(name)] = tensor;
+ else if (tensor?.location === 'gpu-buffer') tensor.dispose();
+ }
+ if (cache) cache.update(present);
+ else cache = new library.DynamicCache(present);
+ }
+ return cache;
+ } catch (error) {
+ await cache?.dispose?.();
+ throw error;
+ }
+}
+
export function normalizeBonsai2Config(config) {
return {
...config,
@@ -1648,15 +1728,28 @@ async function runMultimodalText(payload) {
lastWebGpuDeviceError = '';
lastWebGpuDeviceLost = '';
let outputs;
+ let prefilled = null;
try {
+ if (WEBGPU_BONSAI2_VISION_MODEL_IDS.has(modelId)) {
+ prefilled = await prefillBonsai2InSlices(
+ runtime,
+ inputs,
+ bonsai2PrefillSliceTokens(payload?.options?.prefillSliceTokens),
+ );
+ }
outputs = await runtime.model.generate({
...inputs,
+ ...(prefilled ? { past_key_values: prefilled } : {}),
do_sample: false,
max_new_tokens: maxNewTokens,
});
} catch (error) {
if (isWebGpuExecutionFailure(error)) throw await handleWebGpuExecutionFailure(error);
throw error;
+ } finally {
+ // generate() keeps a caller-supplied cache alive; release its GPU buffers
+ // without masking a generation error.
+ if (prefilled) await prefilled.dispose().catch(() => {});
}
const inputLength = inputs.input_ids.dims.at(-1);
const generated = outputs.slice(null, [inputLength, null]);
diff --git a/src/chrome/src/providers/webgpu.js b/src/chrome/src/providers/webgpu.js
index 2fecec279..1ee047eb5 100644
--- a/src/chrome/src/providers/webgpu.js
+++ b/src/chrome/src/providers/webgpu.js
@@ -195,6 +195,10 @@ export const WEBGPU_MODEL_NOT_READY_ERROR = `${WEBGPU_COMPASS_TINY_V2_MODEL_ID}
// `visionModel` endpoint so enabling the fallback never overwrites a user's
// remote vision credentials or sends a Chromium-only provider type to Firefox.
export const WEBGPU_VISION_ENABLED_KEY = 'webgpuVisionEnabled';
+// Optional chrome.storage.local override for the Bonsai 2 Vision prefill slice
+// (prompt tokens per GPU pass; 0 disables slicing). Smaller slices keep a GPU
+// shared with the display responsive at a small throughput cost.
+export const WEBGPU_PREFILL_SLICE_TOKENS_KEY = 'webgpuPrefillSliceTokens';
// Legacy provenance key retained only so the one-time consent migration can
// remove selections made automatically by older Apocalypse Mode builds.
export const WEBGPU_VISION_AUTO_SELECTED_KEY = 'webgpuVisionAutoSelected';
@@ -419,6 +423,7 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {
if (!download.ready) {
throw new Error(`${webgpuModelDisplayName(this.model)} is not downloaded. Open Settings > Providers > WebGPU or Apocalypse Mode > WebGPU to download it before chatting.`);
}
+ const prefillSliceTokens = await this._prefillSliceTokens();
const response = await this._dispatch({
type: 'webgpu-chat',
model: this.model,
@@ -430,6 +435,7 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {
options: {
maxTokens: options.maxTokens,
tools: Array.isArray(options.tools) ? options.tools : [],
+ ...(prefillSliceTokens == null ? {} : { prefillSliceTokens }),
},
});
if (!response || response.error) {
@@ -452,6 +458,15 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {
};
}
+ async _prefillSliceTokens() {
+ try {
+ const stored = await chrome.storage.local.get(WEBGPU_PREFILL_SLICE_TOKENS_KEY);
+ return stored?.[WEBGPU_PREFILL_SLICE_TOKENS_KEY] ?? null;
+ } catch {
+ return null;
+ }
+ }
+
/** Probe the packaged runtime and adapter without downloading model weights. */
async testConnection() {
return this._testWebGPU();
diff --git a/src/chrome/vendor/transformers/README.md b/src/chrome/vendor/transformers/README.md
index 632e630c6..37f1ec2ab 100644
--- a/src/chrome/vendor/transformers/README.md
+++ b/src/chrome/vendor/transformers/README.md
@@ -40,6 +40,15 @@ single causal graph with its own complete metadata. The Vision presets use
`webbrain-one/Ternary-Bonsai-2-{2B,4B}-ONNX-WebGPU-Vision`, the Qwen processor's
text-first image API, and a bounded image resolution.
+The Vision presets prefill long prompts in 512-token slices that carry the
+recurrent, convolution and KV cache, then let `generate()` continue from that
+cache. A single pass over a multi-thousand-token prompt can occupy a GPU shared
+with the display (Apple silicon, a display-attached card) for seconds; slices
+bound each pass at the cost of roughly one extra decoder run per slice. Greedy
+answers match the single-pass path. `chrome.storage.local` key
+`webgpuPrefillSliceTokens` overrides the slice size (64–8192 tokens; `0`
+disables slicing).
+
For the split Bonsai text and Vision models, the worker corrects Transformers.js
4.2's text-only rotary cache offset; image turns keep the native three-axis
positions. Independent screenshots are processed separately before concatenating
diff --git a/test/run.js b/test/run.js
index c25afac4f..81405ebf4 100644
--- a/test/run.js
+++ b/test/run.js
@@ -67653,6 +67653,102 @@ test('Bonsai 2 Vision worker replaces only a stale cached decoder graph', async
}
});
+test('Bonsai 2 Vision prefill runs long prompts in cache-carrying slices', async () => {
+ const previousSelf = globalThis.self;
+ const ortSymbol = Symbol.for('onnxruntime');
+ const previousOrt = globalThis[ortSymbol];
+ try {
+ globalThis.self = { addEventListener() {}, postMessage() {} };
+ globalThis[ortSymbol] = await import(pathToFileURL(path.join(ROOT, 'src/chrome/vendor/transformers/onnxruntime-common/index.js')).href);
+ const worker = await import(`${pathToFileURL(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js')).href}?bonsai2-prefill-slices`);
+ const { Tensor, DynamicCache } = await import(pathToFileURL(path.join(ROOT, 'src/chrome/vendor/transformers/transformers.web.js')).href);
+ assert.equal(worker.bonsai2PrefillSliceTokens(0), 0, '0 disables slicing');
+ assert.equal(worker.bonsai2PrefillSliceTokens('0'), 0);
+ assert.equal(worker.bonsai2PrefillSliceTokens(undefined), worker.BONSAI2_DEFAULT_PREFILL_SLICE_TOKENS);
+ assert.equal(worker.bonsai2PrefillSliceTokens('nonsense'), worker.BONSAI2_DEFAULT_PREFILL_SLICE_TOKENS);
+ assert.equal(worker.bonsai2PrefillSliceTokens(8), 64);
+ assert.equal(worker.bonsai2PrefillSliceTokens(1e9), 8192);
+ assert.equal(worker.bonsai2PrefillSliceTokens(300), 300);
+
+ const L = 10, H = 3;
+ const ids = new Tensor('int64', BigInt64Array.from({ length: L }, (_, i) => BigInt(100 + i)), [1, L]);
+ const mask = new Tensor('int64', new BigInt64Array(L).fill(1n), [1, L]);
+ const calls = [];
+ const imageCalls = [];
+ const model = {
+ async encode_text({ input_ids }) {
+ const n = input_ids.dims[1];
+ return new Tensor('float32', Float32Array.from({ length: n * H }, (_, i) => i), [1, n, H]);
+ },
+ async encode_image(args) { imageCalls.push(['encode', args.image_grid_thw.dims]); return new Tensor('float32', new Float32Array(2 * H), [2, H]); },
+ _merge_input_ids_with_image_features({ inputs_embeds, attention_mask, image_features }) {
+ imageCalls.push(['merge', image_features.dims]);
+ return { inputs_embeds, attention_mask };
+ },
+ get_rope_index(input_ids, grid, video, attention_mask) {
+ const n = input_ids.dims[1];
+ const data = new BigInt64Array(3 * n);
+ for (let r = 0; r < 3; r++) for (let i = 0; i < n; i++) data[r * n + i] = BigInt(i);
+ return [new Tensor('int64', data, [3, 1, n]), new Tensor('int64', [0n], [1, 1])];
+ },
+ async forward(inputs) {
+ calls.push({
+ embeds: inputs.inputs_embeds.dims.slice(),
+ firstEmbed: inputs.inputs_embeds.data[0],
+ mask: inputs.attention_mask.dims[1],
+ positions: Array.from(inputs.position_ids.data.slice(0, inputs.position_ids.dims[2])).map(Number),
+ past: inputs.past_key_values,
+ });
+ const n = calls.length;
+ return {
+ logits: new Tensor('float32', new Float32Array(1), [1, 1, 1]),
+ 'present_conv.0': new Tensor('float32', [n], [1]),
+ 'present_recurrent.0': new Tensor('float32', [n], [1]),
+ 'present.3.key': new Tensor('float32', new Float32Array(inputs.attention_mask.dims[1]), [1, 1, inputs.attention_mask.dims[1], 1]),
+ };
+ },
+ };
+ const runtime = { model, library: { DynamicCache } };
+ assert.equal(await worker.prefillBonsai2InSlices(runtime, { input_ids: ids, attention_mask: mask }, 0), null);
+ assert.equal(await worker.prefillBonsai2InSlices(runtime, { input_ids: ids, attention_mask: mask }, 9), null,
+ 'a prompt whose prefill fits in one slice keeps the single-pass path');
+ assert.equal(calls.length, 0);
+
+ const cache = await worker.prefillBonsai2InSlices(runtime, { input_ids: ids, attention_mask: mask }, 4);
+ assert.deepEqual(calls.map(c => c.embeds), [[1, 4, H], [1, 4, H], [1, 1, H]],
+ 'all but the last prompt token are prefilled in slices; generate() feeds the last one');
+ assert.deepEqual(calls.map(c => c.firstEmbed), [0, 4 * H, 8 * H], 'slices cover consecutive prompt positions');
+ assert.deepEqual(calls.map(c => c.mask), [4, 8, 9], 'each slice attends to every earlier prompt token');
+ assert.deepEqual(calls.map(c => c.positions[0]), [0, 4, 8]);
+ assert.equal(calls[0].past, null);
+ assert.equal(calls[1].past, cache, 'later slices continue from the shared cache');
+ assert.equal(calls[2].past, cache);
+ assert.deepEqual(Object.keys(cache).sort(), ['past_conv.0', 'past_key_values.3.key', 'past_recurrent.0']);
+ assert.equal(cache['past_conv.0'].data[0], 3, 'the cache holds the final slice state');
+ assert.equal(cache.get_seq_length(), 9);
+
+ calls.length = 0;
+ const grid = new Tensor('int64', [1n, 2n, 2n], [1, 3]);
+ await worker.prefillBonsai2InSlices(runtime, {
+ input_ids: ids, attention_mask: mask,
+ pixel_values: new Tensor('float32', new Float32Array(4), [4, 1]), image_grid_thw: grid,
+ }, 4);
+ assert.deepEqual(imageCalls, [['encode', [1, 3]], ['merge', [2, H]]],
+ 'image features are merged once over the whole prompt before slicing');
+ assert.equal(calls.length, 3);
+
+ const workerSource = fs.readFileSync(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js'), 'utf8');
+ assert.match(workerSource, /past_key_values: prefilled/);
+ assert.match(workerSource, /if \(prefilled\) await prefilled\.dispose\(\)/,
+ 'the caller-supplied cache must be released after generation');
+ } finally {
+ if (previousSelf === undefined) delete globalThis.self;
+ else globalThis.self = previousSelf;
+ if (previousOrt === undefined) delete globalThis[ortSymbol];
+ else globalThis[ortSymbol] = previousOrt;
+ }
+});
+
test('Bonsai 2 worker keeps native rotary positions continuous for text and images', async () => {
const previousSelf = globalThis.self;
const ortSymbol = Symbol.for('onnxruntime');