Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Publish the 2B decoder graph with decision_hidden; eval --refresh path; CPU parity check
Claudef29988ee46aa
diff --git a/docs/webgpu-models.md b/docs/webgpu-models.mdindex 78764a506..adf273b1d 100644--- a/docs/webgpu-models.md+++ b/docs/webgpu-models.md@@ -265,13 +265,13 @@ Results on the held-out test, RTX 5090, through the extension:| | d1 FP32, main | d1 FP16, new runtime | Bonsai 2B readout, no training | Bonsai 2B + probe ||---|---:|---:|---:|---:|-| Categorical | 92/126 | 92/126 | 74/126 | {probe_cat} |-| Completion found | 22/24 | 22/24 | 24/24 | {probe_tp} |-| False completions | 1/48 | 1/48 | 30/48 | {probe_fp} |-| Unknown recognized | 23/24 | 23/24 | 0/24 | {probe_unk} |-| Other choice / yes-no | 10/36, 13/18 | 10/36, 13/18 | 18/36, 14/18 | {probe_aux} |-| Score MAE | 0.574 | 0.573 | 0.459 | {probe_mae} |-| Latency p50 / p95 (s per request) | 0.83 / 0.97 | 0.75 / 0.88 | 1.65 / 2.16 | {probe_lat} |+| Categorical | 92/126 | 92/126 | 74/126 | 110/126 |+| Completion found | 22/24 | 22/24 | 24/24 | 21/24 |+| False completions | 1/48 | 1/48 | 30/48 | 4/48 |+| Unknown recognized | 23/24 | 23/24 | 0/24 | 20/24 |+| Other choice / yes-no | 10/36, 13/18 | 10/36, 13/18 | 18/36, 14/18 | 29/36, 16/18 |+| Score MAE | 0.574 | 0.573 | 0.459 | 0.724 |+| Latency p50 / p95 (s per request) | 0.83 / 0.97 | 0.75 / 0.88 | 1.65 / 2.16 | 2.25 / 2.81 || 16 questions on 1k / 2k / 4k / 8k-token states | ok / fail / fail / fail | all ok | not run | not run |- At WebBrain's 0.9 completion threshold d1 is still the better completion@@ -281,8 +281,14 @@ Results on the held-out test, RTX 5090, through the extension:- All of these are synthetic, AI-labelled screens from one generator; theprobe and d1 were trained on the same train split. Neither resultestablishes real-site quality.-- Bonsai's cost is the prefill (about 1.1 s for a screenshot of about 1,000- tokens) plus about 0.15 s per option.+- Bonsai's cost is the prefill (median 1.2 s; about 1.3 s for an+ 1,100-token state with one screenshot) plus about 0.15 s per option.+- The 2B repository's decoder graph with `decision_hidden` is published+ (Hugging Face commit `b9895bc`, `decision-validation.json`). On the CPU its+ logits and caches are bit-identical to the previous graph+ (`scripts/models/bonsai/check_decision_hidden.py`). Installs that cached+ the previous graph keep it until the model is downloaded again in Settings;+ until then the provider reports that it needs the updated decoder.## Updating a published graphdiff --git a/scripts/check-webgpu-decisions.mjs b/scripts/check-webgpu-decisions.mjsindex ed2070b0f..d68ff385a 100644--- a/scripts/check-webgpu-decisions.mjs+++ b/scripts/check-webgpu-decisions.mjs@@ -19,6 +19,8 @@// --prompt NAME Bonsai decision mode: probe (default, what Settings uses),// readout, strict or verify// --decoder-graph F install a replacement decoder graph in the cache first+// --refresh download a cached model again first, the Settings path+// that replaces a stale decoder graph by hash// --return-hidden with --prompt verify, record each option's hidden stateimport fs from 'node:fs/promises';import path from 'node:path';@@ -145,6 +147,18 @@ try {if (Date.now() - started > 60 * 60_000) throw new Error('Download did not finish within an hour');}}+ if (status?.ready && args.includes('--refresh')) {+ const started = Date.now();+ const refreshed = await message('start_webgpu_download', { model: bonsaiModel });+ if (!refreshed || refreshed.error || refreshed.success === false) throw new Error(`Refresh failed: ${refreshed?.error}`);+ do {+ await sleep(2000);+ status = await message('get_webgpu_download_status', { model: bonsaiModel });+ if (status?.error) throw new Error(`Refresh failed: ${status.error}`);+ if (Date.now() - started > 10 * 60_000) throw new Error('Refresh did not finish within ten minutes');+ } while (!status?.ready || status?.status === 'downloading');+ result.refreshSeconds = (Date.now() - started) / 1000;+ }const graphFile = option('--decoder-graph', '');if (graphFile) {// Replace the cached decoder graph (same external weights) before the@@ -157,6 +171,13 @@ try {}, { url: `https://huggingface.co/${bonsaiModel}/resolve/main/onnx/decoder_model_merged_q2f16.onnx`, data });result.decoderGraph = graphFile;}+ result.cachedDecoderGraphSha256 = await harness.evaluate(async url => {+ const cached = await (await caches.open('transformers-cache')).match(url);+ if (!cached) return null;+ const digest = await crypto.subtle.digest('SHA-256', await cached.arrayBuffer());+ return [...new Uint8Array(digest)].map(byte => byte.toString(16).padStart(2, '0')).join('');+ }, `https://huggingface.co/${bonsaiModel}/resolve/main/onnx/decoder_model_merged_q2f16.onnx`);+ console.log('cached decoder graph', result.cachedDecoderGraphSha256);const temperature = Number(option('--temperature', 1));const promptVariant = option('--prompt', 'probe');const returnHidden = args.includes('--return-hidden');diff --git a/scripts/models/bonsai/README.md b/scripts/models/bonsai/README.mdindex ad3ff86b2..85ef797af 100644--- a/scripts/models/bonsai/README.md+++ b/scripts/models/bonsai/README.md@@ -202,5 +202,8 @@ the external weight file are unchanged). `train_decision_probe.py` trains the`scripts/check-webgpu-decisions.mjs --prompt verify --return-hidden--decoder-graph <graph>` on d1's balanced-V3 train, validation and test splits.The selected weights live in `src/chrome/src/offscreen/bonsai-decision-probe.js`.+`check_decision_hidden.py <dir> --old <previous graph> --new <graph>` checks on+onnxruntime's CPU provider that logits and caches stay bit-identical (the+external `.onnx_data` must sit next to both graphs).A re-exported decoder or a retrained probe must be evaluated again through theextension before its pin or weights change.diff --git a/scripts/models/bonsai/check_decision_hidden.py b/scripts/models/bonsai/check_decision_hidden.pynew file mode 100644index 000000000..2fd011fca--- /dev/null+++ b/scripts/models/bonsai/check_decision_hidden.py@@ -0,0 +1,97 @@+"""Check that adding `decision_hidden` left the Bonsai 2 decoder unchanged.++Runs the previous and the new merged decoder graph on onnxruntime's CPU+provider with the same random `inputs_embeds`: a prefill (one below and one+across a 128-token chunk boundary) and one cached decode step after it.+Logits and every cache output must be bit-identical, and `decision_hidden`+must be a finite [1, 1, hidden] tensor. Both graphs read the same external+`decoder_model_merged_q2f16.onnx_data`, which must sit next to them.++ python scripts/models/bonsai/check_decision_hidden.py <dir> --old dec.onnx --new dec_hidden.onnx+"""+import argparse+import json+from pathlib import Path++import numpy as np+import onnxruntime as ort+++def session(path, threads):+ options = ort.SessionOptions()+ options.intra_op_num_threads = threads+ return ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"])+++def feeds(sess, embeds, past, total):+ out = {}+ for spec in sess.get_inputs():+ if spec.name == "inputs_embeds":+ out[spec.name] = embeds+ elif spec.name == "attention_mask":+ out[spec.name] = np.ones((1, total), np.int64)+ elif spec.name == "position_ids":+ n = embeds.shape[1]+ positions = np.arange(total - n, total, dtype=np.int64)+ out[spec.name] = np.broadcast_to(positions, (3, 1, n)).copy()+ elif past is not None:+ out[spec.name] = past[spec.name]+ else:+ shape = [1 if dim == "batch_size" else (0 if isinstance(dim, str) else dim) for dim in spec.shape]+ out[spec.name] = np.zeros(shape, np.float16)+ return out+++def past_name(name):+ return (name.replace("present_conv", "past_conv")+ .replace("present_recurrent", "past_recurrent")+ .replace("present", "past_key_values"))+++def run(sess, embeds, past, total):+ names = [output.name for output in sess.get_outputs()]+ outputs = dict(zip(names, sess.run(None, feeds(sess, embeds, past, total))))+ return outputs, {past_name(k): v for k, v in outputs.items() if k.startswith("present")}+++def same(left, right):+ return all(np.array_equal(left[key], right[key]) for key in left if key != "decision_hidden")+++def main():+ parser = argparse.ArgumentParser()+ parser.add_argument("dir", type=Path)+ parser.add_argument("--old", default="dec.onnx")+ parser.add_argument("--new", default="dec_hidden.onnx")+ parser.add_argument("--threads", type=int, default=8)+ args = parser.parse_args()+ old, new = session(args.dir / args.old, args.threads), session(args.dir / args.new, args.threads)+ rng = np.random.default_rng(0)+ width = next(spec.shape[2] for spec in old.get_inputs() if spec.name == "inputs_embeds")+ report = {"onnxruntime": ort.__version__, "inputs": "inputs_embeds ~ N(0, 0.02), seed 0", "cases": []}+ for tokens in (37, 200):+ prefill = (rng.standard_normal((1, tokens, width)) * 0.02).astype(np.float32)+ step = (rng.standard_normal((1, 1, width)) * 0.02).astype(np.float32)+ a, past_a = run(old, prefill, None, tokens)+ b, past_b = run(new, prefill, None, tokens)+ hidden = b["decision_hidden"]+ a_step, _ = run(old, step, past_a, tokens + 1)+ b_step, _ = run(new, step, past_b, tokens + 1)+ case = {+ "prefill_tokens": tokens,+ "prefill_logits_and_caches_identical": same(a, b),+ "decode_logits_and_caches_identical": same(a_step, b_step),+ "hidden_shape": list(hidden.shape),+ "hidden_finite": bool(np.isfinite(hidden).all()),+ }+ report["cases"].append(case)+ print(json.dumps(case), flush=True)+ ok = all(c["prefill_logits_and_caches_identical"] and c["decode_logits_and_caches_identical"]+ and c["hidden_finite"] and c["hidden_shape"] == [1, 1, width] for c in report["cases"])+ report["passed"] = ok+ print(json.dumps(report))+ raise SystemExit(0 if ok else 1)+++if __name__ == "__main__":+ main()