Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Publish the 2B decoder graph with decision_hidden; eval --refresh path; CPU parity check

Claudef29988ee46aa
diff --git a/docs/webgpu-models.md b/docs/webgpu-models.md
index 78764a506..adf273b1d 100644
--- a/docs/webgpu-models.md
+++ b/docs/webgpu-models.md
@@ -265,13 +265,13 @@ Results on the held-out test, RTX 5090, through the extension:
| | d1 FP32, main | d1 FP16, new runtime | Bonsai 2B readout, no training | Bonsai 2B + probe |
|---|---:|---:|---:|---:|
-| Categorical | 92/126 | 92/126 | 74/126 | {probe_cat} |
-| Completion found | 22/24 | 22/24 | 24/24 | {probe_tp} |
-| False completions | 1/48 | 1/48 | 30/48 | {probe_fp} |
-| Unknown recognized | 23/24 | 23/24 | 0/24 | {probe_unk} |
-| Other choice / yes-no | 10/36, 13/18 | 10/36, 13/18 | 18/36, 14/18 | {probe_aux} |
-| Score MAE | 0.574 | 0.573 | 0.459 | {probe_mae} |
-| Latency p50 / p95 (s per request) | 0.83 / 0.97 | 0.75 / 0.88 | 1.65 / 2.16 | {probe_lat} |
+| Categorical | 92/126 | 92/126 | 74/126 | 110/126 |
+| Completion found | 22/24 | 22/24 | 24/24 | 21/24 |
+| False completions | 1/48 | 1/48 | 30/48 | 4/48 |
+| Unknown recognized | 23/24 | 23/24 | 0/24 | 20/24 |
+| Other choice / yes-no | 10/36, 13/18 | 10/36, 13/18 | 18/36, 14/18 | 29/36, 16/18 |
+| Score MAE | 0.574 | 0.573 | 0.459 | 0.724 |
+| Latency p50 / p95 (s per request) | 0.83 / 0.97 | 0.75 / 0.88 | 1.65 / 2.16 | 2.25 / 2.81 |
| 16 questions on 1k / 2k / 4k / 8k-token states | ok / fail / fail / fail | all ok | not run | not run |
- At WebBrain's 0.9 completion threshold d1 is still the better completion
@@ -281,8 +281,14 @@ Results on the held-out test, RTX 5090, through the extension:
- All of these are synthetic, AI-labelled screens from one generator; the
probe and d1 were trained on the same train split. Neither result
establishes real-site quality.
-- Bonsai's cost is the prefill (about 1.1 s for a screenshot of about 1,000
- tokens) plus about 0.15 s per option.
+- Bonsai's cost is the prefill (median 1.2 s; about 1.3 s for an
+ 1,100-token state with one screenshot) plus about 0.15 s per option.
+- The 2B repository's decoder graph with `decision_hidden` is published
+ (Hugging Face commit `b9895bc`, `decision-validation.json`). On the CPU its
+ logits and caches are bit-identical to the previous graph
+ (`scripts/models/bonsai/check_decision_hidden.py`). Installs that cached
+ the previous graph keep it until the model is downloaded again in Settings;
+ until then the provider reports that it needs the updated decoder.
## Updating a published graph
diff --git a/scripts/check-webgpu-decisions.mjs b/scripts/check-webgpu-decisions.mjs
index ed2070b0f..d68ff385a 100644
--- a/scripts/check-webgpu-decisions.mjs
+++ b/scripts/check-webgpu-decisions.mjs
@@ -19,6 +19,8 @@
// --prompt NAME Bonsai decision mode: probe (default, what Settings uses),
// readout, strict or verify
// --decoder-graph F install a replacement decoder graph in the cache first
+// --refresh download a cached model again first, the Settings path
+// that replaces a stale decoder graph by hash
// --return-hidden with --prompt verify, record each option's hidden state
import fs from 'node:fs/promises';
import path from 'node:path';
@@ -145,6 +147,18 @@ try {
if (Date.now() - started > 60 * 60_000) throw new Error('Download did not finish within an hour');
}
}
+ if (status?.ready && args.includes('--refresh')) {
+ const started = Date.now();
+ const refreshed = await message('start_webgpu_download', { model: bonsaiModel });
+ if (!refreshed || refreshed.error || refreshed.success === false) throw new Error(`Refresh failed: ${refreshed?.error}`);
+ do {
+ await sleep(2000);
+ status = await message('get_webgpu_download_status', { model: bonsaiModel });
+ if (status?.error) throw new Error(`Refresh failed: ${status.error}`);
+ if (Date.now() - started > 10 * 60_000) throw new Error('Refresh did not finish within ten minutes');
+ } while (!status?.ready || status?.status === 'downloading');
+ result.refreshSeconds = (Date.now() - started) / 1000;
+ }
const graphFile = option('--decoder-graph', '');
if (graphFile) {
// Replace the cached decoder graph (same external weights) before the
@@ -157,6 +171,13 @@ try {
}, { url: `https://huggingface.co/${bonsaiModel}/resolve/main/onnx/decoder_model_merged_q2f16.onnx`, data });
result.decoderGraph = graphFile;
}
+ result.cachedDecoderGraphSha256 = await harness.evaluate(async url => {
+ const cached = await (await caches.open('transformers-cache')).match(url);
+ if (!cached) return null;
+ const digest = await crypto.subtle.digest('SHA-256', await cached.arrayBuffer());
+ return [...new Uint8Array(digest)].map(byte => byte.toString(16).padStart(2, '0')).join('');
+ }, `https://huggingface.co/${bonsaiModel}/resolve/main/onnx/decoder_model_merged_q2f16.onnx`);
+ console.log('cached decoder graph', result.cachedDecoderGraphSha256);
const temperature = Number(option('--temperature', 1));
const promptVariant = option('--prompt', 'probe');
const returnHidden = args.includes('--return-hidden');
diff --git a/scripts/models/bonsai/README.md b/scripts/models/bonsai/README.md
index ad3ff86b2..85ef797af 100644
--- a/scripts/models/bonsai/README.md
+++ b/scripts/models/bonsai/README.md
@@ -202,5 +202,8 @@ the external weight file are unchanged). `train_decision_probe.py` trains the
`scripts/check-webgpu-decisions.mjs --prompt verify --return-hidden
--decoder-graph <graph>` on d1's balanced-V3 train, validation and test splits.
The selected weights live in `src/chrome/src/offscreen/bonsai-decision-probe.js`.
+`check_decision_hidden.py <dir> --old <previous graph> --new <graph>` checks on
+onnxruntime's CPU provider that logits and caches stay bit-identical (the
+external `.onnx_data` must sit next to both graphs).
A re-exported decoder or a retrained probe must be evaluated again through the
extension before its pin or weights change.
diff --git a/scripts/models/bonsai/check_decision_hidden.py b/scripts/models/bonsai/check_decision_hidden.py
new file mode 100644
index 000000000..2fd011fca
--- /dev/null
+++ b/scripts/models/bonsai/check_decision_hidden.py
@@ -0,0 +1,97 @@
+"""Check that adding `decision_hidden` left the Bonsai 2 decoder unchanged.
+
+Runs the previous and the new merged decoder graph on onnxruntime's CPU
+provider with the same random `inputs_embeds`: a prefill (one below and one
+across a 128-token chunk boundary) and one cached decode step after it.
+Logits and every cache output must be bit-identical, and `decision_hidden`
+must be a finite [1, 1, hidden] tensor. Both graphs read the same external
+`decoder_model_merged_q2f16.onnx_data`, which must sit next to them.
+
+ python scripts/models/bonsai/check_decision_hidden.py <dir> --old dec.onnx --new dec_hidden.onnx
+"""
+import argparse
+import json
+from pathlib import Path
+
+import numpy as np
+import onnxruntime as ort
+
+
+def session(path, threads):
+ options = ort.SessionOptions()
+ options.intra_op_num_threads = threads
+ return ort.InferenceSession(str(path), options, providers=["CPUExecutionProvider"])
+
+
+def feeds(sess, embeds, past, total):
+ out = {}
+ for spec in sess.get_inputs():
+ if spec.name == "inputs_embeds":
+ out[spec.name] = embeds
+ elif spec.name == "attention_mask":
+ out[spec.name] = np.ones((1, total), np.int64)
+ elif spec.name == "position_ids":
+ n = embeds.shape[1]
+ positions = np.arange(total - n, total, dtype=np.int64)
+ out[spec.name] = np.broadcast_to(positions, (3, 1, n)).copy()
+ elif past is not None:
+ out[spec.name] = past[spec.name]
+ else:
+ shape = [1 if dim == "batch_size" else (0 if isinstance(dim, str) else dim) for dim in spec.shape]
+ out[spec.name] = np.zeros(shape, np.float16)
+ return out
+
+
+def past_name(name):
+ return (name.replace("present_conv", "past_conv")
+ .replace("present_recurrent", "past_recurrent")
+ .replace("present", "past_key_values"))
+
+
+def run(sess, embeds, past, total):
+ names = [output.name for output in sess.get_outputs()]
+ outputs = dict(zip(names, sess.run(None, feeds(sess, embeds, past, total))))
+ return outputs, {past_name(k): v for k, v in outputs.items() if k.startswith("present")}
+
+
+def same(left, right):
+ return all(np.array_equal(left[key], right[key]) for key in left if key != "decision_hidden")
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("dir", type=Path)
+ parser.add_argument("--old", default="dec.onnx")
+ parser.add_argument("--new", default="dec_hidden.onnx")
+ parser.add_argument("--threads", type=int, default=8)
+ args = parser.parse_args()
+ old, new = session(args.dir / args.old, args.threads), session(args.dir / args.new, args.threads)
+ rng = np.random.default_rng(0)
+ width = next(spec.shape[2] for spec in old.get_inputs() if spec.name == "inputs_embeds")
+ report = {"onnxruntime": ort.__version__, "inputs": "inputs_embeds ~ N(0, 0.02), seed 0", "cases": []}
+ for tokens in (37, 200):
+ prefill = (rng.standard_normal((1, tokens, width)) * 0.02).astype(np.float32)
+ step = (rng.standard_normal((1, 1, width)) * 0.02).astype(np.float32)
+ a, past_a = run(old, prefill, None, tokens)
+ b, past_b = run(new, prefill, None, tokens)
+ hidden = b["decision_hidden"]
+ a_step, _ = run(old, step, past_a, tokens + 1)
+ b_step, _ = run(new, step, past_b, tokens + 1)
+ case = {
+ "prefill_tokens": tokens,
+ "prefill_logits_and_caches_identical": same(a, b),
+ "decode_logits_and_caches_identical": same(a_step, b_step),
+ "hidden_shape": list(hidden.shape),
+ "hidden_finite": bool(np.isfinite(hidden).all()),
+ }
+ report["cases"].append(case)
+ print(json.dumps(case), flush=True)
+ ok = all(c["prefill_logits_and_caches_identical"] and c["decode_logits_and_caches_identical"]
+ and c["hidden_finite"] and c["hidden_shape"] == [1, 1, width] for c in report["cases"])
+ report["passed"] = ok
+ print(json.dumps(report))
+ raise SystemExit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+ main()