Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Render Bonsai prefill model-card claims only from verified evidence

Claude7eec80c87a93
diff --git a/scripts/models/bonsai/README.md b/scripts/models/bonsai/README.md
index d0c86211b..cdcf07edc 100644
--- a/scripts/models/bonsai/README.md
+++ b/scripts/models/bonsai/README.md
@@ -71,9 +71,13 @@ already sealed one without rebuilding the vision tower:
`validate_chunked_gdn.py` compares the two graphs on CPU with real text:
prefills across chunk boundaries, a cached continuation, last-position logits,
-every cache output and the following decode step. Browser evidence recorded
-against the previous graph is not carried over; rerun the WebGPU checks below
-on the new bundle. The worker replaces a stale cached graph for the two
+every cache output and the following decode step; its `--output` report is the
+CPU evidence for `make_chunked_bundle.py --evidence`. Attached reports are
+checked against the bundle's graph hash and FP16 tolerances, and the model card
+states parity or timing results only for reports that pass; otherwise it says
+the evidence is pending, and a failing report stops the build. Browser evidence
+recorded against the previous graph is not carried over; rerun the WebGPU
+checks below on the new bundle. The worker replaces a stale cached graph for the two
webbrain-one Vision repositories by SHA-256, so existing installs only
download the new 3 MB graph file.
diff --git a/scripts/models/bonsai/build_vision.py b/scripts/models/bonsai/build_vision.py
index 83d5d4042..05f0b9ecb 100644
--- a/scripts/models/bonsai/build_vision.py
+++ b/scripts/models/bonsai/build_vision.py
@@ -19,7 +19,7 @@ import onnx
import onnxruntime as ort
from huggingface_hub import snapshot_download
-from chunk_gdn_prefill import MODEL_CARD_SECTION, rewrite_file
+from chunk_gdn_prefill import model_card_section, rewrite_file
from onnx import numpy_helper
from safetensors import safe_open
@@ -339,7 +339,7 @@ optional check. {ocr_limit}
Apache 2.0; see LICENSE and NOTICE.txt. Original context capacity is 262,144 tokens;
WebBrain applies a smaller browser context budget. Model weights and processor
files are downloaded once; inference runs locally.
-""" + MODEL_CARD_SECTION, encoding="utf-8")
+""" + model_card_section([], language_graph["output_sha256"]), encoding="utf-8")
total = seal_manifest(output)
print(json.dumps({"output": str(output.resolve()), "vision_tensors": len(records),
"total_bytes": total, "vision_inference": validation}), flush=True)
diff --git a/scripts/models/bonsai/chunk_gdn_prefill.py b/scripts/models/bonsai/chunk_gdn_prefill.py
index 2fbb535f6..6af975b8e 100644
--- a/scripts/models/bonsai/chunk_gdn_prefill.py
+++ b/scripts/models/bonsai/chunk_gdn_prefill.py
@@ -38,8 +38,10 @@ from onnx import TensorProto, helper, numpy_helper
INT64_MAX = 9223372036854775807
-# Model-card paragraph shared by build_vision.py and make_chunked_bundle.py.
-MODEL_CARD_SECTION = """
+# Model-card text shared by build_vision.py and make_chunked_bundle.py. The
+# validation claims are rendered only from verified evidence (see
+# model_card_section) so an unvalidated build cannot publish them.
+MODEL_CARD_INTRO = """
## Parallel prefill
The upstream language graph computes every GatedDeltaNet prefill with an ONNX
@@ -53,14 +55,83 @@ prefix scan across chunks), built only from ops the WebGPU EP implements, and
returns logits for the last position only. Weights, quantization and the
external data file are byte-identical to the previous revision; decode
(one token at a time) uses the unchanged recurrent path.
-
-`chunked-prefill-validation.json` records the rewrite, CPU comparisons with the
-original graph on real text (logit differences at fp16 rounding level,
-identical top-1 tokens, prefill lengths across chunk boundaries and a cached
-continuation) and WebGPU prefill timings against the original graph.
-Reproduce with `scripts/models/bonsai/chunk_gdn_prefill.py` in the WebBrain
-repository.
"""
+CPU_EVIDENCE_KIND = "cpu-onnxruntime-comparison"
+WEBGPU_EVIDENCE_KIND = "webgpu-prefill-benchmark"
+# Last-position logits may differ by FP16 rounding only.
+MAX_RELATIVE_LOGIT_DIFF = 5e-3
+MAX_ABSOLUTE_LOGIT_DIFF = 0.05
+MAX_KL = 1e-3
+MAX_WEBGPU_KL = 1e-2
+
+
+def _require(condition, message):
+ if not condition:
+ raise ValueError(f"Prefill evidence rejected: {message}")
+
+
+def verify_prefill_evidence(reports, graph_sha256):
+ """Check attached reports and return the verified ones by kind.
+
+ Raises on an unknown report kind or any failed comparison, so a bundle can
+ never be sealed with evidence that contradicts its model card.
+ """
+ verified = {}
+ for report in reports:
+ kind = report.get("kind")
+ if kind == CPU_EVIDENCE_KIND:
+ _require(report.get("rewritten_graph_sha256") == graph_sha256, "CPU report is for a different graph")
+ rows = list(report.get("fresh_prefill") or [])
+ continuation = report.get("continuation_from_cache")
+ _require(rows and continuation, "CPU report needs fresh prefills and a cached continuation")
+ _require(any(row["tokens"] % 128 == 0 for row in rows) and any(row["tokens"] > 128 for row in rows),
+ "CPU report must cover a chunk boundary and a multi-chunk prompt")
+ for row in rows + [continuation]:
+ _require(row.get("top1_equal") is True, f"top-1 differs ({row})")
+ # Without a recorded scale, allow a few FP16 ulps at typical |logit| of 16-32.
+ tolerance = (MAX_RELATIVE_LOGIT_DIFF * max(float(row["last_logits_scale"]), 1.0)
+ if row.get("last_logits_scale") else MAX_ABSOLUTE_LOGIT_DIFF)
+ _require(float(row["last_logits_max_abs_diff"]) <= tolerance,
+ f"logit difference above FP16 tolerance ({row})")
+ _require(float(row["kl_original_to_rewritten"]) <= MAX_KL, f"KL too large ({row})")
+ for row in rows:
+ _require(row.get("decode_step_top1_equal") is True, f"decode step differs ({row})")
+ elif kind == WEBGPU_EVIDENCE_KIND:
+ if "rewritten_graph_sha256" in report:
+ _require(report["rewritten_graph_sha256"] == graph_sha256, "WebGPU report is for a different graph")
+ rewritten = [row for row in report.get("prefill") or [] if row.get("variant") == "rewritten"]
+ original = [row for row in report.get("prefill") or [] if row.get("variant") == "original"]
+ comparisons = list(report.get("webgpu_logit_comparison") or [])
+ _require(rewritten and original and comparisons, "WebGPU report needs both timings and a logit comparison")
+ for row in comparisons:
+ _require(row.get("argmax_equal") is True and float(row["kl"]) <= MAX_WEBGPU_KL,
+ f"WebGPU logits differ ({row})")
+ else:
+ raise ValueError(f"Prefill evidence rejected: unknown report kind {kind!r}")
+ verified[kind] = report
+ return verified
+
+
+def model_card_section(reports, graph_sha256):
+ verified = verify_prefill_evidence(reports, graph_sha256)
+ text = MODEL_CARD_INTRO + "\n"
+ if CPU_EVIDENCE_KIND in verified:
+ text += ("`chunked-prefill-validation.json` records CPU comparisons with the original\n"
+ "graph on real text (prefill lengths across chunk boundaries and a cached\n"
+ "continuation): identical top-1 tokens, including the following decode step, and\n"
+ "logit differences at FP16 rounding level.")
+ if WEBGPU_EVIDENCE_KIND in verified:
+ text += " It also records WebGPU prefill timings against\nthe original graph."
+ text += "\n"
+ elif WEBGPU_EVIDENCE_KIND in verified:
+ text += ("`chunked-prefill-validation.json` records WebGPU prefill timings and logit\n"
+ "comparisons against the original graph; CPU parity evidence is pending.\n")
+ else:
+ text += ("Parity and timing evidence for this build is pending:\n"
+ "`chunked-prefill-validation.json` records the rewrite only. Attach\n"
+ "`validate_chunked_gdn.py` and WebGPU timing reports before publishing.\n")
+ return text + ("Reproduce with `scripts/models/bonsai/chunk_gdn_prefill.py` in the WebBrain\n"
+ "repository.\n")
class Builder:
diff --git a/scripts/models/bonsai/make_chunked_bundle.py b/scripts/models/bonsai/make_chunked_bundle.py
index 9c6c08b57..eb502d2c7 100644
--- a/scripts/models/bonsai/make_chunked_bundle.py
+++ b/scripts/models/bonsai/make_chunked_bundle.py
@@ -24,7 +24,7 @@ import tempfile
from pathlib import Path
from build_vision import digest, link_or_copy, read_json, seal_manifest, write_json
-from chunk_gdn_prefill import MODEL_CARD_SECTION as CARD_SECTION, rewrite_file
+from chunk_gdn_prefill import model_card_section, rewrite_file
GRAPH = "onnx/decoder_model_merged_q2f16.onnx"
# Upstream danyalmalik language graphs this rewrite was validated against.
@@ -93,10 +93,12 @@ def build_bundle(source, output, manifest, args):
else:
scratch.rename(output / GRAPH)
+ evidence = [read_json(path) for path in args.evidence]
+ card_section = model_card_section(evidence, report["output_sha256"]) # rejects failed evidence
write_json(output / DERIVED_MARKER, {
"rewrite": report,
"external_data_unchanged": "onnx/decoder_model_merged_q2f16.onnx_data",
- "evidence": [read_json(path) for path in args.evidence],
+ "evidence": evidence,
})
validation = read_json(output / "export-validation.json")
validation["language_graph"] = {
@@ -110,7 +112,7 @@ def build_bundle(source, output, manifest, args):
card = (output / "README.md").read_text(encoding="utf-8")
card = card.replace(CARD_LANGUAGE_OLD, CARD_LANGUAGE_NEW)
head, sep, tail = card.partition("\n## Hardware validation")
- card = head.rstrip() + "\n" + CARD_SECTION
+ card = head.rstrip() + "\n" + card_section
if sep:
# Keep OCR limitation notes; hardware evidence is re-recorded for this graph.
ocr = tail.partition("\n### Observed OCR limitation")
diff --git a/scripts/models/bonsai/validate_chunked_gdn.py b/scripts/models/bonsai/validate_chunked_gdn.py
index d5f8338b6..3fbd37532 100644
--- a/scripts/models/bonsai/validate_chunked_gdn.py
+++ b/scripts/models/bonsai/validate_chunked_gdn.py
@@ -21,10 +21,14 @@ import json
import time
from pathlib import Path
+import hashlib
+
import numpy as np
import onnxruntime as ort
from tokenizers import Tokenizer
+from chunk_gdn_prefill import CPU_EVIDENCE_KIND
+
TEXT = (
"The nested set model is a technique for representing nested set collections in relational databases. "
"It is based on Nested Intervals, that are immune to hierarchy reorganization problem, and allow answering "
@@ -107,7 +111,8 @@ def main():
p.add_argument("--rewritten", type=Path, required=True)
p.add_argument("--embed", type=Path, required=True)
p.add_argument("--tokenizer", type=Path, required=True)
- p.add_argument("--lengths", default="2,63,64,65,200,700")
+ # Cover one token, both sides of the 128-token chunk boundary and a multi-chunk prompt.
+ p.add_argument("--lengths", default="2,127,128,129,700")
p.add_argument("--output", type=Path)
args = p.parse_args()
@@ -118,7 +123,17 @@ def main():
all_embeds = embed.run(None, {emb_in: np.array([ids], np.int64)})[0]
orig = session(args.original)
new = session(args.rewritten)
- report = {"tokens_available": len(ids), "fresh_prefill": [], "continuation": None}
+ sha = lambda path: hashlib.sha256(path.read_bytes()).hexdigest() # noqa: E731
+ report = {
+ "kind": CPU_EVIDENCE_KIND,
+ "onnxruntime": f"{ort.__version__} (CPUExecutionProvider)",
+ "original_graph_sha256": sha(args.original),
+ "rewritten_graph_sha256": sha(args.rewritten),
+ "input": "Real English text embedded with the bundle's embed_tokens graph; zero initial cache unless noted.",
+ "tokens_available": len(ids),
+ "fresh_prefill": [],
+ "continuation_from_cache": None,
+ }
for S in [int(x) for x in args.lengths.split(",")]:
e = all_embeds[:, :S]
@@ -145,7 +160,7 @@ def main():
a, _ = run(orig, tail, cache, 150)
b, _ = run(new, tail, cache, 150)
r = compare(a, b)
- report["continuation"] = {"cached_tokens": 150, "new_tokens": 300, **r}
+ report["continuation_from_cache"] = {"cached_tokens": 150, "new_tokens": 300, **r}
print(json.dumps({"continuation": {k: r[k] for k in ("last_logits_max_abs_diff", "top1_equal", "kl_original_to_rewritten", "worst_cache_rel_err")}}))
if args.output:
args.output.write_text(json.dumps(report, indent=2))