Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Render Bonsai prefill model-card claims only from verified evidence
Claude7eec80c87a93
diff --git a/scripts/models/bonsai/README.md b/scripts/models/bonsai/README.mdindex d0c86211b..cdcf07edc 100644--- a/scripts/models/bonsai/README.md+++ b/scripts/models/bonsai/README.md@@ -71,9 +71,13 @@ already sealed one without rebuilding the vision tower:`validate_chunked_gdn.py` compares the two graphs on CPU with real text:prefills across chunk boundaries, a cached continuation, last-position logits,-every cache output and the following decode step. Browser evidence recorded-against the previous graph is not carried over; rerun the WebGPU checks below-on the new bundle. The worker replaces a stale cached graph for the two+every cache output and the following decode step; its `--output` report is the+CPU evidence for `make_chunked_bundle.py --evidence`. Attached reports are+checked against the bundle's graph hash and FP16 tolerances, and the model card+states parity or timing results only for reports that pass; otherwise it says+the evidence is pending, and a failing report stops the build. Browser evidence+recorded against the previous graph is not carried over; rerun the WebGPU+checks below on the new bundle. The worker replaces a stale cached graph for the twowebbrain-one Vision repositories by SHA-256, so existing installs onlydownload the new 3 MB graph file.diff --git a/scripts/models/bonsai/build_vision.py b/scripts/models/bonsai/build_vision.pyindex 83d5d4042..05f0b9ecb 100644--- a/scripts/models/bonsai/build_vision.py+++ b/scripts/models/bonsai/build_vision.py@@ -19,7 +19,7 @@ import onnximport onnxruntime as ortfrom huggingface_hub import snapshot_download-from chunk_gdn_prefill import MODEL_CARD_SECTION, rewrite_file+from chunk_gdn_prefill import model_card_section, rewrite_filefrom onnx import numpy_helperfrom safetensors import safe_open@@ -339,7 +339,7 @@ optional check. {ocr_limit}Apache 2.0; see LICENSE and NOTICE.txt. Original context capacity is 262,144 tokens;WebBrain applies a smaller browser context budget. Model weights and processorfiles are downloaded once; inference runs locally.-""" + MODEL_CARD_SECTION, encoding="utf-8")+""" + model_card_section([], language_graph["output_sha256"]), encoding="utf-8")total = seal_manifest(output)print(json.dumps({"output": str(output.resolve()), "vision_tensors": len(records),"total_bytes": total, "vision_inference": validation}), flush=True)diff --git a/scripts/models/bonsai/chunk_gdn_prefill.py b/scripts/models/bonsai/chunk_gdn_prefill.pyindex 2fbb535f6..6af975b8e 100644--- a/scripts/models/bonsai/chunk_gdn_prefill.py+++ b/scripts/models/bonsai/chunk_gdn_prefill.py@@ -38,8 +38,10 @@ from onnx import TensorProto, helper, numpy_helperINT64_MAX = 9223372036854775807-# Model-card paragraph shared by build_vision.py and make_chunked_bundle.py.-MODEL_CARD_SECTION = """+# Model-card text shared by build_vision.py and make_chunked_bundle.py. The+# validation claims are rendered only from verified evidence (see+# model_card_section) so an unvalidated build cannot publish them.+MODEL_CARD_INTRO = """## Parallel prefillThe upstream language graph computes every GatedDeltaNet prefill with an ONNX@@ -53,14 +55,83 @@ prefix scan across chunks), built only from ops the WebGPU EP implements, andreturns logits for the last position only. Weights, quantization and theexternal data file are byte-identical to the previous revision; decode(one token at a time) uses the unchanged recurrent path.--`chunked-prefill-validation.json` records the rewrite, CPU comparisons with the-original graph on real text (logit differences at fp16 rounding level,-identical top-1 tokens, prefill lengths across chunk boundaries and a cached-continuation) and WebGPU prefill timings against the original graph.-Reproduce with `scripts/models/bonsai/chunk_gdn_prefill.py` in the WebBrain-repository."""+CPU_EVIDENCE_KIND = "cpu-onnxruntime-comparison"+WEBGPU_EVIDENCE_KIND = "webgpu-prefill-benchmark"+# Last-position logits may differ by FP16 rounding only.+MAX_RELATIVE_LOGIT_DIFF = 5e-3+MAX_ABSOLUTE_LOGIT_DIFF = 0.05+MAX_KL = 1e-3+MAX_WEBGPU_KL = 1e-2+++def _require(condition, message):+ if not condition:+ raise ValueError(f"Prefill evidence rejected: {message}")+++def verify_prefill_evidence(reports, graph_sha256):+ """Check attached reports and return the verified ones by kind.++ Raises on an unknown report kind or any failed comparison, so a bundle can+ never be sealed with evidence that contradicts its model card.+ """+ verified = {}+ for report in reports:+ kind = report.get("kind")+ if kind == CPU_EVIDENCE_KIND:+ _require(report.get("rewritten_graph_sha256") == graph_sha256, "CPU report is for a different graph")+ rows = list(report.get("fresh_prefill") or [])+ continuation = report.get("continuation_from_cache")+ _require(rows and continuation, "CPU report needs fresh prefills and a cached continuation")+ _require(any(row["tokens"] % 128 == 0 for row in rows) and any(row["tokens"] > 128 for row in rows),+ "CPU report must cover a chunk boundary and a multi-chunk prompt")+ for row in rows + [continuation]:+ _require(row.get("top1_equal") is True, f"top-1 differs ({row})")+ # Without a recorded scale, allow a few FP16 ulps at typical |logit| of 16-32.+ tolerance = (MAX_RELATIVE_LOGIT_DIFF * max(float(row["last_logits_scale"]), 1.0)+ if row.get("last_logits_scale") else MAX_ABSOLUTE_LOGIT_DIFF)+ _require(float(row["last_logits_max_abs_diff"]) <= tolerance,+ f"logit difference above FP16 tolerance ({row})")+ _require(float(row["kl_original_to_rewritten"]) <= MAX_KL, f"KL too large ({row})")+ for row in rows:+ _require(row.get("decode_step_top1_equal") is True, f"decode step differs ({row})")+ elif kind == WEBGPU_EVIDENCE_KIND:+ if "rewritten_graph_sha256" in report:+ _require(report["rewritten_graph_sha256"] == graph_sha256, "WebGPU report is for a different graph")+ rewritten = [row for row in report.get("prefill") or [] if row.get("variant") == "rewritten"]+ original = [row for row in report.get("prefill") or [] if row.get("variant") == "original"]+ comparisons = list(report.get("webgpu_logit_comparison") or [])+ _require(rewritten and original and comparisons, "WebGPU report needs both timings and a logit comparison")+ for row in comparisons:+ _require(row.get("argmax_equal") is True and float(row["kl"]) <= MAX_WEBGPU_KL,+ f"WebGPU logits differ ({row})")+ else:+ raise ValueError(f"Prefill evidence rejected: unknown report kind {kind!r}")+ verified[kind] = report+ return verified+++def model_card_section(reports, graph_sha256):+ verified = verify_prefill_evidence(reports, graph_sha256)+ text = MODEL_CARD_INTRO + "\n"+ if CPU_EVIDENCE_KIND in verified:+ text += ("`chunked-prefill-validation.json` records CPU comparisons with the original\n"+ "graph on real text (prefill lengths across chunk boundaries and a cached\n"+ "continuation): identical top-1 tokens, including the following decode step, and\n"+ "logit differences at FP16 rounding level.")+ if WEBGPU_EVIDENCE_KIND in verified:+ text += " It also records WebGPU prefill timings against\nthe original graph."+ text += "\n"+ elif WEBGPU_EVIDENCE_KIND in verified:+ text += ("`chunked-prefill-validation.json` records WebGPU prefill timings and logit\n"+ "comparisons against the original graph; CPU parity evidence is pending.\n")+ else:+ text += ("Parity and timing evidence for this build is pending:\n"+ "`chunked-prefill-validation.json` records the rewrite only. Attach\n"+ "`validate_chunked_gdn.py` and WebGPU timing reports before publishing.\n")+ return text + ("Reproduce with `scripts/models/bonsai/chunk_gdn_prefill.py` in the WebBrain\n"+ "repository.\n")class Builder:diff --git a/scripts/models/bonsai/make_chunked_bundle.py b/scripts/models/bonsai/make_chunked_bundle.pyindex 9c6c08b57..eb502d2c7 100644--- a/scripts/models/bonsai/make_chunked_bundle.py+++ b/scripts/models/bonsai/make_chunked_bundle.py@@ -24,7 +24,7 @@ import tempfilefrom pathlib import Pathfrom build_vision import digest, link_or_copy, read_json, seal_manifest, write_json-from chunk_gdn_prefill import MODEL_CARD_SECTION as CARD_SECTION, rewrite_file+from chunk_gdn_prefill import model_card_section, rewrite_fileGRAPH = "onnx/decoder_model_merged_q2f16.onnx"# Upstream danyalmalik language graphs this rewrite was validated against.@@ -93,10 +93,12 @@ def build_bundle(source, output, manifest, args):else:scratch.rename(output / GRAPH)+ evidence = [read_json(path) for path in args.evidence]+ card_section = model_card_section(evidence, report["output_sha256"]) # rejects failed evidencewrite_json(output / DERIVED_MARKER, {"rewrite": report,"external_data_unchanged": "onnx/decoder_model_merged_q2f16.onnx_data",- "evidence": [read_json(path) for path in args.evidence],+ "evidence": evidence,})validation = read_json(output / "export-validation.json")validation["language_graph"] = {@@ -110,7 +112,7 @@ def build_bundle(source, output, manifest, args):card = (output / "README.md").read_text(encoding="utf-8")card = card.replace(CARD_LANGUAGE_OLD, CARD_LANGUAGE_NEW)head, sep, tail = card.partition("\n## Hardware validation")- card = head.rstrip() + "\n" + CARD_SECTION+ card = head.rstrip() + "\n" + card_sectionif sep:# Keep OCR limitation notes; hardware evidence is re-recorded for this graph.ocr = tail.partition("\n### Observed OCR limitation")diff --git a/scripts/models/bonsai/validate_chunked_gdn.py b/scripts/models/bonsai/validate_chunked_gdn.pyindex d5f8338b6..3fbd37532 100644--- a/scripts/models/bonsai/validate_chunked_gdn.py+++ b/scripts/models/bonsai/validate_chunked_gdn.py@@ -21,10 +21,14 @@ import jsonimport timefrom pathlib import Path+import hashlib+import numpy as npimport onnxruntime as ortfrom tokenizers import Tokenizer+from chunk_gdn_prefill import CPU_EVIDENCE_KIND+TEXT = ("The nested set model is a technique for representing nested set collections in relational databases. ""It is based on Nested Intervals, that are immune to hierarchy reorganization problem, and allow answering "@@ -107,7 +111,8 @@ def main():p.add_argument("--rewritten", type=Path, required=True)p.add_argument("--embed", type=Path, required=True)p.add_argument("--tokenizer", type=Path, required=True)- p.add_argument("--lengths", default="2,63,64,65,200,700")+ # Cover one token, both sides of the 128-token chunk boundary and a multi-chunk prompt.+ p.add_argument("--lengths", default="2,127,128,129,700")p.add_argument("--output", type=Path)args = p.parse_args()@@ -118,7 +123,17 @@ def main():all_embeds = embed.run(None, {emb_in: np.array([ids], np.int64)})[0]orig = session(args.original)new = session(args.rewritten)- report = {"tokens_available": len(ids), "fresh_prefill": [], "continuation": None}+ sha = lambda path: hashlib.sha256(path.read_bytes()).hexdigest() # noqa: E731+ report = {+ "kind": CPU_EVIDENCE_KIND,+ "onnxruntime": f"{ort.__version__} (CPUExecutionProvider)",+ "original_graph_sha256": sha(args.original),+ "rewritten_graph_sha256": sha(args.rewritten),+ "input": "Real English text embedded with the bundle's embed_tokens graph; zero initial cache unless noted.",+ "tokens_available": len(ids),+ "fresh_prefill": [],+ "continuation_from_cache": None,+ }for S in [int(x) for x in args.lengths.split(",")]:e = all_embeds[:, :S]@@ -145,7 +160,7 @@ def main():a, _ = run(orig, tail, cache, 150)b, _ = run(new, tail, cache, 150)r = compare(a, b)- report["continuation"] = {"cached_tokens": 150, "new_tokens": 300, **r}+ report["continuation_from_cache"] = {"cached_tokens": 150, "new_tokens": 300, **r}print(json.dumps({"continuation": {k: r[k] for k in ("last_logits_max_abs_diff", "top1_equal", "kl_original_to_rewritten", "worst_cache_rel_err")}}))if args.output:args.output.write_text(json.dumps(report, indent=2))