Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Check prefill evidence against the rewrite's own chunk size
Claude28b0f3963bd6
diff --git a/scripts/models/bonsai/chunk_gdn_prefill.py b/scripts/models/bonsai/chunk_gdn_prefill.pyindex 6c2ec23a2..14baab322 100644--- a/scripts/models/bonsai/chunk_gdn_prefill.py+++ b/scripts/models/bonsai/chunk_gdn_prefill.py@@ -75,11 +75,13 @@ def _require(condition, message):raise ValueError(f"Prefill evidence rejected: {message}")-def verify_prefill_evidence(reports, graph_sha256):+def verify_prefill_evidence(reports, graph_sha256, chunk_tokens):"""Check attached reports and return the verified ones by kind.Raises on an unknown report kind or any failed comparison, so a bundle can- never be sealed with evidence that contradicts its model card.+ never be sealed with evidence that contradicts its model card. CPU parity+ must cover the rewrite's own chunk size: a prompt ending exactly on a chunk+ boundary and one spanning several chunks (which exercises the prefix scan)."""verified = {}for report in reports:@@ -89,8 +91,13 @@ def verify_prefill_evidence(reports, graph_sha256):rows = list(report.get("fresh_prefill") or [])continuation = report.get("continuation_from_cache")_require(rows and continuation, "CPU report needs fresh prefills and a cached continuation")- _require(any(row["tokens"] % 128 == 0 for row in rows) and any(row["tokens"] > 128 for row in rows),- "CPU report must cover a chunk boundary and a multi-chunk prompt")+ if "chunk_tokens" in report:+ _require(int(report["chunk_tokens"]) == chunk_tokens, "CPU report used a different chunk size")+ _require(any(row["tokens"] % chunk_tokens == 0 for row in rows)+ and any(row["tokens"] > chunk_tokens for row in rows),+ f"CPU report must cover a {chunk_tokens}-token chunk boundary and a multi-chunk prompt")+ _require(int(continuation.get("new_tokens", 0)) > chunk_tokens,+ f"CPU continuation must span more than one {chunk_tokens}-token chunk")for row in rows + [continuation]:_require(row.get("top1_equal") is True, f"top-1 differs ({row})")_require(row.get("finite") is True, f"non-finite outputs ({row})")@@ -122,8 +129,17 @@ def verify_prefill_evidence(reports, graph_sha256):return verified+def rewritten_chunk_tokens(graph_path):+ """Chunk size of a graph produced by this tool (the cumsum mask is CxC)."""+ model = onnx.load(graph_path, load_external_data=False)+ for tensor in model.graph.initializer:+ if tensor.name == "/webbrain/chunked_gdn/triu1":+ return int(tensor.dims[0])+ raise ValueError(f"{graph_path} was not rewritten by chunk_gdn_prefill.py")++def model_card_section(reports, graph_sha256, chunk_tokens=128):- verified = verify_prefill_evidence(reports, graph_sha256)+ verified = verify_prefill_evidence(reports, graph_sha256, chunk_tokens)text = MODEL_CARD_INTRO.format(chunk=chunk_tokens) + "\n"if CPU_EVIDENCE_KIND in verified:text += ("`chunked-prefill-validation.json` records CPU comparisons with the original\n"diff --git a/scripts/models/bonsai/validate_chunked_gdn.py b/scripts/models/bonsai/validate_chunked_gdn.pyindex 94b5fc3b6..ef01c4006 100644--- a/scripts/models/bonsai/validate_chunked_gdn.py+++ b/scripts/models/bonsai/validate_chunked_gdn.py@@ -27,7 +27,7 @@ import numpy as npimport onnxruntime as ortfrom tokenizers import Tokenizer-from chunk_gdn_prefill import CPU_EVIDENCE_KIND+from chunk_gdn_prefill import CPU_EVIDENCE_KIND, rewritten_chunk_tokensTEXT = ("The nested set model is a technique for representing nested set collections in relational databases. "@@ -119,13 +119,22 @@ def main():p.add_argument("--rewritten", type=Path, required=True)p.add_argument("--embed", type=Path, required=True)p.add_argument("--tokenizer", type=Path, required=True)- # Cover one token, both sides of the 128-token chunk boundary and a multi-chunk prompt.- p.add_argument("--lengths", default="2,127,128,129,700")+ p.add_argument("--lengths", help="comma-separated prompt lengths; default covers one token, both sides "+ "of the graph's chunk boundary and a multi-chunk prompt")p.add_argument("--output", type=Path)args = p.parse_args()+ chunk = rewritten_chunk_tokens(args.rewritten)+ lengths = ([int(x) for x in args.lengths.split(",")] if args.lengths+ else [2, chunk - 1, chunk, chunk + 1, 5 * chunk + 60])+ # The cached prefix and the continuation both cross chunk boundaries.+ cached, continued = chunk + 22, 2 * chunk + 44+ needed = max(max(lengths) + 1, cached + continued)tok = Tokenizer.from_file(str(args.tokenizer))- ids = tok.encode(TEXT * 8).ids+ repeats = 1+ while len(ids := tok.encode(TEXT * repeats).ids) < needed:+ repeats *= 2+ ids = ids[:needed]embed = session(args.embed)emb_in = embed.get_inputs()[0].nameall_embeds = embed.run(None, {emb_in: np.array([ids], np.int64)})[0]@@ -143,7 +152,8 @@ def main():"continuation_from_cache": None,}- for S in [int(x) for x in args.lengths.split(",")]:+ report["chunk_tokens"] = chunk+ for S in lengths:e = all_embeds[:, :S]a, ta = run(orig, e, empty_cache(orig), 0)b, tb = run(new, e, empty_cache(new), 0)@@ -160,15 +170,15 @@ def main():print(json.dumps({k: r[k] for k in ("tokens", "last_logits_max_abs_diff", "last_logits_scale", "top1_equal", "kl_original_to_rewritten","worst_cache_rel_err", "worst_cache_name", "decode_step_top1_equal", "cpu_seconds_original", "cpu_seconds_rewritten")}))- # continuation: 150 tokens of cache from the original graph, then 300 more tokens through both- head = all_embeds[:, :150]+ # continuation: a cache from the original graph, then more tokens through both+ head = all_embeds[:, :cached]a0, _ = run(orig, head, empty_cache(orig), 0)cache = to_cache(a0)- tail = all_embeds[:, 150:450]- a, _ = run(orig, tail, cache, 150)- b, _ = run(new, tail, cache, 150)+ tail = all_embeds[:, cached:cached + continued]+ a, _ = run(orig, tail, cache, cached)+ b, _ = run(new, tail, cache, cached)r = compare(a, b)- report["continuation_from_cache"] = {"cached_tokens": 150, "new_tokens": 300, **r}+ report["continuation_from_cache"] = {"cached_tokens": cached, "new_tokens": continued, **r}print(json.dumps({"continuation": {k: r[k] for k in ("last_logits_max_abs_diff", "top1_equal", "kl_original_to_rewritten", "worst_cache_rel_err")}}))if args.output:args.output.write_text(json.dumps(report, indent=2))diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.jsindex 0d47c2c43..0c87cf35d 100644--- a/src/chrome/src/offscreen/inference-worker.js+++ b/src/chrome/src/offscreen/inference-worker.js@@ -257,6 +257,9 @@ export async function refreshBonsai2DecoderGraph(modelId, {}const fetcher = typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) : nativeFetch;if (!fetcher) return 'unavailable';+ // Known offline (Apocalypse Mode): load the cached graph now and retry on a+ // later load instead of waiting for the network deadline.+ if (globalThis.navigator?.onLine === false) return 'unavailable';checkedBonsai2DecoderGraphs.add(modelId);const controller = new AbortController();const timer = setTimeout(() => controller.abort(), timeoutMs);diff --git a/test/run.js b/test/run.jsindex fa0c8c972..403b4f7ec 100644--- a/test/run.js+++ b/test/run.js@@ -67610,6 +67610,23 @@ test('Bonsai 2 Vision worker replaces only a stale cached decoder graph', asyncassert.ok(Date.now() - started < 2000, 'a stalled refresh must not hold up model loading');assert.equal(entries.get(otherUrl), 'scan graph');+ const knownOffline = await fresh('known-offline');+ const navigatorDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'navigator');+ let fetchCalls = 0;+ globalThis.fetch = async () => { fetchCalls++; return new Response('chunked graph'); };+ try {+ Object.defineProperty(globalThis, 'navigator', { configurable: true, value: { onLine: false } });+ assert.equal(await knownOffline.refreshBonsai2DecoderGraph(other, { expected: sha('chunked graph') }), 'unavailable');+ assert.equal(fetchCalls, 0, 'a known-offline worker must not wait on the network');+ Object.defineProperty(globalThis, 'navigator', { configurable: true, value: { onLine: true } });+ assert.equal(await knownOffline.refreshBonsai2DecoderGraph(other, { expected: sha('chunked graph') }), 'updated',+ 'a refresh skipped while offline is retried once the worker is online');+ } finally {+ if (navigatorDescriptor) Object.defineProperty(globalThis, 'navigator', navigatorDescriptor);+ else delete globalThis.navigator;+ }+ entries.set(otherUrl, 'scan graph');+const current = await fresh('current');entries.set(otherUrl, 'chunked graph');assert.equal(await current.refreshBonsai2DecoderGraph(other, { expected: sha('chunked graph') }), 'current');