Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Check prefill evidence against the rewrite's own chunk size

Claude28b0f3963bd6
diff --git a/scripts/models/bonsai/chunk_gdn_prefill.py b/scripts/models/bonsai/chunk_gdn_prefill.py
index 6c2ec23a2..14baab322 100644
--- a/scripts/models/bonsai/chunk_gdn_prefill.py
+++ b/scripts/models/bonsai/chunk_gdn_prefill.py
@@ -75,11 +75,13 @@ def _require(condition, message):
raise ValueError(f"Prefill evidence rejected: {message}")
-def verify_prefill_evidence(reports, graph_sha256):
+def verify_prefill_evidence(reports, graph_sha256, chunk_tokens):
"""Check attached reports and return the verified ones by kind.
Raises on an unknown report kind or any failed comparison, so a bundle can
- never be sealed with evidence that contradicts its model card.
+ never be sealed with evidence that contradicts its model card. CPU parity
+ must cover the rewrite's own chunk size: a prompt ending exactly on a chunk
+ boundary and one spanning several chunks (which exercises the prefix scan).
"""
verified = {}
for report in reports:
@@ -89,8 +91,13 @@ def verify_prefill_evidence(reports, graph_sha256):
rows = list(report.get("fresh_prefill") or [])
continuation = report.get("continuation_from_cache")
_require(rows and continuation, "CPU report needs fresh prefills and a cached continuation")
- _require(any(row["tokens"] % 128 == 0 for row in rows) and any(row["tokens"] > 128 for row in rows),
- "CPU report must cover a chunk boundary and a multi-chunk prompt")
+ if "chunk_tokens" in report:
+ _require(int(report["chunk_tokens"]) == chunk_tokens, "CPU report used a different chunk size")
+ _require(any(row["tokens"] % chunk_tokens == 0 for row in rows)
+ and any(row["tokens"] > chunk_tokens for row in rows),
+ f"CPU report must cover a {chunk_tokens}-token chunk boundary and a multi-chunk prompt")
+ _require(int(continuation.get("new_tokens", 0)) > chunk_tokens,
+ f"CPU continuation must span more than one {chunk_tokens}-token chunk")
for row in rows + [continuation]:
_require(row.get("top1_equal") is True, f"top-1 differs ({row})")
_require(row.get("finite") is True, f"non-finite outputs ({row})")
@@ -122,8 +129,17 @@ def verify_prefill_evidence(reports, graph_sha256):
return verified
+def rewritten_chunk_tokens(graph_path):
+ """Chunk size of a graph produced by this tool (the cumsum mask is CxC)."""
+ model = onnx.load(graph_path, load_external_data=False)
+ for tensor in model.graph.initializer:
+ if tensor.name == "/webbrain/chunked_gdn/triu1":
+ return int(tensor.dims[0])
+ raise ValueError(f"{graph_path} was not rewritten by chunk_gdn_prefill.py")
+
+
def model_card_section(reports, graph_sha256, chunk_tokens=128):
- verified = verify_prefill_evidence(reports, graph_sha256)
+ verified = verify_prefill_evidence(reports, graph_sha256, chunk_tokens)
text = MODEL_CARD_INTRO.format(chunk=chunk_tokens) + "\n"
if CPU_EVIDENCE_KIND in verified:
text += ("`chunked-prefill-validation.json` records CPU comparisons with the original\n"
diff --git a/scripts/models/bonsai/validate_chunked_gdn.py b/scripts/models/bonsai/validate_chunked_gdn.py
index 94b5fc3b6..ef01c4006 100644
--- a/scripts/models/bonsai/validate_chunked_gdn.py
+++ b/scripts/models/bonsai/validate_chunked_gdn.py
@@ -27,7 +27,7 @@ import numpy as np
import onnxruntime as ort
from tokenizers import Tokenizer
-from chunk_gdn_prefill import CPU_EVIDENCE_KIND
+from chunk_gdn_prefill import CPU_EVIDENCE_KIND, rewritten_chunk_tokens
TEXT = (
"The nested set model is a technique for representing nested set collections in relational databases. "
@@ -119,13 +119,22 @@ def main():
p.add_argument("--rewritten", type=Path, required=True)
p.add_argument("--embed", type=Path, required=True)
p.add_argument("--tokenizer", type=Path, required=True)
- # Cover one token, both sides of the 128-token chunk boundary and a multi-chunk prompt.
- p.add_argument("--lengths", default="2,127,128,129,700")
+ p.add_argument("--lengths", help="comma-separated prompt lengths; default covers one token, both sides "
+ "of the graph's chunk boundary and a multi-chunk prompt")
p.add_argument("--output", type=Path)
args = p.parse_args()
+ chunk = rewritten_chunk_tokens(args.rewritten)
+ lengths = ([int(x) for x in args.lengths.split(",")] if args.lengths
+ else [2, chunk - 1, chunk, chunk + 1, 5 * chunk + 60])
+ # The cached prefix and the continuation both cross chunk boundaries.
+ cached, continued = chunk + 22, 2 * chunk + 44
+ needed = max(max(lengths) + 1, cached + continued)
tok = Tokenizer.from_file(str(args.tokenizer))
- ids = tok.encode(TEXT * 8).ids
+ repeats = 1
+ while len(ids := tok.encode(TEXT * repeats).ids) < needed:
+ repeats *= 2
+ ids = ids[:needed]
embed = session(args.embed)
emb_in = embed.get_inputs()[0].name
all_embeds = embed.run(None, {emb_in: np.array([ids], np.int64)})[0]
@@ -143,7 +152,8 @@ def main():
"continuation_from_cache": None,
}
- for S in [int(x) for x in args.lengths.split(",")]:
+ report["chunk_tokens"] = chunk
+ for S in lengths:
e = all_embeds[:, :S]
a, ta = run(orig, e, empty_cache(orig), 0)
b, tb = run(new, e, empty_cache(new), 0)
@@ -160,15 +170,15 @@ def main():
print(json.dumps({k: r[k] for k in ("tokens", "last_logits_max_abs_diff", "last_logits_scale", "top1_equal", "kl_original_to_rewritten",
"worst_cache_rel_err", "worst_cache_name", "decode_step_top1_equal", "cpu_seconds_original", "cpu_seconds_rewritten")}))
- # continuation: 150 tokens of cache from the original graph, then 300 more tokens through both
- head = all_embeds[:, :150]
+ # continuation: a cache from the original graph, then more tokens through both
+ head = all_embeds[:, :cached]
a0, _ = run(orig, head, empty_cache(orig), 0)
cache = to_cache(a0)
- tail = all_embeds[:, 150:450]
- a, _ = run(orig, tail, cache, 150)
- b, _ = run(new, tail, cache, 150)
+ tail = all_embeds[:, cached:cached + continued]
+ a, _ = run(orig, tail, cache, cached)
+ b, _ = run(new, tail, cache, cached)
r = compare(a, b)
- report["continuation_from_cache"] = {"cached_tokens": 150, "new_tokens": 300, **r}
+ report["continuation_from_cache"] = {"cached_tokens": cached, "new_tokens": continued, **r}
print(json.dumps({"continuation": {k: r[k] for k in ("last_logits_max_abs_diff", "top1_equal", "kl_original_to_rewritten", "worst_cache_rel_err")}}))
if args.output:
args.output.write_text(json.dumps(report, indent=2))
diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.js
index 0d47c2c43..0c87cf35d 100644
--- a/src/chrome/src/offscreen/inference-worker.js
+++ b/src/chrome/src/offscreen/inference-worker.js
@@ -257,6 +257,9 @@ export async function refreshBonsai2DecoderGraph(modelId, {
}
const fetcher = typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) : nativeFetch;
if (!fetcher) return 'unavailable';
+ // Known offline (Apocalypse Mode): load the cached graph now and retry on a
+ // later load instead of waiting for the network deadline.
+ if (globalThis.navigator?.onLine === false) return 'unavailable';
checkedBonsai2DecoderGraphs.add(modelId);
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
diff --git a/test/run.js b/test/run.js
index fa0c8c972..403b4f7ec 100644
--- a/test/run.js
+++ b/test/run.js
@@ -67610,6 +67610,23 @@ test('Bonsai 2 Vision worker replaces only a stale cached decoder graph', async
assert.ok(Date.now() - started < 2000, 'a stalled refresh must not hold up model loading');
assert.equal(entries.get(otherUrl), 'scan graph');
+ const knownOffline = await fresh('known-offline');
+ const navigatorDescriptor = Object.getOwnPropertyDescriptor(globalThis, 'navigator');
+ let fetchCalls = 0;
+ globalThis.fetch = async () => { fetchCalls++; return new Response('chunked graph'); };
+ try {
+ Object.defineProperty(globalThis, 'navigator', { configurable: true, value: { onLine: false } });
+ assert.equal(await knownOffline.refreshBonsai2DecoderGraph(other, { expected: sha('chunked graph') }), 'unavailable');
+ assert.equal(fetchCalls, 0, 'a known-offline worker must not wait on the network');
+ Object.defineProperty(globalThis, 'navigator', { configurable: true, value: { onLine: true } });
+ assert.equal(await knownOffline.refreshBonsai2DecoderGraph(other, { expected: sha('chunked graph') }), 'updated',
+ 'a refresh skipped while offline is retried once the worker is online');
+ } finally {
+ if (navigatorDescriptor) Object.defineProperty(globalThis, 'navigator', navigatorDescriptor);
+ else delete globalThis.navigator;
+ }
+ entries.set(otherUrl, 'scan graph');
+
const current = await fresh('current');
entries.set(otherUrl, 'chunked graph');
assert.equal(await current.refreshBonsai2DecoderGraph(other, { expected: sha('chunked graph') }), 'current');