Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Refresh stale graphs before session reuse; size the prefix scan to the context

Claudef8c3ece1285a
diff --git a/scripts/models/bonsai/build_vision.py b/scripts/models/bonsai/build_vision.py
index 8e7afb762..659e24a54 100644
--- a/scripts/models/bonsai/build_vision.py
+++ b/scripts/models/bonsai/build_vision.py
@@ -236,7 +236,8 @@ def build(size, workspace, offline=False, skip_inference=False):
# Replace the token-sequential GatedDeltaNet Scan with the chunked prefill
# graph. Only the small graph file changes; the weights stay hardlinked.
language_graph = rewrite_file(text / "onnx" / "decoder_model_merged_q2f16.onnx",
- output / "onnx" / "decoder_model_merged_q2f16.onnx")
+ output / "onnx" / "decoder_model_merged_q2f16.onnx",
+ context_tokens=config["text_config"]["max_position_embeddings"])
print(f"{size}: rewrote {language_graph['gdn_layers_rewritten']} GatedDeltaNet prefill scans", flush=True)
shutil.copy2(text / "tokenizer.json", output / "tokenizer.json")
for filename in ("tokenizer_config.json", "preprocessor_config.json", "processor_config.json",
diff --git a/scripts/models/bonsai/chunk_gdn_prefill.py b/scripts/models/bonsai/chunk_gdn_prefill.py
index dba42420a..fcd9eebc2 100644
--- a/scripts/models/bonsai/chunk_gdn_prefill.py
+++ b/scripts/models/bonsai/chunk_gdn_prefill.py
@@ -412,11 +412,30 @@ def slice_last_logits(model: onnx.ModelProto) -> str:
return hidden
-def rewrite_file(source: Path, output: Path, chunk: int = 128, max_steps: int = 12, always_steps: int = 1,
- head_dim: int = 128, last_logits: bool = True) -> dict:
- """Rewrite `source` into `output`; the external data file name is unchanged."""
- if chunk < 2 or chunk & (chunk - 1):
- raise ValueError("chunk must be a power of two")
+# Qwen3.5 / Bonsai 2 text_config.max_position_embeddings.
+DEFAULT_CONTEXT_TOKENS = 262144
+
+
+def prefix_scan_steps(chunk: int, context_tokens: int) -> int:
+ """Fewest Hillis-Steele steps whose (2**steps - 1) chunks cover the context."""
+ steps = 1
+ while ((1 << steps) - 1) * chunk < context_tokens:
+ steps += 1
+ return steps
+
+
+def rewrite_file(source: Path, output: Path, chunk: int = 128, context_tokens: int = DEFAULT_CONTEXT_TOKENS,
+ always_steps: int = 1, head_dim: int = 128, last_logits: bool = True) -> dict:
+ """Rewrite `source` into `output`; the external data file name is unchanged.
+
+ The prefix scan is sized for the full model context, so every prompt the
+ model accepts propagates its recurrent state correctly.
+ """
+ if not 16 <= chunk <= 1024 or chunk & (chunk - 1):
+ raise ValueError("chunk must be a power of two between 16 and 1024")
+ if context_tokens < 1:
+ raise ValueError("context_tokens must be positive")
+ max_steps = prefix_scan_steps(chunk, context_tokens)
source_sha = hashlib.sha256(source.read_bytes()).hexdigest()
model = onnx.load(source, load_external_data=False)
replaced = rewrite_gdn(model, chunk, max_steps, always_steps, head_dim)
@@ -439,6 +458,7 @@ def rewrite_file(source: Path, output: Path, chunk: int = 128, max_steps: int =
"output_bytes": output.stat().st_size,
"gdn_layers_rewritten": replaced,
"chunk_tokens": chunk,
+ "context_tokens": context_tokens,
"prefix_scan_steps": max_steps,
"max_prompt_tokens": ((1 << max_steps) - 1) * chunk,
"logits": "last position only" if last_logits else "every position",
@@ -450,12 +470,13 @@ def main():
parser.add_argument("input", type=Path, help="decoder_model_merged_q2f16.onnx (external data stays alongside)")
parser.add_argument("output", type=Path)
parser.add_argument("--chunk", type=int, default=128)
- parser.add_argument("--max-steps", type=int, default=12, help="prefix-scan steps; supports (2**steps - 1) chunks")
+ parser.add_argument("--context-tokens", type=int, default=DEFAULT_CONTEXT_TOKENS,
+ help="longest prompt the prefix scan must cover (model context)")
parser.add_argument("--always-steps", type=int, default=1, help="leading steps emitted without an If guard")
parser.add_argument("--head-dim", type=int, default=128)
parser.add_argument("--keep-all-logits", action="store_true")
args = parser.parse_args()
- report = rewrite_file(args.input, args.output, args.chunk, args.max_steps, args.always_steps,
+ report = rewrite_file(args.input, args.output, args.chunk, args.context_tokens, args.always_steps,
args.head_dim, not args.keep_all_logits)
print(json.dumps(report, indent=2))
diff --git a/scripts/models/bonsai/make_chunked_bundle.py b/scripts/models/bonsai/make_chunked_bundle.py
index a696545f2..95e31446b 100644
--- a/scripts/models/bonsai/make_chunked_bundle.py
+++ b/scripts/models/bonsai/make_chunked_bundle.py
@@ -109,7 +109,10 @@ def build_bundle(source, output, manifest, args):
shutil.copy2(source / path, target)
scratch = output / "onnx" / ".rewrite.onnx"
- report = rewrite_file(source / GRAPH, scratch, chunk=args.chunk)
+ context = read_json(source / "config.json").get("text_config", {}).get("max_position_embeddings")
+ if not context:
+ raise ValueError("config.json has no text_config.max_position_embeddings to size the prefix scan")
+ report = rewrite_file(source / GRAPH, scratch, chunk=args.chunk, context_tokens=int(context))
if args.graph:
if digest(args.graph) != report["output_sha256"]:
# Protobuf serializers may order bytes differently; require identical graphs.
diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.js
index 65cf8c82c..c1dc032b7 100644
--- a/src/chrome/src/offscreen/inference-worker.js
+++ b/src/chrome/src/offscreen/inference-worker.js
@@ -62,6 +62,8 @@ const BONSAI2_DECODER_GRAPH_SHA256 = new Map([
// worker lifetime, after which the working cached graph is used.
const BONSAI2_DECODER_GRAPH_REFRESH_TIMEOUT_MS = 15_000;
const checkedBonsai2DecoderGraphs = new Set();
+// Cache-only loads remember a stale graph so inference does not re-hash it.
+const staleBonsai2DecoderGraphs = new Set();
// The Qwen3.5 generation_config sidecar omitted by Danyal's split exports.
// Per-request decoding still controls do_sample and the output token budget.
const BONSAI2_GENERATION_CONFIG = {
@@ -240,6 +242,7 @@ export async function refreshBonsai2DecoderGraph(modelId, {
timeoutMs = BONSAI2_DECODER_GRAPH_REFRESH_TIMEOUT_MS,
} = {}) {
if (!expected || checkedBonsai2DecoderGraphs.has(modelId)) return 'skipped';
+ if (!allowNetwork && staleBonsai2DecoderGraphs.has(modelId)) return 'stale';
if (typeof caches === 'undefined' || !globalThis.crypto?.subtle) return 'unavailable';
const url = `https://huggingface.co/${modelId}/resolve/main/${BONSAI2_DECODER_GRAPH_FILE}`;
let cache;
@@ -259,7 +262,11 @@ export async function refreshBonsai2DecoderGraph(modelId, {
return 'unavailable';
}
// Cache-only loads (inference) never download; an explicit download replaces it.
- if (!allowNetwork) return 'stale';
+ if (!allowNetwork) {
+ staleBonsai2DecoderGraphs.add(modelId);
+ console.info(`[webgpu] ${modelId}: the cached decoder graph predates parallel prefill; download the model again to update it.`);
+ return 'stale';
+ }
const fetcher = typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) : nativeFetch;
if (!fetcher) return 'unavailable';
// Known offline (Apocalypse Mode): load the cached graph now and retry on a
@@ -276,6 +283,7 @@ export async function refreshBonsai2DecoderGraph(modelId, {
await cache.put(url, new Response(bytes, {
headers: { 'content-type': 'application/octet-stream', 'content-length': String(bytes.byteLength) },
}));
+ staleBonsai2DecoderGraphs.delete(modelId);
return 'updated';
} catch {
return 'unavailable';
@@ -772,6 +780,13 @@ async function getVisionRuntime(modelId, dtype, device, {
// A device loss flagged between turns leaves both resident sessions poisoned;
// dispose them before the cached gate so we never hand out a dead session.
await disposeWebgpuRuntimesIfDirty();
+ // Before any resident-session shortcut: download flows replace a stale
+ // decoder graph (and drop the session built from it); cache-only loads
+ // only notice it.
+ if (WEBGPU_BONSAI2_VISION_MODEL_IDS.has(modelId)) {
+ const graph = await refreshBonsai2DecoderGraph(modelId, { allowNetwork: !localFilesOnly });
+ if (graph === 'updated' && visionRuntimeKey.startsWith(`vision|${modelId}|`)) await disposeVisionRuntime();
+ }
if (visionRuntime && visionRuntimeKey === key) {
visionRuntimeOwner = owner;
return visionRuntime;
@@ -794,10 +809,6 @@ async function getVisionRuntime(modelId, dtype, device, {
error.code = readiness === 'text' ? 'text_model_not_downloaded' : 'vision_model_not_downloaded';
throw error;
}
- if (WEBGPU_BONSAI2_VISION_MODEL_IDS.has(modelId)
- && await refreshBonsai2DecoderGraph(modelId, { allowNetwork: !localFilesOnly }) === 'stale') {
- console.info(`[webgpu] ${modelId}: the cached decoder graph predates parallel prefill; download the model again to update it.`);
- }
const library = await loadLibrary();
const { AutoModelForImageTextToText, AutoProcessor } = library;
if (!AutoModelForImageTextToText || !AutoProcessor) {
diff --git a/test/run.js b/test/run.js
index f452d6f5c..c25afac4f 100644
--- a/test/run.js
+++ b/test/run.js
@@ -67572,6 +67572,9 @@ test('Bonsai 2 Vision worker replaces only a stale cached decoder graph', async
}
assert.match(workerSource, /refreshBonsai2DecoderGraph\(modelId, \{ allowNetwork: !localFilesOnly \}\)/,
'runtime loads may download the graph only outside cache-only inference');
+ const visionLoader = workerSource.slice(workerSource.indexOf('async function getVisionRuntime('));
+ assert.ok(visionLoader.indexOf('refreshBonsai2DecoderGraph(') < visionLoader.indexOf('if (visionRuntime && visionRuntimeKey === key)'),
+ 'download flows must refresh the graph before reusing a resident session');
assert.match(workerSource, /refreshBonsai2DecoderGraph\(modelId, \{ allowNetwork: true \}\) === 'updated'\s*&& visionRuntimeKey\.startsWith\(`vision\|\$\{modelId\}\|`\)\) \{[^}]*await disposeVisionRuntime\(\);/,
'an explicit update must drop the resident session built from the replaced graph');
@@ -67582,6 +67585,8 @@ test('Bonsai 2 Vision worker replaces only a stale cached decoder graph', async
const worker = await import(`${pathToFileURL(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js')).href}?bonsai2-graph-refresh`);
assert.equal(await worker.refreshBonsai2DecoderGraph(model, { expected: sha('chunked graph') }), 'stale',
'cache-only (inference) loads detect a stale graph without downloading');
+ assert.equal(await worker.refreshBonsai2DecoderGraph(model, { expected: sha('chunked graph') }), 'stale',
+ 'repeated cache-only loads keep the stale verdict');
assert.deepEqual(fetched, [], 'inference must never reach the network');
assert.equal(entries.get(graphUrl), 'scan graph');
assert.equal(await worker.refreshBonsai2DecoderGraph(model, { allowNetwork: true, expected: sha('chunked graph') }), 'updated');