Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Refresh stale graphs before session reuse; size the prefix scan to the context
Claudef8c3ece1285a
diff --git a/scripts/models/bonsai/build_vision.py b/scripts/models/bonsai/build_vision.pyindex 8e7afb762..659e24a54 100644--- a/scripts/models/bonsai/build_vision.py+++ b/scripts/models/bonsai/build_vision.py@@ -236,7 +236,8 @@ def build(size, workspace, offline=False, skip_inference=False):# Replace the token-sequential GatedDeltaNet Scan with the chunked prefill# graph. Only the small graph file changes; the weights stay hardlinked.language_graph = rewrite_file(text / "onnx" / "decoder_model_merged_q2f16.onnx",- output / "onnx" / "decoder_model_merged_q2f16.onnx")+ output / "onnx" / "decoder_model_merged_q2f16.onnx",+ context_tokens=config["text_config"]["max_position_embeddings"])print(f"{size}: rewrote {language_graph['gdn_layers_rewritten']} GatedDeltaNet prefill scans", flush=True)shutil.copy2(text / "tokenizer.json", output / "tokenizer.json")for filename in ("tokenizer_config.json", "preprocessor_config.json", "processor_config.json",diff --git a/scripts/models/bonsai/chunk_gdn_prefill.py b/scripts/models/bonsai/chunk_gdn_prefill.pyindex dba42420a..fcd9eebc2 100644--- a/scripts/models/bonsai/chunk_gdn_prefill.py+++ b/scripts/models/bonsai/chunk_gdn_prefill.py@@ -412,11 +412,30 @@ def slice_last_logits(model: onnx.ModelProto) -> str:return hidden-def rewrite_file(source: Path, output: Path, chunk: int = 128, max_steps: int = 12, always_steps: int = 1,- head_dim: int = 128, last_logits: bool = True) -> dict:- """Rewrite `source` into `output`; the external data file name is unchanged."""- if chunk < 2 or chunk & (chunk - 1):- raise ValueError("chunk must be a power of two")+# Qwen3.5 / Bonsai 2 text_config.max_position_embeddings.+DEFAULT_CONTEXT_TOKENS = 262144+++def prefix_scan_steps(chunk: int, context_tokens: int) -> int:+ """Fewest Hillis-Steele steps whose (2**steps - 1) chunks cover the context."""+ steps = 1+ while ((1 << steps) - 1) * chunk < context_tokens:+ steps += 1+ return steps+++def rewrite_file(source: Path, output: Path, chunk: int = 128, context_tokens: int = DEFAULT_CONTEXT_TOKENS,+ always_steps: int = 1, head_dim: int = 128, last_logits: bool = True) -> dict:+ """Rewrite `source` into `output`; the external data file name is unchanged.++ The prefix scan is sized for the full model context, so every prompt the+ model accepts propagates its recurrent state correctly.+ """+ if not 16 <= chunk <= 1024 or chunk & (chunk - 1):+ raise ValueError("chunk must be a power of two between 16 and 1024")+ if context_tokens < 1:+ raise ValueError("context_tokens must be positive")+ max_steps = prefix_scan_steps(chunk, context_tokens)source_sha = hashlib.sha256(source.read_bytes()).hexdigest()model = onnx.load(source, load_external_data=False)replaced = rewrite_gdn(model, chunk, max_steps, always_steps, head_dim)@@ -439,6 +458,7 @@ def rewrite_file(source: Path, output: Path, chunk: int = 128, max_steps: int ="output_bytes": output.stat().st_size,"gdn_layers_rewritten": replaced,"chunk_tokens": chunk,+ "context_tokens": context_tokens,"prefix_scan_steps": max_steps,"max_prompt_tokens": ((1 << max_steps) - 1) * chunk,"logits": "last position only" if last_logits else "every position",@@ -450,12 +470,13 @@ def main():parser.add_argument("input", type=Path, help="decoder_model_merged_q2f16.onnx (external data stays alongside)")parser.add_argument("output", type=Path)parser.add_argument("--chunk", type=int, default=128)- parser.add_argument("--max-steps", type=int, default=12, help="prefix-scan steps; supports (2**steps - 1) chunks")+ parser.add_argument("--context-tokens", type=int, default=DEFAULT_CONTEXT_TOKENS,+ help="longest prompt the prefix scan must cover (model context)")parser.add_argument("--always-steps", type=int, default=1, help="leading steps emitted without an If guard")parser.add_argument("--head-dim", type=int, default=128)parser.add_argument("--keep-all-logits", action="store_true")args = parser.parse_args()- report = rewrite_file(args.input, args.output, args.chunk, args.max_steps, args.always_steps,+ report = rewrite_file(args.input, args.output, args.chunk, args.context_tokens, args.always_steps,args.head_dim, not args.keep_all_logits)print(json.dumps(report, indent=2))diff --git a/scripts/models/bonsai/make_chunked_bundle.py b/scripts/models/bonsai/make_chunked_bundle.pyindex a696545f2..95e31446b 100644--- a/scripts/models/bonsai/make_chunked_bundle.py+++ b/scripts/models/bonsai/make_chunked_bundle.py@@ -109,7 +109,10 @@ def build_bundle(source, output, manifest, args):shutil.copy2(source / path, target)scratch = output / "onnx" / ".rewrite.onnx"- report = rewrite_file(source / GRAPH, scratch, chunk=args.chunk)+ context = read_json(source / "config.json").get("text_config", {}).get("max_position_embeddings")+ if not context:+ raise ValueError("config.json has no text_config.max_position_embeddings to size the prefix scan")+ report = rewrite_file(source / GRAPH, scratch, chunk=args.chunk, context_tokens=int(context))if args.graph:if digest(args.graph) != report["output_sha256"]:# Protobuf serializers may order bytes differently; require identical graphs.diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.jsindex 65cf8c82c..c1dc032b7 100644--- a/src/chrome/src/offscreen/inference-worker.js+++ b/src/chrome/src/offscreen/inference-worker.js@@ -62,6 +62,8 @@ const BONSAI2_DECODER_GRAPH_SHA256 = new Map([// worker lifetime, after which the working cached graph is used.const BONSAI2_DECODER_GRAPH_REFRESH_TIMEOUT_MS = 15_000;const checkedBonsai2DecoderGraphs = new Set();+// Cache-only loads remember a stale graph so inference does not re-hash it.+const staleBonsai2DecoderGraphs = new Set();// The Qwen3.5 generation_config sidecar omitted by Danyal's split exports.// Per-request decoding still controls do_sample and the output token budget.const BONSAI2_GENERATION_CONFIG = {@@ -240,6 +242,7 @@ export async function refreshBonsai2DecoderGraph(modelId, {timeoutMs = BONSAI2_DECODER_GRAPH_REFRESH_TIMEOUT_MS,} = {}) {if (!expected || checkedBonsai2DecoderGraphs.has(modelId)) return 'skipped';+ if (!allowNetwork && staleBonsai2DecoderGraphs.has(modelId)) return 'stale';if (typeof caches === 'undefined' || !globalThis.crypto?.subtle) return 'unavailable';const url = `https://huggingface.co/${modelId}/resolve/main/${BONSAI2_DECODER_GRAPH_FILE}`;let cache;@@ -259,7 +262,11 @@ export async function refreshBonsai2DecoderGraph(modelId, {return 'unavailable';}// Cache-only loads (inference) never download; an explicit download replaces it.- if (!allowNetwork) return 'stale';+ if (!allowNetwork) {+ staleBonsai2DecoderGraphs.add(modelId);+ console.info(`[webgpu] ${modelId}: the cached decoder graph predates parallel prefill; download the model again to update it.`);+ return 'stale';+ }const fetcher = typeof globalThis.fetch === 'function' ? globalThis.fetch.bind(globalThis) : nativeFetch;if (!fetcher) return 'unavailable';// Known offline (Apocalypse Mode): load the cached graph now and retry on a@@ -276,6 +283,7 @@ export async function refreshBonsai2DecoderGraph(modelId, {await cache.put(url, new Response(bytes, {headers: { 'content-type': 'application/octet-stream', 'content-length': String(bytes.byteLength) },}));+ staleBonsai2DecoderGraphs.delete(modelId);return 'updated';} catch {return 'unavailable';@@ -772,6 +780,13 @@ async function getVisionRuntime(modelId, dtype, device, {// A device loss flagged between turns leaves both resident sessions poisoned;// dispose them before the cached gate so we never hand out a dead session.await disposeWebgpuRuntimesIfDirty();+ // Before any resident-session shortcut: download flows replace a stale+ // decoder graph (and drop the session built from it); cache-only loads+ // only notice it.+ if (WEBGPU_BONSAI2_VISION_MODEL_IDS.has(modelId)) {+ const graph = await refreshBonsai2DecoderGraph(modelId, { allowNetwork: !localFilesOnly });+ if (graph === 'updated' && visionRuntimeKey.startsWith(`vision|${modelId}|`)) await disposeVisionRuntime();+ }if (visionRuntime && visionRuntimeKey === key) {visionRuntimeOwner = owner;return visionRuntime;@@ -794,10 +809,6 @@ async function getVisionRuntime(modelId, dtype, device, {error.code = readiness === 'text' ? 'text_model_not_downloaded' : 'vision_model_not_downloaded';throw error;}- if (WEBGPU_BONSAI2_VISION_MODEL_IDS.has(modelId)- && await refreshBonsai2DecoderGraph(modelId, { allowNetwork: !localFilesOnly }) === 'stale') {- console.info(`[webgpu] ${modelId}: the cached decoder graph predates parallel prefill; download the model again to update it.`);- }const library = await loadLibrary();const { AutoModelForImageTextToText, AutoProcessor } = library;if (!AutoModelForImageTextToText || !AutoProcessor) {diff --git a/test/run.js b/test/run.jsindex f452d6f5c..c25afac4f 100644--- a/test/run.js+++ b/test/run.js@@ -67572,6 +67572,9 @@ test('Bonsai 2 Vision worker replaces only a stale cached decoder graph', async}assert.match(workerSource, /refreshBonsai2DecoderGraph\(modelId, \{ allowNetwork: !localFilesOnly \}\)/,'runtime loads may download the graph only outside cache-only inference');+ const visionLoader = workerSource.slice(workerSource.indexOf('async function getVisionRuntime('));+ assert.ok(visionLoader.indexOf('refreshBonsai2DecoderGraph(') < visionLoader.indexOf('if (visionRuntime && visionRuntimeKey === key)'),+ 'download flows must refresh the graph before reusing a resident session');assert.match(workerSource, /refreshBonsai2DecoderGraph\(modelId, \{ allowNetwork: true \}\) === 'updated'\s*&& visionRuntimeKey\.startsWith\(`vision\|\$\{modelId\}\|`\)\) \{[^}]*await disposeVisionRuntime\(\);/,'an explicit update must drop the resident session built from the replaced graph');@@ -67582,6 +67585,8 @@ test('Bonsai 2 Vision worker replaces only a stale cached decoder graph', asyncconst worker = await import(`${pathToFileURL(path.join(ROOT, 'src/chrome/src/offscreen/inference-worker.js')).href}?bonsai2-graph-refresh`);assert.equal(await worker.refreshBonsai2DecoderGraph(model, { expected: sha('chunked graph') }), 'stale','cache-only (inference) loads detect a stale graph without downloading');+ assert.equal(await worker.refreshBonsai2DecoderGraph(model, { expected: sha('chunked graph') }), 'stale',+ 'repeated cache-only loads keep the stale verdict');assert.deepEqual(fetched, [], 'inference must never reach the network');assert.equal(entries.get(graphUrl), 'scan graph');assert.equal(await worker.refreshBonsai2DecoderGraph(model, { allowNetwork: true, expected: sha('chunked graph') }), 'updated');