Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Decision readout: verify variant and hidden-state probe hooks for the diagnostic path

Claudeeef6ec4ff416
diff --git a/scripts/check-webgpu-decisions.mjs b/scripts/check-webgpu-decisions.mjs
index 23f822872..14ffb00b5 100644
--- a/scripts/check-webgpu-decisions.mjs
+++ b/scripts/check-webgpu-decisions.mjs
@@ -16,7 +16,9 @@
// the case where dense attention memory used to grow
// --limit N first N test requests only
// --temperature T Bonsai readout temperature (default 1)
-// --prompt NAME Bonsai decision prompt: default or strict
+// --prompt NAME Bonsai decision prompt: default, strict or verify
+// --decoder-graph F install a replacement decoder graph in the cache first
+// --return-hidden with --prompt verify, record each option's hidden state
import fs from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
@@ -142,9 +144,22 @@ try {
if (Date.now() - started > 60 * 60_000) throw new Error('Download did not finish within an hour');
}
}
+ const graphFile = option('--decoder-graph', '');
+ if (graphFile) {
+ // Replace the cached decoder graph (same external weights) before the
+ // worker first loads the model, e.g. one that exposes decision_hidden.
+ const data = (await fs.readFile(graphFile)).toString('base64');
+ await harness.evaluate(async ({ url, data }) => {
+ const bytes = Uint8Array.from(atob(data), c => c.charCodeAt(0));
+ const cache = await caches.open('transformers-cache');
+ await cache.put(url, new Response(bytes, { headers: { 'content-type': 'application/octet-stream', 'content-length': String(bytes.length) } }));
+ }, { url: `https://huggingface.co/${bonsaiModel}/resolve/main/onnx/decoder_model_merged_q2f16.onnx`, data });
+ result.decoderGraph = graphFile;
+ }
const temperature = Number(option('--temperature', 1));
const promptVariant = option('--prompt', 'default');
- await run('bonsai', fixture => message('webgpu_decide_fixture', { fixture: { ...fixture, temperature, promptVariant } }));
+ const returnHidden = args.includes('--return-hidden');
+ await run('bonsai', fixture => message('webgpu_decide_fixture', { fixture: { ...fixture, temperature, promptVariant, returnHidden } }));
}
} catch (error) {
result.error = String(error?.stack || error);
diff --git a/scripts/models/bonsai/add_decision_hidden.py b/scripts/models/bonsai/add_decision_hidden.py
new file mode 100644
index 000000000..caa8af9d6
--- /dev/null
+++ b/scripts/models/bonsai/add_decision_hidden.py
@@ -0,0 +1,39 @@
+"""Expose the decoder's last-position hidden state as an extra output.
+
+The Bonsai 2 chunked decoder slices the final-norm hidden state to the last
+position before the LM head. This adds an Identity of that [batch, 1, hidden]
+FP16 tensor as the graph output `decision_hidden`, for decision-readout probes.
+Nothing else changes: logits, caches and the external weight file reference
+stay as they are.
+
+ python scripts/models/bonsai/add_decision_hidden.py <decoder.onnx> <out.onnx>
+"""
+import sys
+
+import onnx
+from onnx import TensorProto, helper
+
+SOURCE_SUFFIX = "/last_position"
+
+
+def main(source, target):
+ model = onnx.load(source, load_external_data=False)
+ graph = model.graph
+ lm_head = [node for node in graph.node if node.output and node.output[0] == "logits"]
+ if len(lm_head) != 1:
+ raise SystemExit("expected one node producing logits")
+ hidden = lm_head[0].input[0]
+ if not hidden.endswith(SOURCE_SUFFIX):
+ raise SystemExit(f"unexpected LM head input {hidden}")
+ if any(output.name == "decision_hidden" for output in graph.output):
+ raise SystemExit("graph already has decision_hidden")
+ graph.node.append(helper.make_node("Identity", [hidden], ["decision_hidden"], name="/webbrain/decision_hidden"))
+ graph.output.append(helper.make_tensor_value_info("decision_hidden", TensorProto.FLOAT16, ["batch_size", 1, None]))
+ # The weights stay in the original external data file next to the graph,
+ # so onnx.checker cannot run here without it; the browser load checks it.
+ onnx.save_model(model, target)
+ print("added decision_hidden from", hidden)
+
+
+if __name__ == "__main__":
+ main(*sys.argv[1:3])
diff --git a/src/chrome/src/background.js b/src/chrome/src/background.js
index 1154c3117..b2a417dce 100644
--- a/src/chrome/src/background.js
+++ b/src/chrome/src/background.js
@@ -4622,7 +4622,7 @@ async function handleMessage(msg, sender) {
if (!fixture || typeof fixture !== 'object' || Array.isArray(fixture)) throw new Error('Invalid decision fixture.');
const provider = providerManager.getProvider('webgpu');
if (typeof provider?.decide !== 'function') throw new Error('The WebGPU provider is unavailable.');
- return { success: true, result: await provider.decide({ state: fixture.state, images: fixture.images || [], questions: fixture.questions, temperature: fixture.temperature, promptVariant: fixture.promptVariant }) };
+ return { success: true, result: await provider.decide({ state: fixture.state, images: fixture.images || [], questions: fixture.questions, temperature: fixture.temperature, promptVariant: fixture.promptVariant, returnHidden: fixture.returnHidden === true }) };
} catch (error) { return { success: false, error: error.message }; }
}
diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.js
index 0ec9bd89b..6f532bc95 100644
--- a/src/chrome/src/offscreen/inference-worker.js
+++ b/src/chrome/src/offscreen/inference-worker.js
@@ -13,6 +13,7 @@ import {
decisionAnswer,
decisionOptions,
decisionQuestionText,
+ decisionVerifyText,
decisionStateText,
normalizeDecisionRequest,
} from './webgpu-decisions.js';
@@ -392,7 +393,7 @@ export async function prefillBonsai2InSlices(runtime, inputs, sliceTokens) {
* prefix cache is only read: the continuation's cache outputs are released,
* so every question starts from the same page state.
*/
-export async function bonsai2ContinuationLogits(runtime, prefixInputs, cache, suffixIds) {
+export async function bonsai2ContinuationLogits(runtime, prefixInputs, cache, suffixIds, { withHidden = false } = {}) {
const { model, library } = runtime;
const prefixLength = prefixInputs.input_ids.dims.at(-1), suffixLength = suffixIds.dims.at(-1);
const total = prefixLength + suffixLength;
@@ -407,7 +408,12 @@ export async function bonsai2ContinuationLogits(runtime, prefixInputs, cache, su
past_key_values: cache,
});
try {
- return Float32Array.from(outputs.logits.slice(null, -1, null).to('float32').data);
+ const logits = Float32Array.from(outputs.logits.slice(null, -1, null).to('float32').data);
+ if (!withHidden) return logits;
+ // Present only in a decoder graph that exposes `decision_hidden`
+ // (scripts/models/bonsai/add_decision_hidden.py).
+ const hidden = outputs.decision_hidden ? Float32Array.from(outputs.decision_hidden.to('float32').data) : null;
+ return { logits, hidden };
} finally {
for (const tensor of Object.values(outputs)) {
if (tensor?.location === 'gpu-buffer') tensor.dispose();
@@ -1802,6 +1808,19 @@ async function runMultimodalText(payload) {
return { content: result.content, reasoningContent: result.reasoningContent };
}
+function encodeFloat16Base64(values) {
+ const half = new Uint16Array(values.length), view = new DataView(new ArrayBuffer(4));
+ for (let i = 0; i < values.length; i++) {
+ view.setFloat32(0, values[i]);
+ const bits = view.getUint32(0), sign = (bits >>> 16) & 0x8000, exponent = ((bits >>> 23) & 0xff) - 112, mantissa = (bits >>> 13) & 0x3ff;
+ half[i] = exponent <= 0 ? sign : exponent >= 31 ? sign | 0x7c00 : sign | (exponent << 10) | mantissa;
+ }
+ let binary = '';
+ const bytes = new Uint8Array(half.buffer);
+ for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
+ return btoa(binary);
+}
+
// Option probabilities read from one forward pass per question (see
// webgpu-decisions.js). Only the Bonsai 2 Vision presets are supported: they
// use the chunked decoder whose cache the continuations share.
@@ -1865,8 +1884,26 @@ async function runMultimodalDecision(payload) {
try {
cache = await prefillBonsai2Cache(runtime, prefix, bonsai2PrefillSliceTokens(payload?.prefillSliceTokens), prefixLength);
const prefillMs = performance.now() - started;
+ const verify = payload?.promptVariant === 'verify';
+ const returnHidden = payload?.returnHidden === true;
+ const yesId = verify ? codeId('Yes') : null, noId = verify ? codeId('No') : null;
for (const [id, question] of request.entries) {
const questionStarted = performance.now();
+ if (verify) {
+ // One continuation per option; the answer compares their Yes margins.
+ const margins = [], hidden = [];
+ const count = decisionOptions(question).names.length;
+ for (let index = 0; index < count; index++) {
+ const suffix = tokenizer(`${decisionVerifyText(question, index)}${tailText}`, { add_special_tokens: false });
+ suffixTokens += suffix.input_ids.dims.at(-1);
+ const output = await bonsai2ContinuationLogits(runtime, prefix, cache, suffix.input_ids, { withHidden: true });
+ margins.push(output.logits[yesId] - output.logits[noId]);
+ if (returnHidden) hidden.push(output.hidden ? encodeFloat16Base64(output.hidden) : null);
+ }
+ answers[id] = { ...decisionAnswer(question, margins, { temperature }), ...(returnHidden ? { margins, hidden } : {}) };
+ questionMs.push(Math.round(performance.now() - questionStarted));
+ continue;
+ }
const suffix = tokenizer(`${decisionQuestionText(question)}${tailText}`, { add_special_tokens: false });
suffixTokens += suffix.input_ids.dims.at(-1);
const logits = await bonsai2ContinuationLogits(runtime, prefix, cache, suffix.input_ids);
diff --git a/src/chrome/src/offscreen/vision-inference-host.js b/src/chrome/src/offscreen/vision-inference-host.js
index 3f3e19360..c743bce3c 100644
--- a/src/chrome/src/offscreen/vision-inference-host.js
+++ b/src/chrome/src/offscreen/vision-inference-host.js
@@ -727,6 +727,7 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
questions: message.questions,
temperature: message.temperature,
promptVariant: message.promptVariant,
+ returnHidden: message.returnHidden === true,
prefillSliceTokens: message.prefillSliceTokens,
}, { exclusive: true, runtime: message.runtime }));
return;
diff --git a/src/chrome/src/offscreen/webgpu-decisions.js b/src/chrome/src/offscreen/webgpu-decisions.js
index c6015bb58..d66e52aa0 100644
--- a/src/chrome/src/offscreen/webgpu-decisions.js
+++ b/src/chrome/src/offscreen/webgpu-decisions.js
@@ -71,6 +71,28 @@ export function decisionQuestionText(question) {
'Answer with one letter.'].join('\n');
}
+// The verify variant asks about one option at a time ("is this proposed
+// answer correct?") and compares the Yes margins across options, so every
+// option gets its own continuation and its own last-position hidden state.
+// Evaluated through the diagnostic fixture path only.
+export function decisionOptionText(question, index) {
+ const { names } = decisionOptions(question);
+ if (question.type === 'noul') {
+ const criteria = question.criteria && typeof question.criteria === 'object' ? question.criteria : {};
+ const detail = index ? (criteria.true ?? criteria.yes) : (criteria.false ?? criteria.no);
+ return `${index ? 'Yes' : 'No'}${detail != null && detail !== '' ? ` (${describe(detail)})` : ''}`;
+ }
+ if (question.type === 'score') return `level ${index}: ${describe(question.criteria[index])}`;
+ const detail = question.criteria[names[index]];
+ return `${names[index]}${detail != null && detail !== '' ? `: ${describe(detail)}` : ''}`;
+}
+
+export function decisionVerifyText(question, index) {
+ return [`Question: ${String(question.instructions || '').trim()}`,
+ `Proposed answer: ${decisionOptionText(question, index)}`,
+ 'Is the proposed answer correct? Answer Yes or No.'].join('\n');
+}
+
export function decisionStateText(state) {
const text = typeof state === 'string' ? state : JSON.stringify(state ?? '');
return `Page state:\n${text}`;
diff --git a/src/chrome/src/providers/webgpu.js b/src/chrome/src/providers/webgpu.js
index e4047af3b..6ca6749c0 100644
--- a/src/chrome/src/providers/webgpu.js
+++ b/src/chrome/src/providers/webgpu.js
@@ -476,7 +476,7 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {
* Decision readout on the in-browser model: option probabilities from one
* forward pass per question, in the d1 answer shape. Bonsai 2 Vision only.
*/
- async decide({ state = '', images = [], questions, temperature, promptVariant } = {}) {
+ async decide({ state = '', images = [], questions, temperature, promptVariant, returnHidden = false } = {}) {
if (webgpuModelRuntime(this.model) !== WEBGPU_RUNTIME_ONNX_VL || !WEBGPU_DECISION_MODEL_IDS.has(this.model)) {
throw new Error(`${webgpuModelDisplayName(this.model)} does not support in-browser decisions.`);
}
@@ -495,7 +495,8 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {
images,
questions,
...(Number(temperature) > 0 ? { temperature: Number(temperature) } : {}),
- ...(promptVariant === 'strict' ? { promptVariant } : {}),
+ ...(['strict', 'verify'].includes(promptVariant) ? { promptVariant } : {}),
+ ...(returnHidden === true ? { returnHidden: true } : {}),
...(prefillSliceTokens == null ? {} : { prefillSliceTokens }),
});
if (!response || response.ok === false || response.error) {