Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Decision readout: verify variant and hidden-state probe hooks for the diagnostic path
Claudeeef6ec4ff416
diff --git a/scripts/check-webgpu-decisions.mjs b/scripts/check-webgpu-decisions.mjsindex 23f822872..14ffb00b5 100644--- a/scripts/check-webgpu-decisions.mjs+++ b/scripts/check-webgpu-decisions.mjs@@ -16,7 +16,9 @@// the case where dense attention memory used to grow// --limit N first N test requests only// --temperature T Bonsai readout temperature (default 1)-// --prompt NAME Bonsai decision prompt: default or strict+// --prompt NAME Bonsai decision prompt: default, strict or verify+// --decoder-graph F install a replacement decoder graph in the cache first+// --return-hidden with --prompt verify, record each option's hidden stateimport fs from 'node:fs/promises';import path from 'node:path';import { fileURLToPath } from 'node:url';@@ -142,9 +144,22 @@ try {if (Date.now() - started > 60 * 60_000) throw new Error('Download did not finish within an hour');}}+ const graphFile = option('--decoder-graph', '');+ if (graphFile) {+ // Replace the cached decoder graph (same external weights) before the+ // worker first loads the model, e.g. one that exposes decision_hidden.+ const data = (await fs.readFile(graphFile)).toString('base64');+ await harness.evaluate(async ({ url, data }) => {+ const bytes = Uint8Array.from(atob(data), c => c.charCodeAt(0));+ const cache = await caches.open('transformers-cache');+ await cache.put(url, new Response(bytes, { headers: { 'content-type': 'application/octet-stream', 'content-length': String(bytes.length) } }));+ }, { url: `https://huggingface.co/${bonsaiModel}/resolve/main/onnx/decoder_model_merged_q2f16.onnx`, data });+ result.decoderGraph = graphFile;+ }const temperature = Number(option('--temperature', 1));const promptVariant = option('--prompt', 'default');- await run('bonsai', fixture => message('webgpu_decide_fixture', { fixture: { ...fixture, temperature, promptVariant } }));+ const returnHidden = args.includes('--return-hidden');+ await run('bonsai', fixture => message('webgpu_decide_fixture', { fixture: { ...fixture, temperature, promptVariant, returnHidden } }));}} catch (error) {result.error = String(error?.stack || error);diff --git a/scripts/models/bonsai/add_decision_hidden.py b/scripts/models/bonsai/add_decision_hidden.pynew file mode 100644index 000000000..caa8af9d6--- /dev/null+++ b/scripts/models/bonsai/add_decision_hidden.py@@ -0,0 +1,39 @@+"""Expose the decoder's last-position hidden state as an extra output.++The Bonsai 2 chunked decoder slices the final-norm hidden state to the last+position before the LM head. This adds an Identity of that [batch, 1, hidden]+FP16 tensor as the graph output `decision_hidden`, for decision-readout probes.+Nothing else changes: logits, caches and the external weight file reference+stay as they are.++ python scripts/models/bonsai/add_decision_hidden.py <decoder.onnx> <out.onnx>+"""+import sys++import onnx+from onnx import TensorProto, helper++SOURCE_SUFFIX = "/last_position"+++def main(source, target):+ model = onnx.load(source, load_external_data=False)+ graph = model.graph+ lm_head = [node for node in graph.node if node.output and node.output[0] == "logits"]+ if len(lm_head) != 1:+ raise SystemExit("expected one node producing logits")+ hidden = lm_head[0].input[0]+ if not hidden.endswith(SOURCE_SUFFIX):+ raise SystemExit(f"unexpected LM head input {hidden}")+ if any(output.name == "decision_hidden" for output in graph.output):+ raise SystemExit("graph already has decision_hidden")+ graph.node.append(helper.make_node("Identity", [hidden], ["decision_hidden"], name="/webbrain/decision_hidden"))+ graph.output.append(helper.make_tensor_value_info("decision_hidden", TensorProto.FLOAT16, ["batch_size", 1, None]))+ # The weights stay in the original external data file next to the graph,+ # so onnx.checker cannot run here without it; the browser load checks it.+ onnx.save_model(model, target)+ print("added decision_hidden from", hidden)+++if __name__ == "__main__":+ main(*sys.argv[1:3])diff --git a/src/chrome/src/background.js b/src/chrome/src/background.jsindex 1154c3117..b2a417dce 100644--- a/src/chrome/src/background.js+++ b/src/chrome/src/background.js@@ -4622,7 +4622,7 @@ async function handleMessage(msg, sender) {if (!fixture || typeof fixture !== 'object' || Array.isArray(fixture)) throw new Error('Invalid decision fixture.');const provider = providerManager.getProvider('webgpu');if (typeof provider?.decide !== 'function') throw new Error('The WebGPU provider is unavailable.');- return { success: true, result: await provider.decide({ state: fixture.state, images: fixture.images || [], questions: fixture.questions, temperature: fixture.temperature, promptVariant: fixture.promptVariant }) };+ return { success: true, result: await provider.decide({ state: fixture.state, images: fixture.images || [], questions: fixture.questions, temperature: fixture.temperature, promptVariant: fixture.promptVariant, returnHidden: fixture.returnHidden === true }) };} catch (error) { return { success: false, error: error.message }; }}diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.jsindex 0ec9bd89b..6f532bc95 100644--- a/src/chrome/src/offscreen/inference-worker.js+++ b/src/chrome/src/offscreen/inference-worker.js@@ -13,6 +13,7 @@ import {decisionAnswer,decisionOptions,decisionQuestionText,+ decisionVerifyText,decisionStateText,normalizeDecisionRequest,} from './webgpu-decisions.js';@@ -392,7 +393,7 @@ export async function prefillBonsai2InSlices(runtime, inputs, sliceTokens) {* prefix cache is only read: the continuation's cache outputs are released,* so every question starts from the same page state.*/-export async function bonsai2ContinuationLogits(runtime, prefixInputs, cache, suffixIds) {+export async function bonsai2ContinuationLogits(runtime, prefixInputs, cache, suffixIds, { withHidden = false } = {}) {const { model, library } = runtime;const prefixLength = prefixInputs.input_ids.dims.at(-1), suffixLength = suffixIds.dims.at(-1);const total = prefixLength + suffixLength;@@ -407,7 +408,12 @@ export async function bonsai2ContinuationLogits(runtime, prefixInputs, cache, supast_key_values: cache,});try {- return Float32Array.from(outputs.logits.slice(null, -1, null).to('float32').data);+ const logits = Float32Array.from(outputs.logits.slice(null, -1, null).to('float32').data);+ if (!withHidden) return logits;+ // Present only in a decoder graph that exposes `decision_hidden`+ // (scripts/models/bonsai/add_decision_hidden.py).+ const hidden = outputs.decision_hidden ? Float32Array.from(outputs.decision_hidden.to('float32').data) : null;+ return { logits, hidden };} finally {for (const tensor of Object.values(outputs)) {if (tensor?.location === 'gpu-buffer') tensor.dispose();@@ -1802,6 +1808,19 @@ async function runMultimodalText(payload) {return { content: result.content, reasoningContent: result.reasoningContent };}+function encodeFloat16Base64(values) {+ const half = new Uint16Array(values.length), view = new DataView(new ArrayBuffer(4));+ for (let i = 0; i < values.length; i++) {+ view.setFloat32(0, values[i]);+ const bits = view.getUint32(0), sign = (bits >>> 16) & 0x8000, exponent = ((bits >>> 23) & 0xff) - 112, mantissa = (bits >>> 13) & 0x3ff;+ half[i] = exponent <= 0 ? sign : exponent >= 31 ? sign | 0x7c00 : sign | (exponent << 10) | mantissa;+ }+ let binary = '';+ const bytes = new Uint8Array(half.buffer);+ for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);+ return btoa(binary);+}+// Option probabilities read from one forward pass per question (see// webgpu-decisions.js). Only the Bonsai 2 Vision presets are supported: they// use the chunked decoder whose cache the continuations share.@@ -1865,8 +1884,26 @@ async function runMultimodalDecision(payload) {try {cache = await prefillBonsai2Cache(runtime, prefix, bonsai2PrefillSliceTokens(payload?.prefillSliceTokens), prefixLength);const prefillMs = performance.now() - started;+ const verify = payload?.promptVariant === 'verify';+ const returnHidden = payload?.returnHidden === true;+ const yesId = verify ? codeId('Yes') : null, noId = verify ? codeId('No') : null;for (const [id, question] of request.entries) {const questionStarted = performance.now();+ if (verify) {+ // One continuation per option; the answer compares their Yes margins.+ const margins = [], hidden = [];+ const count = decisionOptions(question).names.length;+ for (let index = 0; index < count; index++) {+ const suffix = tokenizer(`${decisionVerifyText(question, index)}${tailText}`, { add_special_tokens: false });+ suffixTokens += suffix.input_ids.dims.at(-1);+ const output = await bonsai2ContinuationLogits(runtime, prefix, cache, suffix.input_ids, { withHidden: true });+ margins.push(output.logits[yesId] - output.logits[noId]);+ if (returnHidden) hidden.push(output.hidden ? encodeFloat16Base64(output.hidden) : null);+ }+ answers[id] = { ...decisionAnswer(question, margins, { temperature }), ...(returnHidden ? { margins, hidden } : {}) };+ questionMs.push(Math.round(performance.now() - questionStarted));+ continue;+ }const suffix = tokenizer(`${decisionQuestionText(question)}${tailText}`, { add_special_tokens: false });suffixTokens += suffix.input_ids.dims.at(-1);const logits = await bonsai2ContinuationLogits(runtime, prefix, cache, suffix.input_ids);diff --git a/src/chrome/src/offscreen/vision-inference-host.js b/src/chrome/src/offscreen/vision-inference-host.jsindex 3f3e19360..c743bce3c 100644--- a/src/chrome/src/offscreen/vision-inference-host.js+++ b/src/chrome/src/offscreen/vision-inference-host.js@@ -727,6 +727,7 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {questions: message.questions,temperature: message.temperature,promptVariant: message.promptVariant,+ returnHidden: message.returnHidden === true,prefillSliceTokens: message.prefillSliceTokens,}, { exclusive: true, runtime: message.runtime }));return;diff --git a/src/chrome/src/offscreen/webgpu-decisions.js b/src/chrome/src/offscreen/webgpu-decisions.jsindex c6015bb58..d66e52aa0 100644--- a/src/chrome/src/offscreen/webgpu-decisions.js+++ b/src/chrome/src/offscreen/webgpu-decisions.js@@ -71,6 +71,28 @@ export function decisionQuestionText(question) {'Answer with one letter.'].join('\n');}+// The verify variant asks about one option at a time ("is this proposed+// answer correct?") and compares the Yes margins across options, so every+// option gets its own continuation and its own last-position hidden state.+// Evaluated through the diagnostic fixture path only.+export function decisionOptionText(question, index) {+ const { names } = decisionOptions(question);+ if (question.type === 'noul') {+ const criteria = question.criteria && typeof question.criteria === 'object' ? question.criteria : {};+ const detail = index ? (criteria.true ?? criteria.yes) : (criteria.false ?? criteria.no);+ return `${index ? 'Yes' : 'No'}${detail != null && detail !== '' ? ` (${describe(detail)})` : ''}`;+ }+ if (question.type === 'score') return `level ${index}: ${describe(question.criteria[index])}`;+ const detail = question.criteria[names[index]];+ return `${names[index]}${detail != null && detail !== '' ? `: ${describe(detail)}` : ''}`;+}++export function decisionVerifyText(question, index) {+ return [`Question: ${String(question.instructions || '').trim()}`,+ `Proposed answer: ${decisionOptionText(question, index)}`,+ 'Is the proposed answer correct? Answer Yes or No.'].join('\n');+}+export function decisionStateText(state) {const text = typeof state === 'string' ? state : JSON.stringify(state ?? '');return `Page state:\n${text}`;diff --git a/src/chrome/src/providers/webgpu.js b/src/chrome/src/providers/webgpu.jsindex e4047af3b..6ca6749c0 100644--- a/src/chrome/src/providers/webgpu.js+++ b/src/chrome/src/providers/webgpu.js@@ -476,7 +476,7 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {* Decision readout on the in-browser model: option probabilities from one* forward pass per question, in the d1 answer shape. Bonsai 2 Vision only.*/- async decide({ state = '', images = [], questions, temperature, promptVariant } = {}) {+ async decide({ state = '', images = [], questions, temperature, promptVariant, returnHidden = false } = {}) {if (webgpuModelRuntime(this.model) !== WEBGPU_RUNTIME_ONNX_VL || !WEBGPU_DECISION_MODEL_IDS.has(this.model)) {throw new Error(`${webgpuModelDisplayName(this.model)} does not support in-browser decisions.`);}@@ -495,7 +495,8 @@ export class WebGPUProvider extends WebGPUOffscreenProvider {images,questions,...(Number(temperature) > 0 ? { temperature: Number(temperature) } : {}),- ...(promptVariant === 'strict' ? { promptVariant } : {}),+ ...(['strict', 'verify'].includes(promptVariant) ? { promptVariant } : {}),+ ...(returnHidden === true ? { returnHidden: true } : {}),...(prefillSliceTokens == null ? {} : { prefillSliceTokens }),});if (!response || response.ok === false || response.error) {