Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Tiny Ask screenshot gate: fail closed without the probe's hidden state

Clauded00c22e3585d
diff --git a/docs/webgpu-models.md b/docs/webgpu-models.md
index 018e7f98f..972f216c9 100644
--- a/docs/webgpu-models.md
+++ b/docs/webgpu-models.md
@@ -194,8 +194,10 @@ Use the tools in `scripts/models/bonsai/`:
screenshot alone never suppresses offline references. With an attached
image both images can be sent, when the readout says the page is needed
too.
- - If the readout fails, or the model cannot run it (only Bonsai 2 2B
- Vision can), the turn stays text-only.
+ - If the readout fails, the turn stays text-only. That includes a model
+ that cannot run it (only Bonsai 2 2B Vision can) and a cached decoder
+ graph from before `decision_hidden`. There is no fallback to the
+ untrained readout.
- The capture is the same as automatic screenshots: redaction, size
budget and the per-turn limit. It is ephemeral like page text, and it
never goes to a separate vision model.
diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.js
index 580afbb55..af3c66f10 100644
--- a/src/chrome/src/offscreen/inference-worker.js
+++ b/src/chrome/src/offscreen/inference-worker.js
@@ -2016,9 +2016,14 @@ async function runMultimodalDecision(payload, { isCancelled = () => false } = {}
stopIfCancelled();
const output = await bonsai2ContinuationLogits(runtime, prefix, cache, suffix.input_ids, { withHidden: true });
const margin = output.logits[yesId] - output.logits[noId];
+ const probability = tinyAskGateProbability(modelId, margin, output.hidden);
+ // Collecting training data needs only the margin and hidden state.
+ if (probability === null && !(returnHidden && output.hidden)) {
+ throw new Error('The screenshot gate needs the updated Ternary Bonsai 2 2B Vision decoder. Download the model again in Settings > Providers > WebGPU.');
+ }
answers[id] = {
type: 'noul',
- noul: tinyAskGateProbability(modelId, margin, output.hidden),
+ noul: probability,
...(returnHidden ? { margins: [margin], hidden: [output.hidden ? encodeFloat16Base64(output.hidden) : null] } : {}),
};
questionMs.push(Math.round(performance.now() - questionStarted));
diff --git a/src/chrome/src/offscreen/tiny-ask-gate-probe.js b/src/chrome/src/offscreen/tiny-ask-gate-probe.js
index ab9e06375..943a17928 100644
--- a/src/chrome/src/offscreen/tiny-ask-gate-probe.js
+++ b/src/chrome/src/offscreen/tiny-ask-gate-probe.js
@@ -4,8 +4,10 @@
// margin and the decoder's last-position state (decision_hidden), and
// p(needs a screenshot) = sigmoid(margin * margin_weight + bias + hidden . weights)
// with standardization folded into the weights and bias. Trained by
-// scripts/models/bonsai/train_gate_probe.py; empty weights fall back to the
-// untrained readout, sigmoid(margin).
+// scripts/models/bonsai/train_gate_probe.py. There is no fallback to the
+// untrained readout, which is known to be poor: without weights, another
+// model, or a hidden state of the wrong size (a cached decoder graph that
+// predates decision_hidden) the probability is null and tiny Ask stays text-only.
// Validation (generated templates, 11 languages): 112/117. Test (hand-written, including 7 languages absent from training): 64/72, unseen languages 23/28.
export const TINY_ASK_GATE_PROBE = Object.freeze({
model: 'webbrain-one/Ternary-Bonsai-2-2B-ONNX-WebGPU-Vision',
@@ -32,10 +34,10 @@ function probeWeights() {
const sigmoid = value => 1 / (1 + Math.exp(-value));
-/** p(the question needs a screenshot of the current page). */
+/** p(the question needs a screenshot of the current page), or null when the probe cannot run. */
export function tinyAskGateProbability(modelId, margin, hidden) {
const w = modelId === TINY_ASK_GATE_PROBE.model ? probeWeights() : null;
- if (!w || !hidden || hidden.length !== w.length) return sigmoid(margin);
+ if (!w || !hidden || hidden.length !== w.length || !Number.isFinite(margin)) return null;
let dot = 0;
for (let i = 0; i < w.length; i++) dot += w[i] * hidden[i];
return sigmoid(margin * TINY_ASK_GATE_PROBE.margin + TINY_ASK_GATE_PROBE.bias + dot);
diff --git a/test/webgpu-decisions.mjs b/test/webgpu-decisions.mjs
index 41e42fac6..dfd3039ce 100644
--- a/test/webgpu-decisions.mjs
+++ b/test/webgpu-decisions.mjs
@@ -370,3 +370,17 @@ test('Settings suggests the chat model for decisions when Bonsai chats, with D1
const firefoxHtml = await readFile(new URL('../src/firefox/src/ui/settings.html', import.meta.url), 'utf8');
assert.doesNotMatch(firefoxHtml, /decision-suggestion/, 'Chrome only');
});
+
+test('the screenshot gate probe never falls back to the untrained readout', async () => {
+ const { TINY_ASK_GATE_PROBE, tinyAskGateProbability } = await import('../src/chrome/src/offscreen/tiny-ask-gate-probe.js');
+ assert.ok(TINY_ASK_GATE_PROBE.weightsBase64.length > 1000, 'trained weights ship with the extension');
+ const hidden = new Float32Array(TINY_ASK_GATE_PROBE.hiddenSize).fill(0.01);
+ const p = tinyAskGateProbability(TINY_ASK_GATE_PROBE.model, 1.5, hidden);
+ assert.ok(p > 0 && p < 1);
+ assert.equal(tinyAskGateProbability(TINY_ASK_GATE_PROBE.model, 1.5, null), null, 'a decoder graph without decision_hidden');
+ assert.equal(tinyAskGateProbability(TINY_ASK_GATE_PROBE.model, 1.5, new Float32Array(16)), null, 'a hidden state of the wrong size');
+ assert.equal(tinyAskGateProbability('webbrain-one/Ternary-Bonsai-2-4B-ONNX-WebGPU-Vision', 1.5, hidden), null, 'another model');
+ const worker = await readFile(new URL('../src/chrome/src/offscreen/inference-worker.js', import.meta.url), 'utf8');
+ assert.match(worker, /probability === null && !\(returnHidden && output\.hidden\)\) \{\n\s*throw new Error\('The screenshot gate needs the updated/,
+ 'without the probe the worker fails, so tiny Ask stays text-only');
+});