Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Tiny Ask screenshot gate: fail closed without the probe's hidden state
Clauded00c22e3585d
diff --git a/docs/webgpu-models.md b/docs/webgpu-models.mdindex 018e7f98f..972f216c9 100644--- a/docs/webgpu-models.md+++ b/docs/webgpu-models.md@@ -194,8 +194,10 @@ Use the tools in `scripts/models/bonsai/`:screenshot alone never suppresses offline references. With an attachedimage both images can be sent, when the readout says the page is neededtoo.- - If the readout fails, or the model cannot run it (only Bonsai 2 2B- Vision can), the turn stays text-only.+ - If the readout fails, the turn stays text-only. That includes a model+ that cannot run it (only Bonsai 2 2B Vision can) and a cached decoder+ graph from before `decision_hidden`. There is no fallback to the+ untrained readout.- The capture is the same as automatic screenshots: redaction, sizebudget and the per-turn limit. It is ephemeral like page text, and itnever goes to a separate vision model.diff --git a/src/chrome/src/offscreen/inference-worker.js b/src/chrome/src/offscreen/inference-worker.jsindex 580afbb55..af3c66f10 100644--- a/src/chrome/src/offscreen/inference-worker.js+++ b/src/chrome/src/offscreen/inference-worker.js@@ -2016,9 +2016,14 @@ async function runMultimodalDecision(payload, { isCancelled = () => false } = {}stopIfCancelled();const output = await bonsai2ContinuationLogits(runtime, prefix, cache, suffix.input_ids, { withHidden: true });const margin = output.logits[yesId] - output.logits[noId];+ const probability = tinyAskGateProbability(modelId, margin, output.hidden);+ // Collecting training data needs only the margin and hidden state.+ if (probability === null && !(returnHidden && output.hidden)) {+ throw new Error('The screenshot gate needs the updated Ternary Bonsai 2 2B Vision decoder. Download the model again in Settings > Providers > WebGPU.');+ }answers[id] = {type: 'noul',- noul: tinyAskGateProbability(modelId, margin, output.hidden),+ noul: probability,...(returnHidden ? { margins: [margin], hidden: [output.hidden ? encodeFloat16Base64(output.hidden) : null] } : {}),};questionMs.push(Math.round(performance.now() - questionStarted));diff --git a/src/chrome/src/offscreen/tiny-ask-gate-probe.js b/src/chrome/src/offscreen/tiny-ask-gate-probe.jsindex ab9e06375..943a17928 100644--- a/src/chrome/src/offscreen/tiny-ask-gate-probe.js+++ b/src/chrome/src/offscreen/tiny-ask-gate-probe.js@@ -4,8 +4,10 @@// margin and the decoder's last-position state (decision_hidden), and// p(needs a screenshot) = sigmoid(margin * margin_weight + bias + hidden . weights)// with standardization folded into the weights and bias. Trained by-// scripts/models/bonsai/train_gate_probe.py; empty weights fall back to the-// untrained readout, sigmoid(margin).+// scripts/models/bonsai/train_gate_probe.py. There is no fallback to the+// untrained readout, which is known to be poor: without weights, another+// model, or a hidden state of the wrong size (a cached decoder graph that+// predates decision_hidden) the probability is null and tiny Ask stays text-only.// Validation (generated templates, 11 languages): 112/117. Test (hand-written, including 7 languages absent from training): 64/72, unseen languages 23/28.export const TINY_ASK_GATE_PROBE = Object.freeze({model: 'webbrain-one/Ternary-Bonsai-2-2B-ONNX-WebGPU-Vision',@@ -32,10 +34,10 @@ function probeWeights() {const sigmoid = value => 1 / (1 + Math.exp(-value));-/** p(the question needs a screenshot of the current page). */+/** p(the question needs a screenshot of the current page), or null when the probe cannot run. */export function tinyAskGateProbability(modelId, margin, hidden) {const w = modelId === TINY_ASK_GATE_PROBE.model ? probeWeights() : null;- if (!w || !hidden || hidden.length !== w.length) return sigmoid(margin);+ if (!w || !hidden || hidden.length !== w.length || !Number.isFinite(margin)) return null;let dot = 0;for (let i = 0; i < w.length; i++) dot += w[i] * hidden[i];return sigmoid(margin * TINY_ASK_GATE_PROBE.margin + TINY_ASK_GATE_PROBE.bias + dot);diff --git a/test/webgpu-decisions.mjs b/test/webgpu-decisions.mjsindex 41e42fac6..dfd3039ce 100644--- a/test/webgpu-decisions.mjs+++ b/test/webgpu-decisions.mjs@@ -370,3 +370,17 @@ test('Settings suggests the chat model for decisions when Bonsai chats, with D1const firefoxHtml = await readFile(new URL('../src/firefox/src/ui/settings.html', import.meta.url), 'utf8');assert.doesNotMatch(firefoxHtml, /decision-suggestion/, 'Chrome only');});++test('the screenshot gate probe never falls back to the untrained readout', async () => {+ const { TINY_ASK_GATE_PROBE, tinyAskGateProbability } = await import('../src/chrome/src/offscreen/tiny-ask-gate-probe.js');+ assert.ok(TINY_ASK_GATE_PROBE.weightsBase64.length > 1000, 'trained weights ship with the extension');+ const hidden = new Float32Array(TINY_ASK_GATE_PROBE.hiddenSize).fill(0.01);+ const p = tinyAskGateProbability(TINY_ASK_GATE_PROBE.model, 1.5, hidden);+ assert.ok(p > 0 && p < 1);+ assert.equal(tinyAskGateProbability(TINY_ASK_GATE_PROBE.model, 1.5, null), null, 'a decoder graph without decision_hidden');+ assert.equal(tinyAskGateProbability(TINY_ASK_GATE_PROBE.model, 1.5, new Float32Array(16)), null, 'a hidden state of the wrong size');+ assert.equal(tinyAskGateProbability('webbrain-one/Ternary-Bonsai-2-4B-ONNX-WebGPU-Vision', 1.5, hidden), null, 'another model');+ const worker = await readFile(new URL('../src/chrome/src/offscreen/inference-worker.js', import.meta.url), 'utf8');+ assert.match(worker, /probability === null && !\(returnHidden && output\.hidden\)\) \{\n\s*throw new Error\('The screenshot gate needs the updated/,+ 'without the probe the worker fails, so tiny Ask stays text-only');+});