Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
tiny Ask: bound the screenshot gate and clear its queued trace note per turn
Claude8e2b5e08143a
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex cf487ccf2..80d369962 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -208,7 +208,7 @@ import { shouldAutoGroupTabs } from '../tab-group-preference.js';import {TINY_ASK_ACT_BLOCKED_MESSAGE, TINY_ASK_COMPACT_LINKED_MAX_CHARS, TINY_ASK_COMPACT_PAGE_MAX_CHARS,TINY_ASK_COMPACT_SELECTION_MAX_CHARS, TINY_ASK_READ_MAX_CHARS, TINY_ASK_READ_WINDOW_CHARS, TINY_ASK_SYSTEM_PROMPT,- TINY_ASK_SCREENSHOT_GATE_THRESHOLD, tinyAskLinkedUrl, tinyAskScreenshotGate, tinyAskScreenshotGateProbability, tinyAskScreenshotNote, tinyAskPageContext, tinyAskPageReadable, tinyAskRuntimeLine,+ TINY_ASK_SCREENSHOT_GATE_THRESHOLD, TINY_ASK_SCREENSHOT_GATE_TIMEOUT_MS, tinyAskLinkedUrl, tinyAskScreenshotGate, tinyAskScreenshotGateProbability, tinyAskScreenshotNote, tinyAskPageContext, tinyAskPageReadable, tinyAskRuntimeLine,} from './tiny-ask.js';import { webgpuModelDisplayName } from '../providers/webgpu.js';@@ -12088,6 +12088,38 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dreturn enriched;}+ // The gate's readout, bounded: a stalled offscreen worker must not hold the+ // turn at enrichment. A timeout or the run's own abort cancels the worker+ // operation (WebGPUProvider.decide sends webgpu-decide-cancel) and settles+ // at once; only a run abort propagates.+ async _tinyAskGateDecision(tabId, provider, gate) {+ const controller = new AbortController();+ const parent = this._runAbortSignal(tabId);+ let timer = null;+ let onAbort = null;+ const settled = new Promise((_, reject) => {+ onAbort = () => {+ const reason = parent?.reason;+ controller.abort(reason);+ reject(reason instanceof Error ? reason : Object.assign(new Error('The run was stopped.'), { name: 'AbortError' }));+ };+ if (parent?.aborted) onAbort();+ else parent?.addEventListener?.('abort', onAbort, { once: true });+ timer = setTimeout(() => {+ const error = new Error('The screenshot gate timed out.');+ controller.abort(error);+ reject(error);+ }, Number(this.tinyAskGateTimeoutMs) > 0 ? Number(this.tinyAskGateTimeoutMs) : TINY_ASK_SCREENSHOT_GATE_TIMEOUT_MS);+ });+ settled.catch(() => {});+ try {+ return await Promise.race([provider.decide({ ...gate, promptVariant: 'gate', signal: controller.signal }), settled]);+ } finally {+ clearTimeout(timer);+ parent?.removeEventListener?.('abort', onAbort);+ }+ }+// The visible page for a tiny Ask turn, when the in-browser model reads// images itself and its own readout says the question needs it (see// tinyAskScreenshotGate: language-independent, and blind to page content).@@ -12116,7 +12148,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst gate = tinyAskScreenshotGate({question, pageChars: String(page.text || '').trim().length, imageAttachment, linkedPage,});- const result = await provider.decide({ ...gate, promptVariant: 'gate', signal: this._runAbortSignal(tabId) });+ const result = await this._tinyAskGateDecision(tabId, provider, gate);probability = tinyAskScreenshotGateProbability(result?.answers);} catch (error) {if (error?.name === 'AbortError' || this._checkAbort(tabId)) throw error;@@ -35829,6 +35861,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dthis.pendingVisionRouteTraces.delete(tabId);this.pendingVisionStatusTraces.delete(tabId);this.pendingVisionSubCallTraces.delete(tabId);+ this.pendingTinyAskGateTraces.delete(tabId);this.carouselTraversalStates.delete(tabId);this._chromeProtectedGalleryStates.delete(tabId);void this._clearBackgroundFocusEmulation(tabId);@@ -54129,6 +54162,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dthis.pendingVisionRouteTraces.delete(tabId);this.pendingVisionStatusTraces.delete(tabId);this.pendingVisionSubCallTraces.delete(tabId);+ this.pendingTinyAskGateTraces.delete(tabId);this.visionStatusNotices.delete(tabId);// Reset the per-turn auto-screenshot budget (issue #311) for a fresh turn.this.autoScreenshotBursts.delete(tabId);@@ -55906,6 +55940,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dthis.pendingVisionRouteTraces.delete(tabId);this.pendingVisionStatusTraces.delete(tabId);this.pendingVisionSubCallTraces.delete(tabId);+ this.pendingTinyAskGateTraces.delete(tabId);this.visionStatusNotices.delete(tabId);// Reset the per-turn auto-screenshot budget (issue #311) for a fresh turn.this.autoScreenshotBursts.delete(tabId);diff --git a/src/chrome/src/agent/tiny-ask.js b/src/chrome/src/agent/tiny-ask.jsindex 5a62c3726..f64306c3a 100644--- a/src/chrome/src/agent/tiny-ask.js+++ b/src/chrome/src/agent/tiny-ask.js@@ -58,6 +58,9 @@ export const TINY_ASK_READ_MAX_CHARS = 60000;// steer the decision.export const TINY_ASK_SCREENSHOT_MAX_PAGE_CHARS = 200;export const TINY_ASK_SCREENSHOT_GATE_THRESHOLD = 0.5;+// The readout takes ~0.2 s, plus a model load on the first turn after a+// browser restart; past this the turn goes ahead text-only.+export const TINY_ASK_SCREENSHOT_GATE_TIMEOUT_MS = 10_000;export const TINY_ASK_SCREENSHOT_GATE_QUESTION = 'To answer this message well, does the assistant need to look at a '+ 'screenshot of the current page, because the answer depends on how something on it looks (an image, photo, '+ 'chart, colour, logo, button or layout) or on what a page with almost no text shows?';diff --git a/test/run.js b/test/run.jsindex 270fed719..8807c748f 100644--- a/test/run.js+++ b/test/run.js@@ -41723,6 +41723,29 @@ test('tiny Ask shows a vision-capable in-browser model the visible page when itsassert.ok(requests[0].at(-1).content.some(block => block.text === TINY_ASK_TEXTLESS_SCREENSHOT_NOTE),'a textless page tells the model to answer from the image');}+ {+ // A stalled worker: the readout never answers. The turn goes ahead+ // text-only after the gate's deadline, and the readout is cancelled.+ let gateSignal = null;+ const { agent, requests, captures, run } = setup();+ agent.tinyAskGateTimeoutMs = 30;+ agent._providerForRunOptions(1730, {}).decide = request => { gateSignal = request.signal; return new Promise(() => {}); };+ const started = Date.now();+ assert.equal(await run('What colour is the banner?'), 'Answer.', 'a stalled readout does not hold the turn');+ assert.ok(Date.now() - started < 2000);+ assert.equal(gateSignal?.aborted, true, 'the readout is cancelled at the deadline');+ assert.equal(captures.length, 0);+ assert.deepEqual(imagesOf(requests[0]), []);+ }+ {+ // A gate note left by a turn that ended before its trace started must not+ // reach the next turn's trace.+ const { agent, tabId, decisions, run } = setup({ supportsVision: false });+ agent.pendingTinyAskGateTraces.set(tabId, { probability: 0.42, ms: 7 });+ await run('How much is the blue plan?');+ assert.equal(decisions.length, 0);+ assert.equal(agent.pendingTinyAskGateTraces.has(tabId), false, 'each turn starts without a stale gate note');+ }{const { requests, decisions, captures, run } = setup({ readFails: true });await run('What colour is the banner?');