Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

tiny Ask: bound the screenshot gate and clear its queued trace note per turn

Claude8e2b5e08143a
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index cf487ccf2..80d369962 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -208,7 +208,7 @@ import { shouldAutoGroupTabs } from '../tab-group-preference.js';
import {
TINY_ASK_ACT_BLOCKED_MESSAGE, TINY_ASK_COMPACT_LINKED_MAX_CHARS, TINY_ASK_COMPACT_PAGE_MAX_CHARS,
TINY_ASK_COMPACT_SELECTION_MAX_CHARS, TINY_ASK_READ_MAX_CHARS, TINY_ASK_READ_WINDOW_CHARS, TINY_ASK_SYSTEM_PROMPT,
- TINY_ASK_SCREENSHOT_GATE_THRESHOLD, tinyAskLinkedUrl, tinyAskScreenshotGate, tinyAskScreenshotGateProbability, tinyAskScreenshotNote, tinyAskPageContext, tinyAskPageReadable, tinyAskRuntimeLine,
+ TINY_ASK_SCREENSHOT_GATE_THRESHOLD, TINY_ASK_SCREENSHOT_GATE_TIMEOUT_MS, tinyAskLinkedUrl, tinyAskScreenshotGate, tinyAskScreenshotGateProbability, tinyAskScreenshotNote, tinyAskPageContext, tinyAskPageReadable, tinyAskRuntimeLine,
} from './tiny-ask.js';
import { webgpuModelDisplayName } from '../providers/webgpu.js';
@@ -12088,6 +12088,38 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return enriched;
}
+ // The gate's readout, bounded: a stalled offscreen worker must not hold the
+ // turn at enrichment. A timeout or the run's own abort cancels the worker
+ // operation (WebGPUProvider.decide sends webgpu-decide-cancel) and settles
+ // at once; only a run abort propagates.
+ async _tinyAskGateDecision(tabId, provider, gate) {
+ const controller = new AbortController();
+ const parent = this._runAbortSignal(tabId);
+ let timer = null;
+ let onAbort = null;
+ const settled = new Promise((_, reject) => {
+ onAbort = () => {
+ const reason = parent?.reason;
+ controller.abort(reason);
+ reject(reason instanceof Error ? reason : Object.assign(new Error('The run was stopped.'), { name: 'AbortError' }));
+ };
+ if (parent?.aborted) onAbort();
+ else parent?.addEventListener?.('abort', onAbort, { once: true });
+ timer = setTimeout(() => {
+ const error = new Error('The screenshot gate timed out.');
+ controller.abort(error);
+ reject(error);
+ }, Number(this.tinyAskGateTimeoutMs) > 0 ? Number(this.tinyAskGateTimeoutMs) : TINY_ASK_SCREENSHOT_GATE_TIMEOUT_MS);
+ });
+ settled.catch(() => {});
+ try {
+ return await Promise.race([provider.decide({ ...gate, promptVariant: 'gate', signal: controller.signal }), settled]);
+ } finally {
+ clearTimeout(timer);
+ parent?.removeEventListener?.('abort', onAbort);
+ }
+ }
+
// The visible page for a tiny Ask turn, when the in-browser model reads
// images itself and its own readout says the question needs it (see
// tinyAskScreenshotGate: language-independent, and blind to page content).
@@ -12116,7 +12148,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const gate = tinyAskScreenshotGate({
question, pageChars: String(page.text || '').trim().length, imageAttachment, linkedPage,
});
- const result = await provider.decide({ ...gate, promptVariant: 'gate', signal: this._runAbortSignal(tabId) });
+ const result = await this._tinyAskGateDecision(tabId, provider, gate);
probability = tinyAskScreenshotGateProbability(result?.answers);
} catch (error) {
if (error?.name === 'AbortError' || this._checkAbort(tabId)) throw error;
@@ -35829,6 +35861,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
this.pendingVisionRouteTraces.delete(tabId);
this.pendingVisionStatusTraces.delete(tabId);
this.pendingVisionSubCallTraces.delete(tabId);
+ this.pendingTinyAskGateTraces.delete(tabId);
this.carouselTraversalStates.delete(tabId);
this._chromeProtectedGalleryStates.delete(tabId);
void this._clearBackgroundFocusEmulation(tabId);
@@ -54129,6 +54162,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
this.pendingVisionRouteTraces.delete(tabId);
this.pendingVisionStatusTraces.delete(tabId);
this.pendingVisionSubCallTraces.delete(tabId);
+ this.pendingTinyAskGateTraces.delete(tabId);
this.visionStatusNotices.delete(tabId);
// Reset the per-turn auto-screenshot budget (issue #311) for a fresh turn.
this.autoScreenshotBursts.delete(tabId);
@@ -55906,6 +55940,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
this.pendingVisionRouteTraces.delete(tabId);
this.pendingVisionStatusTraces.delete(tabId);
this.pendingVisionSubCallTraces.delete(tabId);
+ this.pendingTinyAskGateTraces.delete(tabId);
this.visionStatusNotices.delete(tabId);
// Reset the per-turn auto-screenshot budget (issue #311) for a fresh turn.
this.autoScreenshotBursts.delete(tabId);
diff --git a/src/chrome/src/agent/tiny-ask.js b/src/chrome/src/agent/tiny-ask.js
index 5a62c3726..f64306c3a 100644
--- a/src/chrome/src/agent/tiny-ask.js
+++ b/src/chrome/src/agent/tiny-ask.js
@@ -58,6 +58,9 @@ export const TINY_ASK_READ_MAX_CHARS = 60000;
// steer the decision.
export const TINY_ASK_SCREENSHOT_MAX_PAGE_CHARS = 200;
export const TINY_ASK_SCREENSHOT_GATE_THRESHOLD = 0.5;
+// The readout takes ~0.2 s, plus a model load on the first turn after a
+// browser restart; past this the turn goes ahead text-only.
+export const TINY_ASK_SCREENSHOT_GATE_TIMEOUT_MS = 10_000;
export const TINY_ASK_SCREENSHOT_GATE_QUESTION = 'To answer this message well, does the assistant need to look at a '
+ 'screenshot of the current page, because the answer depends on how something on it looks (an image, photo, '
+ 'chart, colour, logo, button or layout) or on what a page with almost no text shows?';
diff --git a/test/run.js b/test/run.js
index 270fed719..8807c748f 100644
--- a/test/run.js
+++ b/test/run.js
@@ -41723,6 +41723,29 @@ test('tiny Ask shows a vision-capable in-browser model the visible page when its
assert.ok(requests[0].at(-1).content.some(block => block.text === TINY_ASK_TEXTLESS_SCREENSHOT_NOTE),
'a textless page tells the model to answer from the image');
}
+ {
+ // A stalled worker: the readout never answers. The turn goes ahead
+ // text-only after the gate's deadline, and the readout is cancelled.
+ let gateSignal = null;
+ const { agent, requests, captures, run } = setup();
+ agent.tinyAskGateTimeoutMs = 30;
+ agent._providerForRunOptions(1730, {}).decide = request => { gateSignal = request.signal; return new Promise(() => {}); };
+ const started = Date.now();
+ assert.equal(await run('What colour is the banner?'), 'Answer.', 'a stalled readout does not hold the turn');
+ assert.ok(Date.now() - started < 2000);
+ assert.equal(gateSignal?.aborted, true, 'the readout is cancelled at the deadline');
+ assert.equal(captures.length, 0);
+ assert.deepEqual(imagesOf(requests[0]), []);
+ }
+ {
+ // A gate note left by a turn that ended before its trace started must not
+ // reach the next turn's trace.
+ const { agent, tabId, decisions, run } = setup({ supportsVision: false });
+ agent.pendingTinyAskGateTraces.set(tabId, { probability: 0.42, ms: 7 });
+ await run('How much is the blue plan?');
+ assert.equal(decisions.length, 0);
+ assert.equal(agent.pendingTinyAskGateTraces.has(tabId), false, 'each turn starts without a stale gate note');
+ }
{
const { requests, decisions, captures, run } = setup({ readFails: true });
await run('What colour is the banner?');