Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: retry an observation after a passing capture failure
Emre Sokullu0d4f7d6deb0e
diff --git a/docs/vision-mode.md b/docs/vision-mode.mdindex dc1924345..9b8f5e2cd 100644--- a/docs/vision-mode.md+++ b/docs/vision-mode.md@@ -61,7 +61,7 @@ A refused point also stops the rest of the model's batch: the calls after it (aTyping goes to whatever has focus, which only the screen shows. While the run has no frame of this page, or the last screen update failed (a capture or encoding error, or a re-check capture that failed; any later frame, patch or "no visible change" clears it), `type_text` and `press_keys` are refused before dispatch (`visionScreenUnverified`) unless a `click_at` earlier in the same response passed its check, in which case they type into the field it focused. The refusal stops the rest of the batch, and the model is told to take a screenshot or click the field first.-Every viewport measurement and screenshot (frames, patches, zooms, re-check captures, the layout check before a batch) is raced against the run's Stop and a 15 s deadline (`_visionBounded`): a hung renderer or debugger ends as an ordinary capture failure (`vision_capture_timeout`, the usual "could not be captured" note), which is not retried, and Stop ends the wait at once.+Every viewport measurement and screenshot (frames, patches, zooms, re-check captures, the layout check before a batch) is raced against the run's Stop and a 15 s deadline (`_visionBounded`): a hung renderer or debugger ends as an ordinary capture failure (`vision_capture_timeout`, the usual "could not be captured" note), which is not retried, and Stop ends the wait at once. An observation (the first frame, a screen update, the frame on returning to vision) whose capture failed for a passing reason, such as a navigation destroying the page context or a page not ready to paint (`classifyVisionCaptureError`), is tried again up to four times within 3 s (`_visionCaptureRetrying`). Input, timeouts, privacy failures and anything else are never retried. A browser page screenshots cannot capture (`chrome://` and similar) is not retried either, and its note tells the model to open a web page with `navigate` or ask the user instead of taking another screenshot.Every run starts on a frame it captured itself: `_beginVisionRun` retires the previous run's screen, so if the first capture fails the model is told so, earlier screenshots leave the request, and coordinates are refused until `take_screenshot` succeeds. Image ids and epochs continue across sessions of one conversation (another model, a restarted worker), and the current screen is the last full frame in the history by position.diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex 40b2e5c20..9f6166e46 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -67,6 +67,13 @@ const REDACTION_SCAN_WAIT_MS = 1500;// error code (never a control-flow one), and is not retried; Stop ends the// wait at once (_visionBounded).const VISION_CAPTURE_DEADLINE_MS = 15000;+// An observation whose capture failed for a passing reason (a navigation+// destroyed the page context, the page was not ready to paint) is tried+// again, within this many attempts and this much time. Input is never+// retried, nor a timeout, a privacy failure or anything not recognized here.+const OBSERVE_RETRY_ATTEMPTS = 4;+const OBSERVE_RETRY_BUDGET_MS = 3000;+const OBSERVE_RETRY_DELAY_MS = 250;const REDACTION_SCAN_POLL_MS = 250;// Redaction outcomes (agent _redactScreenshotDataUrl) that leave the image as// captured although redaction is on. Redaction is best-effort: the image is@@ -125,6 +132,28 @@ function visionScreenUnverifiedResult(name, { noFrame = false } = {}) {};}+/**+ * Why a viewport measurement or screenshot failed, from a closed list:+ * 'restricted' (a browser page screenshots cannot capture), 'transient' (the+ * page context changed or was not ready; worth one more look), or '' (any+ * other failure, a timeout, Stop).+ */+export function classifyVisionCaptureError(error) {+ if (!error || error.name === 'AbortError' || error.code === 'vision_capture_timeout') return '';+ const message = errorText(error);+ if (/cannot access (?:a )?(?:chrome(?:-extension|-untrusted)?|devtools):\/\/ URL\b/i.test(message)) return 'restricted';+ if (/permission|denied|not allowed|not attached|disconnected|no tab|target closed|session closed|tab (?:was |is )?closed/i.test(message)) return '';+ if (/execution context (?:was )?destroyed|cannot find (?:default )?(?:execution )?context|no such (?:realm|frame)|page changed during screenshot capture|unable to capture screenshot|cannot take screenshot with 0 (?:width|height)|page viewport could not be measured/i.test(message)) return 'transient';+ return '';+}++// What the model is told to do after a capture failure.+function captureFailureAdvice(error, retry = 'Call take_screenshot to look again.') {+ return classifyVisionCaptureError(error) === 'restricted'+ ? 'This is a browser page that screenshots cannot capture: open a web page with navigate({url}), or ask the user.'+ : retry;+}+// The page an upload handoff belongs to: its address without the fragment.function pageKey(url) {return String(url || '').split('#')[0];@@ -340,11 +369,11 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {if (!state.active) return null;this._visionSessions?.get(tabId)?.retireKeyframe?.();try {- const frame = await this._visionCaptureSettled(tabId, state);+ const frame = await this._visionCaptureRetrying(tabId, () => this._visionCaptureSettled(tabId, state));return await this._visionPushKeyframe(tabId, messages, onUpdate, { state, frame, reason: 'vision resumed' });} catch (error) {if (isControlFlowError(error)) throw error;- const message = `[SCREEN: the screen could not be captured on returning to screenshots (${errorText(error).slice(0, 160)}). Earlier screenshots show an older screen and cannot be used for coordinates. Call take_screenshot to try again.]`;+ const message = `[SCREEN: the screen could not be captured on returning to screenshots (${errorText(error).slice(0, 160)}). Earlier screenshots show an older screen and cannot be used for coordinates. ${captureFailureAdvice(error, 'Call take_screenshot to try again.')}]`;messages.push(visionUserMessage(message, { epoch: 0, kind: 'retire' }));this._visionTraceNote(tabId, 'vision_frame_error', { stage: 'resume', error: String(error?.message || error).slice(0, 200) });return null;@@ -590,6 +619,26 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {}};+ /**+ * An observation's capture, tried again while it fails for a transient+ * reason (classifyVisionCaptureError), within OBSERVE_RETRY_ATTEMPTS and+ * OBSERVE_RETRY_BUDGET_MS. Only screen reads come through here.+ */+ proto._visionCaptureRetrying = async function _visionCaptureRetrying(tabId, capture) {+ const deadline = Date.now() + OBSERVE_RETRY_BUDGET_MS;+ for (let attempt = 1; ; attempt++) {+ try {+ return await capture();+ } catch (error) {+ if (isControlFlowError(error) || attempt >= OBSERVE_RETRY_ATTEMPTS+ || classifyVisionCaptureError(error) !== 'transient' || Date.now() + OBSERVE_RETRY_DELAY_MS > deadline) throw error;+ this._visionTraceNote(tabId, 'vision_capture_retry', { attempt, code: 'transient' });+ await sleep(OBSERVE_RETRY_DELAY_MS);+ this._throwIfAborted?.(this._runAbortSignal?.(tabId));+ }+ }+ };+/** Waits up to REDACTION_SCAN_WAIT_MS for the page's redaction scan to answer. */proto._visionAwaitRedactionScan = async function _visionAwaitRedactionScan(tabId) {const deadline = Date.now() + REDACTION_SCAN_WAIT_MS;@@ -761,7 +810,8 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {const state = this._visionRunState(tabId, mode);if (!state.active) return null;try {- const keyframe = await this._visionPushKeyframe(tabId, messages, onUpdate, { state, reason });+ const frame = await this._visionCaptureRetrying(tabId, () => this._visionCaptureFrame(tabId, state));+ const keyframe = await this._visionPushKeyframe(tabId, messages, onUpdate, { state, frame, reason });// The opening frame is the run's first look at the page, as a read is on// the tree path: a question answered from it has task evidence.try { this._markPlanExecutionToolCall?.(tabId, 'take_screenshot', { success: true, method: 'image_attach' }); } catch {}@@ -770,7 +820,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {if (isControlFlowError(error)) throw error;// The run starts with no screen (_beginVisionRun retired the last one);// the tag retires the last frame in the history for the model too.- const message = `[SCREEN: the first screenshot could not be captured (${errorText(error).slice(0, 160)}). Earlier screenshots show an older screen and cannot be used for coordinates. Call take_screenshot to try again.]`;+ const message = `[SCREEN: the first screenshot could not be captured (${errorText(error).slice(0, 160)}). Earlier screenshots show an older screen and cannot be used for coordinates. ${captureFailureAdvice(error, 'Call take_screenshot to try again.')}]`;messages.push(visionUserMessage(message, { epoch: 0, kind: 'retire' }));this._visionTraceNote(tabId, 'vision_frame_error', { stage: 'start', error: String(error?.message || error).slice(0, 200) });return null;@@ -834,7 +884,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {// checked against the live screen (_visionTranslateToolCalls).session.forceKeyframe = true;session.observationFailed = true;- messages.push(visionUserMessage(`[SCREEN: the screen could not be ${stage === 'observe' ? 'captured' : 'prepared'} after the action (${errorText(error).slice(0, 160)}). Call take_screenshot to look again.]`,+ messages.push(visionUserMessage(`[SCREEN: the screen could not be ${stage === 'observe' ? 'captured' : 'prepared'} after the action (${errorText(error).slice(0, 160)}). ${captureFailureAdvice(error)}]`,{ epoch: session.epoch, kind: 'error' }));this._visionTraceNote(tabId, 'vision_frame_error', { stage, error: errorText(error) });return { kind: 'error' };@@ -849,8 +899,8 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {await rerouteLateTab();let frame;try {- frame = await this._visionCaptureSettled(tabId, state);- if ((await rerouteLateTab())?.redirected) frame = await this._visionCaptureSettled(tabId, state);+ frame = await this._visionCaptureRetrying(tabId, () => this._visionCaptureSettled(tabId, state));+ if ((await rerouteLateTab())?.redirected) frame = await this._visionCaptureRetrying(tabId, () => this._visionCaptureSettled(tabId, state));} catch (error) {if (isControlFlowError(error)) throw error;return failed('observe', error);diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex 40b2e5c20..9f6166e46 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -67,6 +67,13 @@ const REDACTION_SCAN_WAIT_MS = 1500;// error code (never a control-flow one), and is not retried; Stop ends the// wait at once (_visionBounded).const VISION_CAPTURE_DEADLINE_MS = 15000;+// An observation whose capture failed for a passing reason (a navigation+// destroyed the page context, the page was not ready to paint) is tried+// again, within this many attempts and this much time. Input is never+// retried, nor a timeout, a privacy failure or anything not recognized here.+const OBSERVE_RETRY_ATTEMPTS = 4;+const OBSERVE_RETRY_BUDGET_MS = 3000;+const OBSERVE_RETRY_DELAY_MS = 250;const REDACTION_SCAN_POLL_MS = 250;// Redaction outcomes (agent _redactScreenshotDataUrl) that leave the image as// captured although redaction is on. Redaction is best-effort: the image is@@ -125,6 +132,28 @@ function visionScreenUnverifiedResult(name, { noFrame = false } = {}) {};}+/**+ * Why a viewport measurement or screenshot failed, from a closed list:+ * 'restricted' (a browser page screenshots cannot capture), 'transient' (the+ * page context changed or was not ready; worth one more look), or '' (any+ * other failure, a timeout, Stop).+ */+export function classifyVisionCaptureError(error) {+ if (!error || error.name === 'AbortError' || error.code === 'vision_capture_timeout') return '';+ const message = errorText(error);+ if (/cannot access (?:a )?(?:chrome(?:-extension|-untrusted)?|devtools):\/\/ URL\b/i.test(message)) return 'restricted';+ if (/permission|denied|not allowed|not attached|disconnected|no tab|target closed|session closed|tab (?:was |is )?closed/i.test(message)) return '';+ if (/execution context (?:was )?destroyed|cannot find (?:default )?(?:execution )?context|no such (?:realm|frame)|page changed during screenshot capture|unable to capture screenshot|cannot take screenshot with 0 (?:width|height)|page viewport could not be measured/i.test(message)) return 'transient';+ return '';+}++// What the model is told to do after a capture failure.+function captureFailureAdvice(error, retry = 'Call take_screenshot to look again.') {+ return classifyVisionCaptureError(error) === 'restricted'+ ? 'This is a browser page that screenshots cannot capture: open a web page with navigate({url}), or ask the user.'+ : retry;+}+// The page an upload handoff belongs to: its address without the fragment.function pageKey(url) {return String(url || '').split('#')[0];@@ -340,11 +369,11 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {if (!state.active) return null;this._visionSessions?.get(tabId)?.retireKeyframe?.();try {- const frame = await this._visionCaptureSettled(tabId, state);+ const frame = await this._visionCaptureRetrying(tabId, () => this._visionCaptureSettled(tabId, state));return await this._visionPushKeyframe(tabId, messages, onUpdate, { state, frame, reason: 'vision resumed' });} catch (error) {if (isControlFlowError(error)) throw error;- const message = `[SCREEN: the screen could not be captured on returning to screenshots (${errorText(error).slice(0, 160)}). Earlier screenshots show an older screen and cannot be used for coordinates. Call take_screenshot to try again.]`;+ const message = `[SCREEN: the screen could not be captured on returning to screenshots (${errorText(error).slice(0, 160)}). Earlier screenshots show an older screen and cannot be used for coordinates. ${captureFailureAdvice(error, 'Call take_screenshot to try again.')}]`;messages.push(visionUserMessage(message, { epoch: 0, kind: 'retire' }));this._visionTraceNote(tabId, 'vision_frame_error', { stage: 'resume', error: String(error?.message || error).slice(0, 200) });return null;@@ -590,6 +619,26 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {}};+ /**+ * An observation's capture, tried again while it fails for a transient+ * reason (classifyVisionCaptureError), within OBSERVE_RETRY_ATTEMPTS and+ * OBSERVE_RETRY_BUDGET_MS. Only screen reads come through here.+ */+ proto._visionCaptureRetrying = async function _visionCaptureRetrying(tabId, capture) {+ const deadline = Date.now() + OBSERVE_RETRY_BUDGET_MS;+ for (let attempt = 1; ; attempt++) {+ try {+ return await capture();+ } catch (error) {+ if (isControlFlowError(error) || attempt >= OBSERVE_RETRY_ATTEMPTS+ || classifyVisionCaptureError(error) !== 'transient' || Date.now() + OBSERVE_RETRY_DELAY_MS > deadline) throw error;+ this._visionTraceNote(tabId, 'vision_capture_retry', { attempt, code: 'transient' });+ await sleep(OBSERVE_RETRY_DELAY_MS);+ this._throwIfAborted?.(this._runAbortSignal?.(tabId));+ }+ }+ };+/** Waits up to REDACTION_SCAN_WAIT_MS for the page's redaction scan to answer. */proto._visionAwaitRedactionScan = async function _visionAwaitRedactionScan(tabId) {const deadline = Date.now() + REDACTION_SCAN_WAIT_MS;@@ -761,7 +810,8 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {const state = this._visionRunState(tabId, mode);if (!state.active) return null;try {- const keyframe = await this._visionPushKeyframe(tabId, messages, onUpdate, { state, reason });+ const frame = await this._visionCaptureRetrying(tabId, () => this._visionCaptureFrame(tabId, state));+ const keyframe = await this._visionPushKeyframe(tabId, messages, onUpdate, { state, frame, reason });// The opening frame is the run's first look at the page, as a read is on// the tree path: a question answered from it has task evidence.try { this._markPlanExecutionToolCall?.(tabId, 'take_screenshot', { success: true, method: 'image_attach' }); } catch {}@@ -770,7 +820,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {if (isControlFlowError(error)) throw error;// The run starts with no screen (_beginVisionRun retired the last one);// the tag retires the last frame in the history for the model too.- const message = `[SCREEN: the first screenshot could not be captured (${errorText(error).slice(0, 160)}). Earlier screenshots show an older screen and cannot be used for coordinates. Call take_screenshot to try again.]`;+ const message = `[SCREEN: the first screenshot could not be captured (${errorText(error).slice(0, 160)}). Earlier screenshots show an older screen and cannot be used for coordinates. ${captureFailureAdvice(error, 'Call take_screenshot to try again.')}]`;messages.push(visionUserMessage(message, { epoch: 0, kind: 'retire' }));this._visionTraceNote(tabId, 'vision_frame_error', { stage: 'start', error: String(error?.message || error).slice(0, 200) });return null;@@ -834,7 +884,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {// checked against the live screen (_visionTranslateToolCalls).session.forceKeyframe = true;session.observationFailed = true;- messages.push(visionUserMessage(`[SCREEN: the screen could not be ${stage === 'observe' ? 'captured' : 'prepared'} after the action (${errorText(error).slice(0, 160)}). Call take_screenshot to look again.]`,+ messages.push(visionUserMessage(`[SCREEN: the screen could not be ${stage === 'observe' ? 'captured' : 'prepared'} after the action (${errorText(error).slice(0, 160)}). ${captureFailureAdvice(error)}]`,{ epoch: session.epoch, kind: 'error' }));this._visionTraceNote(tabId, 'vision_frame_error', { stage, error: errorText(error) });return { kind: 'error' };@@ -849,8 +899,8 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {await rerouteLateTab();let frame;try {- frame = await this._visionCaptureSettled(tabId, state);- if ((await rerouteLateTab())?.redirected) frame = await this._visionCaptureSettled(tabId, state);+ frame = await this._visionCaptureRetrying(tabId, () => this._visionCaptureSettled(tabId, state));+ if ((await rerouteLateTab())?.redirected) frame = await this._visionCaptureRetrying(tabId, () => this._visionCaptureSettled(tabId, state));} catch (error) {if (isControlFlowError(error)) throw error;return failed('observe', error);diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjsindex f2c080d8d..a7310b176 100644--- a/test/vision-mode-agent.mjs+++ b/test/vision-mode-agent.mjs@@ -645,6 +645,34 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))assert.equal(hangs, 1, 'a timed-out capture is not tried again');});+ test(`${build}: a navigation that destroys the page context mid-capture costs one more look, not a failed update`, async () => {+ const ctx = runAgent(step => {+ if (step === 1) return { toolCalls: [call('click_at', { x: 300, y: 200 })] };+ if (step === 2) return { toolCalls: [call('done', { summary: 'Done.', outcome: 'success' })] };+ return { content: 'done' };+ }, { onDispatch: (name, _args, scene) => { if (name === 'click') scene.rects.push({ x: 900, y: 600, w: 80, h: 30, color: [0, 120, 0] }); } });+ const capture = ctx.agent._visionPlatformCapture;+ let failed = false;+ ctx.agent._visionPlatformCapture = async (tab, options) => {+ if (ctx.scene.captures >= 1 && !failed) { failed = true; throw new Error('Execution context was destroyed.'); }+ return capture(tab, options);+ };+ await runLoop(ctx.agent, nextTab++, false);+ const update = visionMessages(ctx.requests[1].messages).at(-1);+ assert.equal(update.webbrainVisionFrame.kind, 'patch', textOf(update));+ });++ test(`${build}: a browser page screenshots cannot capture gets advice the model can act on`, async () => {+ const ctx = runAgent(step => (step === 1 ? { toolCalls: [call('done', { summary: 'Could not see.', outcome: 'partial' })] } : { content: 'done' }));+ let attempts = 0;+ ctx.agent._visionPlatformCapture = async () => { attempts++; throw new Error('Cannot access a chrome:// URL'); };+ await runLoop(ctx.agent, nextTab++, false);+ const note = textOf(visionMessages(ctx.requests[0].messages).at(-1));+ assert.match(note, /browser page that screenshots cannot capture: open a web page with navigate\(\{url\}\), or ask the user/);+ assert.doesNotMatch(note, /Call take_screenshot/);+ assert.equal(attempts, 1);+ });+test(`${build}: a new run never shows or uses the previous run's frame when its first capture fails`, async () => {const ctx = runAgent(step => {if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };diff --git a/test/vision-mode-guards.mjs b/test/vision-mode-guards.mjsindex 3eb40d944..9d636e347 100644--- a/test/vision-mode-guards.mjs+++ b/test/vision-mode-guards.mjs@@ -584,6 +584,28 @@ for (const build of ['chrome', 'firefox']) {agent._endVisionRun(tab, null);});+ test(`${build}: an observation is tried again only after a passing capture failure`, async () => {+ const { classifyVisionCaptureError } = await import(`../src/${build}/src/agent/vision-runtime.js`);+ for (const [message, kind] of [+ ['Execution context was destroyed.', 'transient'], ['Cannot find context with specified id', 'transient'],+ ['The page viewport could not be measured.', 'transient'], ['Unable to capture screenshot', 'transient'],+ ['Cannot access a chrome:// URL', 'restricted'], ['Target closed', ''], ['Permission denied', ''], ['boom', ''],+ ]) assert.equal(classifyVisionCaptureError(new Error(message)), kind, message);+ assert.equal(classifyVisionCaptureError(Object.assign(new Error('The screenshot did not finish within 15 s.'), { code: 'vision_capture_timeout' })), '');+ assert.equal(classifyVisionCaptureError(Object.assign(new Error('Execution context was destroyed.'), { name: 'AbortError' })), '');+ const agent = agentFor();+ let calls = 0;+ const fail = message => async () => { calls++; throw new Error(message); };+ await assert.rejects(agent._visionCaptureRetrying(40, fail('Execution context was destroyed.')), /destroyed/);+ assert.equal(calls, 4, 'four attempts at most');+ calls = 0;+ await assert.rejects(agent._visionCaptureRetrying(40, fail('Cannot access a chrome:// URL')), /chrome/);+ assert.equal(calls, 1, 'a restricted page is not tried again');+ calls = 0;+ assert.equal(await agent._visionCaptureRetrying(40, async () => { if (++calls < 2) throw new Error('Execution context was destroyed.'); return 'frame'; }), 'frame');+ assert.equal(calls, 2);+ });+test(`${build}: Gmail's result counter, which takes no target, stays in the vision tool list`, () => {// Offered on a Gmail result list (getGmailResultCountPolicy).const names = getToolsForMode('act', { tier: 'full', gmailResultCounting: true, vision: { coordinates: 'pixel' } }).map(tool => tool.function.name);@@ -636,6 +658,7 @@ for (const build of ['chrome', 'firefox']) {test(`${build}: the opening frame is the run's first look at the page`, async () => {const agent = agentFor();agent._visionRunState = () => ({ active: true, coordinates: 'pixel' });+ agent._visionCaptureFrame = async () => ({});agent._visionPushKeyframe = async () => ({ id: 'f1' });const marked = [];agent._markPlanExecutionToolCall = (_tab, name, result) => marked.push({ name, success: result?.success });