Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: a later run without vision drops the earlier run's frames
Emre Sokullu3e71e55a4690
diff --git a/docs/vision-mode.md b/docs/vision-mode.mdindex cda4193f8..0fe9683c3 100644--- a/docs/vision-mode.md+++ b/docs/vision-mode.md@@ -64,7 +64,7 @@ Typing goes to whatever has focus, which only the screen shows. While the run haEvery viewport measurement and screenshot (frames, patches, zooms, re-check captures, the layout check before a batch) is raced against the run's Stop and a 15 s deadline (`_visionBounded`): a hung renderer or debugger ends as an ordinary capture failure (`vision_capture_timeout`, the usual "could not be captured" note), which is not retried, and Stop ends the wait at once. An observation (the first frame, a screen update, the frame on returning to vision) whose capture failed for a passing reason, such as a navigation destroying the page context or a page not ready to paint (`classifyVisionCaptureError`), is tried again up to four times within 3 s (`_visionCaptureRetrying`). Input, timeouts, privacy failures and anything else are never retried. A browser page screenshots cannot capture (`chrome://` and similar) is not retried either, and its note tells the model to open a web page with `navigate` or ask the user instead of taking another screenshot.-Every run starts on a frame it captured itself: `_beginVisionRun` retires the previous run's screen, so if the first capture fails the model is told so, earlier screenshots leave the request, and coordinates are refused until `take_screenshot` succeeds. Image ids and epochs continue across sessions of one conversation (another model, a restarted worker), and the current screen is the last full frame in the history by position.+Every run starts on a frame it captured itself: `_beginVisionRun` retires the previous run's screen, so if the first capture fails the model is told so, earlier screenshots leave the request, and coordinates are refused until `take_screenshot` succeeds. Image ids and epochs continue across sessions of one conversation (another model, a restarted worker), and the current screen is the last full frame in the history by position. A run that does not start on vision (the setting off, a text-only model, a page that needs the element inventory, Ask) after a vision run in the same conversation pushes a retire marker first (`_visionRetireEarlierFrames`), so the earlier run's last frame and patches are not sent with its requests. Frames within one run are kept across a tree-path interlude, and replaced by the fresh frame on returning to vision.### Capture, retina and zoomdiff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 3f1f668ea..527addc28 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -57285,6 +57285,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!selectionOnly && !standaloneChatRun && this._visionRunState(tabId, mode).active) {await settleCookieBannersForTurn(this._cookieBannerHost, tabId, messages);await this._visionEnsureKeyframe(tabId, messages, onUpdate, { mode });+ } else if (!selectionOnly && !standaloneChatRun) {+ // An earlier vision run's frames are not this run's screen.+ this._visionRetireEarlierFrames(messages);}let visionPromptActive = !!this._visionToolOptions(tabId, mode);@@ -58839,6 +58842,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!selectionOnly && !standaloneChatRun && this._visionRunState(tabId, mode).active) {await settleCookieBannersForTurn(this._cookieBannerHost, tabId, messages);await this._visionEnsureKeyframe(tabId, messages, onUpdate, { mode });+ } else if (!selectionOnly && !standaloneChatRun) {+ // An earlier vision run's frames are not this run's screen.+ this._visionRetireEarlierFrames(messages);}let visionPromptActive = !!this._visionToolOptions(tabId, mode);diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex f3de9591b..2ea74f73f 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -335,6 +335,21 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {void this._visionPlatformLateTab?.(tabId, { cancel: true })?.catch?.(() => {});};+ /**+ * A run that does not start on vision (the setting off, a text-only model,+ * a page that needs the element inventory, Ask) after a vision run in the+ * same conversation: the last frame and its patches would still be sent+ * (pruneVisionImages keeps them). A retire marker drops them; the run's own+ * frames, if vision starts later, come after it. Returns true if pushed.+ */+ proto._visionRetireEarlierFrames = function _visionRetireEarlierFrames(messages) {+ if (!Array.isArray(messages)) return false;+ const lastKey = messages.findLastIndex(message => visionFrameTag(message)?.kind === 'key');+ if (lastKey < 0 || messages.findLastIndex(message => visionFrameTag(message)?.kind === 'retire') > lastKey) return false;+ messages.push(visionUserMessage('[Screenshots from earlier vision steps are not shown: this run works without them. Read the page again before relying on what it showed.]', { epoch: 0, kind: 'retire' }));+ return true;+ };+/*** A vision click_at the file-chooser guard stopped, in either build and on* any dispatch path, hands the next steps to the element tools, upload_filediff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 3e66c6f4b..04b717903 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -48447,6 +48447,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!selectionOnly && !standaloneChatRun && this._visionRunState(tabId, mode).active) {await settleCookieBannersForTurn(this._cookieBannerHost, tabId, messages);await this._visionEnsureKeyframe(tabId, messages, onUpdate, { mode });+ } else if (!selectionOnly && !standaloneChatRun) {+ // An earlier vision run's frames are not this run's screen.+ this._visionRetireEarlierFrames(messages);}let visionPromptActive = !!this._visionToolOptions(tabId, mode);@@ -49812,6 +49815,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (!selectionOnly && !standaloneChatRun && this._visionRunState(tabId, mode).active) {await settleCookieBannersForTurn(this._cookieBannerHost, tabId, messages);await this._visionEnsureKeyframe(tabId, messages, onUpdate, { mode });+ } else if (!selectionOnly && !standaloneChatRun) {+ // An earlier vision run's frames are not this run's screen.+ this._visionRetireEarlierFrames(messages);}let visionPromptActive = !!this._visionToolOptions(tabId, mode);diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex f3de9591b..2ea74f73f 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -335,6 +335,21 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {void this._visionPlatformLateTab?.(tabId, { cancel: true })?.catch?.(() => {});};+ /**+ * A run that does not start on vision (the setting off, a text-only model,+ * a page that needs the element inventory, Ask) after a vision run in the+ * same conversation: the last frame and its patches would still be sent+ * (pruneVisionImages keeps them). A retire marker drops them; the run's own+ * frames, if vision starts later, come after it. Returns true if pushed.+ */+ proto._visionRetireEarlierFrames = function _visionRetireEarlierFrames(messages) {+ if (!Array.isArray(messages)) return false;+ const lastKey = messages.findLastIndex(message => visionFrameTag(message)?.kind === 'key');+ if (lastKey < 0 || messages.findLastIndex(message => visionFrameTag(message)?.kind === 'retire') > lastKey) return false;+ messages.push(visionUserMessage('[Screenshots from earlier vision steps are not shown: this run works without them. Read the page again before relying on what it showed.]', { epoch: 0, kind: 'retire' }));+ return true;+ };+/*** A vision click_at the file-chooser guard stopped, in either build and on* any dispatch path, hands the next steps to the element tools, upload_filediff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjsindex adbb376fb..3284b2ed4 100644--- a/test/vision-mode-agent.mjs+++ b/test/vision-mode-agent.mjs@@ -749,6 +749,24 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))assert.equal(clicks[1].vision.needsStabilityCheck, false);});+ test(`${build}: a later run that does not use vision sends none of the earlier vision run's screenshots`, async () => {+ const ctx = runAgent(step => {+ if (step === 1) return { toolCalls: [call('click_at', { x: 300, y: 200 })] };+ if (step === 2) return { toolCalls: [call('done', { summary: 'Run one.', outcome: 'success' })] };+ if (step === 3) return { toolCalls: [call('done', { summary: 'Run two.', outcome: 'success' })] };+ return { content: 'done' };+ }, { onDispatch: (name, _args, scene) => { if (name === 'click') scene.rects.push({ x: 900, y: 600, w: 80, h: 30, color: [0, 120, 0] }); } });+ const tab = nextTab++;+ await runLoop(ctx.agent, tab, false, 'Run one.');+ assert.ok(totalImages(ctx.requests[1]) >= 1, 'run one saw its screen');+ ctx.agent.visionMode = 'off';+ await runLoop(ctx.agent, tab, false, 'Run two.');+ const runTwo = ctx.requests[2];+ assert.ok(!runTwo.tools.includes('click_at'), 'run two is on the tree path');+ assert.equal(totalImages(runTwo), 0, 'no screenshot from run one is sent');+ assert.ok(runTwo.messages.some(message => /Screenshots from earlier vision steps are not shown/.test(textOf(message))));+ });+test(`${build}: a new run never shows or uses the previous run's frame when its first capture fails`, async () => {const ctx = runAgent(step => {if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };