Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Vision mode: a later run without vision drops the earlier run's frames

Emre Sokullu3e71e55a4690
diff --git a/docs/vision-mode.md b/docs/vision-mode.md
index cda4193f8..0fe9683c3 100644
--- a/docs/vision-mode.md
+++ b/docs/vision-mode.md
@@ -64,7 +64,7 @@ Typing goes to whatever has focus, which only the screen shows. While the run ha
Every viewport measurement and screenshot (frames, patches, zooms, re-check captures, the layout check before a batch) is raced against the run's Stop and a 15 s deadline (`_visionBounded`): a hung renderer or debugger ends as an ordinary capture failure (`vision_capture_timeout`, the usual "could not be captured" note), which is not retried, and Stop ends the wait at once. An observation (the first frame, a screen update, the frame on returning to vision) whose capture failed for a passing reason, such as a navigation destroying the page context or a page not ready to paint (`classifyVisionCaptureError`), is tried again up to four times within 3 s (`_visionCaptureRetrying`). Input, timeouts, privacy failures and anything else are never retried. A browser page screenshots cannot capture (`chrome://` and similar) is not retried either, and its note tells the model to open a web page with `navigate` or ask the user instead of taking another screenshot.
-Every run starts on a frame it captured itself: `_beginVisionRun` retires the previous run's screen, so if the first capture fails the model is told so, earlier screenshots leave the request, and coordinates are refused until `take_screenshot` succeeds. Image ids and epochs continue across sessions of one conversation (another model, a restarted worker), and the current screen is the last full frame in the history by position.
+Every run starts on a frame it captured itself: `_beginVisionRun` retires the previous run's screen, so if the first capture fails the model is told so, earlier screenshots leave the request, and coordinates are refused until `take_screenshot` succeeds. Image ids and epochs continue across sessions of one conversation (another model, a restarted worker), and the current screen is the last full frame in the history by position. A run that does not start on vision (the setting off, a text-only model, a page that needs the element inventory, Ask) after a vision run in the same conversation pushes a retire marker first (`_visionRetireEarlierFrames`), so the earlier run's last frame and patches are not sent with its requests. Frames within one run are kept across a tree-path interlude, and replaced by the fresh frame on returning to vision.
### Capture, retina and zoom
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 3f1f668ea..527addc28 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -57285,6 +57285,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!selectionOnly && !standaloneChatRun && this._visionRunState(tabId, mode).active) {
await settleCookieBannersForTurn(this._cookieBannerHost, tabId, messages);
await this._visionEnsureKeyframe(tabId, messages, onUpdate, { mode });
+ } else if (!selectionOnly && !standaloneChatRun) {
+ // An earlier vision run's frames are not this run's screen.
+ this._visionRetireEarlierFrames(messages);
}
let visionPromptActive = !!this._visionToolOptions(tabId, mode);
@@ -58839,6 +58842,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!selectionOnly && !standaloneChatRun && this._visionRunState(tabId, mode).active) {
await settleCookieBannersForTurn(this._cookieBannerHost, tabId, messages);
await this._visionEnsureKeyframe(tabId, messages, onUpdate, { mode });
+ } else if (!selectionOnly && !standaloneChatRun) {
+ // An earlier vision run's frames are not this run's screen.
+ this._visionRetireEarlierFrames(messages);
}
let visionPromptActive = !!this._visionToolOptions(tabId, mode);
diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.js
index f3de9591b..2ea74f73f 100644
--- a/src/chrome/src/agent/vision-runtime.js
+++ b/src/chrome/src/agent/vision-runtime.js
@@ -335,6 +335,21 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
void this._visionPlatformLateTab?.(tabId, { cancel: true })?.catch?.(() => {});
};
+ /**
+ * A run that does not start on vision (the setting off, a text-only model,
+ * a page that needs the element inventory, Ask) after a vision run in the
+ * same conversation: the last frame and its patches would still be sent
+ * (pruneVisionImages keeps them). A retire marker drops them; the run's own
+ * frames, if vision starts later, come after it. Returns true if pushed.
+ */
+ proto._visionRetireEarlierFrames = function _visionRetireEarlierFrames(messages) {
+ if (!Array.isArray(messages)) return false;
+ const lastKey = messages.findLastIndex(message => visionFrameTag(message)?.kind === 'key');
+ if (lastKey < 0 || messages.findLastIndex(message => visionFrameTag(message)?.kind === 'retire') > lastKey) return false;
+ messages.push(visionUserMessage('[Screenshots from earlier vision steps are not shown: this run works without them. Read the page again before relying on what it showed.]', { epoch: 0, kind: 'retire' }));
+ return true;
+ };
+
/**
* A vision click_at the file-chooser guard stopped, in either build and on
* any dispatch path, hands the next steps to the element tools, upload_file
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 3e66c6f4b..04b717903 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -48447,6 +48447,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!selectionOnly && !standaloneChatRun && this._visionRunState(tabId, mode).active) {
await settleCookieBannersForTurn(this._cookieBannerHost, tabId, messages);
await this._visionEnsureKeyframe(tabId, messages, onUpdate, { mode });
+ } else if (!selectionOnly && !standaloneChatRun) {
+ // An earlier vision run's frames are not this run's screen.
+ this._visionRetireEarlierFrames(messages);
}
let visionPromptActive = !!this._visionToolOptions(tabId, mode);
@@ -49812,6 +49815,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!selectionOnly && !standaloneChatRun && this._visionRunState(tabId, mode).active) {
await settleCookieBannersForTurn(this._cookieBannerHost, tabId, messages);
await this._visionEnsureKeyframe(tabId, messages, onUpdate, { mode });
+ } else if (!selectionOnly && !standaloneChatRun) {
+ // An earlier vision run's frames are not this run's screen.
+ this._visionRetireEarlierFrames(messages);
}
let visionPromptActive = !!this._visionToolOptions(tabId, mode);
diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.js
index f3de9591b..2ea74f73f 100644
--- a/src/firefox/src/agent/vision-runtime.js
+++ b/src/firefox/src/agent/vision-runtime.js
@@ -335,6 +335,21 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
void this._visionPlatformLateTab?.(tabId, { cancel: true })?.catch?.(() => {});
};
+ /**
+ * A run that does not start on vision (the setting off, a text-only model,
+ * a page that needs the element inventory, Ask) after a vision run in the
+ * same conversation: the last frame and its patches would still be sent
+ * (pruneVisionImages keeps them). A retire marker drops them; the run's own
+ * frames, if vision starts later, come after it. Returns true if pushed.
+ */
+ proto._visionRetireEarlierFrames = function _visionRetireEarlierFrames(messages) {
+ if (!Array.isArray(messages)) return false;
+ const lastKey = messages.findLastIndex(message => visionFrameTag(message)?.kind === 'key');
+ if (lastKey < 0 || messages.findLastIndex(message => visionFrameTag(message)?.kind === 'retire') > lastKey) return false;
+ messages.push(visionUserMessage('[Screenshots from earlier vision steps are not shown: this run works without them. Read the page again before relying on what it showed.]', { epoch: 0, kind: 'retire' }));
+ return true;
+ };
+
/**
* A vision click_at the file-chooser guard stopped, in either build and on
* any dispatch path, hands the next steps to the element tools, upload_file
diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjs
index adbb376fb..3284b2ed4 100644
--- a/test/vision-mode-agent.mjs
+++ b/test/vision-mode-agent.mjs
@@ -749,6 +749,24 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))
assert.equal(clicks[1].vision.needsStabilityCheck, false);
});
+ test(`${build}: a later run that does not use vision sends none of the earlier vision run's screenshots`, async () => {
+ const ctx = runAgent(step => {
+ if (step === 1) return { toolCalls: [call('click_at', { x: 300, y: 200 })] };
+ if (step === 2) return { toolCalls: [call('done', { summary: 'Run one.', outcome: 'success' })] };
+ if (step === 3) return { toolCalls: [call('done', { summary: 'Run two.', outcome: 'success' })] };
+ return { content: 'done' };
+ }, { onDispatch: (name, _args, scene) => { if (name === 'click') scene.rects.push({ x: 900, y: 600, w: 80, h: 30, color: [0, 120, 0] }); } });
+ const tab = nextTab++;
+ await runLoop(ctx.agent, tab, false, 'Run one.');
+ assert.ok(totalImages(ctx.requests[1]) >= 1, 'run one saw its screen');
+ ctx.agent.visionMode = 'off';
+ await runLoop(ctx.agent, tab, false, 'Run two.');
+ const runTwo = ctx.requests[2];
+ assert.ok(!runTwo.tools.includes('click_at'), 'run two is on the tree path');
+ assert.equal(totalImages(runTwo), 0, 'no screenshot from run one is sent');
+ assert.ok(runTwo.messages.some(message => /Screenshots from earlier vision steps are not shown/.test(textOf(message))));
+ });
+
test(`${build}: a new run never shows or uses the previous run's frame when its first capture fails`, async () => {
const ctx = runAgent(step => {
if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };