Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Vision mode: refuse points on another page, re-check them after a scroll

Emre Sokullu16c4ab483cd4
diff --git a/docs/vision-mode.md b/docs/vision-mode.md
index e484688f6..88309494c 100644
--- a/docs/vision-mode.md
+++ b/docs/vision-mode.md
@@ -55,6 +55,7 @@ A point is refused before any dispatch when:
- it lies outside the image;
- it is the second or later coordinate action of one model turn, or any coordinate action while the last screen update failed (`session.observationFailed`), and the live screen changed near it since the screen the model last saw, or the screen could not be re-captured to check (`_visionStabilityBlock`, checked right before dispatch, after every gate; a `drag_at` is checked at both ends against one capture);
- the run has no frame of its own yet (`no_frame`);
+- the tab shows another page than its frame did (`page_changed`, `_visionStaleLayoutRefusal`, a navigation page feedback has not reported yet); a full frame follows. A page that only scrolled since the model looked is not refused, but every point of the batch is checked against the live screen first (`session.screenMoved`), so content that scrolled under a point stops it;
- the browser window was resized or zoomed (CSS viewport size or device pixel ratio) after its frame was taken (`viewport_changed`, `_visionStaleLayoutRefusal`): checked once before each batch that uses points, so a resize page feedback has not reported yet is caught too; a full frame follows. A resize page feedback did report while the model decided sets the decision aside for a fresh full frame before that (`visionScreenFeedback`).
A refused point also stops the rest of the model's batch: the calls after it (a `type_text`, say) were planned on that point.
diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.js
index bfba455c5..f3de9591b 100644
--- a/src/chrome/src/agent/vision-runtime.js
+++ b/src/chrome/src/agent/vision-runtime.js
@@ -995,6 +995,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
const zooms = this._visionTakePendingZooms(tabId);
if (plan.kind === 'none') {
session.registerNoChange();
+ session.noteObservedScroll(frame.probe.scrollX, frame.probe.scrollY);
this._visionRefreshZooms(session, zooms, frame);
const note = visionNoChangeNote(session.keyframe);
if (zooms.length) {
@@ -1016,6 +1017,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
encodedPatches.push({ region, encoded });
}
const patches = session.registerPatches(plan.regions.map(regionLabel), frame.raster.pixels);
+ session.noteObservedScroll(frame.probe.scrollX, frame.probe.scrollY);
this._visionRefreshZooms(session, zooms, frame);
const note = visionPatchNote(patches, session.keyframe, { coordinates: session.coordinates });
const images = patches.map((patch, index) => ({ id: patch.id, kind: 'patch', dataUrl: encodedPatches[index].encoded.dataUrl }));
@@ -1078,7 +1080,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
// A point is re-checked against the live screen when an earlier action
// of the batch ran, or when the page changed while the model decided
// (page feedback notes a response it kept despite that).
- const recheck = actedBefore || session.observationFailed === true
+ const recheck = actedBefore || session.observationFailed === true || session.screenMoved === true
|| (!!call && typeof call === 'object' && this._visionRecheckCalls?.has(call) === true);
if (VISION_TRANSLATED_TOOLS.has(name) || VISION_NATIVE_TOOLS.has(name)) {
let args = {};
@@ -1146,7 +1148,26 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
if (isControlFlowError(error)) throw error;
return null;
}
- if (!probe || !session.layoutChanged({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, devicePixelRatio: probe.devicePixelRatio })) return null;
+ if (!probe) return null;
+ // Another page than the one the frame shows (a navigation page feedback
+ // has not reported yet): its points no longer apply.
+ if (probe.url && keyframe.url && String(probe.url) !== keyframe.url) {
+ session.forceKeyframe = true;
+ this._visionTraceNote(tabId, 'vision_layout_changed', { frame: keyframe.id, reason: 'page_changed' });
+ return {
+ success: false, dispatched: false, noDispatch: true, visionCoordinateError: 'page_changed',
+ error: `Not dispatched: the tab shows another page than ${keyframe.id} did, so points read from it no longer apply. A fresh full frame follows; read the point again there.`,
+ };
+ }
+ // Scrolled since the model looked (a scroll anchoring on a feed, an
+ // auto-scroll): not refused, but every point is checked against the live
+ // screen before it goes out (_visionTranslateToolCalls, _visionStabilityBlock).
+ const seen = session.observedScroll;
+ if (seen && (Math.abs((Number(probe.scrollX) || 0) - seen.x) > 1 || Math.abs((Number(probe.scrollY) || 0) - seen.y) > 1)) {
+ session.screenMoved = true;
+ this._visionTraceNote(tabId, 'vision_layout_changed', { frame: keyframe.id, reason: 'scrolled' });
+ }
+ if (!session.layoutChanged({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, devicePixelRatio: probe.devicePixelRatio })) return null;
session.forceKeyframe = true;
const size = (width, height) => `${Math.round(Number(width) || 0)}x${Math.round(Number(height) || 0)}`;
this._visionTraceNote(tabId, 'vision_layout_changed', {
diff --git a/src/chrome/src/agent/vision-session.js b/src/chrome/src/agent/vision-session.js
index 819f536d7..902ef405e 100644
--- a/src/chrome/src/agent/vision-session.js
+++ b/src/chrome/src/agent/vision-session.js
@@ -61,6 +61,12 @@ export class VisionSession {
// last saw may no longer be the screen, until the next frame, patch or
// "no visible change" (vision-runtime.js gates points and typing on it).
this.observationFailed = false;
+ // Where the screen the model last saw was scrolled (keyframe, patch or
+ // "no visible change"), and whether the page has scrolled since
+ // (vision-runtime.js _visionStaleLayoutRefusal): its points are then
+ // re-checked against the live screen.
+ this.observedScroll = null;
+ this.screenMoved = false;
this.pendingZooms = [];
// Bumped each time the model is shown a screen that differs from the one
// it saw (a patch, a new page or layout, a frame with other pixels), never
@@ -104,6 +110,8 @@ export class VisionSession {
this.patchChainLength = 0;
this.forceKeyframe = false;
this.observationFailed = false;
+ this.screenMoved = false;
+ this.observedScroll = { x: finite(meta.scrollX) || 0, y: finite(meta.scrollY) || 0 };
const id = this._nextId('f');
for (const retired of this.retired.values()) if (!retired.replacedBy) retired.replacedBy = id;
this.keyframe = {
@@ -187,6 +195,12 @@ export class VisionSession {
this.observationFailed = false;
}
+ /** A patch or "no visible change" was read at this scroll position. */
+ noteObservedScroll(scrollX, scrollY) {
+ this.observedScroll = { x: finite(scrollX) || 0, y: finite(scrollY) || 0 };
+ this.screenMoved = false;
+ }
+
/**
* Register a zoomed crop. `region` is in keyframe px; `width`/`height` are
* the crop image's own pixels.
diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.js
index bfba455c5..f3de9591b 100644
--- a/src/firefox/src/agent/vision-runtime.js
+++ b/src/firefox/src/agent/vision-runtime.js
@@ -995,6 +995,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
const zooms = this._visionTakePendingZooms(tabId);
if (plan.kind === 'none') {
session.registerNoChange();
+ session.noteObservedScroll(frame.probe.scrollX, frame.probe.scrollY);
this._visionRefreshZooms(session, zooms, frame);
const note = visionNoChangeNote(session.keyframe);
if (zooms.length) {
@@ -1016,6 +1017,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
encodedPatches.push({ region, encoded });
}
const patches = session.registerPatches(plan.regions.map(regionLabel), frame.raster.pixels);
+ session.noteObservedScroll(frame.probe.scrollX, frame.probe.scrollY);
this._visionRefreshZooms(session, zooms, frame);
const note = visionPatchNote(patches, session.keyframe, { coordinates: session.coordinates });
const images = patches.map((patch, index) => ({ id: patch.id, kind: 'patch', dataUrl: encodedPatches[index].encoded.dataUrl }));
@@ -1078,7 +1080,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
// A point is re-checked against the live screen when an earlier action
// of the batch ran, or when the page changed while the model decided
// (page feedback notes a response it kept despite that).
- const recheck = actedBefore || session.observationFailed === true
+ const recheck = actedBefore || session.observationFailed === true || session.screenMoved === true
|| (!!call && typeof call === 'object' && this._visionRecheckCalls?.has(call) === true);
if (VISION_TRANSLATED_TOOLS.has(name) || VISION_NATIVE_TOOLS.has(name)) {
let args = {};
@@ -1146,7 +1148,26 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
if (isControlFlowError(error)) throw error;
return null;
}
- if (!probe || !session.layoutChanged({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, devicePixelRatio: probe.devicePixelRatio })) return null;
+ if (!probe) return null;
+ // Another page than the one the frame shows (a navigation page feedback
+ // has not reported yet): its points no longer apply.
+ if (probe.url && keyframe.url && String(probe.url) !== keyframe.url) {
+ session.forceKeyframe = true;
+ this._visionTraceNote(tabId, 'vision_layout_changed', { frame: keyframe.id, reason: 'page_changed' });
+ return {
+ success: false, dispatched: false, noDispatch: true, visionCoordinateError: 'page_changed',
+ error: `Not dispatched: the tab shows another page than ${keyframe.id} did, so points read from it no longer apply. A fresh full frame follows; read the point again there.`,
+ };
+ }
+ // Scrolled since the model looked (a scroll anchoring on a feed, an
+ // auto-scroll): not refused, but every point is checked against the live
+ // screen before it goes out (_visionTranslateToolCalls, _visionStabilityBlock).
+ const seen = session.observedScroll;
+ if (seen && (Math.abs((Number(probe.scrollX) || 0) - seen.x) > 1 || Math.abs((Number(probe.scrollY) || 0) - seen.y) > 1)) {
+ session.screenMoved = true;
+ this._visionTraceNote(tabId, 'vision_layout_changed', { frame: keyframe.id, reason: 'scrolled' });
+ }
+ if (!session.layoutChanged({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, devicePixelRatio: probe.devicePixelRatio })) return null;
session.forceKeyframe = true;
const size = (width, height) => `${Math.round(Number(width) || 0)}x${Math.round(Number(height) || 0)}`;
this._visionTraceNote(tabId, 'vision_layout_changed', {
diff --git a/src/firefox/src/agent/vision-session.js b/src/firefox/src/agent/vision-session.js
index 819f536d7..902ef405e 100644
--- a/src/firefox/src/agent/vision-session.js
+++ b/src/firefox/src/agent/vision-session.js
@@ -61,6 +61,12 @@ export class VisionSession {
// last saw may no longer be the screen, until the next frame, patch or
// "no visible change" (vision-runtime.js gates points and typing on it).
this.observationFailed = false;
+ // Where the screen the model last saw was scrolled (keyframe, patch or
+ // "no visible change"), and whether the page has scrolled since
+ // (vision-runtime.js _visionStaleLayoutRefusal): its points are then
+ // re-checked against the live screen.
+ this.observedScroll = null;
+ this.screenMoved = false;
this.pendingZooms = [];
// Bumped each time the model is shown a screen that differs from the one
// it saw (a patch, a new page or layout, a frame with other pixels), never
@@ -104,6 +110,8 @@ export class VisionSession {
this.patchChainLength = 0;
this.forceKeyframe = false;
this.observationFailed = false;
+ this.screenMoved = false;
+ this.observedScroll = { x: finite(meta.scrollX) || 0, y: finite(meta.scrollY) || 0 };
const id = this._nextId('f');
for (const retired of this.retired.values()) if (!retired.replacedBy) retired.replacedBy = id;
this.keyframe = {
@@ -187,6 +195,12 @@ export class VisionSession {
this.observationFailed = false;
}
+ /** A patch or "no visible change" was read at this scroll position. */
+ noteObservedScroll(scrollX, scrollY) {
+ this.observedScroll = { x: finite(scrollX) || 0, y: finite(scrollY) || 0 };
+ this.screenMoved = false;
+ }
+
/**
* Register a zoomed crop. `region` is in keyframe px; `width`/`height` are
* the crop image's own pixels.
diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjs
index 86dde0797..adbb376fb 100644
--- a/test/vision-mode-agent.mjs
+++ b/test/vision-mode-agent.mjs
@@ -723,6 +723,32 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))
assert.match(stops[5], /six actions/);
});
+ test(`${build}: a page that changed while the model decided refuses its points; a scroll gets them re-checked`, async () => {
+ const ctx = runAgent(step => {
+ if (step === 1) {
+ // Navigated while the model decided, unreported.
+ ctx.scene.url = `${PAGE}?other`;
+ return { toolCalls: [call('click_at', { x: 300, y: 200 })] };
+ }
+ if (step === 2) {
+ // Scrolled while the model decided (a feed's scroll anchoring).
+ ctx.scene.scrollY = 240;
+ return { toolCalls: [call('click_at', { x: 300, y: 200 })] };
+ }
+ if (step === 3) return { toolCalls: [call('click_at', { x: 320, y: 220 })] };
+ if (step === 4) return { toolCalls: [call('done', { summary: 'Done.', outcome: 'success' })] };
+ return { content: 'done' };
+ });
+ try { await runLoop(ctx.agent, nextTab++, false); } finally { ctx.scene.url = PAGE; ctx.scene.scrollY = 0; }
+ const refusal = toolTexts(ctx.requests[1]).at(-1);
+ assert.match(refusal, /"visionCoordinateError":"page_changed"/, refusal);
+ assert.match(textOf(visionMessages(ctx.requests[1].messages).at(-1)), /UNTRUSTED SCREENSHOT f\d+/);
+ const clicks = ctx.dispatched.filter(entry => entry.name === 'click');
+ assert.equal(clicks.length, 2, 'the scrolled point went out after its re-check, the next one plainly');
+ assert.equal(clicks[0].vision.needsStabilityCheck, true);
+ assert.equal(clicks[1].vision.needsStabilityCheck, false);
+ });
+
test(`${build}: a new run never shows or uses the previous run's frame when its first capture fails`, async () => {
const ctx = runAgent(step => {
if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };