Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: refuse points on another page, re-check them after a scroll
Emre Sokullu16c4ab483cd4
diff --git a/docs/vision-mode.md b/docs/vision-mode.mdindex e484688f6..88309494c 100644--- a/docs/vision-mode.md+++ b/docs/vision-mode.md@@ -55,6 +55,7 @@ A point is refused before any dispatch when:- it lies outside the image;- it is the second or later coordinate action of one model turn, or any coordinate action while the last screen update failed (`session.observationFailed`), and the live screen changed near it since the screen the model last saw, or the screen could not be re-captured to check (`_visionStabilityBlock`, checked right before dispatch, after every gate; a `drag_at` is checked at both ends against one capture);- the run has no frame of its own yet (`no_frame`);+- the tab shows another page than its frame did (`page_changed`, `_visionStaleLayoutRefusal`, a navigation page feedback has not reported yet); a full frame follows. A page that only scrolled since the model looked is not refused, but every point of the batch is checked against the live screen first (`session.screenMoved`), so content that scrolled under a point stops it;- the browser window was resized or zoomed (CSS viewport size or device pixel ratio) after its frame was taken (`viewport_changed`, `_visionStaleLayoutRefusal`): checked once before each batch that uses points, so a resize page feedback has not reported yet is caught too; a full frame follows. A resize page feedback did report while the model decided sets the decision aside for a fresh full frame before that (`visionScreenFeedback`).A refused point also stops the rest of the model's batch: the calls after it (a `type_text`, say) were planned on that point.diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex bfba455c5..f3de9591b 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -995,6 +995,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {const zooms = this._visionTakePendingZooms(tabId);if (plan.kind === 'none') {session.registerNoChange();+ session.noteObservedScroll(frame.probe.scrollX, frame.probe.scrollY);this._visionRefreshZooms(session, zooms, frame);const note = visionNoChangeNote(session.keyframe);if (zooms.length) {@@ -1016,6 +1017,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {encodedPatches.push({ region, encoded });}const patches = session.registerPatches(plan.regions.map(regionLabel), frame.raster.pixels);+ session.noteObservedScroll(frame.probe.scrollX, frame.probe.scrollY);this._visionRefreshZooms(session, zooms, frame);const note = visionPatchNote(patches, session.keyframe, { coordinates: session.coordinates });const images = patches.map((patch, index) => ({ id: patch.id, kind: 'patch', dataUrl: encodedPatches[index].encoded.dataUrl }));@@ -1078,7 +1080,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {// A point is re-checked against the live screen when an earlier action// of the batch ran, or when the page changed while the model decided// (page feedback notes a response it kept despite that).- const recheck = actedBefore || session.observationFailed === true+ const recheck = actedBefore || session.observationFailed === true || session.screenMoved === true|| (!!call && typeof call === 'object' && this._visionRecheckCalls?.has(call) === true);if (VISION_TRANSLATED_TOOLS.has(name) || VISION_NATIVE_TOOLS.has(name)) {let args = {};@@ -1146,7 +1148,26 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {if (isControlFlowError(error)) throw error;return null;}- if (!probe || !session.layoutChanged({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, devicePixelRatio: probe.devicePixelRatio })) return null;+ if (!probe) return null;+ // Another page than the one the frame shows (a navigation page feedback+ // has not reported yet): its points no longer apply.+ if (probe.url && keyframe.url && String(probe.url) !== keyframe.url) {+ session.forceKeyframe = true;+ this._visionTraceNote(tabId, 'vision_layout_changed', { frame: keyframe.id, reason: 'page_changed' });+ return {+ success: false, dispatched: false, noDispatch: true, visionCoordinateError: 'page_changed',+ error: `Not dispatched: the tab shows another page than ${keyframe.id} did, so points read from it no longer apply. A fresh full frame follows; read the point again there.`,+ };+ }+ // Scrolled since the model looked (a scroll anchoring on a feed, an+ // auto-scroll): not refused, but every point is checked against the live+ // screen before it goes out (_visionTranslateToolCalls, _visionStabilityBlock).+ const seen = session.observedScroll;+ if (seen && (Math.abs((Number(probe.scrollX) || 0) - seen.x) > 1 || Math.abs((Number(probe.scrollY) || 0) - seen.y) > 1)) {+ session.screenMoved = true;+ this._visionTraceNote(tabId, 'vision_layout_changed', { frame: keyframe.id, reason: 'scrolled' });+ }+ if (!session.layoutChanged({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, devicePixelRatio: probe.devicePixelRatio })) return null;session.forceKeyframe = true;const size = (width, height) => `${Math.round(Number(width) || 0)}x${Math.round(Number(height) || 0)}`;this._visionTraceNote(tabId, 'vision_layout_changed', {diff --git a/src/chrome/src/agent/vision-session.js b/src/chrome/src/agent/vision-session.jsindex 819f536d7..902ef405e 100644--- a/src/chrome/src/agent/vision-session.js+++ b/src/chrome/src/agent/vision-session.js@@ -61,6 +61,12 @@ export class VisionSession {// last saw may no longer be the screen, until the next frame, patch or// "no visible change" (vision-runtime.js gates points and typing on it).this.observationFailed = false;+ // Where the screen the model last saw was scrolled (keyframe, patch or+ // "no visible change"), and whether the page has scrolled since+ // (vision-runtime.js _visionStaleLayoutRefusal): its points are then+ // re-checked against the live screen.+ this.observedScroll = null;+ this.screenMoved = false;this.pendingZooms = [];// Bumped each time the model is shown a screen that differs from the one// it saw (a patch, a new page or layout, a frame with other pixels), never@@ -104,6 +110,8 @@ export class VisionSession {this.patchChainLength = 0;this.forceKeyframe = false;this.observationFailed = false;+ this.screenMoved = false;+ this.observedScroll = { x: finite(meta.scrollX) || 0, y: finite(meta.scrollY) || 0 };const id = this._nextId('f');for (const retired of this.retired.values()) if (!retired.replacedBy) retired.replacedBy = id;this.keyframe = {@@ -187,6 +195,12 @@ export class VisionSession {this.observationFailed = false;}+ /** A patch or "no visible change" was read at this scroll position. */+ noteObservedScroll(scrollX, scrollY) {+ this.observedScroll = { x: finite(scrollX) || 0, y: finite(scrollY) || 0 };+ this.screenMoved = false;+ }+/*** Register a zoomed crop. `region` is in keyframe px; `width`/`height` are* the crop image's own pixels.diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex bfba455c5..f3de9591b 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -995,6 +995,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {const zooms = this._visionTakePendingZooms(tabId);if (plan.kind === 'none') {session.registerNoChange();+ session.noteObservedScroll(frame.probe.scrollX, frame.probe.scrollY);this._visionRefreshZooms(session, zooms, frame);const note = visionNoChangeNote(session.keyframe);if (zooms.length) {@@ -1016,6 +1017,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {encodedPatches.push({ region, encoded });}const patches = session.registerPatches(plan.regions.map(regionLabel), frame.raster.pixels);+ session.noteObservedScroll(frame.probe.scrollX, frame.probe.scrollY);this._visionRefreshZooms(session, zooms, frame);const note = visionPatchNote(patches, session.keyframe, { coordinates: session.coordinates });const images = patches.map((patch, index) => ({ id: patch.id, kind: 'patch', dataUrl: encodedPatches[index].encoded.dataUrl }));@@ -1078,7 +1080,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {// A point is re-checked against the live screen when an earlier action// of the batch ran, or when the page changed while the model decided// (page feedback notes a response it kept despite that).- const recheck = actedBefore || session.observationFailed === true+ const recheck = actedBefore || session.observationFailed === true || session.screenMoved === true|| (!!call && typeof call === 'object' && this._visionRecheckCalls?.has(call) === true);if (VISION_TRANSLATED_TOOLS.has(name) || VISION_NATIVE_TOOLS.has(name)) {let args = {};@@ -1146,7 +1148,26 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {if (isControlFlowError(error)) throw error;return null;}- if (!probe || !session.layoutChanged({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, devicePixelRatio: probe.devicePixelRatio })) return null;+ if (!probe) return null;+ // Another page than the one the frame shows (a navigation page feedback+ // has not reported yet): its points no longer apply.+ if (probe.url && keyframe.url && String(probe.url) !== keyframe.url) {+ session.forceKeyframe = true;+ this._visionTraceNote(tabId, 'vision_layout_changed', { frame: keyframe.id, reason: 'page_changed' });+ return {+ success: false, dispatched: false, noDispatch: true, visionCoordinateError: 'page_changed',+ error: `Not dispatched: the tab shows another page than ${keyframe.id} did, so points read from it no longer apply. A fresh full frame follows; read the point again there.`,+ };+ }+ // Scrolled since the model looked (a scroll anchoring on a feed, an+ // auto-scroll): not refused, but every point is checked against the live+ // screen before it goes out (_visionTranslateToolCalls, _visionStabilityBlock).+ const seen = session.observedScroll;+ if (seen && (Math.abs((Number(probe.scrollX) || 0) - seen.x) > 1 || Math.abs((Number(probe.scrollY) || 0) - seen.y) > 1)) {+ session.screenMoved = true;+ this._visionTraceNote(tabId, 'vision_layout_changed', { frame: keyframe.id, reason: 'scrolled' });+ }+ if (!session.layoutChanged({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, devicePixelRatio: probe.devicePixelRatio })) return null;session.forceKeyframe = true;const size = (width, height) => `${Math.round(Number(width) || 0)}x${Math.round(Number(height) || 0)}`;this._visionTraceNote(tabId, 'vision_layout_changed', {diff --git a/src/firefox/src/agent/vision-session.js b/src/firefox/src/agent/vision-session.jsindex 819f536d7..902ef405e 100644--- a/src/firefox/src/agent/vision-session.js+++ b/src/firefox/src/agent/vision-session.js@@ -61,6 +61,12 @@ export class VisionSession {// last saw may no longer be the screen, until the next frame, patch or// "no visible change" (vision-runtime.js gates points and typing on it).this.observationFailed = false;+ // Where the screen the model last saw was scrolled (keyframe, patch or+ // "no visible change"), and whether the page has scrolled since+ // (vision-runtime.js _visionStaleLayoutRefusal): its points are then+ // re-checked against the live screen.+ this.observedScroll = null;+ this.screenMoved = false;this.pendingZooms = [];// Bumped each time the model is shown a screen that differs from the one// it saw (a patch, a new page or layout, a frame with other pixels), never@@ -104,6 +110,8 @@ export class VisionSession {this.patchChainLength = 0;this.forceKeyframe = false;this.observationFailed = false;+ this.screenMoved = false;+ this.observedScroll = { x: finite(meta.scrollX) || 0, y: finite(meta.scrollY) || 0 };const id = this._nextId('f');for (const retired of this.retired.values()) if (!retired.replacedBy) retired.replacedBy = id;this.keyframe = {@@ -187,6 +195,12 @@ export class VisionSession {this.observationFailed = false;}+ /** A patch or "no visible change" was read at this scroll position. */+ noteObservedScroll(scrollX, scrollY) {+ this.observedScroll = { x: finite(scrollX) || 0, y: finite(scrollY) || 0 };+ this.screenMoved = false;+ }+/*** Register a zoomed crop. `region` is in keyframe px; `width`/`height` are* the crop image's own pixels.diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjsindex 86dde0797..adbb376fb 100644--- a/test/vision-mode-agent.mjs+++ b/test/vision-mode-agent.mjs@@ -723,6 +723,32 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))assert.match(stops[5], /six actions/);});+ test(`${build}: a page that changed while the model decided refuses its points; a scroll gets them re-checked`, async () => {+ const ctx = runAgent(step => {+ if (step === 1) {+ // Navigated while the model decided, unreported.+ ctx.scene.url = `${PAGE}?other`;+ return { toolCalls: [call('click_at', { x: 300, y: 200 })] };+ }+ if (step === 2) {+ // Scrolled while the model decided (a feed's scroll anchoring).+ ctx.scene.scrollY = 240;+ return { toolCalls: [call('click_at', { x: 300, y: 200 })] };+ }+ if (step === 3) return { toolCalls: [call('click_at', { x: 320, y: 220 })] };+ if (step === 4) return { toolCalls: [call('done', { summary: 'Done.', outcome: 'success' })] };+ return { content: 'done' };+ });+ try { await runLoop(ctx.agent, nextTab++, false); } finally { ctx.scene.url = PAGE; ctx.scene.scrollY = 0; }+ const refusal = toolTexts(ctx.requests[1]).at(-1);+ assert.match(refusal, /"visionCoordinateError":"page_changed"/, refusal);+ assert.match(textOf(visionMessages(ctx.requests[1].messages).at(-1)), /UNTRUSTED SCREENSHOT f\d+/);+ const clicks = ctx.dispatched.filter(entry => entry.name === 'click');+ assert.equal(clicks.length, 2, 'the scrolled point went out after its re-check, the next one plainly');+ assert.equal(clicks[0].vision.needsStabilityCheck, true);+ assert.equal(clicks[1].vision.needsStabilityCheck, false);+ });+test(`${build}: a new run never shows or uses the previous run's frame when its first capture fails`, async () => {const ctx = runAgent(step => {if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };