Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: bound captures by a deadline and Stop
Emre Sokullu67be21b88347
diff --git a/docs/vision-mode.md b/docs/vision-mode.mdindex a6a3df6bf..dc1924345 100644--- a/docs/vision-mode.md+++ b/docs/vision-mode.md@@ -61,6 +61,8 @@ A refused point also stops the rest of the model's batch: the calls after it (aTyping goes to whatever has focus, which only the screen shows. While the run has no frame of this page, or the last screen update failed (a capture or encoding error, or a re-check capture that failed; any later frame, patch or "no visible change" clears it), `type_text` and `press_keys` are refused before dispatch (`visionScreenUnverified`) unless a `click_at` earlier in the same response passed its check, in which case they type into the field it focused. The refusal stops the rest of the batch, and the model is told to take a screenshot or click the field first.+Every viewport measurement and screenshot (frames, patches, zooms, re-check captures, the layout check before a batch) is raced against the run's Stop and a 15 s deadline (`_visionBounded`): a hung renderer or debugger ends as an ordinary capture failure (`vision_capture_timeout`, the usual "could not be captured" note), which is not retried, and Stop ends the wait at once.+Every run starts on a frame it captured itself: `_beginVisionRun` retires the previous run's screen, so if the first capture fails the model is told so, earlier screenshots leave the request, and coordinates are refused until `take_screenshot` succeeds. Image ids and epochs continue across sessions of one conversation (another model, a restarted worker), and the current screen is the last full frame in the history by position.### Capture, retina and zoomdiff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 71dcde163..dd5d4d646 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -20843,6 +20843,9 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dawait cdpClient.attach(tabId);await cdpClient.sendCommand(tabId, 'Page.enable').catch(() => {});await this._preparePageForCapture(tabId);+ // Stop while the page was prepared: no screenshot is taken+ // (vision-runtime.js _visionBounded no longer waits for it).+ this._throwIfAborted(this._runAbortSignal(tabId));const region = cssRegion || { x: 0, y: 0, width: probe.cssWidth, height: probe.cssHeight };const dpr = Math.max(0.25, Number(probe?.devicePixelRatio) || 1);const zoom = Math.max(0.25, Number(probe?.zoom) || 1);diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex a1182d4ce..40b2e5c20 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -62,6 +62,11 @@ const UPLOAD_HANDOFF_DECISIONS = 5;// scan yet. While the tab loads, WebBrain waits this long for one, then// captures again.const REDACTION_SCAN_WAIT_MS = 1500;+// A viewport measurement or screenshot that has not answered by then (a hung+// renderer, a stalled debugger) fails like any capture failure, with its own+// error code (never a control-flow one), and is not retried; Stop ends the+// wait at once (_visionBounded).+const VISION_CAPTURE_DEADLINE_MS = 15000;const REDACTION_SCAN_POLL_MS = 250;// Redaction outcomes (agent _redactScreenshotDataUrl) that leave the image as// captured although redaction is on. Redaction is best-effort: the image is@@ -509,11 +514,11 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {* resamples to the plan so the coordinate map is exact.*/proto._visionCaptureFrame = async function _visionCaptureFrame(tabId, state, { redactionRetry = true } = {}) {- const probe = await this._visionPlatformProbe(tabId);+ const probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId));if (!probe || !(probe.cssWidth > 0) || !(probe.cssHeight > 0)) throw new Error('The page viewport could not be measured.');const profile = VISION_PROFILES[state.profileId] || VISION_PROFILES.default;const size = planVisionImageSize({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, profile, quality: state.quality });- let dataUrl = await this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height });+ let dataUrl = await this._visionBounded(tabId, 'screenshot', () => this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height }));if (!dataUrl) throw new Error('The browser returned no screenshot.');let redaction = null;if (this.screenshotRedaction && typeof this._redactScreenshotDataUrl === 'function') {@@ -554,6 +559,37 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {return encodeCanvasRegion(canvas, region, options);};+ /**+ * Runs one platform read (`what` names it in the error) against the run's+ * Stop and VISION_CAPTURE_DEADLINE_MS (`_visionCaptureDeadlineMs` overrides+ * it). The read itself is not cancelled; its late answer is ignored.+ */+ proto._visionBounded = async function _visionBounded(tabId, what, run) {+ const signal = this._runAbortSignal?.(tabId) || null;+ this._throwIfAborted?.(signal);+ const deadlineMs = Number(this._visionCaptureDeadlineMs) > 0 ? Number(this._visionCaptureDeadlineMs) : VISION_CAPTURE_DEADLINE_MS;+ let timer = null;+ let onAbort = null;+ try {+ return await new Promise((resolve, reject) => {+ timer = setTimeout(() => {+ const error = new Error(`The ${what} did not finish within ${deadlineMs / 1000} s.`);+ error.code = 'vision_capture_timeout';+ reject(error);+ }, deadlineMs);+ if (signal?.addEventListener) {+ onAbort = () => reject(signal.reason instanceof Error ? signal.reason+ : Object.assign(new Error('The operation was aborted.'), { name: 'AbortError' }));+ signal.addEventListener('abort', onAbort, { once: true });+ }+ Promise.resolve().then(run).then(resolve, reject);+ });+ } finally {+ clearTimeout(timer);+ if (onAbort) signal.removeEventListener?.('abort', onAbort);+ }+ };+/** Waits up to REDACTION_SCAN_WAIT_MS for the page's redaction scan to answer. */proto._visionAwaitRedactionScan = async function _visionAwaitRedactionScan(tabId) {const deadline = Date.now() + REDACTION_SCAN_WAIT_MS;@@ -1007,7 +1043,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {});if (!reads) return null;let probe = null;- try { probe = await this._visionPlatformProbe(tabId); }+ try { probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId)); }catch (error) {if (isControlFlowError(error)) throw error;return null;@@ -1174,9 +1210,9 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {}}const profile = VISION_PROFILES[state.profileId] || VISION_PROFILES.default;- const probe = await this._visionPlatformProbe(tabId);+ const probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId));const plan = planZoomImageSize({ regionWidth: region.cssRegion.width, regionHeight: region.cssRegion.height, devicePixelRatio: probe?.devicePixelRatio || 1, profile });- let dataUrl = await this._visionPlatformCapture(tabId, { probe, cssRegion: region.cssRegion, outWidth: plan.width, outHeight: plan.height });+ let dataUrl = await this._visionBounded(tabId, 'zoom screenshot', () => this._visionPlatformCapture(tabId, { probe, cssRegion: region.cssRegion, outWidth: plan.width, outHeight: plan.height }));if (!dataUrl) throw new Error('The browser returned no screenshot.');let redaction = null;if (this.screenshotRedaction && typeof this._redactScreenshotDataUrl === 'function') {@@ -1224,7 +1260,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {} else if (session.keyframe) {point = { x: session.keyframe.cssWidth / 2, y: session.keyframe.cssHeight / 2 };} else {- const probe = await this._visionPlatformProbe(tabId);+ const probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId));point = { x: (probe?.cssWidth || 800) / 2, y: (probe?.cssHeight || 600) / 2 };}// A focused number field under the pointer takes the wheel as +/-.diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex a1182d4ce..40b2e5c20 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -62,6 +62,11 @@ const UPLOAD_HANDOFF_DECISIONS = 5;// scan yet. While the tab loads, WebBrain waits this long for one, then// captures again.const REDACTION_SCAN_WAIT_MS = 1500;+// A viewport measurement or screenshot that has not answered by then (a hung+// renderer, a stalled debugger) fails like any capture failure, with its own+// error code (never a control-flow one), and is not retried; Stop ends the+// wait at once (_visionBounded).+const VISION_CAPTURE_DEADLINE_MS = 15000;const REDACTION_SCAN_POLL_MS = 250;// Redaction outcomes (agent _redactScreenshotDataUrl) that leave the image as// captured although redaction is on. Redaction is best-effort: the image is@@ -509,11 +514,11 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {* resamples to the plan so the coordinate map is exact.*/proto._visionCaptureFrame = async function _visionCaptureFrame(tabId, state, { redactionRetry = true } = {}) {- const probe = await this._visionPlatformProbe(tabId);+ const probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId));if (!probe || !(probe.cssWidth > 0) || !(probe.cssHeight > 0)) throw new Error('The page viewport could not be measured.');const profile = VISION_PROFILES[state.profileId] || VISION_PROFILES.default;const size = planVisionImageSize({ cssWidth: probe.cssWidth, cssHeight: probe.cssHeight, profile, quality: state.quality });- let dataUrl = await this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height });+ let dataUrl = await this._visionBounded(tabId, 'screenshot', () => this._visionPlatformCapture(tabId, { probe, outWidth: size.width, outHeight: size.height }));if (!dataUrl) throw new Error('The browser returned no screenshot.');let redaction = null;if (this.screenshotRedaction && typeof this._redactScreenshotDataUrl === 'function') {@@ -554,6 +559,37 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {return encodeCanvasRegion(canvas, region, options);};+ /**+ * Runs one platform read (`what` names it in the error) against the run's+ * Stop and VISION_CAPTURE_DEADLINE_MS (`_visionCaptureDeadlineMs` overrides+ * it). The read itself is not cancelled; its late answer is ignored.+ */+ proto._visionBounded = async function _visionBounded(tabId, what, run) {+ const signal = this._runAbortSignal?.(tabId) || null;+ this._throwIfAborted?.(signal);+ const deadlineMs = Number(this._visionCaptureDeadlineMs) > 0 ? Number(this._visionCaptureDeadlineMs) : VISION_CAPTURE_DEADLINE_MS;+ let timer = null;+ let onAbort = null;+ try {+ return await new Promise((resolve, reject) => {+ timer = setTimeout(() => {+ const error = new Error(`The ${what} did not finish within ${deadlineMs / 1000} s.`);+ error.code = 'vision_capture_timeout';+ reject(error);+ }, deadlineMs);+ if (signal?.addEventListener) {+ onAbort = () => reject(signal.reason instanceof Error ? signal.reason+ : Object.assign(new Error('The operation was aborted.'), { name: 'AbortError' }));+ signal.addEventListener('abort', onAbort, { once: true });+ }+ Promise.resolve().then(run).then(resolve, reject);+ });+ } finally {+ clearTimeout(timer);+ if (onAbort) signal.removeEventListener?.('abort', onAbort);+ }+ };+/** Waits up to REDACTION_SCAN_WAIT_MS for the page's redaction scan to answer. */proto._visionAwaitRedactionScan = async function _visionAwaitRedactionScan(tabId) {const deadline = Date.now() + REDACTION_SCAN_WAIT_MS;@@ -1007,7 +1043,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {});if (!reads) return null;let probe = null;- try { probe = await this._visionPlatformProbe(tabId); }+ try { probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId)); }catch (error) {if (isControlFlowError(error)) throw error;return null;@@ -1174,9 +1210,9 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {}}const profile = VISION_PROFILES[state.profileId] || VISION_PROFILES.default;- const probe = await this._visionPlatformProbe(tabId);+ const probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId));const plan = planZoomImageSize({ regionWidth: region.cssRegion.width, regionHeight: region.cssRegion.height, devicePixelRatio: probe?.devicePixelRatio || 1, profile });- let dataUrl = await this._visionPlatformCapture(tabId, { probe, cssRegion: region.cssRegion, outWidth: plan.width, outHeight: plan.height });+ let dataUrl = await this._visionBounded(tabId, 'zoom screenshot', () => this._visionPlatformCapture(tabId, { probe, cssRegion: region.cssRegion, outWidth: plan.width, outHeight: plan.height }));if (!dataUrl) throw new Error('The browser returned no screenshot.');let redaction = null;if (this.screenshotRedaction && typeof this._redactScreenshotDataUrl === 'function') {@@ -1224,7 +1260,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {} else if (session.keyframe) {point = { x: session.keyframe.cssWidth / 2, y: session.keyframe.cssHeight / 2 };} else {- const probe = await this._visionPlatformProbe(tabId);+ const probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId));point = { x: (probe?.cssWidth || 800) / 2, y: (probe?.cssHeight || 600) / 2 };}// A focused number field under the pointer takes the wheel as +/-.diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjsindex e3dce500f..f2c080d8d 100644--- a/test/vision-mode-agent.mjs+++ b/test/vision-mode-agent.mjs@@ -625,6 +625,26 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))assert.equal(agent._guardSkipped('done-verification-debt', 1), true);});+ test(`${build}: a screen update that times out leaves a note and is not retried`, async () => {+ const ctx = runAgent(step => {+ if (step === 1) return { toolCalls: [call('click_at', { x: 300, y: 200 })] };+ if (step === 2) return { toolCalls: [call('done', { summary: 'Stopped.', outcome: 'partial' })] };+ return { content: 'done' };+ });+ const capture = ctx.agent._visionPlatformCapture;+ let hangs = 0;+ ctx.agent._visionCaptureDeadlineMs = 50;+ ctx.agent._visionPlatformCapture = async (tab, options) => {+ // The first frame is captured; the update after the click hangs.+ if (ctx.scene.captures >= 1) { hangs++; return new Promise(() => {}); }+ return capture(tab, options);+ };+ await runLoop(ctx.agent, nextTab++, false);+ assert.equal(ctx.requests.length, 2);+ assert.match(textOf(visionMessages(ctx.requests[1].messages).at(-1)), /could not be captured after the action \(The screenshot did not finish within 0\.05 s\.\)/);+ assert.equal(hangs, 1, 'a timed-out capture is not tried again');+ });+test(`${build}: a new run never shows or uses the previous run's frame when its first capture fails`, async () => {const ctx = runAgent(step => {if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };diff --git a/test/vision-mode-guards.mjs b/test/vision-mode-guards.mjsindex 0b7f43dae..3eb40d944 100644--- a/test/vision-mode-guards.mjs+++ b/test/vision-mode-guards.mjs@@ -555,6 +555,35 @@ for (const build of ['chrome', 'firefox']) {}});+ test(`${build}: a capture that hangs fails at its deadline, and Stop ends the wait at once`, async () => {+ const agent = agentFor();+ agent.visionMode = 'auto';+ const tab = 39;+ agent._effectiveRunMode = () => 'act';+ agent._beginVisionRun(tab);+ const state = agent._visionRunState(tab, 'act');+ agent._visionPlatformProbe = async () => ({ cssWidth: 100, cssHeight: 50, devicePixelRatio: 1, scrollX: 0, scrollY: 0, url: 'https://owned.example/' });+ agent._visionPlatformCapture = () => new Promise(() => {});+ agent._visionCaptureDeadlineMs = 40;+ let started = Date.now();+ await assert.rejects(agent._visionCaptureFrame(tab, state), error => error.code === 'vision_capture_timeout' && /screenshot did not finish within 0\.04 s/.test(error.message));+ assert.ok(Date.now() - started < 1000);+ // Not a control-flow error: the caller's capture-failure path handles it.+ agent._visionCaptureDeadlineMs = 60000;+ const stop = new AbortController();+ agent._runAbortSignal = () => stop.signal;+ started = Date.now();+ setTimeout(() => stop.abort(), 20);+ await assert.rejects(agent._visionCaptureFrame(tab, state), error => error.name === 'AbortError');+ assert.ok(Date.now() - started < 1000, 'Stop did not wait for the capture');+ // A hung measurement is bounded the same way.+ agent._runAbortSignal = () => null;+ agent._visionCaptureDeadlineMs = 40;+ agent._visionPlatformProbe = () => new Promise(() => {});+ await assert.rejects(agent._visionCaptureFrame(tab, state), error => /viewport measurement did not finish/.test(error.message));+ agent._endVisionRun(tab, null);+ });+test(`${build}: Gmail's result counter, which takes no target, stays in the vision tool list`, () => {// Offered on a Gmail result list (getGmailResultCountPolicy).const names = getToolsForMode('act', { tier: 'full', gmailResultCounting: true, vision: { coordinates: 'pixel' } }).map(tool => tool.function.name);