Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: check Stop right before scroll, hover and drag input
Emre Sokullua7877b47203c
diff --git a/docs/vision-mode.md b/docs/vision-mode.mdindex 054df9d60..a6a3df6bf 100644--- a/docs/vision-mode.md+++ b/docs/vision-mode.md@@ -138,7 +138,7 @@ Right before input goes out, both builds read what is under the pointer with `vi- `drag_at` is refused when it is shorter than 6 CSS px, when the press and release would click a control (their nearest common ancestor is or is inside a button, link, label, similar control or pointer-cursor element, unless the drag starts on a slider, scrollbar, resizer, draggable item or grab handle), when either end is inside a frame it cannot inspect (another site's frame, or one drawn rotated, skewed, flipped or in 3D), when the target cannot be read, or when the click submit probe reads its start point as a submit, send, delete or payment control (a drag never gets the confirmation a click gets). On a protected messaging page `drag_at` is blocked outright, like `iframe_click`.- `scroll_at` is refused over a focused number field, which takes the wheel as +/-.-Every CDP input passes `cdpClient`'s page-agent fence and every content action passes `beforePageAgentDispatch`, so the page monitor attributes the events to the agent. `drag_at` is gated as a click (`TOOL_CAPABILITY`); captures and pointer results are untrusted page content (`UNTRUSTED_CONTENT_TOOLS`), and every image label says that text in it is data, never instructions. Screenshot redaction (Settings) applies to frames, patches (cut from the redacted frame) and zooms, and is best-effort there as everywhere: an image it could not run on is still sent. Right after a navigation the page's scan is often not ready (its content script starts when the document is idle), so while the tab is loading WebBrain waits up to 1.5 s for the scan and captures again. When an image does go out unredacted (no scan, an undecodable capture, or a region that could not be pixelated, which fails the whole redaction), the trace records `vision_redaction_skipped` with the reason (`code`) and the image kind (`context`), and the side panel warns once per page and run (`_visionReportUnredacted`).+Every CDP input passes `cdpClient`'s page-agent fence and every content action passes `beforePageAgentDispatch`, so the page monitor attributes the events to the agent. Stop is checked again right before every input goes out, after the probes and captures that precede it (`_visionThrowIfStopped` for `scroll_at`, `hover_at` and `drag_at`; Firefox's `_visionContentInput` before each trusted or content-script input; `_visionDispatchClick` for clicks). `drag_at` is gated as a click (`TOOL_CAPABILITY`); captures and pointer results are untrusted page content (`UNTRUSTED_CONTENT_TOOLS`), and every image label says that text in it is data, never instructions. Screenshot redaction (Settings) applies to frames, patches (cut from the redacted frame) and zooms, and is best-effort there as everywhere: an image it could not run on is still sent. Right after a navigation the page's scan is often not ready (its content script starts when the document is idle), so while the tab is loading WebBrain waits up to 1.5 s for the scan and captures again. When an image does go out unredacted (no scan, an undecodable capture, or a region that could not be pixelated, which fails the whole redaction), the trace records `vision_redaction_skipped` with the reason (`code`) and the image kind (`context`), and the side panel warns once per page and run (`_visionReportUnredacted`).## Guards in a vision rundiff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex ea49c9e52..a1182d4ce 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -1133,6 +1133,13 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {}};+ // Stop pressed while the checks before an input ran (probes, captures, a+ // drag's submit test): no input goes out.+ proto._visionThrowIfStopped = function _visionThrowIfStopped(tabId, context = null) {+ this._throwIfAborted?.(context?._contentActionAbortSignal);+ this._throwIfAborted?.(this._runAbortSignal?.(tabId));+ };+proto._executeVisionToolInner = async function _executeVisionToolInner(tabId, name, args, context, dispatch) {const vision = context?._vision || null;const state = this._visionRunState(tabId);@@ -1226,6 +1233,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {const distance = steps * VISION_SCROLL_STEP_CSS_PX;const deltaX = direction === 'left' ? -distance : direction === 'right' ? distance : 0;const deltaY = direction === 'up' ? -distance : direction === 'down' ? distance : 0;+ this._visionThrowIfStopped(tabId, context);dispatch.started = true;const result = await this._visionPlatformWheel(tabId, { x: point.x, y: point.y, deltaX, deltaY });return { success: result?.success !== false, dispatched: result?.dispatched !== false, direction, amount: steps, at: point, ...(result || {}) };@@ -1234,6 +1242,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {const resolved = await this._visionResolveForTool(tabId, { x: args.x, y: args.y, image: args.image }, vision);if (resolved.error) return resolved.error;const point = { x: resolved.resolved.cssX, y: resolved.resolved.cssY };+ this._visionThrowIfStopped(tabId, context);dispatch.started = true;const result = await this._visionPlatformHover(tabId, point);return { success: result?.success !== false, at: point, ...(result || {}) };@@ -1262,6 +1271,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {error: 'Not dispatched: the drag starts on a control WebBrain reads as a submit, send, delete or payment action, and a drag never gets the confirmation a click gets. Use click_at on it.',};}+ this._visionThrowIfStopped(tabId, context);dispatch.started = true;const result = await this._visionPlatformDrag(tabId, { from, to, steps: 12 });return { success: result?.success !== false, from, to, ...(result || {}) };diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex a6ce0c3aa..36ea724e9 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -17230,6 +17230,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}async _visionContentInput(tabId, params, kind = 'click') {+ // Stop pressed while the checks before this input ran: nothing is sent.+ this._throwIfAborted(this._runAbortSignal(tabId));if (firefoxBidi.runs.has(tabId)) {let trusted;try {@@ -17245,6 +17247,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dawait beforePageAgentDispatch(browser, tabId, {kind, frameId: 0, navigationCandidate: kind === 'click' && params.type === 'click', fenceOnly: true,});+ this._throwIfAborted(this._runAbortSignal(tabId));let response;try {response = await browser.tabs.sendMessage(tabId, { target: 'content', action: 'vision_input', params }, { frameId: 0 });diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex ea49c9e52..a1182d4ce 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -1133,6 +1133,13 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {}};+ // Stop pressed while the checks before an input ran (probes, captures, a+ // drag's submit test): no input goes out.+ proto._visionThrowIfStopped = function _visionThrowIfStopped(tabId, context = null) {+ this._throwIfAborted?.(context?._contentActionAbortSignal);+ this._throwIfAborted?.(this._runAbortSignal?.(tabId));+ };+proto._executeVisionToolInner = async function _executeVisionToolInner(tabId, name, args, context, dispatch) {const vision = context?._vision || null;const state = this._visionRunState(tabId);@@ -1226,6 +1233,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {const distance = steps * VISION_SCROLL_STEP_CSS_PX;const deltaX = direction === 'left' ? -distance : direction === 'right' ? distance : 0;const deltaY = direction === 'up' ? -distance : direction === 'down' ? distance : 0;+ this._visionThrowIfStopped(tabId, context);dispatch.started = true;const result = await this._visionPlatformWheel(tabId, { x: point.x, y: point.y, deltaX, deltaY });return { success: result?.success !== false, dispatched: result?.dispatched !== false, direction, amount: steps, at: point, ...(result || {}) };@@ -1234,6 +1242,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {const resolved = await this._visionResolveForTool(tabId, { x: args.x, y: args.y, image: args.image }, vision);if (resolved.error) return resolved.error;const point = { x: resolved.resolved.cssX, y: resolved.resolved.cssY };+ this._visionThrowIfStopped(tabId, context);dispatch.started = true;const result = await this._visionPlatformHover(tabId, point);return { success: result?.success !== false, at: point, ...(result || {}) };@@ -1262,6 +1271,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {error: 'Not dispatched: the drag starts on a control WebBrain reads as a submit, send, delete or payment action, and a drag never gets the confirmation a click gets. Use click_at on it.',};}+ this._visionThrowIfStopped(tabId, context);dispatch.started = true;const result = await this._visionPlatformDrag(tabId, { from, to, steps: 12 });return { success: result?.success !== false, from, to, ...(result || {}) };diff --git a/test/vision-mode-guards.mjs b/test/vision-mode-guards.mjsindex 95dcf6e64..0b7f43dae 100644--- a/test/vision-mode-guards.mjs+++ b/test/vision-mode-guards.mjs@@ -507,6 +507,54 @@ for (const build of ['chrome', 'firefox']) {assert.equal(await redact(snapshot([{ kind: 'input', rect: { x: 10, y: 10, w: 20, h: 10 } }]), {}), 'decode_failed');});+ test(`${build}: Stop pressed during the checks before a scroll, hover or drag sends no input`, async () => {+ const agent = agentFor();+ agent.visionMode = 'auto';+ const tab = 38;+ agent._effectiveRunMode = () => 'act';+ agent._beginVisionRun(tab);+ const session = agent._visionSessionFor(tab, agent._visionRunState(tab, 'act'));+ session.registerKeyframe({ imageWidth: 100, imageHeight: 50, cssWidth: 100, cssHeight: 50 }, new Uint8ClampedArray(100 * 50 * 4));+ const sent = [];+ for (const kind of ['Wheel', 'Hover', 'Drag']) agent[`_visionPlatform${kind}`] = async () => { sent.push(kind); return { success: true, dispatched: true }; };+ agent._visionResolveForTool = async () => ({ resolved: { cssX: 20, cssY: 20 } });+ agent._visionResolvePointsForTool = async () => ({ resolved: [{ cssX: 10, cssY: 10 }, { cssX: 80, cssY: 40 }] });+ agent._visionSubmitAt = async () => false;+ let stop;+ // The person presses Stop while the last check before the input runs.+ agent._visionDomProbe = async (_tab, request) => {+ stop.abort();+ return { points: request.points.map(() => ({ found: true, tag: 'div', clickable: false, opaqueFrame: false })), ...(request.pair ? { pair: { known: true, clickEquivalent: false } } : {}) };+ };+ const hoverProbe = agent._visionResolveForTool;+ for (const [name, args] of [['scroll_at', { direction: 'down' }], ['hover_at', { x: 20, y: 20 }], ['drag_at', { start_x: 10, start_y: 10, end_x: 80, end_y: 40 }]]) {+ stop = new AbortController();+ if (name === 'hover_at') agent._visionResolveForTool = async (...callArgs) => { const resolved = await hoverProbe(...callArgs); stop.abort(); return resolved; };+ await assert.rejects(agent._executeVisionTool(tab, name, args, { _contentActionAbortSignal: stop.signal, _vision: { tool: name } }), error => error?.name === 'AbortError', name);+ }+ assert.deepEqual(sent, [], 'no input went out');+ agent._endVisionRun(tab, null);+ if (build === 'firefox') {+ // Every Firefox vision input goes through the content input, which checks Stop first.+ const hadBrowser = 'browser' in globalThis;+ const api = globalThis.browser ??= { tabs: {} };+ api.tabs ??= {};+ const sendMessage = api.tabs.sendMessage;+ const messages = [];+ api.tabs.sendMessage = async (_tab, message) => { messages.push(message); return { success: true }; };+ const stopped = new AbortController();+ stopped.abort();+ agent._runAbortSignal = () => stopped.signal;+ try {+ await assert.rejects(agent._visionContentInput(tab, { type: 'click', x: 5, y: 5 }), error => error?.name === 'AbortError');+ assert.equal(messages.filter(message => message.action === 'vision_input').length, 0);+ } finally {+ api.tabs.sendMessage = sendMessage;+ if (!hadBrowser) delete globalThis.browser;+ }+ }+ });+test(`${build}: Gmail's result counter, which takes no target, stays in the vision tool list`, () => {// Offered on a Gmail result list (getGmailResultCountPolicy).const names = getToolsForMode('act', { tier: 'full', gmailResultCounting: true, vision: { coordinates: 'pixel' } }).map(tool => tool.function.name);