Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: release a Firefox vision drag in the frame's own coordinates
Emre Sokulluce88eb97b4be
diff --git a/docs/vision-mode.md b/docs/vision-mode.mdindex 118268a7c..f65b105d2 100644--- a/docs/vision-mode.md+++ b/docs/vision-mode.md@@ -118,7 +118,7 @@ With an instant model the MV3 fixture's four steps (click and type, send, scroll`click_at` is rewritten into an ordinary coordinate `click` (`translateVisionClick`) before the batch pipeline runs, so the permission gate, the submit probe and every hard guard (recipient, deletion, money, credentials, publishing) inspect it like any other click. Only the final dispatch differs:- **Chrome**: trusted CDP pointer events at exactly the point (`_visionDispatchClick`), with the coordinate path's dispatch-time protections kept: a native `<select>` is focused for `type_text` instead of opening an OS popup no screenshot can show, the file-chooser guard, the recipient dispatch binding, `target=_blank` redirection into the run tab (also for a tab opened after the click returned, see Latency) and click-progress annotation. The legacy coordinate path's semantic re-targeting and nearby-input redirect are skipped: a vision click lands where the model pointed. Iframes, including cross-origin ones, receive the events natively.-- **Firefox**: no CDP, so `content.js` `visionInput` dispatches a synthetic pointer sequence at the exact point with real client coordinates, reproducing a press's focus and a click's activation, selecting text for double/triple clicks, and entering same-origin iframes. The click runs inside the same native file-chooser guard as every agent click (`clickWithoutNativeFilePicker`), and the agent settles that guard afterwards, so a chooser opened a moment later is caught too. A click that must pass the recipient binding keeps the content click path. Cross-origin iframes receive the click on the frame element.+- **Firefox**: no CDP, so `content.js` `visionInput` dispatches a synthetic pointer sequence at the exact point with real client coordinates, reproducing a press's focus and a click's activation, selecting text for double/triple clicks, and entering same-origin iframes. The click runs inside the same native file-chooser guard as every agent click (`clickWithoutNativeFilePicker`), and the agent settles that guard afterwards, so a chooser opened a moment later is caught too. A drag gives every move and the release in the coordinates of the document under the point, so a slider in a same-origin frame reads one consistent path. A click that must pass the recipient binding keeps the content click path. Cross-origin iframes receive the click on the frame element.Right before input goes out, both builds read what is under the pointer with `visionDomProbe` (`vision-dom-probe.js`), in an isolated world (Chrome, CDP) or the content-script world (Firefox), so page scripts cannot redefine the DOM methods it calls:diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 444d008d1..071f6acce 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -1810,19 +1810,31 @@const steps = Math.min(40, Math.max(2, Math.floor(Number(params.steps) || 12)));hover(el);pointer(el, 'pointerdown', { button: 0, buttons: 1 }); mouse(el, 'mousedown', { button: 0, buttons: 1, detail: 1 });+ // Every point, the release included, is given in the coordinates of+ // the document under it (a same-origin frame has its own), so the+ // events a slider or canvas reads all agree. A point with nothing+ // under it keeps the offset of the last frame that had something.+ let offsetX = x - cx;+ let offsetY = y - cy;+ const pointAt = (at, px, py) => {+ if (at) { offsetX = px - at.cx; offsetY = py - at.cy; }+ return { clientX: px - offsetX, clientY: py - offsetY, screenX: px + (window.screenX || 0), screenY: py + (window.screenY || 0) };+ };let last = el;for (let i = 1; i <= steps; i++) {const px = x + (toX - x) * (i / steps);const py = y + (toY - y) * (i / steps);const at = _visionTargetAt(px, py);last = at?.el || last;- const moveInit = { button: 0, buttons: 1, clientX: at ? at.cx : px, clientY: at ? at.cy : py };+ const moveInit = { button: 0, buttons: 1, ...pointAt(at, px, py) };last.dispatchEvent(new PointerCtor('pointermove', init({ pointerId: 1, pointerType: 'mouse', isPrimary: true, ...moveInit })));last.dispatchEvent(new view.MouseEvent('mousemove', init(moveInit)));}- const endInit = { button: 0, buttons: 0, clientX: toX, clientY: toY };- last.dispatchEvent(new PointerCtor('pointerup', init({ pointerId: 1, pointerType: 'mouse', isPrimary: true, ...endInit })));- last.dispatchEvent(new view.MouseEvent('mouseup', init({ ...endInit, detail: 1 })));+ const end = _visionTargetAt(toX, toY);+ const releaseTarget = end?.el || last;+ const endInit = { button: 0, buttons: 0, ...pointAt(end, toX, toY) };+ releaseTarget.dispatchEvent(new PointerCtor('pointerup', init({ pointerId: 1, pointerType: 'mouse', isPrimary: true, ...endInit })));+ releaseTarget.dispatchEvent(new view.MouseEvent('mouseup', init({ ...endInit, detail: 1 })));return { success: true, dispatched: true, method: 'synthetic-vision-drag', target: describe };}hover(el);diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex 90b681361..1671c2f34 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -2104,19 +2104,31 @@const steps = Math.min(40, Math.max(2, Math.floor(Number(params.steps) || 12)));hover(el);pointer(el, 'pointerdown', { button: 0, buttons: 1 }); mouse(el, 'mousedown', { button: 0, buttons: 1, detail: 1 });+ // Every point, the release included, is given in the coordinates of+ // the document under it (a same-origin frame has its own), so the+ // events a slider or canvas reads all agree. A point with nothing+ // under it keeps the offset of the last frame that had something.+ let offsetX = x - cx;+ let offsetY = y - cy;+ const pointAt = (at, px, py) => {+ if (at) { offsetX = px - at.cx; offsetY = py - at.cy; }+ return { clientX: px - offsetX, clientY: py - offsetY, screenX: px + (window.screenX || 0), screenY: py + (window.screenY || 0) };+ };let last = el;for (let i = 1; i <= steps; i++) {const px = x + (toX - x) * (i / steps);const py = y + (toY - y) * (i / steps);const at = _visionTargetAt(px, py);last = at?.el || last;- const moveInit = { button: 0, buttons: 1, clientX: at ? at.cx : px, clientY: at ? at.cy : py };+ const moveInit = { button: 0, buttons: 1, ...pointAt(at, px, py) };last.dispatchEvent(new PointerCtor('pointermove', init({ pointerId: 1, pointerType: 'mouse', isPrimary: true, ...moveInit })));last.dispatchEvent(new view.MouseEvent('mousemove', init(moveInit)));}- const endInit = { button: 0, buttons: 0, clientX: toX, clientY: toY };- last.dispatchEvent(new PointerCtor('pointerup', init({ pointerId: 1, pointerType: 'mouse', isPrimary: true, ...endInit })));- last.dispatchEvent(new view.MouseEvent('mouseup', init({ ...endInit, detail: 1 })));+ const end = _visionTargetAt(toX, toY);+ const releaseTarget = end?.el || last;+ const endInit = { button: 0, buttons: 0, ...pointAt(end, toX, toY) };+ releaseTarget.dispatchEvent(new PointerCtor('pointerup', init({ pointerId: 1, pointerType: 'mouse', isPrimary: true, ...endInit })));+ releaseTarget.dispatchEvent(new view.MouseEvent('mouseup', init({ ...endInit, detail: 1 })));return { success: true, dispatched: true, method: 'synthetic-vision-drag', target: describe };}hover(el);diff --git a/test/vision-mode.mjs b/test/vision-mode.mjsindex bdd8da9ed..95fa429ac 100644--- a/test/vision-mode.mjs+++ b/test/vision-mode.mjs@@ -610,6 +610,17 @@ test('a blocked file chooser is reported in vision terms in both builds', () =>}});+test('content scripts: the vision input path is the same in both builds', () => {+ const visionBlock = file => {+ const source = read(file);+ const start = source.indexOf(' function _visionTargetAt(');+ const end = source.indexOf('\n }\n', source.indexOf(' function visionInput(')) + 4;+ assert.ok(start > 0 && end > start, file);+ return source.slice(start, end);+ };+ assert.equal(visionBlock('src/firefox/src/content/content.js'), visionBlock('src/chrome/src/content/content.js'));+});+test('micro screen changes: a caret, a clock or a spinner settle; a menu or a toast does not', () => {const { isMicroScreenChange } = runtimeModule;const width = 1296, height = 810;diff --git a/test/vision-pipeline-browser.mjs b/test/vision-pipeline-browser.mjsindex 3fa60eeb3..04622bbc6 100644--- a/test/vision-pipeline-browser.mjs+++ b/test/vision-pipeline-browser.mjs@@ -226,6 +226,46 @@ test('content-script vision input (Firefox path) clicks, double-clicks, focuses,}});+test('content-script vision drag (Firefox path) moves and releases in a same-origin frame\'s own coordinates', async () => {+ const browser = await chromium.launch({ channel: 'chromium' });+ try {+ const page = await browser.newPage({ viewport: { width: 1000, height: 700 } });+ // A slider inside a same-origin frame at (500, 400) of the page, with a 2 px frame border.+ const frame = `<style>body{margin:0}#s{position:absolute;left:20px;top:20px;width:300px;height:30px;background:#ddd}</style><div id="s"></div>+ <script>window.events=[];const s=document.getElementById('s');let down=false;+ s.addEventListener('mousedown',e=>{down=true;events.push(['down',e.clientX,e.clientY])});+ document.addEventListener('mousemove',e=>{if(down)events.push(['move',e.clientX,e.clientY])});+ document.addEventListener('pointerup',e=>events.push(['pointerup',e.clientX,e.clientY]));+ document.addEventListener('mouseup',e=>{down=false;events.push(['up',e.clientX,e.clientY])});<\/script>`;+ await page.setContent(`<!doctype html><style>body{margin:0}iframe{position:absolute;left:500px;top:400px;width:400px;height:150px;border:2px solid #000}</style>+ <iframe srcdoc="${frame.replace(/"/g, '"')}"></iframe>`);+ await page.waitForFunction(() => document.querySelector('iframe').contentWindow?.events);+ const source = fs.readFileSync(path.join(ROOT, 'src/firefox/src/content/content.js'), 'utf8');+ const block = source.slice(source.indexOf(' function _visionTargetAt('), source.indexOf(' let _lastClickIdent = null;'));+ const guardBlock = source.slice(source.indexOf(' function uniqueFileInputSelector(input) {'), source.indexOf(' /**\n * Click an element by selector or coordinates.'));+ await page.evaluate(({ code, guardBlock }) => {+ // eslint-disable-next-line no-new-func+ window.__visionInput = new Function(`function withLocalPageDispatch(callback) { return callback(); }+ function _uniqueFileInputSelector(input) { return input.id ? '#' + input.id : ''; }+ function _deepSelectorMatches(selector) { return [...document.querySelectorAll(selector)]; }+ ${guardBlock}\n${code}\nreturn visionInput;`)();+ }, { code: block, guardBlock });+ // Page point (532, 437) is frame point (30, 35): 500 + 2 border + 30.+ const result = await page.evaluate(p => window.__visionInput(p), { type: 'drag', x: 532, y: 437, toX: 782, toY: 437, steps: 5 });+ assert.equal(result.success, true, JSON.stringify(result));+ const events = await page.evaluate(() => document.querySelector('iframe').contentWindow.events);+ const moves = events.filter(event => event[0] === 'move');+ const up = events.find(event => event[0] === 'up');+ const pointerUp = events.find(event => event[0] === 'pointerup');+ assert.deepEqual(events[0], ['down', 30, 35], JSON.stringify(events));+ assert.deepEqual(moves.at(-1), ['move', 280, 35], JSON.stringify(events));+ assert.deepEqual(up, ['up', 280, 35], `the release is where the last move was, in frame coordinates: ${JSON.stringify(events)}`);+ assert.deepEqual(pointerUp, ['pointerup', 280, 35], JSON.stringify(events));+ } finally {+ await browser.close();+ }+});+test('content-script vision clicks (Firefox path) never open a native file chooser', async () => {const browser = await chromium.launch({ channel: 'chromium' });try {