Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Attribute agent-driven window resizes
Barack Sokullu7d1b3446d84a
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 51825b24f..0bac6d0e3 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -33363,7 +33363,10 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaiif (args.left != null && Number.isFinite(Number(args.left))) update.left = Math.round(Number(args.left));if (args.top != null && Number.isFinite(Number(args.top))) update.top = Math.round(Number(args.top));+ let resizeMarker = null;+ let expectedViewport = null;try {+ resizeMarker = await this._beginPageAgentResize(tabId);const win = await chrome.windows.get(tab.windowId);if (win?.state && win.state !== 'normal') {await chrome.windows.update(tab.windowId, { state: 'normal' });@@ -33372,6 +33375,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaiawait chrome.windows.update(tab.windowId, update);await new Promise(r => setTimeout(r, 250));const info = await this._getWindowInfo(tabId);+ expectedViewport = info?.viewport || null;return {...info,resized: true,@@ -33380,6 +33384,8 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai};} catch (e) {return { success: false, error: `resize_window failed: ${e.message}` };+ } finally {+ await this._finishPageAgentResize(tabId, resizeMarker, expectedViewport);}}diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.jsindex 901c7156d..4a269972b 100644--- a/src/chrome/src/agent/page-feedback.js+++ b/src/chrome/src/agent/page-feedback.js@@ -145,6 +145,25 @@ export const pageFeedbackMethods = {});},+ async _beginPageAgentResize(tabId) {+ const run = this._pageFeedbackRuns?.get(tabId);+ if (!run) return null;+ const marker = { runToken: run.token, operationId: token() };+ try {+ const response = await apiFor().tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_resize_begin',+ params: marker }, { frameId: 0 });+ return response?.ready === true && this._pageFeedbackRuns?.get(tabId) === run ? marker : null;+ } catch { return null; }+ },++ async _finishPageAgentResize(tabId, marker, expectedViewport) {+ if (!marker) return;+ try {+ await apiFor().tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_resize_finish',+ params: { ...marker, expectedViewport: expectedViewport || null } }, { frameId: 0 });+ } catch { /* A protected or navigated document cannot report resize attribution. */ }+ },+_finishPageFeedbackRun(tabId) {const run = this._pageFeedbackRuns?.get(tabId);if (!run) return;@@ -192,6 +211,12 @@ export const pageFeedbackMethods = {|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };frame.seq = seq;if (feedback.source === 'agent') {+ if (feedback.kind === 'resize') {+ const session = this._jevSessions?.get(tabId);+ if (session) { session.disabled = true; session.queue = []; session.snapshot = null; }+ this.screenshotCaptures?.delete(tabId);+ this.clearLastTypeFieldIdent?.(tabId);+ }if (['click', 'submit'].includes(feedback.operation)) {let url = '';try {diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex 5baa82d28..10f07b87c 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -30,6 +30,7 @@let preparedOperationCursor = 0;let requestGeneration = 0, unreported = 0, pendingDOM = null;let lastViewport = '';+ let agentResize = null;let lastFeedbackDelivery = Promise.resolve();let agentTurn = null, userTurn = null, lastUserTarget = null;let matchedEvents = new WeakMap();@@ -123,6 +124,35 @@return delivery;} catch { stop(); }};+ const currentViewport = () => ({ width: window.innerWidth, height: window.innerHeight,+ visualWidth: window.visualViewport?.width ?? window.innerWidth, visualHeight: window.visualViewport?.height ?? window.innerHeight,+ scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });+ function flushAgentResize(source, expected = agentResize) {+ if (!expected || agentResize !== expected) return Promise.resolve();+ clearTimeout(expected.timer); agentResize = null;+ return Promise.allSettled(expected.events.map(viewport => send({ kind: 'resize', source, target: 'viewport', viewport })));+ }+ function beginAgentResize(params) {+ if (!active || params.runToken !== runToken || typeof params.operationId !== 'string' || !params.operationId) return false;+ if (agentResize) void flushAgentResize('unknown');+ const marker = { runToken, operationId: params.operationId, userAt: lastUserAt, events: [], timer: null };+ marker.timer = setTimeout(() => { void flushAgentResize('unknown', marker); }, 5000);+ agentResize = marker;+ return true;+ }+ async function finishAgentResize(params) {+ const marker = agentResize;+ if (!marker || marker.runToken !== runToken || params.runToken !== runToken || marker.operationId !== params.operationId) {+ return { ready: false };+ }+ const finalViewport = marker.events.at(-1);+ const expected = params.expectedViewport;+ const attributed = marker.userAt === lastUserAt && finalViewport+ && Number.isFinite(expected?.width) && Number.isFinite(expected?.height)+ && finalViewport.width === expected.width && finalViewport.height === expected.height;+ await flushAgentResize(attributed ? 'agent' : 'unknown', marker);+ return { ready: true, attributed: !!attributed };+ }const interact = (kind, el, { completeGesture = false, ...extra } = {}) => {const isIgnored = ignored(el);if (!active || (isIgnored && !completeGesture)) return;@@ -1001,14 +1031,16 @@const el = elementFor(event);if (event.isTrusted && !expected('scroll', el, event)) interact('activity', el);});- const viewport = () => ({ width: innerWidth, height: innerHeight,- visualWidth: window.visualViewport?.width ?? innerWidth, visualHeight: window.visualViewport?.height ?? innerHeight,- scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });- lastViewport = JSON.stringify(viewport());+ lastViewport = JSON.stringify(currentViewport());const onResize = () => {- const current = viewport(), key = JSON.stringify(current);+ const current = currentViewport(), key = JSON.stringify(current);if (!active || key === lastViewport) return;- lastViewport = key; revision++;+ lastViewport = key; revision++; publishRevision();+ if (agentResize?.runToken === runToken && agentResize.events.length < 16) {+ agentResize.events.push(current);+ return;+ }+ if (agentResize) void flushAgentResize('unknown');send({ kind: 'resize', source: window === window.top ? 'unknown' : 'page', target: 'viewport', viewport: current });};listen(window, 'resize', onResize);@@ -1052,6 +1084,7 @@}function stop() {active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;+ clearTimeout(agentResize?.timer); agentResize = null;lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlIterator = null; preparedOperationCursor = 0;document.documentElement?.removeAttribute(fenceAttribute);for (const el of nativeTargets) {@@ -1232,6 +1265,10 @@if (msg.active) { void requestState().then(() => respond({ ready: true })); return true; }else if (!msg.runToken || msg.runToken === runToken) { requestGeneration++; stop(); respond({ ready: true }); }} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }+ else if (msg.action === 'page_monitor_resize_begin') { respond({ ready: beginAgentResize(msg.params || {}) }); }+ else if (msg.action === 'page_monitor_resize_finish') {+ void finishAgentResize(msg.params || {}).then(respond, () => respond({ ready: false })); return true;+ }else if (msg.action === 'page_monitor_validate') { respond({ ready: validateNativeDispatch(msg.params || {}) }); }else if (msg.action === 'page_monitor_dispatch') {const params = msg.params || {};diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 07b36c11b..3e150481f 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -30950,7 +30950,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (args.left != null && Number.isFinite(Number(args.left))) update.left = Math.round(Number(args.left));if (args.top != null && Number.isFinite(Number(args.top))) update.top = Math.round(Number(args.top));+ let resizeMarker = null;+ let expectedViewport = null;try {+ resizeMarker = await this._beginPageAgentResize(tabId);const win = await browser.windows.get(tab.windowId);if (win?.state && win.state !== 'normal') {await browser.windows.update(tab.windowId, { state: 'normal' });@@ -30959,6 +30962,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dawait browser.windows.update(tab.windowId, update);await new Promise(r => setTimeout(r, 250));const info = await this._getWindowInfo(tabId);+ expectedViewport = info?.viewport || null;return {...info,resized: true,@@ -30967,6 +30971,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d};} catch (e) {return { success: false, error: `resize_window failed: ${e.message}` };+ } finally {+ await this._finishPageAgentResize(tabId, resizeMarker, expectedViewport);}}diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.jsindex 901c7156d..4a269972b 100644--- a/src/firefox/src/agent/page-feedback.js+++ b/src/firefox/src/agent/page-feedback.js@@ -145,6 +145,25 @@ export const pageFeedbackMethods = {});},+ async _beginPageAgentResize(tabId) {+ const run = this._pageFeedbackRuns?.get(tabId);+ if (!run) return null;+ const marker = { runToken: run.token, operationId: token() };+ try {+ const response = await apiFor().tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_resize_begin',+ params: marker }, { frameId: 0 });+ return response?.ready === true && this._pageFeedbackRuns?.get(tabId) === run ? marker : null;+ } catch { return null; }+ },++ async _finishPageAgentResize(tabId, marker, expectedViewport) {+ if (!marker) return;+ try {+ await apiFor().tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_resize_finish',+ params: { ...marker, expectedViewport: expectedViewport || null } }, { frameId: 0 });+ } catch { /* A protected or navigated document cannot report resize attribution. */ }+ },+_finishPageFeedbackRun(tabId) {const run = this._pageFeedbackRuns?.get(tabId);if (!run) return;@@ -192,6 +211,12 @@ export const pageFeedbackMethods = {|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };frame.seq = seq;if (feedback.source === 'agent') {+ if (feedback.kind === 'resize') {+ const session = this._jevSessions?.get(tabId);+ if (session) { session.disabled = true; session.queue = []; session.snapshot = null; }+ this.screenshotCaptures?.delete(tabId);+ this.clearLastTypeFieldIdent?.(tabId);+ }if (['click', 'submit'].includes(feedback.operation)) {let url = '';try {diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex 5baa82d28..10f07b87c 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -30,6 +30,7 @@let preparedOperationCursor = 0;let requestGeneration = 0, unreported = 0, pendingDOM = null;let lastViewport = '';+ let agentResize = null;let lastFeedbackDelivery = Promise.resolve();let agentTurn = null, userTurn = null, lastUserTarget = null;let matchedEvents = new WeakMap();@@ -123,6 +124,35 @@return delivery;} catch { stop(); }};+ const currentViewport = () => ({ width: window.innerWidth, height: window.innerHeight,+ visualWidth: window.visualViewport?.width ?? window.innerWidth, visualHeight: window.visualViewport?.height ?? window.innerHeight,+ scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });+ function flushAgentResize(source, expected = agentResize) {+ if (!expected || agentResize !== expected) return Promise.resolve();+ clearTimeout(expected.timer); agentResize = null;+ return Promise.allSettled(expected.events.map(viewport => send({ kind: 'resize', source, target: 'viewport', viewport })));+ }+ function beginAgentResize(params) {+ if (!active || params.runToken !== runToken || typeof params.operationId !== 'string' || !params.operationId) return false;+ if (agentResize) void flushAgentResize('unknown');+ const marker = { runToken, operationId: params.operationId, userAt: lastUserAt, events: [], timer: null };+ marker.timer = setTimeout(() => { void flushAgentResize('unknown', marker); }, 5000);+ agentResize = marker;+ return true;+ }+ async function finishAgentResize(params) {+ const marker = agentResize;+ if (!marker || marker.runToken !== runToken || params.runToken !== runToken || marker.operationId !== params.operationId) {+ return { ready: false };+ }+ const finalViewport = marker.events.at(-1);+ const expected = params.expectedViewport;+ const attributed = marker.userAt === lastUserAt && finalViewport+ && Number.isFinite(expected?.width) && Number.isFinite(expected?.height)+ && finalViewport.width === expected.width && finalViewport.height === expected.height;+ await flushAgentResize(attributed ? 'agent' : 'unknown', marker);+ return { ready: true, attributed: !!attributed };+ }const interact = (kind, el, { completeGesture = false, ...extra } = {}) => {const isIgnored = ignored(el);if (!active || (isIgnored && !completeGesture)) return;@@ -1001,14 +1031,16 @@const el = elementFor(event);if (event.isTrusted && !expected('scroll', el, event)) interact('activity', el);});- const viewport = () => ({ width: innerWidth, height: innerHeight,- visualWidth: window.visualViewport?.width ?? innerWidth, visualHeight: window.visualViewport?.height ?? innerHeight,- scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });- lastViewport = JSON.stringify(viewport());+ lastViewport = JSON.stringify(currentViewport());const onResize = () => {- const current = viewport(), key = JSON.stringify(current);+ const current = currentViewport(), key = JSON.stringify(current);if (!active || key === lastViewport) return;- lastViewport = key; revision++;+ lastViewport = key; revision++; publishRevision();+ if (agentResize?.runToken === runToken && agentResize.events.length < 16) {+ agentResize.events.push(current);+ return;+ }+ if (agentResize) void flushAgentResize('unknown');send({ kind: 'resize', source: window === window.top ? 'unknown' : 'page', target: 'viewport', viewport: current });};listen(window, 'resize', onResize);@@ -1052,6 +1084,7 @@}function stop() {active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;+ clearTimeout(agentResize?.timer); agentResize = null;lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlIterator = null; preparedOperationCursor = 0;document.documentElement?.removeAttribute(fenceAttribute);for (const el of nativeTargets) {@@ -1232,6 +1265,10 @@if (msg.active) { void requestState().then(() => respond({ ready: true })); return true; }else if (!msg.runToken || msg.runToken === runToken) { requestGeneration++; stop(); respond({ ready: true }); }} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }+ else if (msg.action === 'page_monitor_resize_begin') { respond({ ready: beginAgentResize(msg.params || {}) }); }+ else if (msg.action === 'page_monitor_resize_finish') {+ void finishAgentResize(msg.params || {}).then(respond, () => respond({ ready: false })); return true;+ }else if (msg.action === 'page_monitor_validate') { respond({ ready: validateNativeDispatch(msg.params || {}) }); }else if (msg.action === 'page_monitor_dispatch') {const params = msg.params || {};diff --git a/test/page-feedback.mjs b/test/page-feedback.mjsindex b57efda25..374ec9b50 100644--- a/test/page-feedback.mjs+++ b/test/page-feedback.mjs@@ -365,6 +365,54 @@ for (const build of ['chrome', 'firefox']) {} finally { agent._releaseRunEntry(tab); }});+ test(`${build}: agent resize feedback invalidates caches without queueing or idle delay`, async () => {+ const agent = setup(Agent), tab = nextTab++;+ await agent._claimRunEntry(tab, 'interactive');+ try {+ const binding = bind(agent, tab);+ agent._jevSessions = new Map([[tab, { disabled: false, queue: ['prepared'], snapshot: {} }]]);+ agent.screenshotCaptures.set(tab, { captureId: 'stale-resize' });+ let clearedTypeField = false;+ agent.clearLastTypeFieldIdent = id => { if (id === tab) clearedTypeField = true; };+ assert.equal(binding.send({ kind: 'resize', source: 'agent', target: 'viewport',+ viewport: { width: 900, height: 600, scale: 1 } }).accepted, true);+ const run = agent._pageFeedbackRuns.get(tab);+ assert.equal(run.events.size, 0, 'Agent geometry changes do not become page feedback');+ assert.equal(run.lastActivityAt, 0, 'Agent geometry changes do not add an idle delay');+ assert.equal(agent.screenshotCaptures.has(tab), false);+ assert.equal(agent._jevSessions.get(tab).disabled, true);+ assert.deepEqual(agent._jevSessions.get(tab).queue, []);+ assert.equal(clearedTypeField, true);+ } finally { agent._releaseRunEntry(tab); }+ });++ test(`${build}: resize_window brackets browser updates with the active monitor marker`, async () => {+ const agent = setup(Agent), tab = nextTab++;+ const previousGet = api.tabs.get, previousSend = api.tabs.sendMessage, previousWindows = api.windows;+ const messages = [];+ api.tabs.get = async id => ({ id, url: 'https://example.com/new', windowId: 23 });+ api.tabs.sendMessage = async (_id, message) => { messages.push(message); return { ready: true }; };+ api.windows = { get: async () => ({ state: 'normal' }), update: async () => ({}) };+ agent._getWindowInfo = async () => ({ success: true, viewport: { width: 900, height: 600 }, note: 'test' });+ await agent._claimRunEntry(tab, 'interactive');+ try {+ await agent._beginPageFeedbackRun(tab, 'interactive');+ const result = await agent._resizeWindow(tab, { width: 900, height: 600 });+ assert.equal(result.resized, true);+ const markers = messages.filter(message => ['page_monitor_resize_begin', 'page_monitor_resize_finish'].includes(message.action));+ assert.deepEqual(markers.map(message => message.action), ['page_monitor_resize_begin', 'page_monitor_resize_finish']);+ assert.equal(markers[0].params.runToken, markers[1].params.runToken);+ assert.equal(markers[0].params.operationId, markers[1].params.operationId);+ assert.deepEqual(markers[1].params.expectedViewport, { width: 900, height: 600 });+ } finally {+ if (agent._pageFeedbackRuns.has(tab)) agent._finishPageFeedbackRun(tab);+ agent._releaseRunEntry(tab);+ api.tabs.get = previousGet;+ api.tabs.sendMessage = previousSend;+ if (previousWindows === undefined) delete api.windows; else api.windows = previousWindows;+ }+ });+test(`${build}: navigation arriving during a page read gets its own feedback ID`, async () => {const agent = setup(Agent), tab = nextTab++, entered = deferred(), release = deferred();const updates = [], messages = [];diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 222f89a9e..61af6516f 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -1804,6 +1804,47 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: agent resize attribution waits for a matching viewport and preserves concurrent input`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ const agentResize = { operationId: 'agent-resize', runToken: 'test-run' };+ await page.evaluate(marker => {+ feedback = [];+ deliver('page_monitor_resize_begin', marker);+ deliver('page_monitor_prepare', { operationId: 'resize-click', tool: 'click', selector: '#agent' });+ }, agentResize);+ assert.equal(await page.evaluate(() => lastMonitorResponse.ready), true);+ await page.setViewportSize({ width: 1000, height: 700 });+ await page.waitForTimeout(50);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Resize feedback stays buffered until its result is verified');+ await page.evaluate(marker => deliver('page_monitor_resize_finish', {+ ...marker, expectedViewport: { width: 1000, height: 700 },+ }), agentResize);+ await page.waitForFunction(() => feedback.some(event => event.kind === 'resize'), null, { timeout: 1000 });+ const attributed = await page.evaluate(() => feedback.find(event => event.kind === 'resize'));+ assert.equal(attributed.source, 'agent');+ assert.equal(attributed.viewport.width, 1000);+ const rejected = await page.evaluate(async () => {+ deliver('page_monitor_dispatch', { operationId: 'resize-click', kind: 'click', selector: '#agent' });+ await new Promise(resolve => setTimeout(resolve, 0)); return lastMonitorResponse;+ });+ assert.equal(rejected.pageFeedbackPending, true, 'The agent resize still invalidates prepared coordinates');++ const overlap = { operationId: 'agent-resize-overlap', runToken: 'test-run' };+ await page.evaluate(marker => { feedback = []; deliver('page_monitor_resize_begin', marker); }, overlap);+ await page.setViewportSize({ width: 1100, height: 750 });+ await page.waitForTimeout(50);+ await page.locator('#human').click();+ await page.evaluate(marker => deliver('page_monitor_resize_finish', {+ ...marker, expectedViewport: { width: 1100, height: 750 },+ }), overlap);+ await page.waitForFunction(() => feedback.some(event => event.kind === 'resize')+ && feedback.some(event => event.kind === 'click' && event.source === 'user'), null, { timeout: 1000 });+ const overlappingResize = await page.evaluate(() => feedback.find(event => event.kind === 'resize'));+ assert.equal(overlappingResize.source, 'unknown', 'Concurrent trusted input prevents agent attribution');+ } finally { await browser.close(); }+ });+test(`${build}: contenteditable fallback attributes its cancellable gate and native edits`, async () => {const { browser, page } = await fixture(engine, build);try {