Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Attribute agent-driven window resizes

Barack Sokullu7d1b3446d84a
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 51825b24f..0bac6d0e3 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -33363,7 +33363,10 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
if (args.left != null && Number.isFinite(Number(args.left))) update.left = Math.round(Number(args.left));
if (args.top != null && Number.isFinite(Number(args.top))) update.top = Math.round(Number(args.top));
+ let resizeMarker = null;
+ let expectedViewport = null;
try {
+ resizeMarker = await this._beginPageAgentResize(tabId);
const win = await chrome.windows.get(tab.windowId);
if (win?.state && win.state !== 'normal') {
await chrome.windows.update(tab.windowId, { state: 'normal' });
@@ -33372,6 +33375,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
await chrome.windows.update(tab.windowId, update);
await new Promise(r => setTimeout(r, 250));
const info = await this._getWindowInfo(tabId);
+ expectedViewport = info?.viewport || null;
return {
...info,
resized: true,
@@ -33380,6 +33384,8 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
};
} catch (e) {
return { success: false, error: `resize_window failed: ${e.message}` };
+ } finally {
+ await this._finishPageAgentResize(tabId, resizeMarker, expectedViewport);
}
}
diff --git a/src/chrome/src/agent/page-feedback.js b/src/chrome/src/agent/page-feedback.js
index 901c7156d..4a269972b 100644
--- a/src/chrome/src/agent/page-feedback.js
+++ b/src/chrome/src/agent/page-feedback.js
@@ -145,6 +145,25 @@ export const pageFeedbackMethods = {
});
},
+ async _beginPageAgentResize(tabId) {
+ const run = this._pageFeedbackRuns?.get(tabId);
+ if (!run) return null;
+ const marker = { runToken: run.token, operationId: token() };
+ try {
+ const response = await apiFor().tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_resize_begin',
+ params: marker }, { frameId: 0 });
+ return response?.ready === true && this._pageFeedbackRuns?.get(tabId) === run ? marker : null;
+ } catch { return null; }
+ },
+
+ async _finishPageAgentResize(tabId, marker, expectedViewport) {
+ if (!marker) return;
+ try {
+ await apiFor().tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_resize_finish',
+ params: { ...marker, expectedViewport: expectedViewport || null } }, { frameId: 0 });
+ } catch { /* A protected or navigated document cannot report resize attribution. */ }
+ },
+
_finishPageFeedbackRun(tabId) {
const run = this._pageFeedbackRuns?.get(tabId);
if (!run) return;
@@ -192,6 +211,12 @@ export const pageFeedbackMethods = {
|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };
frame.seq = seq;
if (feedback.source === 'agent') {
+ if (feedback.kind === 'resize') {
+ const session = this._jevSessions?.get(tabId);
+ if (session) { session.disabled = true; session.queue = []; session.snapshot = null; }
+ this.screenshotCaptures?.delete(tabId);
+ this.clearLastTypeFieldIdent?.(tabId);
+ }
if (['click', 'submit'].includes(feedback.operation)) {
let url = '';
try {
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index 5baa82d28..10f07b87c 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -30,6 +30,7 @@
let preparedOperationCursor = 0;
let requestGeneration = 0, unreported = 0, pendingDOM = null;
let lastViewport = '';
+ let agentResize = null;
let lastFeedbackDelivery = Promise.resolve();
let agentTurn = null, userTurn = null, lastUserTarget = null;
let matchedEvents = new WeakMap();
@@ -123,6 +124,35 @@
return delivery;
} catch { stop(); }
};
+ const currentViewport = () => ({ width: window.innerWidth, height: window.innerHeight,
+ visualWidth: window.visualViewport?.width ?? window.innerWidth, visualHeight: window.visualViewport?.height ?? window.innerHeight,
+ scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });
+ function flushAgentResize(source, expected = agentResize) {
+ if (!expected || agentResize !== expected) return Promise.resolve();
+ clearTimeout(expected.timer); agentResize = null;
+ return Promise.allSettled(expected.events.map(viewport => send({ kind: 'resize', source, target: 'viewport', viewport })));
+ }
+ function beginAgentResize(params) {
+ if (!active || params.runToken !== runToken || typeof params.operationId !== 'string' || !params.operationId) return false;
+ if (agentResize) void flushAgentResize('unknown');
+ const marker = { runToken, operationId: params.operationId, userAt: lastUserAt, events: [], timer: null };
+ marker.timer = setTimeout(() => { void flushAgentResize('unknown', marker); }, 5000);
+ agentResize = marker;
+ return true;
+ }
+ async function finishAgentResize(params) {
+ const marker = agentResize;
+ if (!marker || marker.runToken !== runToken || params.runToken !== runToken || marker.operationId !== params.operationId) {
+ return { ready: false };
+ }
+ const finalViewport = marker.events.at(-1);
+ const expected = params.expectedViewport;
+ const attributed = marker.userAt === lastUserAt && finalViewport
+ && Number.isFinite(expected?.width) && Number.isFinite(expected?.height)
+ && finalViewport.width === expected.width && finalViewport.height === expected.height;
+ await flushAgentResize(attributed ? 'agent' : 'unknown', marker);
+ return { ready: true, attributed: !!attributed };
+ }
const interact = (kind, el, { completeGesture = false, ...extra } = {}) => {
const isIgnored = ignored(el);
if (!active || (isIgnored && !completeGesture)) return;
@@ -1001,14 +1031,16 @@
const el = elementFor(event);
if (event.isTrusted && !expected('scroll', el, event)) interact('activity', el);
});
- const viewport = () => ({ width: innerWidth, height: innerHeight,
- visualWidth: window.visualViewport?.width ?? innerWidth, visualHeight: window.visualViewport?.height ?? innerHeight,
- scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });
- lastViewport = JSON.stringify(viewport());
+ lastViewport = JSON.stringify(currentViewport());
const onResize = () => {
- const current = viewport(), key = JSON.stringify(current);
+ const current = currentViewport(), key = JSON.stringify(current);
if (!active || key === lastViewport) return;
- lastViewport = key; revision++;
+ lastViewport = key; revision++; publishRevision();
+ if (agentResize?.runToken === runToken && agentResize.events.length < 16) {
+ agentResize.events.push(current);
+ return;
+ }
+ if (agentResize) void flushAgentResize('unknown');
send({ kind: 'resize', source: window === window.top ? 'unknown' : 'page', target: 'viewport', viewport: current });
};
listen(window, 'resize', onResize);
@@ -1052,6 +1084,7 @@
}
function stop() {
active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;
+ clearTimeout(agentResize?.timer); agentResize = null;
lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlIterator = null; preparedOperationCursor = 0;
document.documentElement?.removeAttribute(fenceAttribute);
for (const el of nativeTargets) {
@@ -1232,6 +1265,10 @@
if (msg.active) { void requestState().then(() => respond({ ready: true })); return true; }
else if (!msg.runToken || msg.runToken === runToken) { requestGeneration++; stop(); respond({ ready: true }); }
} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }
+ else if (msg.action === 'page_monitor_resize_begin') { respond({ ready: beginAgentResize(msg.params || {}) }); }
+ else if (msg.action === 'page_monitor_resize_finish') {
+ void finishAgentResize(msg.params || {}).then(respond, () => respond({ ready: false })); return true;
+ }
else if (msg.action === 'page_monitor_validate') { respond({ ready: validateNativeDispatch(msg.params || {}) }); }
else if (msg.action === 'page_monitor_dispatch') {
const params = msg.params || {};
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 07b36c11b..3e150481f 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -30950,7 +30950,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (args.left != null && Number.isFinite(Number(args.left))) update.left = Math.round(Number(args.left));
if (args.top != null && Number.isFinite(Number(args.top))) update.top = Math.round(Number(args.top));
+ let resizeMarker = null;
+ let expectedViewport = null;
try {
+ resizeMarker = await this._beginPageAgentResize(tabId);
const win = await browser.windows.get(tab.windowId);
if (win?.state && win.state !== 'normal') {
await browser.windows.update(tab.windowId, { state: 'normal' });
@@ -30959,6 +30962,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
await browser.windows.update(tab.windowId, update);
await new Promise(r => setTimeout(r, 250));
const info = await this._getWindowInfo(tabId);
+ expectedViewport = info?.viewport || null;
return {
...info,
resized: true,
@@ -30967,6 +30971,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
};
} catch (e) {
return { success: false, error: `resize_window failed: ${e.message}` };
+ } finally {
+ await this._finishPageAgentResize(tabId, resizeMarker, expectedViewport);
}
}
diff --git a/src/firefox/src/agent/page-feedback.js b/src/firefox/src/agent/page-feedback.js
index 901c7156d..4a269972b 100644
--- a/src/firefox/src/agent/page-feedback.js
+++ b/src/firefox/src/agent/page-feedback.js
@@ -145,6 +145,25 @@ export const pageFeedbackMethods = {
});
},
+ async _beginPageAgentResize(tabId) {
+ const run = this._pageFeedbackRuns?.get(tabId);
+ if (!run) return null;
+ const marker = { runToken: run.token, operationId: token() };
+ try {
+ const response = await apiFor().tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_resize_begin',
+ params: marker }, { frameId: 0 });
+ return response?.ready === true && this._pageFeedbackRuns?.get(tabId) === run ? marker : null;
+ } catch { return null; }
+ },
+
+ async _finishPageAgentResize(tabId, marker, expectedViewport) {
+ if (!marker) return;
+ try {
+ await apiFor().tabs.sendMessage(tabId, { target: 'content', action: 'page_monitor_resize_finish',
+ params: { ...marker, expectedViewport: expectedViewport || null } }, { frameId: 0 });
+ } catch { /* A protected or navigated document cannot report resize attribution. */ }
+ },
+
_finishPageFeedbackRun(tabId) {
const run = this._pageFeedbackRuns?.get(tabId);
if (!run) return;
@@ -192,6 +211,12 @@ export const pageFeedbackMethods = {
|| !['user', 'agent', 'page', 'unknown'].includes(feedback.source)) return { accepted: false, reason: 'invalid-observation' };
frame.seq = seq;
if (feedback.source === 'agent') {
+ if (feedback.kind === 'resize') {
+ const session = this._jevSessions?.get(tabId);
+ if (session) { session.disabled = true; session.queue = []; session.snapshot = null; }
+ this.screenshotCaptures?.delete(tabId);
+ this.clearLastTypeFieldIdent?.(tabId);
+ }
if (['click', 'submit'].includes(feedback.operation)) {
let url = '';
try {
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index 5baa82d28..10f07b87c 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -30,6 +30,7 @@
let preparedOperationCursor = 0;
let requestGeneration = 0, unreported = 0, pendingDOM = null;
let lastViewport = '';
+ let agentResize = null;
let lastFeedbackDelivery = Promise.resolve();
let agentTurn = null, userTurn = null, lastUserTarget = null;
let matchedEvents = new WeakMap();
@@ -123,6 +124,35 @@
return delivery;
} catch { stop(); }
};
+ const currentViewport = () => ({ width: window.innerWidth, height: window.innerHeight,
+ visualWidth: window.visualViewport?.width ?? window.innerWidth, visualHeight: window.visualViewport?.height ?? window.innerHeight,
+ scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });
+ function flushAgentResize(source, expected = agentResize) {
+ if (!expected || agentResize !== expected) return Promise.resolve();
+ clearTimeout(expected.timer); agentResize = null;
+ return Promise.allSettled(expected.events.map(viewport => send({ kind: 'resize', source, target: 'viewport', viewport })));
+ }
+ function beginAgentResize(params) {
+ if (!active || params.runToken !== runToken || typeof params.operationId !== 'string' || !params.operationId) return false;
+ if (agentResize) void flushAgentResize('unknown');
+ const marker = { runToken, operationId: params.operationId, userAt: lastUserAt, events: [], timer: null };
+ marker.timer = setTimeout(() => { void flushAgentResize('unknown', marker); }, 5000);
+ agentResize = marker;
+ return true;
+ }
+ async function finishAgentResize(params) {
+ const marker = agentResize;
+ if (!marker || marker.runToken !== runToken || params.runToken !== runToken || marker.operationId !== params.operationId) {
+ return { ready: false };
+ }
+ const finalViewport = marker.events.at(-1);
+ const expected = params.expectedViewport;
+ const attributed = marker.userAt === lastUserAt && finalViewport
+ && Number.isFinite(expected?.width) && Number.isFinite(expected?.height)
+ && finalViewport.width === expected.width && finalViewport.height === expected.height;
+ await flushAgentResize(attributed ? 'agent' : 'unknown', marker);
+ return { ready: true, attributed: !!attributed };
+ }
const interact = (kind, el, { completeGesture = false, ...extra } = {}) => {
const isIgnored = ignored(el);
if (!active || (isIgnored && !completeGesture)) return;
@@ -1001,14 +1031,16 @@
const el = elementFor(event);
if (event.isTrusted && !expected('scroll', el, event)) interact('activity', el);
});
- const viewport = () => ({ width: innerWidth, height: innerHeight,
- visualWidth: window.visualViewport?.width ?? innerWidth, visualHeight: window.visualViewport?.height ?? innerHeight,
- scale: window.visualViewport?.scale ?? 1, offsetX: window.visualViewport?.offsetLeft ?? 0, offsetY: window.visualViewport?.offsetTop ?? 0 });
- lastViewport = JSON.stringify(viewport());
+ lastViewport = JSON.stringify(currentViewport());
const onResize = () => {
- const current = viewport(), key = JSON.stringify(current);
+ const current = currentViewport(), key = JSON.stringify(current);
if (!active || key === lastViewport) return;
- lastViewport = key; revision++;
+ lastViewport = key; revision++; publishRevision();
+ if (agentResize?.runToken === runToken && agentResize.events.length < 16) {
+ agentResize.events.push(current);
+ return;
+ }
+ if (agentResize) void flushAgentResize('unknown');
send({ kind: 'resize', source: window === window.top ? 'unknown' : 'page', target: 'viewport', viewport: current });
};
listen(window, 'resize', onResize);
@@ -1052,6 +1084,7 @@
}
function stop() {
active = false; runToken = ''; activePointers.clear(); pointerHeld = false; composing = false;
+ clearTimeout(agentResize?.timer); agentResize = null;
lastUserAt = 0; lastViewport = ''; userTurn = null; lastUserTarget = null; matchedEvents = new WeakMap(); controlIterator = null; preparedOperationCursor = 0;
document.documentElement?.removeAttribute(fenceAttribute);
for (const el of nativeTargets) {
@@ -1232,6 +1265,10 @@
if (msg.active) { void requestState().then(() => respond({ ready: true })); return true; }
else if (!msg.runToken || msg.runToken === runToken) { requestGeneration++; stop(); respond({ ready: true }); }
} else if (msg.action === 'page_monitor_prepare') { prepare(msg.params || {}); respond({ ready: true }); }
+ else if (msg.action === 'page_monitor_resize_begin') { respond({ ready: beginAgentResize(msg.params || {}) }); }
+ else if (msg.action === 'page_monitor_resize_finish') {
+ void finishAgentResize(msg.params || {}).then(respond, () => respond({ ready: false })); return true;
+ }
else if (msg.action === 'page_monitor_validate') { respond({ ready: validateNativeDispatch(msg.params || {}) }); }
else if (msg.action === 'page_monitor_dispatch') {
const params = msg.params || {};
diff --git a/test/page-feedback.mjs b/test/page-feedback.mjs
index b57efda25..374ec9b50 100644
--- a/test/page-feedback.mjs
+++ b/test/page-feedback.mjs
@@ -365,6 +365,54 @@ for (const build of ['chrome', 'firefox']) {
} finally { agent._releaseRunEntry(tab); }
});
+ test(`${build}: agent resize feedback invalidates caches without queueing or idle delay`, async () => {
+ const agent = setup(Agent), tab = nextTab++;
+ await agent._claimRunEntry(tab, 'interactive');
+ try {
+ const binding = bind(agent, tab);
+ agent._jevSessions = new Map([[tab, { disabled: false, queue: ['prepared'], snapshot: {} }]]);
+ agent.screenshotCaptures.set(tab, { captureId: 'stale-resize' });
+ let clearedTypeField = false;
+ agent.clearLastTypeFieldIdent = id => { if (id === tab) clearedTypeField = true; };
+ assert.equal(binding.send({ kind: 'resize', source: 'agent', target: 'viewport',
+ viewport: { width: 900, height: 600, scale: 1 } }).accepted, true);
+ const run = agent._pageFeedbackRuns.get(tab);
+ assert.equal(run.events.size, 0, 'Agent geometry changes do not become page feedback');
+ assert.equal(run.lastActivityAt, 0, 'Agent geometry changes do not add an idle delay');
+ assert.equal(agent.screenshotCaptures.has(tab), false);
+ assert.equal(agent._jevSessions.get(tab).disabled, true);
+ assert.deepEqual(agent._jevSessions.get(tab).queue, []);
+ assert.equal(clearedTypeField, true);
+ } finally { agent._releaseRunEntry(tab); }
+ });
+
+ test(`${build}: resize_window brackets browser updates with the active monitor marker`, async () => {
+ const agent = setup(Agent), tab = nextTab++;
+ const previousGet = api.tabs.get, previousSend = api.tabs.sendMessage, previousWindows = api.windows;
+ const messages = [];
+ api.tabs.get = async id => ({ id, url: 'https://example.com/new', windowId: 23 });
+ api.tabs.sendMessage = async (_id, message) => { messages.push(message); return { ready: true }; };
+ api.windows = { get: async () => ({ state: 'normal' }), update: async () => ({}) };
+ agent._getWindowInfo = async () => ({ success: true, viewport: { width: 900, height: 600 }, note: 'test' });
+ await agent._claimRunEntry(tab, 'interactive');
+ try {
+ await agent._beginPageFeedbackRun(tab, 'interactive');
+ const result = await agent._resizeWindow(tab, { width: 900, height: 600 });
+ assert.equal(result.resized, true);
+ const markers = messages.filter(message => ['page_monitor_resize_begin', 'page_monitor_resize_finish'].includes(message.action));
+ assert.deepEqual(markers.map(message => message.action), ['page_monitor_resize_begin', 'page_monitor_resize_finish']);
+ assert.equal(markers[0].params.runToken, markers[1].params.runToken);
+ assert.equal(markers[0].params.operationId, markers[1].params.operationId);
+ assert.deepEqual(markers[1].params.expectedViewport, { width: 900, height: 600 });
+ } finally {
+ if (agent._pageFeedbackRuns.has(tab)) agent._finishPageFeedbackRun(tab);
+ agent._releaseRunEntry(tab);
+ api.tabs.get = previousGet;
+ api.tabs.sendMessage = previousSend;
+ if (previousWindows === undefined) delete api.windows; else api.windows = previousWindows;
+ }
+ });
+
test(`${build}: navigation arriving during a page read gets its own feedback ID`, async () => {
const agent = setup(Agent), tab = nextTab++, entered = deferred(), release = deferred();
const updates = [], messages = [];
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 222f89a9e..61af6516f 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -1804,6 +1804,47 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: agent resize attribution waits for a matching viewport and preserves concurrent input`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ const agentResize = { operationId: 'agent-resize', runToken: 'test-run' };
+ await page.evaluate(marker => {
+ feedback = [];
+ deliver('page_monitor_resize_begin', marker);
+ deliver('page_monitor_prepare', { operationId: 'resize-click', tool: 'click', selector: '#agent' });
+ }, agentResize);
+ assert.equal(await page.evaluate(() => lastMonitorResponse.ready), true);
+ await page.setViewportSize({ width: 1000, height: 700 });
+ await page.waitForTimeout(50);
+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Resize feedback stays buffered until its result is verified');
+ await page.evaluate(marker => deliver('page_monitor_resize_finish', {
+ ...marker, expectedViewport: { width: 1000, height: 700 },
+ }), agentResize);
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'resize'), null, { timeout: 1000 });
+ const attributed = await page.evaluate(() => feedback.find(event => event.kind === 'resize'));
+ assert.equal(attributed.source, 'agent');
+ assert.equal(attributed.viewport.width, 1000);
+ const rejected = await page.evaluate(async () => {
+ deliver('page_monitor_dispatch', { operationId: 'resize-click', kind: 'click', selector: '#agent' });
+ await new Promise(resolve => setTimeout(resolve, 0)); return lastMonitorResponse;
+ });
+ assert.equal(rejected.pageFeedbackPending, true, 'The agent resize still invalidates prepared coordinates');
+
+ const overlap = { operationId: 'agent-resize-overlap', runToken: 'test-run' };
+ await page.evaluate(marker => { feedback = []; deliver('page_monitor_resize_begin', marker); }, overlap);
+ await page.setViewportSize({ width: 1100, height: 750 });
+ await page.waitForTimeout(50);
+ await page.locator('#human').click();
+ await page.evaluate(marker => deliver('page_monitor_resize_finish', {
+ ...marker, expectedViewport: { width: 1100, height: 750 },
+ }), overlap);
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'resize')
+ && feedback.some(event => event.kind === 'click' && event.source === 'user'), null, { timeout: 1000 });
+ const overlappingResize = await page.evaluate(() => feedback.find(event => event.kind === 'resize'));
+ assert.equal(overlappingResize.source, 'unknown', 'Concurrent trusted input prevents agent attribution');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: contenteditable fallback attributes its cancellable gate and native edits`, async () => {
const { browser, page } = await fixture(engine, build);
try {