Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Close page feedback review gaps

Barack Sokulluba017515dcc8
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjs
index 6e73ebfc4..ad1afe80c 100644
--- a/firefox-companion/session.mjs
+++ b/firefox-companion/session.mjs
@@ -338,6 +338,14 @@ export class BidiSession {
};
const point = action === 'click' && payload.point != null ? payload.point : null;
if (point && (!Number.isInteger(point.x) || !Number.isInteger(point.y))) throw new Error('Invalid click coordinates');
+ const focusGuard = payload.pageFeedbackGuard;
+ if (['type', 'field', 'key'].includes(action) && focusGuard
+ && (typeof validatePageDispatch !== 'function'
+ || !await validatePageDispatch(id, focusGuard, 'input', false))) {
+ const error = new Error('Page changed during native preparation; no focus or input sent');
+ error.code = 'page_feedback_pending'; throw error;
+ }
+ assertLive();
const check = await this.call(match, `(el, token, action, x, y) => {
if (!el.isConnected || el.getAttribute('data-webbrain-bidi') !== token || el.disabled) return false;
el.removeAttribute('data-webbrain-bidi');
diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.js
index fd2868ba7..5baa82d28 100644
--- a/src/chrome/src/content/page-monitor.js
+++ b/src/chrome/src/content/page-monitor.js
@@ -710,6 +710,7 @@
}
const editableTextMutation = editable(el) && (record.type === 'characterData'
|| (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));
+ const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);
if (ignored(el)) continue;
if (record.type === 'attributes' && record.attributeName === 'data-selected'
&& record.oldValue !== el.getAttribute('data-selected')) {
@@ -720,6 +721,10 @@
// DOM writes are attributed to its dispatch. Keep those paths deduplicated
// while observing independent page-script changes to editable text.
if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;
+ if (accessibleNameContentMutation && el.tagName === 'OPTION' && el.selected) {
+ const select = el.closest('select');
+ if (select && !ignored(select)) noteChange(select, record.agentUserAt);
+ }
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
&& deferAnimationCheck(record, el)) continue;
if (record.type === 'layout' && deferAnimationCheck(record, el)) continue;
@@ -739,7 +744,6 @@
if (nextControl !== previousControl) noteChange(el, record.agentUserAt);
continue;
}
- const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);
const nativeLabelAssociationMutation = record.type === 'attributes'
&& record.attributeName === 'for' && el.tagName === 'LABEL';
const hiddenAriaLabelIdMutation = record.type === 'attributes' && record.attributeName === 'id';
@@ -987,9 +991,11 @@
listen(document, 'keydown', event => {
const el = elementFor(event);
if (!event.isTrusted || expected('input', el, event)) return;
- // Key names identify scrolling and focus movement locally, never transmitted.
- if (editable(el)) interact('activity', el);
- else if (['Tab', 'Escape', 'ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' '].includes(event.key)) interact('activity', el);
+ // Modifier-only presses do not activate page shortcuts. Other trusted
+ // keydowns can change page state even when no DOM mutation follows; key
+ // values are used only for this local filter and never enter feedback.
+ if (!['Shift', 'Control', 'Alt', 'Meta', 'AltGraph', 'CapsLock', 'NumLock', 'ScrollLock', 'Fn', 'FnLock', 'Hyper', 'Super']
+ .includes(event.key)) interact('activity', el);
});
for (const name of ['wheel', 'touchmove']) listen(document, name, event => {
const el = elementFor(event);
diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.js
index fd2868ba7..5baa82d28 100644
--- a/src/firefox/src/content/page-monitor.js
+++ b/src/firefox/src/content/page-monitor.js
@@ -710,6 +710,7 @@
}
const editableTextMutation = editable(el) && (record.type === 'characterData'
|| (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));
+ const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);
if (ignored(el)) continue;
if (record.type === 'attributes' && record.attributeName === 'data-selected'
&& record.oldValue !== el.getAttribute('data-selected')) {
@@ -720,6 +721,10 @@
// DOM writes are attributed to its dispatch. Keep those paths deduplicated
// while observing independent page-script changes to editable text.
if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;
+ if (accessibleNameContentMutation && el.tagName === 'OPTION' && el.selected) {
+ const select = el.closest('select');
+ if (select && !ignored(select)) noteChange(select, record.agentUserAt);
+ }
if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)
&& deferAnimationCheck(record, el)) continue;
if (record.type === 'layout' && deferAnimationCheck(record, el)) continue;
@@ -739,7 +744,6 @@
if (nextControl !== previousControl) noteChange(el, record.agentUserAt);
continue;
}
- const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);
const nativeLabelAssociationMutation = record.type === 'attributes'
&& record.attributeName === 'for' && el.tagName === 'LABEL';
const hiddenAriaLabelIdMutation = record.type === 'attributes' && record.attributeName === 'id';
@@ -987,9 +991,11 @@
listen(document, 'keydown', event => {
const el = elementFor(event);
if (!event.isTrusted || expected('input', el, event)) return;
- // Key names identify scrolling and focus movement locally, never transmitted.
- if (editable(el)) interact('activity', el);
- else if (['Tab', 'Escape', 'ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' '].includes(event.key)) interact('activity', el);
+ // Modifier-only presses do not activate page shortcuts. Other trusted
+ // keydowns can change page state even when no DOM mutation follows; key
+ // values are used only for this local filter and never enter feedback.
+ if (!['Shift', 'Control', 'Alt', 'Meta', 'AltGraph', 'CapsLock', 'NumLock', 'ScrollLock', 'Fn', 'FnLock', 'Hyper', 'Super']
+ .includes(event.key)) interact('activity', el);
});
for (const name of ['wheel', 'touchmove']) listen(document, name, event => {
const el = elementFor(event);
diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjs
index 63250bf33..bfbddbd89 100644
--- a/test/firefox-bidi.mjs
+++ b/test/firefox-bidi.mjs
@@ -206,7 +206,7 @@ test('BiDi navigation revalidates the page feedback guard before native dispatch
async (_id, _guard, kind) => kind === 'navigate');
assert.equal(sent.filter(call => call.method === 'browsingContext.navigate').length, 1);
});
-for (const failAt of [1, 2, 3]) {
+for (const failAt of [1, 2, 3, 4]) {
test(`page feedback change before native input ${failAt} preserves dispatch evidence`, async () => {
const session = new BidiSession(), runId = id();
session.runs.set(runId, { context: 'a' });
@@ -217,15 +217,34 @@ for (const failAt of [1, 2, 3]) {
await assert.rejects(session.perform(runId, 'type', { text: 'ab', clear: false,
pageFeedbackGuard: { documentToken: 'doc', revision: 7, operationId: 'op' } }, async () => ++checks < failAt), error => {
assert.equal(error.code, 'page_feedback_pending');
- assert.equal(error.dispatchState.noDispatch, failAt < 3);
- assert.equal(error.dispatchState.dispatched, failAt === 3);
- assert.equal(error.dispatchState.outcomeUnknown, failAt === 3);
+ assert.equal(error.dispatchState.noDispatch, failAt < 4);
+ assert.equal(error.dispatchState.dispatched, failAt === 4);
+ assert.equal(error.dispatchState.outcomeUnknown, failAt === 4);
return true;
});
- assert.equal(inputs, Math.max(0, failAt - 2), 'No subsequent character may be sent after the intervention');
+ assert.equal(inputs, failAt === 4 ? 1 : 0, 'No subsequent character may be sent after the intervention');
});
}
+test('feedback during BiDi target lookup blocks focus before the focus-capable target check', async () => {
+ const session = new BidiSession(), runId = id();
+ session.runs.set(runId, { context: 'a' });
+ let intervened = false, targetChecks = 0;
+ session.locate = async () => {
+ intervened = true;
+ return { context: 'a', node: { sharedId: 'el' } };
+ };
+ session.call = async () => { targetChecks++; return { result: { value: true } }; };
+ await assert.rejects(session.perform(runId, 'type', { text: 'stale', clear: false,
+ pageFeedbackGuard: { documentToken: 'doc', revision: 7, operationId: 'op' } },
+ async () => !intervened), error => {
+ assert.equal(error.code, 'page_feedback_pending');
+ assert.equal(error.dispatchState.noDispatch, true);
+ return true;
+ });
+ assert.equal(targetChecks, 0, 'A stale page must be rejected before the target check calls focus()');
+});
+
test('page feedback after native marker installation clears attribution and blocks input', async () => {
const session = new BidiSession(), runId = id(), sent = [];
session.runs.set(runId, { context: 'a' });
@@ -261,7 +280,7 @@ for (const platform of ['Win32', 'MacIntel']) {
: declaration.includes('el.innerText : el.value') ? '' : true } });
session.send = async (method, params) => { if (method === 'input.performActions') sent.push({ marks: checks, actions: params.actions[0].actions }); return {}; };
await session.perform(runId, 'field', { text: 'x', pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => { checks++; return true; });
- assert.deepEqual(sent.map(item => item.marks), [2, 4, 6, 8]);
+ assert.deepEqual(sent.map(item => item.marks), [3, 5, 7, 9]);
assert.deepEqual(sent[0].actions, [{ type: 'keyDown', value: platform === 'MacIntel' ? '\uE03D' : '\uE009' }]);
assert.equal(sent[1].actions[0].value, 'a');
assert.equal(sent[2].actions[0].value, '\uE003');
@@ -298,6 +317,7 @@ test('repeated native Tab validates and rebinds to the focused target before eac
});
assert.equal(result.success, true);
assert.deepEqual(validations, [
+ { kind: 'input', rebindFocus: false },
{ kind: 'input', rebindFocus: false },
{ kind: 'input', rebindFocus: false },
{ kind: 'input', rebindFocus: true },
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 48ad98b7b..78a4ca257 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -779,6 +779,35 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: selected native option label edits invalidate the owning select`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const select = document.createElement('select'); select.id = 'native-option-label-select';
+ select.innerHTML = '<option value="stable-choice" selected>Initial choice</option><option value="other">Other choice</option>';
+ document.body.append(select);
+ });
+ await page.waitForTimeout(300);
+ const guard = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'native-option-label', tool: 'click', selector: '#native-option-label-select' });
+ deliver('page_monitor_dispatch', { operationId: 'native-option-label', kind: 'click', selector: '#native-option-label-select', fenceOnly: true });
+ document.querySelector('#native-option-label-select option:checked').textContent = 'Updated choice';
+ return lastMonitorResponse.guard;
+ });
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'
+ && event.target === 'select#native-option-label-select'), null, { timeout: 1000 });
+ assert.equal(await page.locator('#native-option-label-select').evaluate(select => select.value), 'stable-choice',
+ 'The native value remains stable while the model-visible selected label changes');
+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('Updated choice'), false,
+ 'Selected option labels must not be copied into feedback');
+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {
+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), cancelable: true,
+ })), guard);
+ assert.equal(accepted, false, 'Editing the selected option label must stale the prepared select action');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: fenced iframe clicks reject transport-gap input and attribute the dispatched click`, async () => {
const { browser, page } = await fixture(engine, build);
try {
@@ -2099,6 +2128,39 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
} finally { await browser.close(); }
});
+ test(`${build}: non-editable site shortcut keys invalidate actions without exposing keys`, async () => {
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const surface = document.createElement('div'); surface.id = 'shortcut-surface'; surface.tabIndex = 0;
+ document.body.prepend(surface);
+ });
+ await page.waitForTimeout(250);
+ await page.locator('#shortcut-surface').focus();
+ await page.waitForTimeout(150);
+ const dispatch = await page.evaluate(() => {
+ feedback = [];
+ deliver('page_monitor_prepare', { operationId: 'site-shortcut', tool: 'click', selector: '#agent' });
+ deliver('page_monitor_dispatch', { operationId: 'site-shortcut', kind: 'click', selector: '#agent', fenceOnly: true });
+ return lastMonitorResponse;
+ });
+ assert.equal(dispatch.ready, true, JSON.stringify(dispatch));
+ await page.locator('#shortcut-surface').press('Shift');
+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Modifier-only presses do not produce shortcut activity');
+ await page.locator('#shortcut-surface').press('j');
+ await page.waitForFunction(() => feedback.some(event => event.kind === 'activity' && event.source === 'user'), null, { timeout: 1000 });
+ const events = await page.evaluate(() => feedback);
+ assert.equal(events.some(event => event.kind === 'activity' && event.source === 'user'), true, JSON.stringify(events));
+ assert.equal(events.some(event => Object.hasOwn(event, 'key') || Object.hasOwn(event, 'value')), false,
+ 'Keyboard contents must never be included in page feedback');
+ const blocked = await page.evaluate(() => {
+ try { __wbPageMonitor.activatePreparedDispatch({ operationId: 'site-shortcut', kind: 'click' }); return null; }
+ catch (error) { return error.code; }
+ });
+ assert.equal(blocked, 'page_feedback_pending', 'A site shortcut must invalidate the prepared action');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: aria-modal changes invalidate prepared page context`, async () => {
const { browser, page } = await fixture(engine, build);
try {