Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Close page feedback review gaps
Barack Sokulluba017515dcc8
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex 6e73ebfc4..ad1afe80c 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -338,6 +338,14 @@ export class BidiSession {};const point = action === 'click' && payload.point != null ? payload.point : null;if (point && (!Number.isInteger(point.x) || !Number.isInteger(point.y))) throw new Error('Invalid click coordinates');+ const focusGuard = payload.pageFeedbackGuard;+ if (['type', 'field', 'key'].includes(action) && focusGuard+ && (typeof validatePageDispatch !== 'function'+ || !await validatePageDispatch(id, focusGuard, 'input', false))) {+ const error = new Error('Page changed during native preparation; no focus or input sent');+ error.code = 'page_feedback_pending'; throw error;+ }+ assertLive();const check = await this.call(match, `(el, token, action, x, y) => {if (!el.isConnected || el.getAttribute('data-webbrain-bidi') !== token || el.disabled) return false;el.removeAttribute('data-webbrain-bidi');diff --git a/src/chrome/src/content/page-monitor.js b/src/chrome/src/content/page-monitor.jsindex fd2868ba7..5baa82d28 100644--- a/src/chrome/src/content/page-monitor.js+++ b/src/chrome/src/content/page-monitor.js@@ -710,6 +710,7 @@}const editableTextMutation = editable(el) && (record.type === 'characterData'|| (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));+ const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);if (ignored(el)) continue;if (record.type === 'attributes' && record.attributeName === 'data-selected'&& record.oldValue !== el.getAttribute('data-selected')) {@@ -720,6 +721,10 @@// DOM writes are attributed to its dispatch. Keep those paths deduplicated// while observing independent page-script changes to editable text.if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;+ if (accessibleNameContentMutation && el.tagName === 'OPTION' && el.selected) {+ const select = el.closest('select');+ if (select && !ignored(select)) noteChange(select, record.agentUserAt);+ }if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)&& deferAnimationCheck(record, el)) continue;if (record.type === 'layout' && deferAnimationCheck(record, el)) continue;@@ -739,7 +744,6 @@if (nextControl !== previousControl) noteChange(el, record.agentUserAt);continue;}- const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);const nativeLabelAssociationMutation = record.type === 'attributes'&& record.attributeName === 'for' && el.tagName === 'LABEL';const hiddenAriaLabelIdMutation = record.type === 'attributes' && record.attributeName === 'id';@@ -987,9 +991,11 @@listen(document, 'keydown', event => {const el = elementFor(event);if (!event.isTrusted || expected('input', el, event)) return;- // Key names identify scrolling and focus movement locally, never transmitted.- if (editable(el)) interact('activity', el);- else if (['Tab', 'Escape', 'ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' '].includes(event.key)) interact('activity', el);+ // Modifier-only presses do not activate page shortcuts. Other trusted+ // keydowns can change page state even when no DOM mutation follows; key+ // values are used only for this local filter and never enter feedback.+ if (!['Shift', 'Control', 'Alt', 'Meta', 'AltGraph', 'CapsLock', 'NumLock', 'ScrollLock', 'Fn', 'FnLock', 'Hyper', 'Super']+ .includes(event.key)) interact('activity', el);});for (const name of ['wheel', 'touchmove']) listen(document, name, event => {const el = elementFor(event);diff --git a/src/firefox/src/content/page-monitor.js b/src/firefox/src/content/page-monitor.jsindex fd2868ba7..5baa82d28 100644--- a/src/firefox/src/content/page-monitor.js+++ b/src/firefox/src/content/page-monitor.js@@ -710,6 +710,7 @@}const editableTextMutation = editable(el) && (record.type === 'characterData'|| (record.type === 'childList' && [...record.addedNodes, ...record.removedNodes].some(node => node.nodeType === 3)));+ const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);if (ignored(el)) continue;if (record.type === 'attributes' && record.attributeName === 'data-selected'&& record.oldValue !== el.getAttribute('data-selected')) {@@ -720,6 +721,10 @@// DOM writes are attributed to its dispatch. Keep those paths deduplicated// while observing independent page-script changes to editable text.if (editableTextMutation && (userTurn || (agentTurn && lastUserAt <= agentTurn.userAt))) continue;+ if (accessibleNameContentMutation && el.tagName === 'OPTION' && el.selected) {+ const select = el.closest('select');+ if (select && !ignored(select)) noteChange(select, record.agentUserAt);+ }if (record.type === 'attributes' && ['class', 'style'].includes(record.attributeName)&& deferAnimationCheck(record, el)) continue;if (record.type === 'layout' && deferAnimationCheck(record, el)) continue;@@ -739,7 +744,6 @@if (nextControl !== previousControl) noteChange(el, record.agentUserAt);continue;}- const accessibleNameContentMutation = ['characterData', 'childList'].includes(record.type);const nativeLabelAssociationMutation = record.type === 'attributes'&& record.attributeName === 'for' && el.tagName === 'LABEL';const hiddenAriaLabelIdMutation = record.type === 'attributes' && record.attributeName === 'id';@@ -987,9 +991,11 @@listen(document, 'keydown', event => {const el = elementFor(event);if (!event.isTrusted || expected('input', el, event)) return;- // Key names identify scrolling and focus movement locally, never transmitted.- if (editable(el)) interact('activity', el);- else if (['Tab', 'Escape', 'ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' '].includes(event.key)) interact('activity', el);+ // Modifier-only presses do not activate page shortcuts. Other trusted+ // keydowns can change page state even when no DOM mutation follows; key+ // values are used only for this local filter and never enter feedback.+ if (!['Shift', 'Control', 'Alt', 'Meta', 'AltGraph', 'CapsLock', 'NumLock', 'ScrollLock', 'Fn', 'FnLock', 'Hyper', 'Super']+ .includes(event.key)) interact('activity', el);});for (const name of ['wheel', 'touchmove']) listen(document, name, event => {const el = elementFor(event);diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex 63250bf33..bfbddbd89 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -206,7 +206,7 @@ test('BiDi navigation revalidates the page feedback guard before native dispatchasync (_id, _guard, kind) => kind === 'navigate');assert.equal(sent.filter(call => call.method === 'browsingContext.navigate').length, 1);});-for (const failAt of [1, 2, 3]) {+for (const failAt of [1, 2, 3, 4]) {test(`page feedback change before native input ${failAt} preserves dispatch evidence`, async () => {const session = new BidiSession(), runId = id();session.runs.set(runId, { context: 'a' });@@ -217,15 +217,34 @@ for (const failAt of [1, 2, 3]) {await assert.rejects(session.perform(runId, 'type', { text: 'ab', clear: false,pageFeedbackGuard: { documentToken: 'doc', revision: 7, operationId: 'op' } }, async () => ++checks < failAt), error => {assert.equal(error.code, 'page_feedback_pending');- assert.equal(error.dispatchState.noDispatch, failAt < 3);- assert.equal(error.dispatchState.dispatched, failAt === 3);- assert.equal(error.dispatchState.outcomeUnknown, failAt === 3);+ assert.equal(error.dispatchState.noDispatch, failAt < 4);+ assert.equal(error.dispatchState.dispatched, failAt === 4);+ assert.equal(error.dispatchState.outcomeUnknown, failAt === 4);return true;});- assert.equal(inputs, Math.max(0, failAt - 2), 'No subsequent character may be sent after the intervention');+ assert.equal(inputs, failAt === 4 ? 1 : 0, 'No subsequent character may be sent after the intervention');});}+test('feedback during BiDi target lookup blocks focus before the focus-capable target check', async () => {+ const session = new BidiSession(), runId = id();+ session.runs.set(runId, { context: 'a' });+ let intervened = false, targetChecks = 0;+ session.locate = async () => {+ intervened = true;+ return { context: 'a', node: { sharedId: 'el' } };+ };+ session.call = async () => { targetChecks++; return { result: { value: true } }; };+ await assert.rejects(session.perform(runId, 'type', { text: 'stale', clear: false,+ pageFeedbackGuard: { documentToken: 'doc', revision: 7, operationId: 'op' } },+ async () => !intervened), error => {+ assert.equal(error.code, 'page_feedback_pending');+ assert.equal(error.dispatchState.noDispatch, true);+ return true;+ });+ assert.equal(targetChecks, 0, 'A stale page must be rejected before the target check calls focus()');+});+test('page feedback after native marker installation clears attribution and blocks input', async () => {const session = new BidiSession(), runId = id(), sent = [];session.runs.set(runId, { context: 'a' });@@ -261,7 +280,7 @@ for (const platform of ['Win32', 'MacIntel']) {: declaration.includes('el.innerText : el.value') ? '' : true } });session.send = async (method, params) => { if (method === 'input.performActions') sent.push({ marks: checks, actions: params.actions[0].actions }); return {}; };await session.perform(runId, 'field', { text: 'x', pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => { checks++; return true; });- assert.deepEqual(sent.map(item => item.marks), [2, 4, 6, 8]);+ assert.deepEqual(sent.map(item => item.marks), [3, 5, 7, 9]);assert.deepEqual(sent[0].actions, [{ type: 'keyDown', value: platform === 'MacIntel' ? '\uE03D' : '\uE009' }]);assert.equal(sent[1].actions[0].value, 'a');assert.equal(sent[2].actions[0].value, '\uE003');@@ -298,6 +317,7 @@ test('repeated native Tab validates and rebinds to the focused target before eac});assert.equal(result.success, true);assert.deepEqual(validations, [+ { kind: 'input', rebindFocus: false },{ kind: 'input', rebindFocus: false },{ kind: 'input', rebindFocus: false },{ kind: 'input', rebindFocus: true },diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 48ad98b7b..78a4ca257 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -779,6 +779,35 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: selected native option label edits invalidate the owning select`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const select = document.createElement('select'); select.id = 'native-option-label-select';+ select.innerHTML = '<option value="stable-choice" selected>Initial choice</option><option value="other">Other choice</option>';+ document.body.append(select);+ });+ await page.waitForTimeout(300);+ const guard = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'native-option-label', tool: 'click', selector: '#native-option-label-select' });+ deliver('page_monitor_dispatch', { operationId: 'native-option-label', kind: 'click', selector: '#native-option-label-select', fenceOnly: true });+ document.querySelector('#native-option-label-select option:checked').textContent = 'Updated choice';+ return lastMonitorResponse.guard;+ });+ await page.waitForFunction(() => feedback.some(event => event.kind === 'dom'+ && event.target === 'select#native-option-label-select'), null, { timeout: 1000 });+ assert.equal(await page.locator('#native-option-label-select').evaluate(select => select.value), 'stable-choice',+ 'The native value remains stable while the model-visible selected label changes');+ assert.equal(JSON.stringify(await page.evaluate(() => feedback)).includes('Updated choice'), false,+ 'Selected option labels must not be copied into feedback');+ const accepted = await page.evaluate(value => window.dispatchEvent(new CustomEvent('webbrain-agent-dom-dispatch', {+ detail: JSON.stringify({ ...value, dispatchPhase: 'click' }), cancelable: true,+ })), guard);+ assert.equal(accepted, false, 'Editing the selected option label must stale the prepared select action');+ } finally { await browser.close(); }+ });+test(`${build}: fenced iframe clicks reject transport-gap input and attribute the dispatched click`, async () => {const { browser, page } = await fixture(engine, build);try {@@ -2099,6 +2128,39 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {} finally { await browser.close(); }});+ test(`${build}: non-editable site shortcut keys invalidate actions without exposing keys`, async () => {+ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const surface = document.createElement('div'); surface.id = 'shortcut-surface'; surface.tabIndex = 0;+ document.body.prepend(surface);+ });+ await page.waitForTimeout(250);+ await page.locator('#shortcut-surface').focus();+ await page.waitForTimeout(150);+ const dispatch = await page.evaluate(() => {+ feedback = [];+ deliver('page_monitor_prepare', { operationId: 'site-shortcut', tool: 'click', selector: '#agent' });+ deliver('page_monitor_dispatch', { operationId: 'site-shortcut', kind: 'click', selector: '#agent', fenceOnly: true });+ return lastMonitorResponse;+ });+ assert.equal(dispatch.ready, true, JSON.stringify(dispatch));+ await page.locator('#shortcut-surface').press('Shift');+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Modifier-only presses do not produce shortcut activity');+ await page.locator('#shortcut-surface').press('j');+ await page.waitForFunction(() => feedback.some(event => event.kind === 'activity' && event.source === 'user'), null, { timeout: 1000 });+ const events = await page.evaluate(() => feedback);+ assert.equal(events.some(event => event.kind === 'activity' && event.source === 'user'), true, JSON.stringify(events));+ assert.equal(events.some(event => Object.hasOwn(event, 'key') || Object.hasOwn(event, 'value')), false,+ 'Keyboard contents must never be included in page feedback');+ const blocked = await page.evaluate(() => {+ try { __wbPageMonitor.activatePreparedDispatch({ operationId: 'site-shortcut', kind: 'click' }); return null; }+ catch (error) { return error.code; }+ });+ assert.equal(blocked, 'page_feedback_pending', 'A site shortcut must invalidate the prepared action');+ } finally { await browser.close(); }+ });+test(`${build}: aria-modal changes invalidate prepared page context`, async () => {const { browser, page } = await fixture(engine, build);try {