Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Fence accessibility focus and BiDi dispatch
Barack Sokullud5eee3e196f9
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex 4efbeae76..127a895d3 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -301,7 +301,17 @@ export class BidiSession {const error = new Error('Page changed during native preparation; no further input sent');error.code = 'page_feedback_pending'; throw error;}- assertLive();+ try {+ assertLive();+ if (!await validatePageDispatch(id, guard, kind, rebindFocus === true)) {+ const error = new Error('Page changed during native preparation; no further input sent');+ error.code = 'page_feedback_pending'; throw error;+ }+ assertLive();+ } catch (error) {+ await clearNativeDispatch(marker);+ throw error;+ }return marker;};const clearNativeDispatch = async marker => {diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex bdf9a177a..82bc8de28 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -7217,6 +7217,7 @@if (!_isFullyVisibleForInteraction(el)) {try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }}+ window.__wbPageMonitor?.beforeLocalDispatch({ preparation: true, kind: 'focus', target: el });try { el.focus({ preventScroll: true }); } catch {}const rect = el.getBoundingClientRect();if (!el.isConnected || rect.width < 1 || rect.height < 1) {@@ -7605,6 +7606,7 @@if (actionDeadlineExpired()) return deadlineFailure();try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }if (actionDeadlineExpired()) return deadlineFailure();+ window.__wbPageMonitor?.beforeLocalDispatch({ preparation: true, kind: 'focus', target: el });try { el.focus({ preventScroll: true }); } catch {}if (actionDeadlineExpired()) return deadlineFailure();const rect = el.getBoundingClientRect();diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex 81b27cf71..f14334bc0 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -6319,6 +6319,7 @@_scrollElementIntoClearView(el);} catch {}}+ window.__wbPageMonitor?.beforeLocalDispatch({ preparation: true, kind: 'focus', target: el });try { el.focus({ preventScroll: true }); } catch {}const rect = el.getBoundingClientRect();if (!el.isConnected || rect.width < 1 || rect.height < 1) {@@ -6645,6 +6646,7 @@if (actionDeadlineExpired()) return deadlineFailure();try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }if (actionDeadlineExpired()) return deadlineFailure();+ window.__wbPageMonitor?.beforeLocalDispatch({ preparation: true, kind: 'focus', target: el });try { el.focus({ preventScroll: true }); } catch {}if (actionDeadlineExpired()) return deadlineFailure();const rect = el.getBoundingClientRect();diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex f64bb5240..63250bf33 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -206,7 +206,7 @@ test('BiDi navigation revalidates the page feedback guard before native dispatchasync (_id, _guard, kind) => kind === 'navigate');assert.equal(sent.filter(call => call.method === 'browsingContext.navigate').length, 1);});-for (const failAt of [1, 2]) {+for (const failAt of [1, 2, 3]) {test(`page feedback change before native input ${failAt} preserves dispatch evidence`, async () => {const session = new BidiSession(), runId = id();session.runs.set(runId, { context: 'a' });@@ -217,14 +217,39 @@ for (const failAt of [1, 2]) {await assert.rejects(session.perform(runId, 'type', { text: 'ab', clear: false,pageFeedbackGuard: { documentToken: 'doc', revision: 7, operationId: 'op' } }, async () => ++checks < failAt), error => {assert.equal(error.code, 'page_feedback_pending');- assert.equal(error.dispatchState.noDispatch, failAt === 1);- assert.equal(error.dispatchState.dispatched, failAt === 2);- assert.equal(error.dispatchState.outcomeUnknown, failAt === 2);+ assert.equal(error.dispatchState.noDispatch, failAt < 3);+ assert.equal(error.dispatchState.dispatched, failAt === 3);+ assert.equal(error.dispatchState.outcomeUnknown, failAt === 3);return true;});- assert.equal(inputs, failAt - 1, 'No subsequent character may be sent after the intervention');+ assert.equal(inputs, Math.max(0, failAt - 2), 'No subsequent character may be sent after the intervention');});}++test('page feedback after native marker installation clears attribution and blocks input', async () => {+ const session = new BidiSession(), runId = id(), sent = [];+ session.runs.set(runId, { context: 'a' });+ session.locate = async () => ({ context: 'a', node: { sharedId: 'el' } });+ let markerInstalled = false, markerCleared = false, validations = 0;+ session.call = async (_match, declaration) => {+ if (declaration.includes('target.setAttribute')) { markerInstalled = true; return { result: { value: true } }; }+ if (declaration.includes('removeAttribute')) { markerCleared = true; markerInstalled = false; return { result: { value: true } }; }+ return { result: { value: true } };+ };+ session.send = async method => { sent.push(method); return {}; };+ await assert.rejects(session.perform(runId, 'click', { token: 'target', url: 'https://example.com',+ pageFeedbackGuard: { documentToken: 'doc', revision: 3, operationId: 'op' } },+ async () => ++validations === 1), error => {+ assert.equal(error.code, 'page_feedback_pending');+ assert.equal(error.dispatchState.noDispatch, true);+ assert.equal(error.dispatchState.outcomeUnknown, false);+ return true;+ });+ assert.equal(validations, 2, 'Validate again after awaiting installation of the native marker');+ assert.equal(markerInstalled, false);+ assert.equal(markerCleared, true);+ assert.equal(sent.includes('input.performActions'), false);+});for (const platform of ['Win32', 'MacIntel']) {test(`field clearing renews native attribution between modifier and A on ${platform}`, async () => {const session = new BidiSession(), runId = id(), sent = [];@@ -236,7 +261,7 @@ for (const platform of ['Win32', 'MacIntel']) {: declaration.includes('el.innerText : el.value') ? '' : true } });session.send = async (method, params) => { if (method === 'input.performActions') sent.push({ marks: checks, actions: params.actions[0].actions }); return {}; };await session.perform(runId, 'field', { text: 'x', pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => { checks++; return true; });- assert.deepEqual(sent.map(item => item.marks), [1, 2, 3, 4]);+ assert.deepEqual(sent.map(item => item.marks), [2, 4, 6, 8]);assert.deepEqual(sent[0].actions, [{ type: 'keyDown', value: platform === 'MacIntel' ? '\uE03D' : '\uE009' }]);assert.equal(sent[1].actions[0].value, 'a');assert.equal(sent[2].actions[0].value, '\uE003');@@ -252,7 +277,7 @@ test('human intervention after the clear modifier blocks A and releases held key: declaration.includes('el.innerText : el.value') ? '' : true } });session.send = async method => { sent.push(method); return {}; };await assert.rejects(session.perform(runId, 'field', { text: 'x',- pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => ++checks < 2), error => {+ pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => ++checks < 4), error => {assert.equal(error.code, 'page_feedback_pending');assert.equal(error.dispatchState.dispatched, true);assert.equal(error.dispatchState.outcomeUnknown, true);@@ -274,6 +299,9 @@ test('repeated native Tab validates and rebinds to the focused target before eacassert.equal(result.success, true);assert.deepEqual(validations, [{ kind: 'input', rebindFocus: false },+ { kind: 'input', rebindFocus: false },+ { kind: 'input', rebindFocus: true },+ { kind: 'input', rebindFocus: true },{ kind: 'input', rebindFocus: true },{ kind: 'input', rebindFocus: true },]);diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjsindex 5aebbc22f..2065b709a 100644--- a/test/page-monitor-ui.mjs+++ b/test/page-monitor-ui.mjs@@ -451,6 +451,47 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {}});+ test(`${build}: accessibility click actions attribute preparatory focus before focusin`, async () => {+ const source = read(build, 'content/content.js');+ const clickAxStart = source.indexOf("'click_ax':");+ const checkedStart = source.indexOf("'set_checked':", clickAxStart);+ const typeAxStart = source.indexOf("'type_ax':", checkedStart);+ assert.ok(clickAxStart >= 0 && checkedStart > clickAxStart && typeAxStart > checkedStart);+ for (const body of [source.slice(clickAxStart, checkedStart), source.slice(checkedStart, typeAxStart)]) {+ assert.match(body, /beforeLocalDispatch\(\{ preparation: true, kind: 'focus', target: el \}\);\s*try \{ el\.focus\(\{ preventScroll: true \}\); \} catch \{\}/,+ `${build}: click_ax and set_checked must fence focus before invoking it`);+ }++ const { browser, page } = await fixture(engine, build);+ try {+ await page.evaluate(() => {+ const checkbox = document.createElement('input'); checkbox.type = 'checkbox'; checkbox.id = 'prepared-checkbox';+ document.body.append(checkbox);+ });+ await page.waitForTimeout(160);+ const results = await page.evaluate(() => {+ feedback = [];+ const actions = [+ ['click_ax', document.getElementById('human')],+ ['set_checked', document.getElementById('prepared-checkbox')],+ ];+ return actions.map(([action, target]) => {+ const finish = __wbPageMonitor.beginContentAction(action, { selector: '#' + target.id });+ let code = null;+ try {+ __wbPageMonitor.beforeLocalDispatch({ preparation: true, kind: 'focus', target });+ target.focus({ preventScroll: true });+ __wbPageMonitor.beforeLocalDispatch();+ } catch (error) { code = error.code || error.message; }+ finally { finish(); }+ return { action, code };+ });+ });+ assert.deepEqual(results, [{ action: 'click_ax', code: null }, { action: 'set_checked', code: null }]);+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Programmatic focus preparation must not create page feedback');+ } finally { await browser.close(); }+ });+test(`${build}: content-script recovery preserves a live page monitor`, async () => {const { browser, page } = await fixture(engine, build);try {