Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fence accessibility focus and BiDi dispatch

Barack Sokullud5eee3e196f9
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjs
index 4efbeae76..127a895d3 100644
--- a/firefox-companion/session.mjs
+++ b/firefox-companion/session.mjs
@@ -301,7 +301,17 @@ export class BidiSession {
const error = new Error('Page changed during native preparation; no further input sent');
error.code = 'page_feedback_pending'; throw error;
}
- assertLive();
+ try {
+ assertLive();
+ if (!await validatePageDispatch(id, guard, kind, rebindFocus === true)) {
+ const error = new Error('Page changed during native preparation; no further input sent');
+ error.code = 'page_feedback_pending'; throw error;
+ }
+ assertLive();
+ } catch (error) {
+ await clearNativeDispatch(marker);
+ throw error;
+ }
return marker;
};
const clearNativeDispatch = async marker => {
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index bdf9a177a..82bc8de28 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -7217,6 +7217,7 @@
if (!_isFullyVisibleForInteraction(el)) {
try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
}
+ window.__wbPageMonitor?.beforeLocalDispatch({ preparation: true, kind: 'focus', target: el });
try { el.focus({ preventScroll: true }); } catch {}
const rect = el.getBoundingClientRect();
if (!el.isConnected || rect.width < 1 || rect.height < 1) {
@@ -7605,6 +7606,7 @@
if (actionDeadlineExpired()) return deadlineFailure();
try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
if (actionDeadlineExpired()) return deadlineFailure();
+ window.__wbPageMonitor?.beforeLocalDispatch({ preparation: true, kind: 'focus', target: el });
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
const rect = el.getBoundingClientRect();
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index 81b27cf71..f14334bc0 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -6319,6 +6319,7 @@
_scrollElementIntoClearView(el);
} catch {}
}
+ window.__wbPageMonitor?.beforeLocalDispatch({ preparation: true, kind: 'focus', target: el });
try { el.focus({ preventScroll: true }); } catch {}
const rect = el.getBoundingClientRect();
if (!el.isConnected || rect.width < 1 || rect.height < 1) {
@@ -6645,6 +6646,7 @@
if (actionDeadlineExpired()) return deadlineFailure();
try { _scrollElementIntoClearView(el); } catch (error) { if (error?.code === 'page_feedback_pending') throw error; }
if (actionDeadlineExpired()) return deadlineFailure();
+ window.__wbPageMonitor?.beforeLocalDispatch({ preparation: true, kind: 'focus', target: el });
try { el.focus({ preventScroll: true }); } catch {}
if (actionDeadlineExpired()) return deadlineFailure();
const rect = el.getBoundingClientRect();
diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjs
index f64bb5240..63250bf33 100644
--- a/test/firefox-bidi.mjs
+++ b/test/firefox-bidi.mjs
@@ -206,7 +206,7 @@ test('BiDi navigation revalidates the page feedback guard before native dispatch
async (_id, _guard, kind) => kind === 'navigate');
assert.equal(sent.filter(call => call.method === 'browsingContext.navigate').length, 1);
});
-for (const failAt of [1, 2]) {
+for (const failAt of [1, 2, 3]) {
test(`page feedback change before native input ${failAt} preserves dispatch evidence`, async () => {
const session = new BidiSession(), runId = id();
session.runs.set(runId, { context: 'a' });
@@ -217,14 +217,39 @@ for (const failAt of [1, 2]) {
await assert.rejects(session.perform(runId, 'type', { text: 'ab', clear: false,
pageFeedbackGuard: { documentToken: 'doc', revision: 7, operationId: 'op' } }, async () => ++checks < failAt), error => {
assert.equal(error.code, 'page_feedback_pending');
- assert.equal(error.dispatchState.noDispatch, failAt === 1);
- assert.equal(error.dispatchState.dispatched, failAt === 2);
- assert.equal(error.dispatchState.outcomeUnknown, failAt === 2);
+ assert.equal(error.dispatchState.noDispatch, failAt < 3);
+ assert.equal(error.dispatchState.dispatched, failAt === 3);
+ assert.equal(error.dispatchState.outcomeUnknown, failAt === 3);
return true;
});
- assert.equal(inputs, failAt - 1, 'No subsequent character may be sent after the intervention');
+ assert.equal(inputs, Math.max(0, failAt - 2), 'No subsequent character may be sent after the intervention');
});
}
+
+test('page feedback after native marker installation clears attribution and blocks input', async () => {
+ const session = new BidiSession(), runId = id(), sent = [];
+ session.runs.set(runId, { context: 'a' });
+ session.locate = async () => ({ context: 'a', node: { sharedId: 'el' } });
+ let markerInstalled = false, markerCleared = false, validations = 0;
+ session.call = async (_match, declaration) => {
+ if (declaration.includes('target.setAttribute')) { markerInstalled = true; return { result: { value: true } }; }
+ if (declaration.includes('removeAttribute')) { markerCleared = true; markerInstalled = false; return { result: { value: true } }; }
+ return { result: { value: true } };
+ };
+ session.send = async method => { sent.push(method); return {}; };
+ await assert.rejects(session.perform(runId, 'click', { token: 'target', url: 'https://example.com',
+ pageFeedbackGuard: { documentToken: 'doc', revision: 3, operationId: 'op' } },
+ async () => ++validations === 1), error => {
+ assert.equal(error.code, 'page_feedback_pending');
+ assert.equal(error.dispatchState.noDispatch, true);
+ assert.equal(error.dispatchState.outcomeUnknown, false);
+ return true;
+ });
+ assert.equal(validations, 2, 'Validate again after awaiting installation of the native marker');
+ assert.equal(markerInstalled, false);
+ assert.equal(markerCleared, true);
+ assert.equal(sent.includes('input.performActions'), false);
+});
for (const platform of ['Win32', 'MacIntel']) {
test(`field clearing renews native attribution between modifier and A on ${platform}`, async () => {
const session = new BidiSession(), runId = id(), sent = [];
@@ -236,7 +261,7 @@ for (const platform of ['Win32', 'MacIntel']) {
: declaration.includes('el.innerText : el.value') ? '' : true } });
session.send = async (method, params) => { if (method === 'input.performActions') sent.push({ marks: checks, actions: params.actions[0].actions }); return {}; };
await session.perform(runId, 'field', { text: 'x', pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => { checks++; return true; });
- assert.deepEqual(sent.map(item => item.marks), [1, 2, 3, 4]);
+ assert.deepEqual(sent.map(item => item.marks), [2, 4, 6, 8]);
assert.deepEqual(sent[0].actions, [{ type: 'keyDown', value: platform === 'MacIntel' ? '\uE03D' : '\uE009' }]);
assert.equal(sent[1].actions[0].value, 'a');
assert.equal(sent[2].actions[0].value, '\uE003');
@@ -252,7 +277,7 @@ test('human intervention after the clear modifier blocks A and releases held key
: declaration.includes('el.innerText : el.value') ? '' : true } });
session.send = async method => { sent.push(method); return {}; };
await assert.rejects(session.perform(runId, 'field', { text: 'x',
- pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => ++checks < 2), error => {
+ pageFeedbackGuard: { documentToken: 'doc', revision: 0, operationId: 'op' } }, async () => ++checks < 4), error => {
assert.equal(error.code, 'page_feedback_pending');
assert.equal(error.dispatchState.dispatched, true);
assert.equal(error.dispatchState.outcomeUnknown, true);
@@ -274,6 +299,9 @@ test('repeated native Tab validates and rebinds to the focused target before eac
assert.equal(result.success, true);
assert.deepEqual(validations, [
{ kind: 'input', rebindFocus: false },
+ { kind: 'input', rebindFocus: false },
+ { kind: 'input', rebindFocus: true },
+ { kind: 'input', rebindFocus: true },
{ kind: 'input', rebindFocus: true },
{ kind: 'input', rebindFocus: true },
]);
diff --git a/test/page-monitor-ui.mjs b/test/page-monitor-ui.mjs
index 5aebbc22f..2065b709a 100644
--- a/test/page-monitor-ui.mjs
+++ b/test/page-monitor-ui.mjs
@@ -451,6 +451,47 @@ for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
}
});
+ test(`${build}: accessibility click actions attribute preparatory focus before focusin`, async () => {
+ const source = read(build, 'content/content.js');
+ const clickAxStart = source.indexOf("'click_ax':");
+ const checkedStart = source.indexOf("'set_checked':", clickAxStart);
+ const typeAxStart = source.indexOf("'type_ax':", checkedStart);
+ assert.ok(clickAxStart >= 0 && checkedStart > clickAxStart && typeAxStart > checkedStart);
+ for (const body of [source.slice(clickAxStart, checkedStart), source.slice(checkedStart, typeAxStart)]) {
+ assert.match(body, /beforeLocalDispatch\(\{ preparation: true, kind: 'focus', target: el \}\);\s*try \{ el\.focus\(\{ preventScroll: true \}\); \} catch \{\}/,
+ `${build}: click_ax and set_checked must fence focus before invoking it`);
+ }
+
+ const { browser, page } = await fixture(engine, build);
+ try {
+ await page.evaluate(() => {
+ const checkbox = document.createElement('input'); checkbox.type = 'checkbox'; checkbox.id = 'prepared-checkbox';
+ document.body.append(checkbox);
+ });
+ await page.waitForTimeout(160);
+ const results = await page.evaluate(() => {
+ feedback = [];
+ const actions = [
+ ['click_ax', document.getElementById('human')],
+ ['set_checked', document.getElementById('prepared-checkbox')],
+ ];
+ return actions.map(([action, target]) => {
+ const finish = __wbPageMonitor.beginContentAction(action, { selector: '#' + target.id });
+ let code = null;
+ try {
+ __wbPageMonitor.beforeLocalDispatch({ preparation: true, kind: 'focus', target });
+ target.focus({ preventScroll: true });
+ __wbPageMonitor.beforeLocalDispatch();
+ } catch (error) { code = error.code || error.message; }
+ finally { finish(); }
+ return { action, code };
+ });
+ });
+ assert.deepEqual(results, [{ action: 'click_ax', code: null }, { action: 'set_checked', code: null }]);
+ assert.deepEqual(await page.evaluate(() => feedback), [], 'Programmatic focus preparation must not create page feedback');
+ } finally { await browser.close(); }
+ });
+
test(`${build}: content-script recovery preserves a live page monitor`, async () => {
const { browser, page } = await fixture(engine, build);
try {