Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Guard policy: a Firefox editor model read back as another value ends the batch

Barack Sokullu6745b52bc0ba
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjs
index cdf493149..8db8c8700 100644
--- a/firefox-companion/session.mjs
+++ b/firefox-companion/session.mjs
@@ -93,13 +93,23 @@ function readDraftJsText(el) {
return texts.join('\n');
}
-function readRichTextFieldValue(el, semanticKind = '') {
+// The field's text and the reader that produced it: the editor's own model
+// (DraftJS, ProseMirror), a native <input> or <textarea>'s exact value, or
+// an editor's rendered text (innerText), whose spacing is layout.
+function readRichTextFieldReading(el, semanticKind = '') {
if (!el?.isConnected) return null;
const draftJs = semanticKind === 'draftjs' || isDraftJsEditor(el);
const proseMirror = semanticKind === 'prosemirror' || el.classList?.contains('ProseMirror');
const semantic = draftJs ? readDraftJsText(el) : readProseMirrorText(el);
if ((draftJs || proseMirror) && semantic === null) return null;
- return semantic !== null ? semantic : (el.isContentEditable ? el.innerText : el.value);
+ if (semantic !== null) return { value: semantic, reader: 'semantic' };
+ if (el.isContentEditable) return { value: el.innerText, reader: 'rendered' };
+ return { value: el.value, reader: ['INPUT', 'TEXTAREA'].includes(String(el.tagName || '').toUpperCase()) ? 'exact' : 'unknown' };
+}
+
+function readRichTextFieldValue(el, semanticKind = '') {
+ const reading = readRichTextFieldReading(el, semanticKind);
+ return reading ? reading.value : null;
}
function proseMirrorSelectionMatches(el, bound) {
@@ -116,7 +126,7 @@ function proseMirrorSelectionMatches(el, bound) {
&& range.endContainer === bound.endNode && range.endOffset === bound.endOffset;
}
-const FIELD_TEXT_READBACK_SOURCE = [readProseMirrorText, isDraftJsEditor, readDraftJsText, readRichTextFieldValue]
+const FIELD_TEXT_READBACK_SOURCE = [readProseMirrorText, isDraftJsEditor, readDraftJsText, readRichTextFieldReading, readRichTextFieldValue]
.map(fn => fn.toString()).join('\n');
const sipMask = (1n << 64n) - 1n;
@@ -815,17 +825,32 @@ export class BidiSession {
await new Promise(resolve => setTimeout(resolve, 100));
assertLive();
const expected = clear ? payload.text : String(beforeSnapshot.value || '') + payload.text;
- const verified = await this.call(match, `(el, expected, semanticKind, bound) => {
+ // The settled readback names the reader that compared the value, so
+ // the extension can tell a field known to hold another value (an exact
+ // or semantic reading) from one whose rendered text only differs, or
+ // one that could not be read.
+ const settled = await this.call(match, `(el, expected, semanticKind, bound) => {
${FIELD_TEXT_READBACK_SOURCE}
- if (bound && (el !== bound.editor || el.ownerDocument !== bound.document || el.getRootNode() !== bound.root)) return false;
- const actual = readRichTextFieldValue(el, semanticKind);
- if (actual === null) return false;
- if (!el.isContentEditable) return actual === expected;
+ if (bound && (el !== bound.editor || el.ownerDocument !== bound.document || el.getRootNode() !== bound.root))
+ return { verified: false, compared: false, reason: 'element_replaced' };
+ const settledReading = readRichTextFieldReading(el, semanticKind);
+ if (settledReading === null) return { verified: false, compared: false, reason: 'readback_unavailable' };
+ const actual = settledReading.value;
+ if (!el.isContentEditable) return { verified: actual === expected, compared: true, reader: settledReading.reader };
const normalize = value => String(value).replace(/\\r\\n?/g, '\\n');
- return normalize(actual) === normalize(expected);
+ return { verified: normalize(actual) === normalize(expected), compared: true, reader: settledReading.reader };
}`, [{ type: 'string', value: expected }, { type: 'string', value: beforeSnapshot.semanticKind || '' },
proseMirrorBinding ? { handle: proseMirrorBinding } : { type: 'null' }]);
- if (verified.result?.value !== true) return { success: false, dispatched: true, verified: false, submitted: false, outcomeUnknown: true, retryable: false, error: 'Trusted typing did not match the requested value; inspect before another write.' };
+ const readback = unpackRemoteObjectValue(settled.result?.type ? settled.result : settled.result?.value);
+ if (readback !== true && readback?.verified !== true) {
+ const reading = readback && typeof readback === 'object' ? { readback: {
+ compared: readback.compared === true,
+ ...(['exact', 'semantic', 'rendered'].includes(readback.reader) ? { reader: readback.reader } : {}),
+ ...(['element_replaced', 'readback_unavailable'].includes(readback.reason) ? { reason: readback.reason } : {}),
+ } } : {};
+ return { success: false, dispatched: true, verified: false, submitted: false, outcomeUnknown: true, retryable: false, ...reading,
+ error: 'Trusted typing did not match the requested value; inspect before another write.' };
+ }
if (action === 'field' && payload.submit === true) {
assertLive();
await press('\uE007');
diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.js
index a39066f9e..67e0b664f 100644
--- a/src/firefox/src/bidi/client.js
+++ b/src/firefox/src/bidi/client.js
@@ -6,6 +6,30 @@ import { beforePageAgentDispatch, validateNativePageDispatch } from '../agent/pa
const REQUEST_NOT_SENT = new WeakSet();
const notSent = error => { if (error && typeof error === 'object') REQUEST_NOT_SENT.add(error); return error; };
export const isBidiRequestNotSentError = error => !!error && typeof error === 'object' && REQUEST_NOT_SENT.has(error);
+// What a failed settled readback of trusted typing says about the field.
+// The companion names the reader that compared the value. A native <input>
+// or <textarea>'s exact value, or an editor's own model (DraftJS,
+// ProseMirror), read as another value is known: the field holds another
+// value, and a batched submit must not run on it (browser-batch.js
+// textWriteRefuted). An editor's rendered text (innerText), or a field that
+// could not be read, leaves the agent's policy to decide whether that is a
+// failure (Strict checks) or a noted, unverified success. An older companion
+// names no reader: a native field is then known by its exact value.
+function trustedTypingReadback(readback, fieldMeta) {
+ if (!readback || typeof readback !== 'object') {
+ const nativeField = !!fieldMeta && fieldMeta.contentEditable !== true
+ && ['input', 'textarea'].includes(String(fieldMeta.tag || '').toLowerCase());
+ return nativeField
+ ? { unverified: { reason: 'readback_mismatch', reader: 'exact' }, readbackExact: true }
+ : { unverified: { reason: 'readback_mismatch', reader: 'unknown' } };
+ }
+ const reader = readback.compared === true ? readback.reader : null;
+ if (reader === 'exact' || reader === 'semantic') return { unverified: { reason: 'readback_mismatch', reader }, readbackExact: true };
+ if (reader === 'rendered') return { unverified: { reason: 'readback_mismatch', reader }, readbackExact: false };
+ if (readback.compared === true) return { unverified: { reason: 'readback_unproven', reader: 'unknown' } };
+ return { unverified: { reason: readback.reason === 'element_replaced' ? 'element_replaced' : 'readback_unavailable', reader: 'none' },
+ readbackExact: false };
+}
export class FirefoxBidiClient {
constructor(api) { this.apiOverride = api; this.pending = new Map(); this.runs = new Map(); this.captures = new Map(); this.sequence = 0; this.connectionEpoch = 0; }
get api() { return this.apiOverride || globalThis.browser; }
@@ -210,21 +234,10 @@ export class FirefoxBidiClient {
deadlineAt: message.actionDeadlineAt || 0,
}, { onDispatch });
// The companion answers verified:false only from its settled readback
- // after trusted typing went out. A native <input> or <textarea> it
- // reads back by its exact value, so there a mismatch is known: the
- // field holds another value, and a batched submit must not run on it
- // (browser-batch.js textWriteRefuted). A contenteditable editor it
- // reads through the editor's model or, failing that, its rendered text
- // (innerText), so there the agent's policy decides whether that is a
- // failure (Strict checks) or a noted, unverified success.
- const meta = prepared.fieldMeta || null;
- const exactValue = !!meta && meta.contentEditable !== true
- && ['input', 'textarea'].includes(String(meta.tag || '').toLowerCase());
+ // after trusted typing went out (trustedTypingReadback).
const unverified = (action === 'type' || action === 'field') && performed?.success === false
&& performed.dispatched === true && performed.verified === false && !performed.unverified
- ? (exactValue
- ? { unverified: { reason: 'readback_mismatch', reader: 'exact' }, readbackExact: true }
- : { unverified: { reason: 'readback_mismatch', reader: 'unknown' } })
+ ? trustedTypingReadback(performed.readback, prepared.fieldMeta)
: {};
return { ...metadata, ...performed, ...unverified, ...(prepared.rect ? { rect: prepared.rect } : {}), ...(prepared._filePickerGuardId ? { _filePickerGuardId: prepared._filePickerGuardId } : {}) };
} catch (error) {
diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjs
index 62e03c25f..85095724b 100644
--- a/test/firefox-bidi.mjs
+++ b/test/firefox-bidi.mjs
@@ -516,6 +516,61 @@ test('literal newline insertion never dispatches an Enter key', async () => {
assert.deepEqual(dispatched.flatMap(p=>p.actions[0].actions.map(a=>a.value)), ['a','a','b','b']);
});
+test('the settled typing readback names the reader that compared the value', async () => {
+ const session = new BidiSession(); const runId = id(); const scripts = [];
+ session.runs.set(runId, {context:'tab'});
+ session.locate = async () => ({context:'tab',node:{sharedId:'editor'}});
+ const marker = 'const settledReading = readRichTextFieldReading(el, semanticKind)';
+ let settled = { verified: false, compared: true, reader: 'semantic' };
+ session.call = async (_match, fn) => {
+ scripts.push(fn);
+ if (!fn.includes(marker)) return {result:{value:true}};
+ return {result:{type:'object', value:Object.entries(settled).map(([key, value]) => [key, {type:typeof value, value}])}};
+ };
+ session.send = async () => ({});
+ const mismatch = await session.perform(runId, 'field', {text:'ab', submit:true});
+ assert.equal(mismatch.success, false);
+ assert.equal(mismatch.verified, false);
+ assert.equal(mismatch.submitted, false);
+ assert.deepEqual(mismatch.readback, {compared:true, reader:'semantic'});
+ // The extension reads an editor model's mismatch as known.
+ const client = new FirefoxBidiClient({tabs:{sendMessage:async () => ({bidiPrepared:true, url:'https://example.com/',
+ fieldMeta:{tag:'div', contentEditable:true}})}});
+ client.runs.set(1, {runId, bound:true});
+ client.bindRun = async () => {};
+ client.perform = async () => mismatch;
+ const known = await client.sendContent(1, {action:'set_field', params:{text:'ab'}});
+ assert.deepEqual(known.unverified, {reason:'readback_mismatch', reader:'semantic'});
+ assert.equal(known.readbackExact, true);
+ settled = { verified: false, compared: false, reason: 'element_replaced' };
+ assert.deepEqual((await session.perform(runId, 'field', {text:'ab'})).readback, {compared:false, reason:'element_replaced'});
+ settled = { verified: true, compared: true, reader: 'exact' };
+ assert.equal((await session.perform(runId, 'field', {text:'ab'})).verified, true);
+
+ // The readback itself, against a minimal DOM.
+ const declaration = scripts.find(fn => fn.includes(marker));
+ const settle = (...args) => ({ ...vm.runInNewContext(`(${declaration})`)(...args) });
+ const style = {display:'block', visibility:'visible', opacity:'1', contentVisibility:'visible'};
+ const doc = {defaultView:{getComputedStyle:() => style}};
+ const element = ({classes = [], ...props}) => ({nodeType:1, isConnected:true, shadowRoot:null, hidden:false, ownerDocument:doc,
+ getAttribute:() => null, classList:{contains:name => classes.includes(name)}, children:[], childNodes:[], getRootNode:() => doc, ...props});
+ const input = element({tagName:'INPUT', isContentEditable:false, value:'ab'});
+ assert.deepEqual(settle(input, 'ab', '', null), {verified:true, compared:true, reader:'exact'});
+ input.value = 'a';
+ assert.deepEqual(settle(input, 'ab', '', null), {verified:false, compared:true, reader:'exact'});
+ const paragraph = element({tagName:'P', isContentEditable:true, childNodes:[{nodeType:3, nodeValue:'Original'}]});
+ const proseMirror = element({tagName:'DIV', isContentEditable:true, classes:['ProseMirror'], childNodes:[paragraph], innerText:'Original'});
+ assert.deepEqual(settle(proseMirror, 'Original', 'prosemirror', null), {verified:true, compared:true, reader:'semantic'});
+ assert.deepEqual(settle(proseMirror, 'First\n\nLast', 'prosemirror', null), {verified:false, compared:true, reader:'semantic'});
+ const editor = element({tagName:'DIV', isContentEditable:true, innerText:'Original'});
+ assert.deepEqual(settle(editor, 'First', '', null), {verified:false, compared:true, reader:'rendered'});
+ const other = element({tagName:'DIV', isContentEditable:false});
+ assert.deepEqual(settle(other, 'ab', '', null), {verified:false, compared:true, reader:'unknown'});
+ assert.deepEqual(settle(input, 'a', '', {editor:proseMirror, document:doc, root:doc}), {verified:false, compared:false, reason:'element_replaced'});
+ proseMirror.classList = {contains:() => false};
+ assert.deepEqual(settle(proseMirror, 'Original', 'prosemirror', null), {verified:false, compared:false, reason:'readback_unavailable'});
+});
+
test('validation failures preserve no-dispatch status across the native client', async () => {
const session = new BidiSession(); const runId = id();
session.runs.set(runId, {context:'tab'});
diff --git a/test/firefox-rich-text-readback.mjs b/test/firefox-rich-text-readback.mjs
index 901eb5e64..9dd318caf 100644
--- a/test/firefox-rich-text-readback.mjs
+++ b/test/firefox-rich-text-readback.mjs
@@ -136,10 +136,12 @@ test('Firefox: production BiDi typing uses exact DraftJS model text for empty ap
assert.equal(native.commands.some(command => command.params?.actions?.[0]?.actions.some(action => action.value === '\uE007')), false, 'Literal line breaks must not send Enter.');
assert.equal(await page.evaluate(() => qaEditor.childNodes.length), 1, 'Append stays inside the original marked document root.');
if (scenario.name === 'append inside last leaf') {
- const declaration = native.declarations.find(source => source.includes('const actual = readRichTextFieldValue'));
+ const declaration = native.declarations.find(source => source.includes('const settledReading = readRichTextFieldReading(el, semanticKind)'));
const readback = expected => page.evaluate(({ declaration, expected }) => (0, eval)(`(${declaration})`)(qaEditor, expected), { declaration, expected });
- assert.equal(await readback('First\r\n\r\nLast suffix'), true, 'CRLF normalization preserves authored document separators.');
- assert.equal(await readback('First\n\n\nLast suffix'), false, 'Visual spacing cannot excuse an extra authored line break.');
+ assert.deepEqual(await readback('First\r\n\r\nLast suffix'), { verified: true, compared: true, reader: 'semantic' },
+ 'CRLF normalization preserves authored document separators.');
+ assert.deepEqual(await readback('First\n\n\nLast suffix'), { verified: false, compared: true, reader: 'semantic' },
+ 'Visual spacing cannot excuse an extra authored line break.');
}
});
} finally { await browser.close(); }
@@ -321,6 +323,7 @@ test('Firefox: unsupported DraftJS preflight sends no text, and unsupported sett
assert.equal(result.outcomeUnknown, true);
assert.equal(result.retryable, false);
assert.equal(result.submitted, false);
+ assert.deepEqual(result.readback, { compared: false, reason: 'readback_unavailable' }, 'An unreadable document is not a known mismatch.');
assert.equal(await page.evaluate(() => qaModel), 'aB');
assert.equal(native.commands.filter(command => command.method === 'input.performActions').length, 2, 'Each character is sent once; verification must not dispatch a second edit.');
});
@@ -371,6 +374,7 @@ test('Firefox: DraftJS plain-text paste writes a complete multiline document exa
assert.equal(result.retryable, false);
assert.equal(result.submitted, false);
assert.equal(await page.evaluate(() => qaModel), '');
+ assert.deepEqual(result.readback, { compared: true, reader: 'semantic' }, 'The editor model holds another document: a known mismatch.');
const keyDowns = native.commands.filter(command => command.method === 'input.performActions')
.flatMap(command => command.params.actions[0].actions.filter(action => action.type === 'keyDown').map(action => action.value));
assert.equal(keyDowns.includes('\uE007'), false, 'Literal LF insertion cannot send a submit shortcut.');
@@ -444,6 +448,7 @@ test('Firefox: actual ProseMirror writes one complete plain-text edit without in
assert.equal(result.retryable, false);
assert.equal(result.submitted, false);
assert.equal(await page.evaluate(() => qaModelText), 'Original');
+ assert.deepEqual(result.readback, { compared: true, reader: 'semantic' }, 'The editor model holds another document: a known mismatch.');
assert.equal(native.commands.some(command => command.method === 'input.performActions'), false);
assert.equal(native.declarations.filter(source => source.includes("new view.ClipboardEvent('paste'")).length, 1);
});
@@ -512,6 +517,7 @@ test('Firefox: ProseMirror paste retains complete document and original selectio
assert.equal(result.submitted, false);
assert.equal(result.outcomeUnknown, true);
assert.equal(result.retryable, false);
+ assert.deepEqual(result.readback, { compared: false, reason: 'readback_unavailable' }, 'An unreadable document is not a known mismatch.');
assert.equal(await page.evaluate(() => qaModelText), 'First\nLast');
assert.deepEqual(await page.evaluate(() => qaPastes), ['First\nLast']);
assert.equal(native.commands.some(command => command.method === 'input.performActions'), false);
diff --git a/test/guard-advisory-writes.mjs b/test/guard-advisory-writes.mjs
index efd3a78d1..7dba7d248 100644
--- a/test/guard-advisory-writes.mjs
+++ b/test/guard-advisory-writes.mjs
@@ -862,6 +862,45 @@ test('firefox: a trusted BiDi mismatch in a native input or textarea is known an
}
});
+test('firefox: the companion names the reader of a trusted BiDi readback; an editor model read as another value ends the batch', async () => {
+ // An exact or semantic (DraftJS, ProseMirror) reading of another value is
+ // known; a rendered one, or a field that could not be read, is not.
+ const { Agent, batch } = builds.find(item => item.build === 'firefox');
+ const tab = 79;
+ const editor = { tag: 'div', type: 'div', contentEditable: true, name: 'post' };
+ const input = { tag: 'input', type: 'text', contentEditable: false, name: 'title' };
+ for (const [label, fieldMeta, readback, unverified, readbackExact, known, note] of [
+ ['semantic', editor, { compared: true, reader: 'semantic' }, { reason: 'readback_mismatch', reader: 'semantic' }, true, true,
+ /reads back a different value/],
+ ['exact', input, { compared: true, reader: 'exact' }, { reason: 'readback_mismatch', reader: 'exact' }, true, true,
+ /reads back a different value/],
+ ['rendered', editor, { compared: true, reader: 'rendered' }, { reason: 'readback_mismatch', reader: 'rendered' }, false, false,
+ /only reads back its rendered layout/],
+ ['replaced', editor, { compared: false, reason: 'element_replaced' }, { reason: 'element_replaced', reader: 'none' }, false, false,
+ /re-rendered before it could be read back/],
+ ['unreadable', input, { compared: false, reason: 'readback_unavailable' }, { reason: 'readback_unavailable', reader: 'none' }, false, false,
+ /editor could not be read back\./],
+ ['no reader', input, { compared: true }, { reason: 'readback_unproven', reader: 'unknown' }, undefined, false,
+ /could not be read back exactly/],
+ ]) {
+ const client = new FirefoxBidiClient({ tabs: { sendMessage: async () => ({ bidiPrepared: true, fieldMeta, url: scope.pageUrl }) } });
+ client.runs.set(tab, { runId: 'run-1', bound: true });
+ client.bindRun = async () => {};
+ client.perform = async () => ({ success: false, dispatched: true, verified: false, submitted: false, outcomeUnknown: true,
+ retryable: false, readback, error: 'Trusted typing did not match the requested value; inspect before another write.' });
+ const result = await client.sendContent(tab, { target: 'content', action: 'set_field', params: { ...args } });
+ assert.deepEqual(result.unverified, unverified, label);
+ assert.equal(result.readbackExact, readbackExact, label);
+ const agent = setup(Agent);
+ const advisory = await agent._finalizeTextMutationResult(tabId, 'set_field', args, result);
+ assert.equal(advisory.success, true, label);
+ assert.equal(advisory.unverifiedWrite, true, label);
+ assert.match(advisoryOf(advisory, 'text-write-readback-unverified')?.note || '', note, label);
+ assert.equal(batch.textWriteRefuted(advisory), known, label);
+ assert.equal(batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(advisory) }).failed, known, label);
+ }
+});
+
// The real content-script readback handlers, run against a minimal DOM.
function functionSource(source, name) {
let start = source.indexOf(`function ${name}(`);