Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Guard policy: a Firefox editor model read back as another value ends the batch
Barack Sokullu6745b52bc0ba
diff --git a/firefox-companion/session.mjs b/firefox-companion/session.mjsindex cdf493149..8db8c8700 100644--- a/firefox-companion/session.mjs+++ b/firefox-companion/session.mjs@@ -93,13 +93,23 @@ function readDraftJsText(el) {return texts.join('\n');}-function readRichTextFieldValue(el, semanticKind = '') {+// The field's text and the reader that produced it: the editor's own model+// (DraftJS, ProseMirror), a native <input> or <textarea>'s exact value, or+// an editor's rendered text (innerText), whose spacing is layout.+function readRichTextFieldReading(el, semanticKind = '') {if (!el?.isConnected) return null;const draftJs = semanticKind === 'draftjs' || isDraftJsEditor(el);const proseMirror = semanticKind === 'prosemirror' || el.classList?.contains('ProseMirror');const semantic = draftJs ? readDraftJsText(el) : readProseMirrorText(el);if ((draftJs || proseMirror) && semantic === null) return null;- return semantic !== null ? semantic : (el.isContentEditable ? el.innerText : el.value);+ if (semantic !== null) return { value: semantic, reader: 'semantic' };+ if (el.isContentEditable) return { value: el.innerText, reader: 'rendered' };+ return { value: el.value, reader: ['INPUT', 'TEXTAREA'].includes(String(el.tagName || '').toUpperCase()) ? 'exact' : 'unknown' };+}++function readRichTextFieldValue(el, semanticKind = '') {+ const reading = readRichTextFieldReading(el, semanticKind);+ return reading ? reading.value : null;}function proseMirrorSelectionMatches(el, bound) {@@ -116,7 +126,7 @@ function proseMirrorSelectionMatches(el, bound) {&& range.endContainer === bound.endNode && range.endOffset === bound.endOffset;}-const FIELD_TEXT_READBACK_SOURCE = [readProseMirrorText, isDraftJsEditor, readDraftJsText, readRichTextFieldValue]+const FIELD_TEXT_READBACK_SOURCE = [readProseMirrorText, isDraftJsEditor, readDraftJsText, readRichTextFieldReading, readRichTextFieldValue].map(fn => fn.toString()).join('\n');const sipMask = (1n << 64n) - 1n;@@ -815,17 +825,32 @@ export class BidiSession {await new Promise(resolve => setTimeout(resolve, 100));assertLive();const expected = clear ? payload.text : String(beforeSnapshot.value || '') + payload.text;- const verified = await this.call(match, `(el, expected, semanticKind, bound) => {+ // The settled readback names the reader that compared the value, so+ // the extension can tell a field known to hold another value (an exact+ // or semantic reading) from one whose rendered text only differs, or+ // one that could not be read.+ const settled = await this.call(match, `(el, expected, semanticKind, bound) => {${FIELD_TEXT_READBACK_SOURCE}- if (bound && (el !== bound.editor || el.ownerDocument !== bound.document || el.getRootNode() !== bound.root)) return false;- const actual = readRichTextFieldValue(el, semanticKind);- if (actual === null) return false;- if (!el.isContentEditable) return actual === expected;+ if (bound && (el !== bound.editor || el.ownerDocument !== bound.document || el.getRootNode() !== bound.root))+ return { verified: false, compared: false, reason: 'element_replaced' };+ const settledReading = readRichTextFieldReading(el, semanticKind);+ if (settledReading === null) return { verified: false, compared: false, reason: 'readback_unavailable' };+ const actual = settledReading.value;+ if (!el.isContentEditable) return { verified: actual === expected, compared: true, reader: settledReading.reader };const normalize = value => String(value).replace(/\\r\\n?/g, '\\n');- return normalize(actual) === normalize(expected);+ return { verified: normalize(actual) === normalize(expected), compared: true, reader: settledReading.reader };}`, [{ type: 'string', value: expected }, { type: 'string', value: beforeSnapshot.semanticKind || '' },proseMirrorBinding ? { handle: proseMirrorBinding } : { type: 'null' }]);- if (verified.result?.value !== true) return { success: false, dispatched: true, verified: false, submitted: false, outcomeUnknown: true, retryable: false, error: 'Trusted typing did not match the requested value; inspect before another write.' };+ const readback = unpackRemoteObjectValue(settled.result?.type ? settled.result : settled.result?.value);+ if (readback !== true && readback?.verified !== true) {+ const reading = readback && typeof readback === 'object' ? { readback: {+ compared: readback.compared === true,+ ...(['exact', 'semantic', 'rendered'].includes(readback.reader) ? { reader: readback.reader } : {}),+ ...(['element_replaced', 'readback_unavailable'].includes(readback.reason) ? { reason: readback.reason } : {}),+ } } : {};+ return { success: false, dispatched: true, verified: false, submitted: false, outcomeUnknown: true, retryable: false, ...reading,+ error: 'Trusted typing did not match the requested value; inspect before another write.' };+ }if (action === 'field' && payload.submit === true) {assertLive();await press('\uE007');diff --git a/src/firefox/src/bidi/client.js b/src/firefox/src/bidi/client.jsindex a39066f9e..67e0b664f 100644--- a/src/firefox/src/bidi/client.js+++ b/src/firefox/src/bidi/client.js@@ -6,6 +6,30 @@ import { beforePageAgentDispatch, validateNativePageDispatch } from '../agent/paconst REQUEST_NOT_SENT = new WeakSet();const notSent = error => { if (error && typeof error === 'object') REQUEST_NOT_SENT.add(error); return error; };export const isBidiRequestNotSentError = error => !!error && typeof error === 'object' && REQUEST_NOT_SENT.has(error);+// What a failed settled readback of trusted typing says about the field.+// The companion names the reader that compared the value. A native <input>+// or <textarea>'s exact value, or an editor's own model (DraftJS,+// ProseMirror), read as another value is known: the field holds another+// value, and a batched submit must not run on it (browser-batch.js+// textWriteRefuted). An editor's rendered text (innerText), or a field that+// could not be read, leaves the agent's policy to decide whether that is a+// failure (Strict checks) or a noted, unverified success. An older companion+// names no reader: a native field is then known by its exact value.+function trustedTypingReadback(readback, fieldMeta) {+ if (!readback || typeof readback !== 'object') {+ const nativeField = !!fieldMeta && fieldMeta.contentEditable !== true+ && ['input', 'textarea'].includes(String(fieldMeta.tag || '').toLowerCase());+ return nativeField+ ? { unverified: { reason: 'readback_mismatch', reader: 'exact' }, readbackExact: true }+ : { unverified: { reason: 'readback_mismatch', reader: 'unknown' } };+ }+ const reader = readback.compared === true ? readback.reader : null;+ if (reader === 'exact' || reader === 'semantic') return { unverified: { reason: 'readback_mismatch', reader }, readbackExact: true };+ if (reader === 'rendered') return { unverified: { reason: 'readback_mismatch', reader }, readbackExact: false };+ if (readback.compared === true) return { unverified: { reason: 'readback_unproven', reader: 'unknown' } };+ return { unverified: { reason: readback.reason === 'element_replaced' ? 'element_replaced' : 'readback_unavailable', reader: 'none' },+ readbackExact: false };+}export class FirefoxBidiClient {constructor(api) { this.apiOverride = api; this.pending = new Map(); this.runs = new Map(); this.captures = new Map(); this.sequence = 0; this.connectionEpoch = 0; }get api() { return this.apiOverride || globalThis.browser; }@@ -210,21 +234,10 @@ export class FirefoxBidiClient {deadlineAt: message.actionDeadlineAt || 0,}, { onDispatch });// The companion answers verified:false only from its settled readback- // after trusted typing went out. A native <input> or <textarea> it- // reads back by its exact value, so there a mismatch is known: the- // field holds another value, and a batched submit must not run on it- // (browser-batch.js textWriteRefuted). A contenteditable editor it- // reads through the editor's model or, failing that, its rendered text- // (innerText), so there the agent's policy decides whether that is a- // failure (Strict checks) or a noted, unverified success.- const meta = prepared.fieldMeta || null;- const exactValue = !!meta && meta.contentEditable !== true- && ['input', 'textarea'].includes(String(meta.tag || '').toLowerCase());+ // after trusted typing went out (trustedTypingReadback).const unverified = (action === 'type' || action === 'field') && performed?.success === false&& performed.dispatched === true && performed.verified === false && !performed.unverified- ? (exactValue- ? { unverified: { reason: 'readback_mismatch', reader: 'exact' }, readbackExact: true }- : { unverified: { reason: 'readback_mismatch', reader: 'unknown' } })+ ? trustedTypingReadback(performed.readback, prepared.fieldMeta): {};return { ...metadata, ...performed, ...unverified, ...(prepared.rect ? { rect: prepared.rect } : {}), ...(prepared._filePickerGuardId ? { _filePickerGuardId: prepared._filePickerGuardId } : {}) };} catch (error) {diff --git a/test/firefox-bidi.mjs b/test/firefox-bidi.mjsindex 62e03c25f..85095724b 100644--- a/test/firefox-bidi.mjs+++ b/test/firefox-bidi.mjs@@ -516,6 +516,61 @@ test('literal newline insertion never dispatches an Enter key', async () => {assert.deepEqual(dispatched.flatMap(p=>p.actions[0].actions.map(a=>a.value)), ['a','a','b','b']);});+test('the settled typing readback names the reader that compared the value', async () => {+ const session = new BidiSession(); const runId = id(); const scripts = [];+ session.runs.set(runId, {context:'tab'});+ session.locate = async () => ({context:'tab',node:{sharedId:'editor'}});+ const marker = 'const settledReading = readRichTextFieldReading(el, semanticKind)';+ let settled = { verified: false, compared: true, reader: 'semantic' };+ session.call = async (_match, fn) => {+ scripts.push(fn);+ if (!fn.includes(marker)) return {result:{value:true}};+ return {result:{type:'object', value:Object.entries(settled).map(([key, value]) => [key, {type:typeof value, value}])}};+ };+ session.send = async () => ({});+ const mismatch = await session.perform(runId, 'field', {text:'ab', submit:true});+ assert.equal(mismatch.success, false);+ assert.equal(mismatch.verified, false);+ assert.equal(mismatch.submitted, false);+ assert.deepEqual(mismatch.readback, {compared:true, reader:'semantic'});+ // The extension reads an editor model's mismatch as known.+ const client = new FirefoxBidiClient({tabs:{sendMessage:async () => ({bidiPrepared:true, url:'https://example.com/',+ fieldMeta:{tag:'div', contentEditable:true}})}});+ client.runs.set(1, {runId, bound:true});+ client.bindRun = async () => {};+ client.perform = async () => mismatch;+ const known = await client.sendContent(1, {action:'set_field', params:{text:'ab'}});+ assert.deepEqual(known.unverified, {reason:'readback_mismatch', reader:'semantic'});+ assert.equal(known.readbackExact, true);+ settled = { verified: false, compared: false, reason: 'element_replaced' };+ assert.deepEqual((await session.perform(runId, 'field', {text:'ab'})).readback, {compared:false, reason:'element_replaced'});+ settled = { verified: true, compared: true, reader: 'exact' };+ assert.equal((await session.perform(runId, 'field', {text:'ab'})).verified, true);++ // The readback itself, against a minimal DOM.+ const declaration = scripts.find(fn => fn.includes(marker));+ const settle = (...args) => ({ ...vm.runInNewContext(`(${declaration})`)(...args) });+ const style = {display:'block', visibility:'visible', opacity:'1', contentVisibility:'visible'};+ const doc = {defaultView:{getComputedStyle:() => style}};+ const element = ({classes = [], ...props}) => ({nodeType:1, isConnected:true, shadowRoot:null, hidden:false, ownerDocument:doc,+ getAttribute:() => null, classList:{contains:name => classes.includes(name)}, children:[], childNodes:[], getRootNode:() => doc, ...props});+ const input = element({tagName:'INPUT', isContentEditable:false, value:'ab'});+ assert.deepEqual(settle(input, 'ab', '', null), {verified:true, compared:true, reader:'exact'});+ input.value = 'a';+ assert.deepEqual(settle(input, 'ab', '', null), {verified:false, compared:true, reader:'exact'});+ const paragraph = element({tagName:'P', isContentEditable:true, childNodes:[{nodeType:3, nodeValue:'Original'}]});+ const proseMirror = element({tagName:'DIV', isContentEditable:true, classes:['ProseMirror'], childNodes:[paragraph], innerText:'Original'});+ assert.deepEqual(settle(proseMirror, 'Original', 'prosemirror', null), {verified:true, compared:true, reader:'semantic'});+ assert.deepEqual(settle(proseMirror, 'First\n\nLast', 'prosemirror', null), {verified:false, compared:true, reader:'semantic'});+ const editor = element({tagName:'DIV', isContentEditable:true, innerText:'Original'});+ assert.deepEqual(settle(editor, 'First', '', null), {verified:false, compared:true, reader:'rendered'});+ const other = element({tagName:'DIV', isContentEditable:false});+ assert.deepEqual(settle(other, 'ab', '', null), {verified:false, compared:true, reader:'unknown'});+ assert.deepEqual(settle(input, 'a', '', {editor:proseMirror, document:doc, root:doc}), {verified:false, compared:false, reason:'element_replaced'});+ proseMirror.classList = {contains:() => false};+ assert.deepEqual(settle(proseMirror, 'Original', 'prosemirror', null), {verified:false, compared:false, reason:'readback_unavailable'});+});+test('validation failures preserve no-dispatch status across the native client', async () => {const session = new BidiSession(); const runId = id();session.runs.set(runId, {context:'tab'});diff --git a/test/firefox-rich-text-readback.mjs b/test/firefox-rich-text-readback.mjsindex 901eb5e64..9dd318caf 100644--- a/test/firefox-rich-text-readback.mjs+++ b/test/firefox-rich-text-readback.mjs@@ -136,10 +136,12 @@ test('Firefox: production BiDi typing uses exact DraftJS model text for empty apassert.equal(native.commands.some(command => command.params?.actions?.[0]?.actions.some(action => action.value === '\uE007')), false, 'Literal line breaks must not send Enter.');assert.equal(await page.evaluate(() => qaEditor.childNodes.length), 1, 'Append stays inside the original marked document root.');if (scenario.name === 'append inside last leaf') {- const declaration = native.declarations.find(source => source.includes('const actual = readRichTextFieldValue'));+ const declaration = native.declarations.find(source => source.includes('const settledReading = readRichTextFieldReading(el, semanticKind)'));const readback = expected => page.evaluate(({ declaration, expected }) => (0, eval)(`(${declaration})`)(qaEditor, expected), { declaration, expected });- assert.equal(await readback('First\r\n\r\nLast suffix'), true, 'CRLF normalization preserves authored document separators.');- assert.equal(await readback('First\n\n\nLast suffix'), false, 'Visual spacing cannot excuse an extra authored line break.');+ assert.deepEqual(await readback('First\r\n\r\nLast suffix'), { verified: true, compared: true, reader: 'semantic' },+ 'CRLF normalization preserves authored document separators.');+ assert.deepEqual(await readback('First\n\n\nLast suffix'), { verified: false, compared: true, reader: 'semantic' },+ 'Visual spacing cannot excuse an extra authored line break.');}});} finally { await browser.close(); }@@ -321,6 +323,7 @@ test('Firefox: unsupported DraftJS preflight sends no text, and unsupported settassert.equal(result.outcomeUnknown, true);assert.equal(result.retryable, false);assert.equal(result.submitted, false);+ assert.deepEqual(result.readback, { compared: false, reason: 'readback_unavailable' }, 'An unreadable document is not a known mismatch.');assert.equal(await page.evaluate(() => qaModel), 'aB');assert.equal(native.commands.filter(command => command.method === 'input.performActions').length, 2, 'Each character is sent once; verification must not dispatch a second edit.');});@@ -371,6 +374,7 @@ test('Firefox: DraftJS plain-text paste writes a complete multiline document exaassert.equal(result.retryable, false);assert.equal(result.submitted, false);assert.equal(await page.evaluate(() => qaModel), '');+ assert.deepEqual(result.readback, { compared: true, reader: 'semantic' }, 'The editor model holds another document: a known mismatch.');const keyDowns = native.commands.filter(command => command.method === 'input.performActions').flatMap(command => command.params.actions[0].actions.filter(action => action.type === 'keyDown').map(action => action.value));assert.equal(keyDowns.includes('\uE007'), false, 'Literal LF insertion cannot send a submit shortcut.');@@ -444,6 +448,7 @@ test('Firefox: actual ProseMirror writes one complete plain-text edit without inassert.equal(result.retryable, false);assert.equal(result.submitted, false);assert.equal(await page.evaluate(() => qaModelText), 'Original');+ assert.deepEqual(result.readback, { compared: true, reader: 'semantic' }, 'The editor model holds another document: a known mismatch.');assert.equal(native.commands.some(command => command.method === 'input.performActions'), false);assert.equal(native.declarations.filter(source => source.includes("new view.ClipboardEvent('paste'")).length, 1);});@@ -512,6 +517,7 @@ test('Firefox: ProseMirror paste retains complete document and original selectioassert.equal(result.submitted, false);assert.equal(result.outcomeUnknown, true);assert.equal(result.retryable, false);+ assert.deepEqual(result.readback, { compared: false, reason: 'readback_unavailable' }, 'An unreadable document is not a known mismatch.');assert.equal(await page.evaluate(() => qaModelText), 'First\nLast');assert.deepEqual(await page.evaluate(() => qaPastes), ['First\nLast']);assert.equal(native.commands.some(command => command.method === 'input.performActions'), false);diff --git a/test/guard-advisory-writes.mjs b/test/guard-advisory-writes.mjsindex efd3a78d1..7dba7d248 100644--- a/test/guard-advisory-writes.mjs+++ b/test/guard-advisory-writes.mjs@@ -862,6 +862,45 @@ test('firefox: a trusted BiDi mismatch in a native input or textarea is known an}});+test('firefox: the companion names the reader of a trusted BiDi readback; an editor model read as another value ends the batch', async () => {+ // An exact or semantic (DraftJS, ProseMirror) reading of another value is+ // known; a rendered one, or a field that could not be read, is not.+ const { Agent, batch } = builds.find(item => item.build === 'firefox');+ const tab = 79;+ const editor = { tag: 'div', type: 'div', contentEditable: true, name: 'post' };+ const input = { tag: 'input', type: 'text', contentEditable: false, name: 'title' };+ for (const [label, fieldMeta, readback, unverified, readbackExact, known, note] of [+ ['semantic', editor, { compared: true, reader: 'semantic' }, { reason: 'readback_mismatch', reader: 'semantic' }, true, true,+ /reads back a different value/],+ ['exact', input, { compared: true, reader: 'exact' }, { reason: 'readback_mismatch', reader: 'exact' }, true, true,+ /reads back a different value/],+ ['rendered', editor, { compared: true, reader: 'rendered' }, { reason: 'readback_mismatch', reader: 'rendered' }, false, false,+ /only reads back its rendered layout/],+ ['replaced', editor, { compared: false, reason: 'element_replaced' }, { reason: 'element_replaced', reader: 'none' }, false, false,+ /re-rendered before it could be read back/],+ ['unreadable', input, { compared: false, reason: 'readback_unavailable' }, { reason: 'readback_unavailable', reader: 'none' }, false, false,+ /editor could not be read back\./],+ ['no reader', input, { compared: true }, { reason: 'readback_unproven', reader: 'unknown' }, undefined, false,+ /could not be read back exactly/],+ ]) {+ const client = new FirefoxBidiClient({ tabs: { sendMessage: async () => ({ bidiPrepared: true, fieldMeta, url: scope.pageUrl }) } });+ client.runs.set(tab, { runId: 'run-1', bound: true });+ client.bindRun = async () => {};+ client.perform = async () => ({ success: false, dispatched: true, verified: false, submitted: false, outcomeUnknown: true,+ retryable: false, readback, error: 'Trusted typing did not match the requested value; inspect before another write.' });+ const result = await client.sendContent(tab, { target: 'content', action: 'set_field', params: { ...args } });+ assert.deepEqual(result.unverified, unverified, label);+ assert.equal(result.readbackExact, readbackExact, label);+ const agent = setup(Agent);+ const advisory = await agent._finalizeTextMutationResult(tabId, 'set_field', args, result);+ assert.equal(advisory.success, true, label);+ assert.equal(advisory.unverifiedWrite, true, label);+ assert.match(advisoryOf(advisory, 'text-write-readback-unverified')?.note || '', note, label);+ assert.equal(batch.textWriteRefuted(advisory), known, label);+ assert.equal(batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(advisory) }).failed, known, label);+ }+});+// The real content-script readback handlers, run against a minimal DOM.function functionSource(source, name) {let start = source.indexOf(`function ${name}(`);