Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Guard policy: a Chrome native field read back as another value ends the batch
Barack Sokullue8e436bcbf1b
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 36581d4c8..02036c081 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -48469,6 +48469,28 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dthrow new Error('Focused typing verification returned no response.');}tokenConsumed = true;+ // A native field's value read back as another value is a known+ // mismatch: reported as cdpClient.typeText reports one, so it+ // ends a batch before a submit runs on it.+ if (verification.verified !== true && verification.compared === true && verification.reader === 'exact') {+ return {+ success: false,+ dispatched: true,+ verified: false,+ mutationMayHaveOccurred: true,+ unverified: { reason: 'readback_mismatch', reader: 'exact' },+ readbackExact: true,+ method: 'cdp-insert-focused',+ focusedField: {+ tag: prepared.tag,+ type: prepared.type,+ name: prepared.name,+ contentEditable: prepared.contentEditable === true,+ },+ fieldMeta: verification.fieldMeta || null,+ error: 'Text was inserted, but the focused field reads back a different value.',+ };+ }const fieldIdent = `focused:${prepared.tag}|${prepared.name}`;let warning;@@ -48769,13 +48791,38 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dawait sendTrustedCdpInput(options => cdpClient.sendCommand(tabId, 'Input.insertText',{ text: nativeText }, '', options), { beforeSend: throwIfEarlyCdpAborted, onSent: markEarlyCdpDispatched });throwIfEarlyCdpAborted();- const verified = await cdpClient.verifyTextEntry(tabId, {+ const readback = await cdpClient.verifyTextEntry(tabId, {focused: true,text: nativeText,clear: !!args.clear,beforeSignature,+ details: true,});+ const verification = readback && typeof readback === 'object' ? readback : { verified: readback === true };+ const verified = verification.verified === true;throwIfEarlyCdpAborted();+ // A native field's value, or an editor's own model, read back as+ // another value is a known mismatch: reported as cdpClient.typeText+ // reports one, so it ends a batch before a submit runs on it.+ if (!verified && verification.found && verification.compared+ && (verification.reader === 'exact' || verification.reader === 'semantic')) {+ return {+ success: false,+ dispatched: true,+ verified: false,+ mutationMayHaveOccurred: true,+ unverified: { reason: 'readback_mismatch', reader: verification.reader },+ readbackExact: true,+ method: 'cdp-insert-focused',+ focusedField: {+ tag: focus.tag,+ type: focus.type,+ name: focus.name,+ contentEditable: focus.contentEditable === true,+ },+ error: 'Text was inserted, but the focused field reads back a different value.',+ };+ }// Track field for duplicate-typing detectionconst fieldIdent = `focused:${focus.tag}|${focus.name}`;diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.jsindex 9f3ada339..4e06afa13 100644--- a/src/chrome/src/cdp/cdp-client.js+++ b/src/chrome/src/cdp/cdp-client.js@@ -5726,11 +5726,23 @@ export class CDPClient {* fails it too. Reporting those as unproven keeps them out of the failure* path; only a positive `true` is ever asserted.*/+ // Positive proof of a text write: true, else null. With `details`, an+ // object instead: { found, verified, reader, compared }. `reader` is+ // 'exact' for a native field's value, 'semantic' for an editor's own model+ // (ProseMirror, DraftJS) and 'rendered' for textContent; `compared` says+ // the readback was actually compared with the expected value, so a+ // compared, unverified exact or semantic reading is a known mismatch.async verifyTextEntry(tabId, {selector = '', nodeId = null, text = '', clear = false, focused = false, beforeSignature = null,+ details = false,} = {}) {await new Promise(resolve => setTimeout(resolve, 30));const expected = String(text || '');+ const outcome = value => (details+ ? { found: value?.found === true, verified: value?.verified === true,+ reader: ['exact', 'semantic', 'rendered'].includes(value?.reader) ? value.reader : 'unknown',+ compared: value?.compared === true }+ : (value?.verified === true ? true : null));const verifyFunction = `function (expected, shouldClear, beforeSignature) {const el = this;if (!el || el.nodeType !== 1 || !el.isConnected) return { found: false, verified: false };@@ -5759,6 +5771,9 @@ export class CDPClient {};return {found: true,+ reader: semantic !== null ? 'semantic' : (el.isContentEditable ? 'rendered' : 'exact'),+ compared: shouldClear || (!!beforeSignature && /^\\d+:/.test(beforeSignature) && !!expected+ && value.length <= ${TEXT_ENTRY_PROOF_MAX_CHARS}),verified: shouldClear? value === expected && !(expected === '' && semantic === null && el.isContentEditable&& (el.innerText !== '' || el.querySelector('br')))@@ -5772,7 +5787,7 @@ export class CDPClient {await this.sendCommand(tabId, 'DOM.enable');const resolved = await this.sendCommand(tabId, 'DOM.resolveNode', { nodeId });objectId = resolved?.object?.objectId || null;- if (!objectId) return null;+ if (!objectId) return outcome(null);const result = await this.sendCommand(tabId, 'Runtime.callFunctionOn', {objectId,returnByValue: true,@@ -5783,9 +5798,9 @@ export class CDPClient {{ value: typeof beforeSignature === 'string' ? beforeSignature : '' },],});- return result?.result?.value?.verified === true ? true : null;+ return outcome(result?.result?.value);} catch {- return null;+ return outcome(null);} finally {if (objectId) {this.releaseObject(tabId, objectId);@@ -5846,6 +5861,9 @@ export class CDPClient {};return {found: true,+ reader: semantic !== null ? 'semantic' : (el.isContentEditable ? 'rendered' : 'exact'),+ compared: shouldClear || (!!beforeSignature && /^\\d+:/.test(beforeSignature) && !!expected+ && value.length <= ${TEXT_ENTRY_PROOF_MAX_CHARS}),verified: shouldClear? value === expected && !(expected === '' && semantic === null && el.isContentEditable&& (el.innerText !== '' || el.querySelector('br')))@@ -5853,7 +5871,7 @@ export class CDPClient {};})()`).catch(() => null);- return result?.result?.value?.verified === true ? true : null;+ return outcome(result?.result?.value);}async _cleanupRichTextToolbarTargetMarker(tabId, attribute, marker, { includeClosed = false } = {}) {@@ -6631,25 +6649,34 @@ export class CDPClient {};}- const verified = await this.verifyTextEntry(tabId, {+ const readback = await this.verifyTextEntry(tabId, {selector,nodeId: info.nodeId,text: insertionText,clear,beforeSignature,+ details: true,});+ const verification = readback && typeof readback === 'object' ? readback : { verified: readback === true };// The text went out in both cases below; `unverified` lets the agent's// policy report it as a failure (Strict checks) or a noted success.// verifyTextEntry reads textContent for generic editors, which drops- // block line breaks, so its doubt is not proof of a wrong value.- if (verified !== true) {+ // block line breaks, so its doubt is not proof of a wrong value. A+ // native field's value, or an editor's own model, read back as another+ // value is: readbackExact ends a batch before a submit runs on it.+ if (verification.verified !== true) {+ const exact = verification.found && verification.compared+ && (verification.reader === 'exact' || verification.reader === 'semantic');return {success: false,dispatched: true,verified: false,mutationMayHaveOccurred: true,- unverified: { reason: 'readback_unproven', reader: 'unknown' },- error: 'Text was inserted, but the complete settled field value could not be verified exactly.',+ unverified: exact ? { reason: 'readback_mismatch', reader: verification.reader }+ : { reason: 'readback_unproven', reader: 'unknown' },+ ...(exact ? { readbackExact: true } : {}),+ error: exact ? 'Text was inserted, but the field reads back a different value.'+ : 'Text was inserted, but the complete settled field value could not be verified exactly.',};}if (finishInput && !await finishInput()) {diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex c4fde701e..05472d3b3 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -6526,6 +6526,11 @@if (!el.isConnected || record.pageUrl !== location.href) return { success: true, verified: false };const tag = String(el.tagName || '').toUpperCase();let verified = false;+ // Which reader compared the value: a native field's own value is exact,+ // an editor's textContent is its rendered text. `compared` says the+ // value was actually compared, so a compared exact mismatch is known.+ let reader = 'exact';+ let compared = true;if (tag === 'SELECT') {const option = el.options?.[el.selectedIndex];verified = String(el.value || '') === String(params.targetValue || '')@@ -6533,21 +6538,24 @@} else {const expected = String(params.text || '');const value = String(el.isContentEditable ? (el.textContent || '') : (el.value || ''));+ if (el.isContentEditable) reader = 'rendered';+ else if (!['INPUT', 'TEXTAREA'].includes(tag)) reader = 'unknown';if (params.clear === true) {verified = value === expected;} else {- verified = _richTextToolbarExactInsertion(- null,- value,- expected,- String(params.beforeSignature || ''),- );+ const beforeSignature = String(params.beforeSignature || '');+ // The insertion proof gives up above its size cap without comparing.+ compared = /^\d+:/.test(beforeSignature) && !!expected+ && value.length <= RICH_TEXT_TOOLBAR_PROOF_MAX_CHARS;+ verified = _richTextToolbarExactInsertion(null, value, expected, beforeSignature);}}const rect = el.getBoundingClientRect();return {success: true,verified,+ reader,+ compared,fieldMeta: _fieldMeta(el),rect: {x: Math.round(rect.x),diff --git a/test/guard-advisory-writes.mjs b/test/guard-advisory-writes.mjsindex 5300926db..efd3a78d1 100644--- a/test/guard-advisory-writes.mjs+++ b/test/guard-advisory-writes.mjs@@ -675,6 +675,165 @@ test('firefox: a trusted BiDi readback mismatch is an unverified write, not debtassert.equal(agent._uncertainTextMutations.get(tabId)?.size, 1);});+test('chrome: a CDP readback of a native field as another value is known and ends the batch', async () => {+ const { Agent, batch } = builds.find(item => item.build === 'chrome');+ const { CDPClient } = await import('../src/chrome/src/cdp/cdp-client.js');+ // verifyTextEntry's details say which reader read the field and whether it+ // compared the value; its default answer stays true or null.+ const reader = new CDPClient();+ const field = { nodeType: 1, isConnected: true, tagName: 'INPUT', isContentEditable: false, value: 'Release notes',+ classList: { contains: () => false }, querySelector: () => null, querySelectorAll: () => [], closest: () => null,+ getAttribute: () => null, hasAttribute: () => false, matches: () => false, childNodes: [], children: [], textContent: '' };+ reader.evaluate = async (_tab, code) => ({ result: { value: vm.runInNewContext(code, {+ document: { querySelector: () => field, activeElement: field, createTreeWalker: () => ({ currentNode: null, nextNode: () => null }) },+ NodeFilter: { SHOW_ELEMENT: 1 } }) } });+ const check = details => reader.verifyTextEntry(1, { selector: '#title', text: 'Release notes', clear: true, details });+ assert.deepEqual(await check(true), { found: true, verified: true, reader: 'exact', compared: true });+ field.value = 'Release note';+ assert.deepEqual(await check(true), { found: true, verified: false, reader: 'exact', compared: true });+ assert.equal(await check(false), null, 'the default answer is unchanged');+ // typeText reports a compared exact or semantic mismatch as known, so the+ // write ends its batch; a rendered readback stays unproven.+ const typing = new CDPClient();+ typing.resolveSelector = async () => ({ inViewport: false, hitOk: false, nodeId: null, tag: 'INPUT', x: 10, y: 20, width: 30, height: 40 });+ typing.sendCommand = async () => ({});+ typing.evaluate = async () => ({ result: { value: null } });+ for (const [readerName, compared, known] of [['exact', true, true], ['semantic', true, true], ['rendered', true, false], ['exact', false, false]]) {+ typing.verifyTextEntry = async (_tab, options) => (options.details+ ? { found: true, verified: false, reader: readerName, compared } : null);+ const label = `${readerName} compared=${compared}`;+ const result = await typing.typeText(42, '#title', 'Release notes', false);+ assert.equal(result.success, false, label);+ assert.equal(result.dispatched, true, label);+ assert.deepEqual(result.unverified, known ? { reason: 'readback_mismatch', reader: readerName }+ : { reason: 'readback_unproven', reader: 'unknown' }, label);+ assert.equal(result.readbackExact, known ? true : undefined, label);+ const agent = setup(Agent);+ const advisory = await agent._finalizeTextMutationResult(tabId, 'set_field', args, result);+ assert.equal(advisory.unverifiedWrite, true, label);+ assert.equal(batch.textWriteRefuted(advisory), known, label);+ assert.equal(batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(advisory) }).failed, known, label);+ }+});++test('chrome: a bound focused type_text read back as another value is known and ends the batch', async () => {+ const { Agent, batch } = builds.find(item => item.build === 'chrome');+ // The content script's check says which reader compared the value.+ const source = fs.readFileSync(new URL('../src/chrome/src/content/content.js', import.meta.url), 'utf8');+ const bindings = new Map();+ const context = vm.createContext({ _dispatchBindings: bindings, location: { href: 'https://example.com/form' },+ setTimeout, clearTimeout, _fieldMeta: el => ({ tag: String(el.tagName).toLowerCase() }) });+ vm.runInContext(functionSource(source, '_verifyFocusedTypeDispatch'), context);+ const rect = () => ({ x: 0, y: 0, width: 10, height: 10 });+ const check = (el, params) => {+ bindings.set('focused-token', { el, prepared: true, pageUrl: 'https://example.com/form' });+ return context._verifyFocusedTypeDispatch({ dispatchBinding: { token: 'focused-token' }, ...params });+ };+ const input = { isConnected: true, tagName: 'INPUT', isContentEditable: false, value: 'Release note', getBoundingClientRect: rect };+ const editor = { isConnected: true, tagName: 'DIV', isContentEditable: true, textContent: 'Release note', getBoundingClientRect: rect };+ const pick = result => ({ verified: result.verified, reader: result.reader, compared: result.compared });+ assert.deepEqual(pick(await check(input, { text: 'Release notes', clear: true })), { verified: false, reader: 'exact', compared: true });+ assert.deepEqual(pick(await check(editor, { text: 'Release notes', clear: true })), { verified: false, reader: 'rendered', compared: true });+ input.value = 'Release notes';+ assert.deepEqual(pick(await check(input, { text: 'Release notes', clear: true })), { verified: true, reader: 'exact', compared: true });+ // An append is compared through the insertion proof, within its size cap.+ vm.runInContext(functionSource(source, '_richTextToolbarValueSignature'), context);+ vm.runInContext(functionSource(source, '_richTextToolbarExactInsertion'), context);+ const cap = Number(/const RICH_TEXT_TOOLBAR_PROOF_MAX_CHARS = (\d+);/.exec(source)?.[1]);+ assert(cap > 0);+ context.RICH_TEXT_TOOLBAR_PROOF_MAX_CHARS = cap;+ const empty = context._richTextToolbarValueSignature('');+ assert.deepEqual(pick(await check(input, { text: 'Release notes', beforeSignature: empty })), { verified: true, reader: 'exact', compared: true });+ input.value = 'Release note';+ assert.deepEqual(pick(await check(input, { text: 'Release notes', beforeSignature: empty })), { verified: false, reader: 'exact', compared: true });+ assert.deepEqual(pick(await check(input, { text: 'Release notes', beforeSignature: '' })), { verified: false, reader: 'exact', compared: false });+ const long = 'x'.repeat(cap);+ input.value = `${long}y`;+ assert.deepEqual(pick(await check(input, { text: 'y', beforeSignature: context._richTextToolbarValueSignature(long) })),+ { verified: false, reader: 'exact', compared: false }, 'a value above the cap was not compared');+ // The agent reports a compared exact mismatch as known, so it ends the+ // batch; the rendered text of an editor leaves the write unproven.+ const originalAttach = cdpClient.attach;+ const originalSendCommand = cdpClient.sendCommand;+ const originalSendMessage = api.tabs.sendMessage;+ try {+ cdpClient.attach = async () => ({ attached: true });+ cdpClient.sendCommand = async () => ({});+ for (const [reader, known] of [['exact', true], ['rendered', false]]) {+ api.tabs.sendMessage = async (_tab, message) => {+ if (message.action === 'prepare_focused_type_dispatch') {+ return { success: true, tag: reader === 'exact' ? 'INPUT' : 'DIV', type: reader === 'exact' ? 'text' : '', name: 'title',+ contentEditable: reader !== 'exact', beforeSignature: '12:cbf29ce484222325', rect: { x: 0, y: 0, w: 10, h: 10 } };+ }+ if (message.action === 'verify_focused_type_dispatch') {+ return { success: true, verified: false, reader, compared: true, fieldMeta: { tag: reader === 'exact' ? 'input' : 'div' } };+ }+ return { success: true };+ };+ const agent = new Agent({});+ // executeTool returns the finalized write: an advised success either way.+ const result = await agent.executeTool(tabId, 'type_text', { text: 'Release notes' }, null,+ { dispatchBinding: { token: 'focused-token', frameId: 0 } });+ assert.equal(result.success, true, reader);+ assert.equal(result.unverifiedWrite, true, reader);+ if (known) {+ assert.deepEqual(result.unverified, { reason: 'readback_mismatch', reader: 'exact' });+ assert.equal(result.readbackExact, true);+ assert.match(advisoryOf(result, 'text-write-readback-unverified')?.note || '', /reads back a different value/);+ } else {+ assert.notEqual(result.readbackExact, true, 'an unproven write is not refuted');+ }+ assert.equal(batch.textWriteRefuted(result), known, reader);+ assert.equal(batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(result) }).failed, known, reader);+ }+ } finally {+ cdpClient.attach = originalAttach;+ cdpClient.sendCommand = originalSendCommand;+ api.tabs.sendMessage = originalSendMessage;+ }+});++test('chrome: an unbound focused type_text read back as another value is known and ends the batch', async () => {+ // With no bound field, type_text types into the focused field and reads it+ // back with verifyTextEntry's details.+ const { Agent, batch } = builds.find(item => item.build === 'chrome');+ const saved = Object.fromEntries(['attach', 'sendCommand', 'evaluate', 'textEntrySignature', 'verifyTextEntry']+ .map(name => [name, cdpClient[name]]));+ try {+ cdpClient.attach = async () => ({ attached: true });+ cdpClient.sendCommand = async () => ({});+ cdpClient.evaluate = async () => ({ result: { value: { focused: true, editable: true, contentEditable: false, tag: 'INPUT',+ type: 'text', name: 'title', value: '', rect: { x: 0, y: 0, w: 10, h: 10 } } } });+ cdpClient.textEntrySignature = async () => '0:811c9dc5';+ for (const [reader, compared, known] of [['exact', true, true], ['semantic', true, true], ['rendered', true, false], ['exact', false, false]]) {+ const label = `${reader} compared=${compared}`;+ const reads = [];+ cdpClient.verifyTextEntry = async (_tab, options) => {+ reads.push(options);+ return options.details ? { found: true, verified: false, reader, compared } : null;+ };+ const agent = new Agent({});+ const result = await agent.executeTool(tabId, 'type_text', { text: 'Release notes' });+ assert.deepEqual(reads.map(options => [options.focused, options.details, options.beforeSignature]),+ [[true, true, '0:811c9dc5']], label);+ // executeTool returns the finalized write: an advised success either way.+ assert.equal(result.success, true, label);+ assert.equal(result.unverifiedWrite, true, label);+ if (known) {+ assert.deepEqual(result.unverified, { reason: 'readback_mismatch', reader }, label);+ assert.equal(result.readbackExact, true, label);+ assert.match(advisoryOf(result, 'text-write-readback-unverified')?.note || '', /reads back a different value/, label);+ } else {+ assert.notEqual(result.readbackExact, true, `${label}: an unproven write is not refuted`);+ }+ assert.equal(batch.textWriteRefuted(result), known, label);+ assert.equal(batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(result) }).failed, known, label);+ }+ } finally {+ Object.assign(cdpClient, saved);+ }+});+test('firefox: a trusted BiDi mismatch in a native input or textarea is known and ends the batch', async () => {// The companion reads a native field back by its exact value: a mismatch// there means the field holds another value, so a batched submit must not