Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Guard policy: a Chrome native field read back as another value ends the batch

Barack Sokullue8e436bcbf1b
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 36581d4c8..02036c081 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -48469,6 +48469,28 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
throw new Error('Focused typing verification returned no response.');
}
tokenConsumed = true;
+ // A native field's value read back as another value is a known
+ // mismatch: reported as cdpClient.typeText reports one, so it
+ // ends a batch before a submit runs on it.
+ if (verification.verified !== true && verification.compared === true && verification.reader === 'exact') {
+ return {
+ success: false,
+ dispatched: true,
+ verified: false,
+ mutationMayHaveOccurred: true,
+ unverified: { reason: 'readback_mismatch', reader: 'exact' },
+ readbackExact: true,
+ method: 'cdp-insert-focused',
+ focusedField: {
+ tag: prepared.tag,
+ type: prepared.type,
+ name: prepared.name,
+ contentEditable: prepared.contentEditable === true,
+ },
+ fieldMeta: verification.fieldMeta || null,
+ error: 'Text was inserted, but the focused field reads back a different value.',
+ };
+ }
const fieldIdent = `focused:${prepared.tag}|${prepared.name}`;
let warning;
@@ -48769,13 +48791,38 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
await sendTrustedCdpInput(options => cdpClient.sendCommand(tabId, 'Input.insertText',
{ text: nativeText }, '', options), { beforeSend: throwIfEarlyCdpAborted, onSent: markEarlyCdpDispatched });
throwIfEarlyCdpAborted();
- const verified = await cdpClient.verifyTextEntry(tabId, {
+ const readback = await cdpClient.verifyTextEntry(tabId, {
focused: true,
text: nativeText,
clear: !!args.clear,
beforeSignature,
+ details: true,
});
+ const verification = readback && typeof readback === 'object' ? readback : { verified: readback === true };
+ const verified = verification.verified === true;
throwIfEarlyCdpAborted();
+ // A native field's value, or an editor's own model, read back as
+ // another value is a known mismatch: reported as cdpClient.typeText
+ // reports one, so it ends a batch before a submit runs on it.
+ if (!verified && verification.found && verification.compared
+ && (verification.reader === 'exact' || verification.reader === 'semantic')) {
+ return {
+ success: false,
+ dispatched: true,
+ verified: false,
+ mutationMayHaveOccurred: true,
+ unverified: { reason: 'readback_mismatch', reader: verification.reader },
+ readbackExact: true,
+ method: 'cdp-insert-focused',
+ focusedField: {
+ tag: focus.tag,
+ type: focus.type,
+ name: focus.name,
+ contentEditable: focus.contentEditable === true,
+ },
+ error: 'Text was inserted, but the focused field reads back a different value.',
+ };
+ }
// Track field for duplicate-typing detection
const fieldIdent = `focused:${focus.tag}|${focus.name}`;
diff --git a/src/chrome/src/cdp/cdp-client.js b/src/chrome/src/cdp/cdp-client.js
index 9f3ada339..4e06afa13 100644
--- a/src/chrome/src/cdp/cdp-client.js
+++ b/src/chrome/src/cdp/cdp-client.js
@@ -5726,11 +5726,23 @@ export class CDPClient {
* fails it too. Reporting those as unproven keeps them out of the failure
* path; only a positive `true` is ever asserted.
*/
+ // Positive proof of a text write: true, else null. With `details`, an
+ // object instead: { found, verified, reader, compared }. `reader` is
+ // 'exact' for a native field's value, 'semantic' for an editor's own model
+ // (ProseMirror, DraftJS) and 'rendered' for textContent; `compared` says
+ // the readback was actually compared with the expected value, so a
+ // compared, unverified exact or semantic reading is a known mismatch.
async verifyTextEntry(tabId, {
selector = '', nodeId = null, text = '', clear = false, focused = false, beforeSignature = null,
+ details = false,
} = {}) {
await new Promise(resolve => setTimeout(resolve, 30));
const expected = String(text || '');
+ const outcome = value => (details
+ ? { found: value?.found === true, verified: value?.verified === true,
+ reader: ['exact', 'semantic', 'rendered'].includes(value?.reader) ? value.reader : 'unknown',
+ compared: value?.compared === true }
+ : (value?.verified === true ? true : null));
const verifyFunction = `function (expected, shouldClear, beforeSignature) {
const el = this;
if (!el || el.nodeType !== 1 || !el.isConnected) return { found: false, verified: false };
@@ -5759,6 +5771,9 @@ export class CDPClient {
};
return {
found: true,
+ reader: semantic !== null ? 'semantic' : (el.isContentEditable ? 'rendered' : 'exact'),
+ compared: shouldClear || (!!beforeSignature && /^\\d+:/.test(beforeSignature) && !!expected
+ && value.length <= ${TEXT_ENTRY_PROOF_MAX_CHARS}),
verified: shouldClear
? value === expected && !(expected === '' && semantic === null && el.isContentEditable
&& (el.innerText !== '' || el.querySelector('br')))
@@ -5772,7 +5787,7 @@ export class CDPClient {
await this.sendCommand(tabId, 'DOM.enable');
const resolved = await this.sendCommand(tabId, 'DOM.resolveNode', { nodeId });
objectId = resolved?.object?.objectId || null;
- if (!objectId) return null;
+ if (!objectId) return outcome(null);
const result = await this.sendCommand(tabId, 'Runtime.callFunctionOn', {
objectId,
returnByValue: true,
@@ -5783,9 +5798,9 @@ export class CDPClient {
{ value: typeof beforeSignature === 'string' ? beforeSignature : '' },
],
});
- return result?.result?.value?.verified === true ? true : null;
+ return outcome(result?.result?.value);
} catch {
- return null;
+ return outcome(null);
} finally {
if (objectId) {
this.releaseObject(tabId, objectId);
@@ -5846,6 +5861,9 @@ export class CDPClient {
};
return {
found: true,
+ reader: semantic !== null ? 'semantic' : (el.isContentEditable ? 'rendered' : 'exact'),
+ compared: shouldClear || (!!beforeSignature && /^\\d+:/.test(beforeSignature) && !!expected
+ && value.length <= ${TEXT_ENTRY_PROOF_MAX_CHARS}),
verified: shouldClear
? value === expected && !(expected === '' && semantic === null && el.isContentEditable
&& (el.innerText !== '' || el.querySelector('br')))
@@ -5853,7 +5871,7 @@ export class CDPClient {
};
})()
`).catch(() => null);
- return result?.result?.value?.verified === true ? true : null;
+ return outcome(result?.result?.value);
}
async _cleanupRichTextToolbarTargetMarker(tabId, attribute, marker, { includeClosed = false } = {}) {
@@ -6631,25 +6649,34 @@ export class CDPClient {
};
}
- const verified = await this.verifyTextEntry(tabId, {
+ const readback = await this.verifyTextEntry(tabId, {
selector,
nodeId: info.nodeId,
text: insertionText,
clear,
beforeSignature,
+ details: true,
});
+ const verification = readback && typeof readback === 'object' ? readback : { verified: readback === true };
// The text went out in both cases below; `unverified` lets the agent's
// policy report it as a failure (Strict checks) or a noted success.
// verifyTextEntry reads textContent for generic editors, which drops
- // block line breaks, so its doubt is not proof of a wrong value.
- if (verified !== true) {
+ // block line breaks, so its doubt is not proof of a wrong value. A
+ // native field's value, or an editor's own model, read back as another
+ // value is: readbackExact ends a batch before a submit runs on it.
+ if (verification.verified !== true) {
+ const exact = verification.found && verification.compared
+ && (verification.reader === 'exact' || verification.reader === 'semantic');
return {
success: false,
dispatched: true,
verified: false,
mutationMayHaveOccurred: true,
- unverified: { reason: 'readback_unproven', reader: 'unknown' },
- error: 'Text was inserted, but the complete settled field value could not be verified exactly.',
+ unverified: exact ? { reason: 'readback_mismatch', reader: verification.reader }
+ : { reason: 'readback_unproven', reader: 'unknown' },
+ ...(exact ? { readbackExact: true } : {}),
+ error: exact ? 'Text was inserted, but the field reads back a different value.'
+ : 'Text was inserted, but the complete settled field value could not be verified exactly.',
};
}
if (finishInput && !await finishInput()) {
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index c4fde701e..05472d3b3 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -6526,6 +6526,11 @@
if (!el.isConnected || record.pageUrl !== location.href) return { success: true, verified: false };
const tag = String(el.tagName || '').toUpperCase();
let verified = false;
+ // Which reader compared the value: a native field's own value is exact,
+ // an editor's textContent is its rendered text. `compared` says the
+ // value was actually compared, so a compared exact mismatch is known.
+ let reader = 'exact';
+ let compared = true;
if (tag === 'SELECT') {
const option = el.options?.[el.selectedIndex];
verified = String(el.value || '') === String(params.targetValue || '')
@@ -6533,21 +6538,24 @@
} else {
const expected = String(params.text || '');
const value = String(el.isContentEditable ? (el.textContent || '') : (el.value || ''));
+ if (el.isContentEditable) reader = 'rendered';
+ else if (!['INPUT', 'TEXTAREA'].includes(tag)) reader = 'unknown';
if (params.clear === true) {
verified = value === expected;
} else {
- verified = _richTextToolbarExactInsertion(
- null,
- value,
- expected,
- String(params.beforeSignature || ''),
- );
+ const beforeSignature = String(params.beforeSignature || '');
+ // The insertion proof gives up above its size cap without comparing.
+ compared = /^\d+:/.test(beforeSignature) && !!expected
+ && value.length <= RICH_TEXT_TOOLBAR_PROOF_MAX_CHARS;
+ verified = _richTextToolbarExactInsertion(null, value, expected, beforeSignature);
}
}
const rect = el.getBoundingClientRect();
return {
success: true,
verified,
+ reader,
+ compared,
fieldMeta: _fieldMeta(el),
rect: {
x: Math.round(rect.x),
diff --git a/test/guard-advisory-writes.mjs b/test/guard-advisory-writes.mjs
index 5300926db..efd3a78d1 100644
--- a/test/guard-advisory-writes.mjs
+++ b/test/guard-advisory-writes.mjs
@@ -675,6 +675,165 @@ test('firefox: a trusted BiDi readback mismatch is an unverified write, not debt
assert.equal(agent._uncertainTextMutations.get(tabId)?.size, 1);
});
+test('chrome: a CDP readback of a native field as another value is known and ends the batch', async () => {
+ const { Agent, batch } = builds.find(item => item.build === 'chrome');
+ const { CDPClient } = await import('../src/chrome/src/cdp/cdp-client.js');
+ // verifyTextEntry's details say which reader read the field and whether it
+ // compared the value; its default answer stays true or null.
+ const reader = new CDPClient();
+ const field = { nodeType: 1, isConnected: true, tagName: 'INPUT', isContentEditable: false, value: 'Release notes',
+ classList: { contains: () => false }, querySelector: () => null, querySelectorAll: () => [], closest: () => null,
+ getAttribute: () => null, hasAttribute: () => false, matches: () => false, childNodes: [], children: [], textContent: '' };
+ reader.evaluate = async (_tab, code) => ({ result: { value: vm.runInNewContext(code, {
+ document: { querySelector: () => field, activeElement: field, createTreeWalker: () => ({ currentNode: null, nextNode: () => null }) },
+ NodeFilter: { SHOW_ELEMENT: 1 } }) } });
+ const check = details => reader.verifyTextEntry(1, { selector: '#title', text: 'Release notes', clear: true, details });
+ assert.deepEqual(await check(true), { found: true, verified: true, reader: 'exact', compared: true });
+ field.value = 'Release note';
+ assert.deepEqual(await check(true), { found: true, verified: false, reader: 'exact', compared: true });
+ assert.equal(await check(false), null, 'the default answer is unchanged');
+ // typeText reports a compared exact or semantic mismatch as known, so the
+ // write ends its batch; a rendered readback stays unproven.
+ const typing = new CDPClient();
+ typing.resolveSelector = async () => ({ inViewport: false, hitOk: false, nodeId: null, tag: 'INPUT', x: 10, y: 20, width: 30, height: 40 });
+ typing.sendCommand = async () => ({});
+ typing.evaluate = async () => ({ result: { value: null } });
+ for (const [readerName, compared, known] of [['exact', true, true], ['semantic', true, true], ['rendered', true, false], ['exact', false, false]]) {
+ typing.verifyTextEntry = async (_tab, options) => (options.details
+ ? { found: true, verified: false, reader: readerName, compared } : null);
+ const label = `${readerName} compared=${compared}`;
+ const result = await typing.typeText(42, '#title', 'Release notes', false);
+ assert.equal(result.success, false, label);
+ assert.equal(result.dispatched, true, label);
+ assert.deepEqual(result.unverified, known ? { reason: 'readback_mismatch', reader: readerName }
+ : { reason: 'readback_unproven', reader: 'unknown' }, label);
+ assert.equal(result.readbackExact, known ? true : undefined, label);
+ const agent = setup(Agent);
+ const advisory = await agent._finalizeTextMutationResult(tabId, 'set_field', args, result);
+ assert.equal(advisory.unverifiedWrite, true, label);
+ assert.equal(batch.textWriteRefuted(advisory), known, label);
+ assert.equal(batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(advisory) }).failed, known, label);
+ }
+});
+
+test('chrome: a bound focused type_text read back as another value is known and ends the batch', async () => {
+ const { Agent, batch } = builds.find(item => item.build === 'chrome');
+ // The content script's check says which reader compared the value.
+ const source = fs.readFileSync(new URL('../src/chrome/src/content/content.js', import.meta.url), 'utf8');
+ const bindings = new Map();
+ const context = vm.createContext({ _dispatchBindings: bindings, location: { href: 'https://example.com/form' },
+ setTimeout, clearTimeout, _fieldMeta: el => ({ tag: String(el.tagName).toLowerCase() }) });
+ vm.runInContext(functionSource(source, '_verifyFocusedTypeDispatch'), context);
+ const rect = () => ({ x: 0, y: 0, width: 10, height: 10 });
+ const check = (el, params) => {
+ bindings.set('focused-token', { el, prepared: true, pageUrl: 'https://example.com/form' });
+ return context._verifyFocusedTypeDispatch({ dispatchBinding: { token: 'focused-token' }, ...params });
+ };
+ const input = { isConnected: true, tagName: 'INPUT', isContentEditable: false, value: 'Release note', getBoundingClientRect: rect };
+ const editor = { isConnected: true, tagName: 'DIV', isContentEditable: true, textContent: 'Release note', getBoundingClientRect: rect };
+ const pick = result => ({ verified: result.verified, reader: result.reader, compared: result.compared });
+ assert.deepEqual(pick(await check(input, { text: 'Release notes', clear: true })), { verified: false, reader: 'exact', compared: true });
+ assert.deepEqual(pick(await check(editor, { text: 'Release notes', clear: true })), { verified: false, reader: 'rendered', compared: true });
+ input.value = 'Release notes';
+ assert.deepEqual(pick(await check(input, { text: 'Release notes', clear: true })), { verified: true, reader: 'exact', compared: true });
+ // An append is compared through the insertion proof, within its size cap.
+ vm.runInContext(functionSource(source, '_richTextToolbarValueSignature'), context);
+ vm.runInContext(functionSource(source, '_richTextToolbarExactInsertion'), context);
+ const cap = Number(/const RICH_TEXT_TOOLBAR_PROOF_MAX_CHARS = (\d+);/.exec(source)?.[1]);
+ assert(cap > 0);
+ context.RICH_TEXT_TOOLBAR_PROOF_MAX_CHARS = cap;
+ const empty = context._richTextToolbarValueSignature('');
+ assert.deepEqual(pick(await check(input, { text: 'Release notes', beforeSignature: empty })), { verified: true, reader: 'exact', compared: true });
+ input.value = 'Release note';
+ assert.deepEqual(pick(await check(input, { text: 'Release notes', beforeSignature: empty })), { verified: false, reader: 'exact', compared: true });
+ assert.deepEqual(pick(await check(input, { text: 'Release notes', beforeSignature: '' })), { verified: false, reader: 'exact', compared: false });
+ const long = 'x'.repeat(cap);
+ input.value = `${long}y`;
+ assert.deepEqual(pick(await check(input, { text: 'y', beforeSignature: context._richTextToolbarValueSignature(long) })),
+ { verified: false, reader: 'exact', compared: false }, 'a value above the cap was not compared');
+ // The agent reports a compared exact mismatch as known, so it ends the
+ // batch; the rendered text of an editor leaves the write unproven.
+ const originalAttach = cdpClient.attach;
+ const originalSendCommand = cdpClient.sendCommand;
+ const originalSendMessage = api.tabs.sendMessage;
+ try {
+ cdpClient.attach = async () => ({ attached: true });
+ cdpClient.sendCommand = async () => ({});
+ for (const [reader, known] of [['exact', true], ['rendered', false]]) {
+ api.tabs.sendMessage = async (_tab, message) => {
+ if (message.action === 'prepare_focused_type_dispatch') {
+ return { success: true, tag: reader === 'exact' ? 'INPUT' : 'DIV', type: reader === 'exact' ? 'text' : '', name: 'title',
+ contentEditable: reader !== 'exact', beforeSignature: '12:cbf29ce484222325', rect: { x: 0, y: 0, w: 10, h: 10 } };
+ }
+ if (message.action === 'verify_focused_type_dispatch') {
+ return { success: true, verified: false, reader, compared: true, fieldMeta: { tag: reader === 'exact' ? 'input' : 'div' } };
+ }
+ return { success: true };
+ };
+ const agent = new Agent({});
+ // executeTool returns the finalized write: an advised success either way.
+ const result = await agent.executeTool(tabId, 'type_text', { text: 'Release notes' }, null,
+ { dispatchBinding: { token: 'focused-token', frameId: 0 } });
+ assert.equal(result.success, true, reader);
+ assert.equal(result.unverifiedWrite, true, reader);
+ if (known) {
+ assert.deepEqual(result.unverified, { reason: 'readback_mismatch', reader: 'exact' });
+ assert.equal(result.readbackExact, true);
+ assert.match(advisoryOf(result, 'text-write-readback-unverified')?.note || '', /reads back a different value/);
+ } else {
+ assert.notEqual(result.readbackExact, true, 'an unproven write is not refuted');
+ }
+ assert.equal(batch.textWriteRefuted(result), known, reader);
+ assert.equal(batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(result) }).failed, known, reader);
+ }
+ } finally {
+ cdpClient.attach = originalAttach;
+ cdpClient.sendCommand = originalSendCommand;
+ api.tabs.sendMessage = originalSendMessage;
+ }
+});
+
+test('chrome: an unbound focused type_text read back as another value is known and ends the batch', async () => {
+ // With no bound field, type_text types into the focused field and reads it
+ // back with verifyTextEntry's details.
+ const { Agent, batch } = builds.find(item => item.build === 'chrome');
+ const saved = Object.fromEntries(['attach', 'sendCommand', 'evaluate', 'textEntrySignature', 'verifyTextEntry']
+ .map(name => [name, cdpClient[name]]));
+ try {
+ cdpClient.attach = async () => ({ attached: true });
+ cdpClient.sendCommand = async () => ({});
+ cdpClient.evaluate = async () => ({ result: { value: { focused: true, editable: true, contentEditable: false, tag: 'INPUT',
+ type: 'text', name: 'title', value: '', rect: { x: 0, y: 0, w: 10, h: 10 } } } });
+ cdpClient.textEntrySignature = async () => '0:811c9dc5';
+ for (const [reader, compared, known] of [['exact', true, true], ['semantic', true, true], ['rendered', true, false], ['exact', false, false]]) {
+ const label = `${reader} compared=${compared}`;
+ const reads = [];
+ cdpClient.verifyTextEntry = async (_tab, options) => {
+ reads.push(options);
+ return options.details ? { found: true, verified: false, reader, compared } : null;
+ };
+ const agent = new Agent({});
+ const result = await agent.executeTool(tabId, 'type_text', { text: 'Release notes' });
+ assert.deepEqual(reads.map(options => [options.focused, options.details, options.beforeSignature]),
+ [[true, true, '0:811c9dc5']], label);
+ // executeTool returns the finalized write: an advised success either way.
+ assert.equal(result.success, true, label);
+ assert.equal(result.unverifiedWrite, true, label);
+ if (known) {
+ assert.deepEqual(result.unverified, { reason: 'readback_mismatch', reader }, label);
+ assert.equal(result.readbackExact, true, label);
+ assert.match(advisoryOf(result, 'text-write-readback-unverified')?.note || '', /reads back a different value/, label);
+ } else {
+ assert.notEqual(result.readbackExact, true, `${label}: an unproven write is not refuted`);
+ }
+ assert.equal(batch.textWriteRefuted(result), known, label);
+ assert.equal(batch.browserBatchResultStatus({ role: 'tool', content: JSON.stringify(result) }).failed, known, label);
+ }
+ } finally {
+ Object.assign(cdpClient, saved);
+ }
+});
+
test('firefox: a trusted BiDi mismatch in a native input or textarea is known and ends the batch', async () => {
// The companion reads a native field back by its exact value: a mismatch
// there means the field holds another value, so a batched submit must not