Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix: distinguish unresolved controls from recipient failures
Barack Sokulluc1afcdd14438
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex cdb29e56a..ab87f172d 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -15341,7 +15341,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (runId) trace.recordToolCall(runId, step, {name: fnName, args: fnArgs, result: messageRecipientBlock, latencyMs: 0,});- onUpdate('warning', { message: 'Message send blocked until the active recipient is verified.' });+ onUpdate('warning', {+ message: messageRecipientBlock.reasonCode === 'message_target_unresolved'+ ? 'The requested control could not be resolved. No action was dispatched.'+ : 'Message send blocked until the active recipient is verified.',+ });if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}@@ -28067,6 +28071,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d? (probe?.reasonCode === 'message_commit_without_composer'? 'Message send blocked: this control commits a message (for example "Send" / "Send without a note") but no active composer or recipient could be verified. Changing click targeting methods will not help; ask the user to confirm the recipient or open the intended conversation first.': 'Message send blocked: no message composer is visible, so this send cannot be verified. Changing click targeting methods will not resolve this classification failure. Re-read the page to find a supported navigation or composer-opening control; if none is available, report the blocker instead of repeating the click.')+ : probe?.reasonCode === 'message_target_unresolved'+ ? 'The requested control could not be resolved in the current page or frame, so no action was dispatched. This does not establish a recipient mismatch or an attempted Send. Read fresh page evidence and use an exact visible control or fresh ref_id; do not guess hidden menu labels or repeat the same unresolved target. If the control is unavailable, report that blocker.': inconclusive? 'Message action blocked: WebBrain could not conclusively resolve the target control and active composer. Re-read the page and retry with an exact visible control or fresh ref_id.': targetdiff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 60b8050ec..1ff3a351f 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -7463,7 +7463,11 @@messageSend = args.submit === true;} else if (tool === 'click' || tool === 'click_ax') {if (!targetResolved || !target) {- return { success: true, messageSend: null, conclusive: false, identityCandidates: [] };+ return {+ success: true, messageSend: null, conclusive: false,+ targetResolved: false, reasonCode: 'message_target_unresolved',+ identityCandidates: [],+ };}const control = target.closest?.('button,[role="button"],input[type="submit"],input[type="button"],[data-action]') || target;const modal = _findTopmostBlockingModal();diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex b44d23f5a..1e3ba73bc 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -13913,7 +13913,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dif (runId) trace.recordToolCall(runId, step, {name: fnName, args: fnArgs, result: messageRecipientBlock, latencyMs: 0,});- onUpdate('warning', { message: 'Message send blocked until the active recipient is verified.' });+ onUpdate('warning', {+ message: messageRecipientBlock.reasonCode === 'message_target_unresolved'+ ? 'The requested control could not be resolved. No action was dispatched.'+ : 'Message send blocked until the active recipient is verified.',+ });if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}@@ -25547,10 +25551,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dmessageRecipientGuard: true,...(missingLinkedInComposer ? { retryable: false } : {}),reasonCode: probe?.success !== true || probe?.conclusive !== true || probe?.messageSend !== true- ? 'message_send_classification_inconclusive'+ ? (probe?.reasonCode || 'message_send_classification_inconclusive'): (target ? 'active_recipient_unverified' : 'authorized_recipient_missing'),error: missingLinkedInComposer? 'Message action blocked: no message composer is visible and this control is not a verified non-send action. Changing click targeting methods will not resolve this classification failure. Re-read the page to find a supported navigation or composer-opening control; if none is available, report the blocker instead of repeating the click.'+ : probe?.reasonCode === 'message_target_unresolved'+ ? 'The requested control could not be resolved in the current page or frame, so no action was dispatched. This does not establish a recipient mismatch or an attempted Send. Read fresh page evidence and use an exact visible control or fresh ref_id; do not guess hidden menu labels or repeat the same unresolved target. If the control is unavailable, report that blocker.': probe?.success !== true || probe?.conclusive !== true || probe?.messageSend !== true? 'Message action blocked: WebBrain could not conclusively resolve the target control and active composer. Re-read the page and retry with an exact visible control or fresh ref_id.': targetdiff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex bd32403ca..d3e99abcf 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -6585,7 +6585,11 @@messageSend = args.submit === true;} else if (tool === 'click' || tool === 'click_ax') {if (!targetResolved || !target) {- return { success: true, messageSend: null, conclusive: false, identityCandidates: [] };+ return {+ success: true, messageSend: null, conclusive: false,+ targetResolved: false, reasonCode: 'message_target_unresolved',+ identityCandidates: [],+ };}const control = target.closest?.('button,[role="button"],input[type="submit"],input[type="button"],[data-action]') || target;const modal = _findTopmostBlockingModal();diff --git a/test/message-recipient-gmail.mjs b/test/message-recipient-gmail.mjsindex bb5229c94..2a2c4609d 100644--- a/test/message-recipient-gmail.mjs+++ b/test/message-recipient-gmail.mjs@@ -29,6 +29,48 @@ async function call(page, action, params = {}) {}for (const [kind, engine] of [['chrome', chromium], ['firefox', firefox]]) {+ test(kind + ': unresolved Google account iframe controls are target errors, while Send remains blocked', async () => {+ const browser = await engine.launch();+ try {+ const page = await browser.newPage();+ const top = '<!doctype html><style>body{margin:0}a{display:block;width:150px;height:30px}iframe{width:600px;height:400px}</style>'+ + '<header><a id="account" role="button" aria-label="Google Account: Fixture Owner">Account</a></header>'+ + '<iframe src="https://ogs.google.com/u/0/widget/account"></iframe>';+ await page.route('**/*', route => route.fulfill({ contentType: 'text/html', body: route.request().url().startsWith('https://ogs.google.com/')+ ? '<!doctype html><a href="https://myaccount.google.com/">Manage your Google Account</a><a href="https://accounts.google.com/AddSession">Add another account</a>'+ : top }));+ await page.goto('https://mail.google.com/mail/u/0/#inbox');+ const runtime = kind === 'firefox' ? 'browser' : 'chrome';+ await page.addScriptTag({ content: 'window.' + runtime + '={runtime:{onMessage:{addListener(handler){window.__wb_handler=handler}}}}' });+ for (const name of ['accessibility-tree', 'rich-text-toolbar-heuristic', 'content'])+ await page.addScriptTag({ content: await readFile(new URL('../src/' + kind + '/src/content/' + name + '.js', import.meta.url), 'utf8') });+ await page.evaluate(() => {+ window.fixtureAccountOpens = 0;+ document.querySelector('#account').addEventListener('click', () => window.fixtureAccountOpens++);+ });+ const { Agent } = await import('../src/' + kind + '/src/agent/agent.js');+ const agent = new Agent({ getActive: () => ({ supportsVision: false }) });+ agent._messageRecipientContentProbe = (_, params) => call(page, 'probe_message_recipient_guard', params);+ agent._planExecutionGuards.set(1, { messaging: { target_kind: 'named', recipients: [{ identity: address, role: 'to' }] } });+ assert.equal(await agent._messageRecipientGuardBlock(1, 'click', { selector: '#account' }, page.url()), null);+ assert.equal((await call(page, 'click', { selector: '#account' })).success, true);+ assert.equal(await page.evaluate(() => window.fixtureAccountOpens), 1);+ for (const text of ['Manage your Google Account', 'Add another account', 'Send']) {+ const probe = await call(page, 'probe_message_recipient_guard', {+ tool: 'click', args: { text }, adapterName: 'gmail', supportsRecipientSets: true,+ });+ assert.equal(probe.reasonCode, 'message_target_unresolved');+ assert.equal(probe.targetResolved, false);+ const blocked = await agent._messageRecipientGuardBlock(1, 'click', { text }, page.url());+ assert.equal(blocked?.noDispatch, true);+ assert.equal(blocked?.dispatched, false);+ assert.equal(blocked?.reasonCode, 'message_target_unresolved');+ assert.match(blocked?.error, /control could not be resolved/);+ assert.doesNotMatch(blocked?.error, /not the recipient authorized|active conversation does not exactly match/);+ }+ assert.equal(await page.evaluate(() => window.fixtureAccountOpens), 1);+ } finally { await browser.close(); }+ });test(`${kind}: only observed Gmail compose drafts bypass email reservation; Send and other fields stay protected`, async () => {const browser = await engine.launch();try {