Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Guard policy: a coordinate click's own target assertion still refuses
Barack Sokullua83a249122b1
diff --git a/docs/guards.md b/docs/guards.mdindex 4638d7936..e78730841 100644--- a/docs/guards.md+++ b/docs/guards.md@@ -55,8 +55,9 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):modes and budgets (`permission_gate`); same-site, frame and task-authorityboundaries, CAPTCHA and access walls, and a person acting in the same tab(`host_check`); loop and observation limits (`budget`); calls that cannot- run as given (`infra`); and Stop, corrections and cancelled cards- (`user_control`).+ run as given (`infra`), such as a coordinate click whose own+ `expected_name` or `expected_role` no longer matches the point; and Stop,+ corrections and cancelled cards (`user_control`).- **Strict checks.** Settings β General β "Strict checks" (`strictGuards`)makes every quality guard block again, with the old forced verificationturns and the failure-only `done`.diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 02036c081..11c233810 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -19989,18 +19989,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst expectedRole = normalized(messageRecipientContext.expectedRole);const intentMismatch = (expectedName && normalized(target?.name) !== expectedName)|| (expectedRole && normalized(target?.role) !== expectedRole);- // click-coordinate-intent (quality): the model's own expected_name /- // expected_role no longer matches what the point resolves to. Without- // Strict checks the raw point is clicked, exactly as a coordinate click- // without those hints would be; recipient, deletion and publication checks- // still apply to that click.- if (intentMismatch && this._guardAdvisesFor(messageRecipientContext.guardAdvisories, 'click-coordinate-intent',- 'The point no longer resolves to the expected_name/expected_role you gave, so WebBrain could not confirm it hits the intended control. The point was clicked anyway; read the page to check what it activated.')) {- return {- result: null,- diagnostic: this._coordinateReconciliationDiagnostic(point, resolution, 'coordinate-fallback', 'intent-mismatch'),- };- }+ // click-coordinate-intent (infra): expected_name / expected_role are the+ // call's own safety assertions, which the tool contract checks before+ // dispatch. A point that no longer resolves to them may now hit another+ // control, so the call cannot run as given: nothing is clicked.if (intentMismatch) {return {result: {diff --git a/src/chrome/src/agent/guard-policy.js b/src/chrome/src/agent/guard-policy.jsindex 47f8f61e5..0b17e4ffd 100644--- a/src/chrome/src/agent/guard-policy.js+++ b/src/chrome/src/agent/guard-policy.js@@ -286,8 +286,8 @@ export const GUARDS = Object.freeze({summary: "solve_captcha arguments conflict with the detected widget (type, invisibility, enterprise flag, pageAction, rqdata, metadata), token injection has no detected frame, or the hCaptcha frame User-Agent cannot be read: refused before any paid dispatch." },"carousel-traversal-guards": { class: "quality", family: "tool", decides: "mixed", onDecided: "advise", onCannotDecide: "advise",summary: "A forward carousel scan asks for an index at or below the highest verified one (reverse order assumed only on a sure reading), or Instagram did not honor img_index / the media did not change (post-dispatch failure)." },- "click-coordinate-intent": { class: "quality", family: "tool", decides: "knows", onDecided: "advise", onCannotDecide: "advise",- summary: "A coordinate click with expected_name/expected_role whose point no longer resolves to that semantic target (resolver errors fall back to the raw point)." },+ "click-coordinate-intent": { class: "infra", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "block",+ summary: "A coordinate click whose own expected_name/expected_role assertion does not match what the point resolves to (or nothing resolves there) is refused before dispatch, as the tool contract promises. Without an assertion, a resolver error falls back to the raw point." },"click-duplicate-submit": { class: "unrequested_publication", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "ask",summary: "A second click within 45 s on a control whose caption reads as commit/publish, on the same unchanged URL, is refused (model escape _allowResubmit:true)." },"click-native-os-ui": { class: "infra", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "block",diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 5cf5d57aa..0104914f8 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -16136,18 +16136,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst expectedRole = normalized(messageRecipientContext.expectedRole);const intentMismatch = (expectedName && normalized(target?.name) !== expectedName)|| (expectedRole && normalized(target?.role) !== expectedRole);- // click-coordinate-intent (quality): the model's own expected_name /- // expected_role no longer matches what the point resolves to. Without- // Strict checks the raw point is clicked, exactly as a coordinate click- // without those hints would be; recipient, deletion and publication checks- // still apply to that click.- if (intentMismatch && this._guardAdvisesFor(messageRecipientContext.guardAdvisories, 'click-coordinate-intent',- 'The point no longer resolves to the expected_name/expected_role you gave, so WebBrain could not confirm it hits the intended control. The point was clicked anyway; read the page to check what it activated.')) {- return {- result: null,- diagnostic: this._coordinateReconciliationDiagnostic(point, resolution, 'coordinate-fallback', 'intent-mismatch'),- };- }+ // click-coordinate-intent (infra): expected_name / expected_role are the+ // call's own safety assertions, which the tool contract checks before+ // dispatch. A point that no longer resolves to them may now hit another+ // control, so the call cannot run as given: nothing is clicked.if (intentMismatch) {return {result: {diff --git a/src/firefox/src/agent/guard-policy.js b/src/firefox/src/agent/guard-policy.jsindex 47f8f61e5..0b17e4ffd 100644--- a/src/firefox/src/agent/guard-policy.js+++ b/src/firefox/src/agent/guard-policy.js@@ -286,8 +286,8 @@ export const GUARDS = Object.freeze({summary: "solve_captcha arguments conflict with the detected widget (type, invisibility, enterprise flag, pageAction, rqdata, metadata), token injection has no detected frame, or the hCaptcha frame User-Agent cannot be read: refused before any paid dispatch." },"carousel-traversal-guards": { class: "quality", family: "tool", decides: "mixed", onDecided: "advise", onCannotDecide: "advise",summary: "A forward carousel scan asks for an index at or below the highest verified one (reverse order assumed only on a sure reading), or Instagram did not honor img_index / the media did not change (post-dispatch failure)." },- "click-coordinate-intent": { class: "quality", family: "tool", decides: "knows", onDecided: "advise", onCannotDecide: "advise",- summary: "A coordinate click with expected_name/expected_role whose point no longer resolves to that semantic target (resolver errors fall back to the raw point)." },+ "click-coordinate-intent": { class: "infra", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "block",+ summary: "A coordinate click whose own expected_name/expected_role assertion does not match what the point resolves to (or nothing resolves there) is refused before dispatch, as the tool contract promises. Without an assertion, a resolver error falls back to the raw point." },"click-duplicate-submit": { class: "unrequested_publication", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "ask",summary: "A second click within 45 s on a control whose caption reads as commit/publish, on the same unchanged URL, is refused (model escape _allowResubmit:true)." },"click-native-os-ui": { class: "infra", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "block",diff --git a/test/guard-advisory-actions.mjs b/test/guard-advisory-actions.mjsindex 29cd5cb93..26bbc793d 100644--- a/test/guard-advisory-actions.mjs+++ b/test/guard-advisory-actions.mjs@@ -494,27 +494,39 @@ for (const build of ['chrome', 'firefox']) {});// ββ click-coordinate-intent βββββββββββββββββββββββββββββββββββββββββββββ+ // expected_name / expected_role are the call's own safety assertions, which+ // the tool contract checks before dispatch: a point that no longer resolves+ // to them is never clicked, by default or with Strict checks.for (const [mode, strict] of MODES) {- test(`${build}: click-coordinate-intent ${mode} mode ${strict ? 'refuses a mismatched point' : 'clicks the raw point with a note'}`, async () => {+ test(`${build}: click-coordinate-intent ${mode} mode refuses a point that no longer resolves to the expected target`, async () => {const agent = new Agent({ getVisionProvider: async () => null });agent.strictGuards = strict;agent._resolveCoordinateVisualTarget = async () => ({ success: true, documentToken: 'doc', refScopeUrl: PAGE_URL,semanticTarget: { name: 'Language', role: 'button', eligibility: 'semantic-button', ref_id: 'ref_9' } });agent._dispatchClickAx = async () => assert.fail('A mismatched semantic target is never clicked by ref');- const advisories = [];- const reconciled = await agent._reconcileCoordinateClick(nextTab++, { x: 40, y: 50 },- { expectedName: 'Send', guardAdvisories: advisories });- if (strict) {+ for (const assertion of [{ expectedName: 'Send' }, { expectedRole: 'link' }]) {+ const advisories = [];+ const reconciled = await agent._reconcileCoordinateClick(nextTab++, { x: 40, y: 50 },+ { ...assertion, guardAdvisories: advisories });+ // A null result would make the click handler dispatch the raw point.+ assert.notEqual(reconciled.result, null, JSON.stringify(assertion));assert.equal(reconciled.result.targetMismatch, true);+ assert.equal(reconciled.result.dispatched, false);assert.equal(reconciled.result.noDispatch, true);assert.deepEqual(advisories, []);- } else {- // result:null makes the click handler dispatch the raw point.- assert.equal(reconciled.result, null);- assert.equal(reconciled.diagnostic.clickPath, 'coordinate-fallback');- assert.equal(reconciled.diagnostic.fallbackReason, 'intent-mismatch');- assert.deepEqual(guardsOf(advisories), ['click-coordinate-intent']);}+ // A point that resolves to nothing cannot satisfy an assertion either.+ agent._resolveCoordinateVisualTarget = async () => ({ success: false });+ const unresolved = await agent._reconcileCoordinateClick(nextTab++, { x: 40, y: 50 }, { expectedName: 'Send', guardAdvisories: [] });+ assert.equal(unresolved.result?.targetMismatch, true);+ // A matching assertion clicks the semantic target by its ref.+ agent._resolveCoordinateVisualTarget = async () => ({ success: true, documentToken: 'doc', refScopeUrl: PAGE_URL,+ semanticTarget: { name: 'Send', role: 'button', eligibility: 'semantic-button', ref_id: 'ref_9' } });+ const clicked = [];+ agent._dispatchClickAx = async (_tab, args) => { clicked.push(args.ref_id); return { success: true, dispatched: true }; };+ const matched = await agent._reconcileCoordinateClick(nextTab++, { x: 40, y: 50 }, { expectedName: 'send', guardAdvisories: [] });+ assert.equal(matched.result.success, true);+ assert.deepEqual(clicked, ['ref_9']);});}diff --git a/test/run.js b/test/run.jsindex 4f5c1e26d..40868e2ac 100644--- a/test/run.js+++ b/test/run.js@@ -20800,7 +20800,6 @@ async function runCoordinateSemanticCase({privateRestoreAttempt = null,privateDraftPreparationAttempt = false,privateDraftBindingAttempt = null,- strictGuards = false,}) {const previousChrome = globalThis.chrome;const previousBrowser = globalThis.browser;@@ -20833,7 +20832,6 @@ async function runCoordinateSemanticCase({: { ...(previousBrowser || {}), tabs: { ...(previousBrowser?.tabs || {}), ...tabs } };try {const agent = new AgentClass({ getVisionProvider: async () => null });- agent.strictGuards = strictGuards;const tabId = label === 'chrome' ? 8811 : 8812;agent._isPdfTab = async () => false;agent._richTextToolbarToolBlock = async () => null;@@ -21095,8 +21093,6 @@ test('screenshot coordinate assertions reject an unrelated language selector witglobalKey,expectedName: 'Next',expectedRole: 'button',- // Strict checks: the guard's refusal itself is under test.- strictGuards: true,resolverResponse: {success: true,semanticTarget: {