Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Guard policy: a coordinate click's own target assertion still refuses

Barack Sokullua83a249122b1
diff --git a/docs/guards.md b/docs/guards.md
index 4638d7936..e78730841 100644
--- a/docs/guards.md
+++ b/docs/guards.md
@@ -55,8 +55,9 @@ the action is wrong (`onDecided`) and when it cannot tell (`onCannotDecide`):
modes and budgets (`permission_gate`); same-site, frame and task-authority
boundaries, CAPTCHA and access walls, and a person acting in the same tab
(`host_check`); loop and observation limits (`budget`); calls that cannot
- run as given (`infra`); and Stop, corrections and cancelled cards
- (`user_control`).
+ run as given (`infra`), such as a coordinate click whose own
+ `expected_name` or `expected_role` no longer matches the point; and Stop,
+ corrections and cancelled cards (`user_control`).
- **Strict checks.** Settings β†’ General β†’ "Strict checks" (`strictGuards`)
makes every quality guard block again, with the old forced verification
turns and the failure-only `done`.
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 02036c081..11c233810 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -19989,18 +19989,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const expectedRole = normalized(messageRecipientContext.expectedRole);
const intentMismatch = (expectedName && normalized(target?.name) !== expectedName)
|| (expectedRole && normalized(target?.role) !== expectedRole);
- // click-coordinate-intent (quality): the model's own expected_name /
- // expected_role no longer matches what the point resolves to. Without
- // Strict checks the raw point is clicked, exactly as a coordinate click
- // without those hints would be; recipient, deletion and publication checks
- // still apply to that click.
- if (intentMismatch && this._guardAdvisesFor(messageRecipientContext.guardAdvisories, 'click-coordinate-intent',
- 'The point no longer resolves to the expected_name/expected_role you gave, so WebBrain could not confirm it hits the intended control. The point was clicked anyway; read the page to check what it activated.')) {
- return {
- result: null,
- diagnostic: this._coordinateReconciliationDiagnostic(point, resolution, 'coordinate-fallback', 'intent-mismatch'),
- };
- }
+ // click-coordinate-intent (infra): expected_name / expected_role are the
+ // call's own safety assertions, which the tool contract checks before
+ // dispatch. A point that no longer resolves to them may now hit another
+ // control, so the call cannot run as given: nothing is clicked.
if (intentMismatch) {
return {
result: {
diff --git a/src/chrome/src/agent/guard-policy.js b/src/chrome/src/agent/guard-policy.js
index 47f8f61e5..0b17e4ffd 100644
--- a/src/chrome/src/agent/guard-policy.js
+++ b/src/chrome/src/agent/guard-policy.js
@@ -286,8 +286,8 @@ export const GUARDS = Object.freeze({
summary: "solve_captcha arguments conflict with the detected widget (type, invisibility, enterprise flag, pageAction, rqdata, metadata), token injection has no detected frame, or the hCaptcha frame User-Agent cannot be read: refused before any paid dispatch." },
"carousel-traversal-guards": { class: "quality", family: "tool", decides: "mixed", onDecided: "advise", onCannotDecide: "advise",
summary: "A forward carousel scan asks for an index at or below the highest verified one (reverse order assumed only on a sure reading), or Instagram did not honor img_index / the media did not change (post-dispatch failure)." },
- "click-coordinate-intent": { class: "quality", family: "tool", decides: "knows", onDecided: "advise", onCannotDecide: "advise",
- summary: "A coordinate click with expected_name/expected_role whose point no longer resolves to that semantic target (resolver errors fall back to the raw point)." },
+ "click-coordinate-intent": { class: "infra", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "block",
+ summary: "A coordinate click whose own expected_name/expected_role assertion does not match what the point resolves to (or nothing resolves there) is refused before dispatch, as the tool contract promises. Without an assertion, a resolver error falls back to the raw point." },
"click-duplicate-submit": { class: "unrequested_publication", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "ask",
summary: "A second click within 45 s on a control whose caption reads as commit/publish, on the same unchanged URL, is refused (model escape _allowResubmit:true)." },
"click-native-os-ui": { class: "infra", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "block",
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 5cf5d57aa..0104914f8 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -16136,18 +16136,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const expectedRole = normalized(messageRecipientContext.expectedRole);
const intentMismatch = (expectedName && normalized(target?.name) !== expectedName)
|| (expectedRole && normalized(target?.role) !== expectedRole);
- // click-coordinate-intent (quality): the model's own expected_name /
- // expected_role no longer matches what the point resolves to. Without
- // Strict checks the raw point is clicked, exactly as a coordinate click
- // without those hints would be; recipient, deletion and publication checks
- // still apply to that click.
- if (intentMismatch && this._guardAdvisesFor(messageRecipientContext.guardAdvisories, 'click-coordinate-intent',
- 'The point no longer resolves to the expected_name/expected_role you gave, so WebBrain could not confirm it hits the intended control. The point was clicked anyway; read the page to check what it activated.')) {
- return {
- result: null,
- diagnostic: this._coordinateReconciliationDiagnostic(point, resolution, 'coordinate-fallback', 'intent-mismatch'),
- };
- }
+ // click-coordinate-intent (infra): expected_name / expected_role are the
+ // call's own safety assertions, which the tool contract checks before
+ // dispatch. A point that no longer resolves to them may now hit another
+ // control, so the call cannot run as given: nothing is clicked.
if (intentMismatch) {
return {
result: {
diff --git a/src/firefox/src/agent/guard-policy.js b/src/firefox/src/agent/guard-policy.js
index 47f8f61e5..0b17e4ffd 100644
--- a/src/firefox/src/agent/guard-policy.js
+++ b/src/firefox/src/agent/guard-policy.js
@@ -286,8 +286,8 @@ export const GUARDS = Object.freeze({
summary: "solve_captcha arguments conflict with the detected widget (type, invisibility, enterprise flag, pageAction, rqdata, metadata), token injection has no detected frame, or the hCaptcha frame User-Agent cannot be read: refused before any paid dispatch." },
"carousel-traversal-guards": { class: "quality", family: "tool", decides: "mixed", onDecided: "advise", onCannotDecide: "advise",
summary: "A forward carousel scan asks for an index at or below the highest verified one (reverse order assumed only on a sure reading), or Instagram did not honor img_index / the media did not change (post-dispatch failure)." },
- "click-coordinate-intent": { class: "quality", family: "tool", decides: "knows", onDecided: "advise", onCannotDecide: "advise",
- summary: "A coordinate click with expected_name/expected_role whose point no longer resolves to that semantic target (resolver errors fall back to the raw point)." },
+ "click-coordinate-intent": { class: "infra", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "block",
+ summary: "A coordinate click whose own expected_name/expected_role assertion does not match what the point resolves to (or nothing resolves there) is refused before dispatch, as the tool contract promises. Without an assertion, a resolver error falls back to the raw point." },
"click-duplicate-submit": { class: "unrequested_publication", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "ask",
summary: "A second click within 45 s on a control whose caption reads as commit/publish, on the same unchanged URL, is refused (model escape _allowResubmit:true)." },
"click-native-os-ui": { class: "infra", family: "tool", decides: "knows", onDecided: "block", onCannotDecide: "block",
diff --git a/test/guard-advisory-actions.mjs b/test/guard-advisory-actions.mjs
index 29cd5cb93..26bbc793d 100644
--- a/test/guard-advisory-actions.mjs
+++ b/test/guard-advisory-actions.mjs
@@ -494,27 +494,39 @@ for (const build of ['chrome', 'firefox']) {
});
// ── click-coordinate-intent ─────────────────────────────────────────────
+ // expected_name / expected_role are the call's own safety assertions, which
+ // the tool contract checks before dispatch: a point that no longer resolves
+ // to them is never clicked, by default or with Strict checks.
for (const [mode, strict] of MODES) {
- test(`${build}: click-coordinate-intent ${mode} mode ${strict ? 'refuses a mismatched point' : 'clicks the raw point with a note'}`, async () => {
+ test(`${build}: click-coordinate-intent ${mode} mode refuses a point that no longer resolves to the expected target`, async () => {
const agent = new Agent({ getVisionProvider: async () => null });
agent.strictGuards = strict;
agent._resolveCoordinateVisualTarget = async () => ({ success: true, documentToken: 'doc', refScopeUrl: PAGE_URL,
semanticTarget: { name: 'Language', role: 'button', eligibility: 'semantic-button', ref_id: 'ref_9' } });
agent._dispatchClickAx = async () => assert.fail('A mismatched semantic target is never clicked by ref');
- const advisories = [];
- const reconciled = await agent._reconcileCoordinateClick(nextTab++, { x: 40, y: 50 },
- { expectedName: 'Send', guardAdvisories: advisories });
- if (strict) {
+ for (const assertion of [{ expectedName: 'Send' }, { expectedRole: 'link' }]) {
+ const advisories = [];
+ const reconciled = await agent._reconcileCoordinateClick(nextTab++, { x: 40, y: 50 },
+ { ...assertion, guardAdvisories: advisories });
+ // A null result would make the click handler dispatch the raw point.
+ assert.notEqual(reconciled.result, null, JSON.stringify(assertion));
assert.equal(reconciled.result.targetMismatch, true);
+ assert.equal(reconciled.result.dispatched, false);
assert.equal(reconciled.result.noDispatch, true);
assert.deepEqual(advisories, []);
- } else {
- // result:null makes the click handler dispatch the raw point.
- assert.equal(reconciled.result, null);
- assert.equal(reconciled.diagnostic.clickPath, 'coordinate-fallback');
- assert.equal(reconciled.diagnostic.fallbackReason, 'intent-mismatch');
- assert.deepEqual(guardsOf(advisories), ['click-coordinate-intent']);
}
+ // A point that resolves to nothing cannot satisfy an assertion either.
+ agent._resolveCoordinateVisualTarget = async () => ({ success: false });
+ const unresolved = await agent._reconcileCoordinateClick(nextTab++, { x: 40, y: 50 }, { expectedName: 'Send', guardAdvisories: [] });
+ assert.equal(unresolved.result?.targetMismatch, true);
+ // A matching assertion clicks the semantic target by its ref.
+ agent._resolveCoordinateVisualTarget = async () => ({ success: true, documentToken: 'doc', refScopeUrl: PAGE_URL,
+ semanticTarget: { name: 'Send', role: 'button', eligibility: 'semantic-button', ref_id: 'ref_9' } });
+ const clicked = [];
+ agent._dispatchClickAx = async (_tab, args) => { clicked.push(args.ref_id); return { success: true, dispatched: true }; };
+ const matched = await agent._reconcileCoordinateClick(nextTab++, { x: 40, y: 50 }, { expectedName: 'send', guardAdvisories: [] });
+ assert.equal(matched.result.success, true);
+ assert.deepEqual(clicked, ['ref_9']);
});
}
diff --git a/test/run.js b/test/run.js
index 4f5c1e26d..40868e2ac 100644
--- a/test/run.js
+++ b/test/run.js
@@ -20800,7 +20800,6 @@ async function runCoordinateSemanticCase({
privateRestoreAttempt = null,
privateDraftPreparationAttempt = false,
privateDraftBindingAttempt = null,
- strictGuards = false,
}) {
const previousChrome = globalThis.chrome;
const previousBrowser = globalThis.browser;
@@ -20833,7 +20832,6 @@ async function runCoordinateSemanticCase({
: { ...(previousBrowser || {}), tabs: { ...(previousBrowser?.tabs || {}), ...tabs } };
try {
const agent = new AgentClass({ getVisionProvider: async () => null });
- agent.strictGuards = strictGuards;
const tabId = label === 'chrome' ? 8811 : 8812;
agent._isPdfTab = async () => false;
agent._richTextToolbarToolBlock = async () => null;
@@ -21095,8 +21093,6 @@ test('screenshot coordinate assertions reject an unrelated language selector wit
globalKey,
expectedName: 'Next',
expectedRole: 'button',
- // Strict checks: the guard's refusal itself is under test.
- strictGuards: true,
resolverResponse: {
success: true,
semanticTarget: {