Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Guard generic click submissions with observed bot memory targets

Barack Sokullufbb495a72db3
diff --git a/package.json b/package.json
index 53b7ca9d1..091617ab6 100644
--- a/package.json
+++ b/package.json
@@ -19,7 +19,7 @@
"test:attachment-drop": "node test/attachment-drop.mjs",
"test:workflow-editor": "node --test test/workflow-editor.mjs",
"test:security": "node test/security/injection-corpus.mjs",
- "test:cloud-bridge-approval": "node --test test/cloud-bridge-approval.mjs test/bot-memory.mjs test/bot-memory-registries.mjs test/bot-memory-cloud-controller.mjs test/cloud-conversation-isolation.mjs test/cloud-run-outcomes.mjs test/cloud-act-completion.mjs test/cloud-model-recovery.mjs test/private-results.mjs",
+ "test:cloud-bridge-approval": "node --test test/cloud-bridge-approval.mjs test/bot-memory.mjs test/bot-memory-click-guard.mjs test/bot-memory-registries.mjs test/bot-memory-cloud-controller.mjs test/cloud-conversation-isolation.mjs test/cloud-run-outcomes.mjs test/cloud-act-completion.mjs test/cloud-model-recovery.mjs test/private-results.mjs",
"test:discord": "node --test test/discord-guard.mjs",
"test:toolbar-guard": "node test/rich-text-toolbar-guard.mjs",
"test:pdf-read": "node test/pdf-read.mjs",
@@ -87,8 +87,8 @@
"test:chat-draft-recovery": "node --test test/native-chat-pre-enter-draft.mjs test/chat-draft-dispatch-ownership.mjs && node test/quality/native-chat-presubmit-mv3.mjs",
"test:upload-owner": "node --test test/page-monitor-upload-owner.mjs",
"test:shard:quick": "npm run test:cms && npm run test:markdown && npm run test:generative-media && npm run test:systemone && npm run test:systemone:fast && node --test test/browser-dialogs.mjs test/capsolver-cloud-broker.mjs test/captcha-providers.mjs test/captcha-weighted-fallback.mjs test/captcha-hcaptcha-providers.mjs test/captcha-native-providers.mjs test/captcha-runtime-policy.mjs test/captcha-callback-bridge.mjs test/captcha-bridge-registration.mjs test/captcha-aws-recovery.mjs test/captcha-documented-contracts.mjs test/huggingface-signup-recovery.mjs test/trace-full-export.mjs test/feedback-traces.mjs && npm run test:firefox-bidi && npm run test:billing-notice && npm run test:compass-quota && npm run test:provider-limits && npm run test:provider-catalog && npm run test:provider-instances:ui && npm run test:openrouter-capabilities && npm run test:anthropic && npm run test:accessibility-tree-benchmark && npm run test:toolbar-guard && npm run test:cloud-bridge-approval && npm run test:discord && npm run test:pdf-read && npm run test:pdf-selection && npm run test:social-contract && npm run test:safesocial && npm run test:build-unpacked && npm run test:attachment-drop && npm run test:workflow-editor && node --test test/profile-sync-stats.mjs && node --test test/memcode-recall.mjs",
- "test:shard:runtime-unit": "node --test test/agent-lifecycle.mjs test/runtime-startup.mjs test/run-ui-scheduled-lifecycle.mjs test/transport-deadlines.mjs test/cdp-worker-restart.mjs test/bot-memory.mjs test/bot-memory-registries.mjs test/bot-memory-cloud-controller.mjs test/cdp-clear-receipt.mjs test/cdp-field-submit-recovery.mjs test/conversation-persistence.mjs test/context-live-tool-group.mjs test/integration-races.mjs test/watch-completion-checkpoint.mjs test/runtime-model-identity.mjs test/provider-message-privacy.mjs test/runtime-task-authority.mjs test/planner-repair-authority.mjs test/runtime-progress-continuation.mjs test/chat-resume-chain.mjs test/chat-resume-completion.mjs test/native-chat-execution-ledger.mjs test/chat-send-dispatch-result.mjs test/chat-reply-dedup-race.mjs test/memcode-lifecycle.mjs test/runtime-peer-review.mjs test/read-continuation-schema.mjs test/completion-submit-acknowledgement.mjs test/completion-live-status.mjs test/completion-iframe-submit.mjs test/completion-semantic-commit.mjs test/completion-diagnostics.mjs test/mastodon-publication.mjs test/calculator.mjs test/chat-recovery-protocol.mjs test/chat-pending-reconciliation.mjs test/semantic-page-retry.mjs test/pdf-detection.mjs test/runtime-lifecycle.mjs test/collection-read-progress.mjs test/network-lifecycle.mjs test/text-write-cancellation.mjs test/trusted-field-preparation.mjs test/draftjs-text.mjs test/rich-text-frameworks.mjs test/firefox-rich-text-readback.mjs test/contenteditable-beforeinput.mjs test/trusted-field-preparation-ownership.mjs test/chat-steering.mjs test/steering-revision-recovery.mjs test/pending-clarification-hydration.mjs test/permission-steering-wakeup.mjs test/pure-calculation-reply.mjs test/pure-calculation-tools.mjs test/page-feedback.mjs test/page-feedback-progress.mjs test/page-feedback-policy.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/telegram-read-preparation-agent.mjs test/media-download-agent-handler.mjs test/code-scanning-url-recipient.mjs test/apocalypse-comm-parsing.mjs test/apocalypse-scalar-decoding.mjs test/bitgpu-chat-security.mjs test/social-media-url-upgrades.mjs test/downloaded-file-saved-bytes.mjs test/downloaded-file-local-probe.mjs test/formless-verification-recovery.mjs test/file-upload-accept.mjs test/file-upload-accept-agent.mjs test/workflow-route-generic-completion.mjs test/completion-current-execution-evidence.mjs",
- "test:shard:browser-a": "node test/bot-memory-dom.mjs && node test/ui-content-lifecycle.mjs && node --test test/native-telegram-chat.mjs test/native-telegram-recipient.mjs test/telegram-native-baseline.mjs test/telegram-native-latest.mjs test/native-chat-pre-enter-draft.mjs test/chat-draft-dispatch-ownership.mjs test/native-telegram-alias-preparation.mjs test/telegram-read-preparation-content.mjs test/queued-composer-ownership.mjs test/onboarding-provider-ready.mjs test/workflow-native-targets.mjs test/native-whatsapp-chat.mjs test/native-chat-context-identity.mjs test/native-whatsapp-contact.mjs test/whatsapp-native-bootstrap.mjs test/whatsapp-drawer-identity.mjs test/whatsapp-read-latest.mjs test/whatsapp-read-preparation.mjs test/whatsapp-chat-completion.mjs test/whatsapp-chat-restart.mjs test/chat-goal-lifecycle.mjs test/chat-completion.mjs test/chat-generic-dispatch.mjs test/completion-live-status-browser.mjs test/native-confirmation-visibility.mjs test/native-button-preparation.mjs test/read-continuation-browser.mjs test/iframe-preparation-focus.mjs test/contenteditable-literal.mjs test/native-chat-empty-editor.mjs test/native-chat-trusted-clear.mjs test/chat-steering-ui.mjs test/assistant-message-visibility-ui.mjs test/page-monitor-ui.mjs test/page-monitor-dialog.mjs test/page-monitor-dialog-native.mjs test/page-monitor-inline-post.mjs test/bluesky-composer-context.mjs test/page-monitor-anchored.mjs test/page-monitor-upload-owner.mjs test/page-monitor-native-preparation.mjs test/native-preparation-cold.mjs test/page-monitor-navigation.mjs test/page-monitor-main-owner.mjs test/page-monitor-collateral-controls.mjs test/accessibility-tree-scoped-budget.mjs test/ax-reference-origin.mjs test/page-monitor-decorative-hit.mjs test/social-ui-navigation.mjs test/dropdown-menu-inventory.mjs test/page-monitor-passive-progress.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs test/social-media-downloader-reinjection.mjs test/upload-file-local-state-guidance.mjs",
+ "test:shard:runtime-unit": "node --test test/agent-lifecycle.mjs test/runtime-startup.mjs test/run-ui-scheduled-lifecycle.mjs test/transport-deadlines.mjs test/cdp-worker-restart.mjs test/bot-memory.mjs test/bot-memory-click-guard.mjs test/bot-memory-registries.mjs test/bot-memory-cloud-controller.mjs test/cdp-clear-receipt.mjs test/cdp-field-submit-recovery.mjs test/conversation-persistence.mjs test/context-live-tool-group.mjs test/integration-races.mjs test/watch-completion-checkpoint.mjs test/runtime-model-identity.mjs test/provider-message-privacy.mjs test/runtime-task-authority.mjs test/planner-repair-authority.mjs test/runtime-progress-continuation.mjs test/chat-resume-chain.mjs test/chat-resume-completion.mjs test/native-chat-execution-ledger.mjs test/chat-send-dispatch-result.mjs test/chat-reply-dedup-race.mjs test/memcode-lifecycle.mjs test/runtime-peer-review.mjs test/read-continuation-schema.mjs test/completion-submit-acknowledgement.mjs test/completion-live-status.mjs test/completion-iframe-submit.mjs test/completion-semantic-commit.mjs test/completion-diagnostics.mjs test/mastodon-publication.mjs test/calculator.mjs test/chat-recovery-protocol.mjs test/chat-pending-reconciliation.mjs test/semantic-page-retry.mjs test/pdf-detection.mjs test/runtime-lifecycle.mjs test/collection-read-progress.mjs test/network-lifecycle.mjs test/text-write-cancellation.mjs test/trusted-field-preparation.mjs test/draftjs-text.mjs test/rich-text-frameworks.mjs test/firefox-rich-text-readback.mjs test/contenteditable-beforeinput.mjs test/trusted-field-preparation-ownership.mjs test/chat-steering.mjs test/steering-revision-recovery.mjs test/pending-clarification-hydration.mjs test/permission-steering-wakeup.mjs test/pure-calculation-reply.mjs test/pure-calculation-tools.mjs test/page-feedback.mjs test/page-feedback-progress.mjs test/page-feedback-policy.mjs test/page-feedback-transport-races.mjs test/page-feedback-recovery.mjs test/telegram-read-preparation-agent.mjs test/media-download-agent-handler.mjs test/code-scanning-url-recipient.mjs test/apocalypse-comm-parsing.mjs test/apocalypse-scalar-decoding.mjs test/bitgpu-chat-security.mjs test/social-media-url-upgrades.mjs test/downloaded-file-saved-bytes.mjs test/downloaded-file-local-probe.mjs test/formless-verification-recovery.mjs test/file-upload-accept.mjs test/file-upload-accept-agent.mjs test/workflow-route-generic-completion.mjs test/completion-current-execution-evidence.mjs",
+ "test:shard:browser-a": "node test/bot-memory-dom.mjs && node test/bot-memory-click-dom.mjs && node test/ui-content-lifecycle.mjs && node --test test/native-telegram-chat.mjs test/native-telegram-recipient.mjs test/telegram-native-baseline.mjs test/telegram-native-latest.mjs test/native-chat-pre-enter-draft.mjs test/chat-draft-dispatch-ownership.mjs test/native-telegram-alias-preparation.mjs test/telegram-read-preparation-content.mjs test/queued-composer-ownership.mjs test/onboarding-provider-ready.mjs test/workflow-native-targets.mjs test/native-whatsapp-chat.mjs test/native-chat-context-identity.mjs test/native-whatsapp-contact.mjs test/whatsapp-native-bootstrap.mjs test/whatsapp-drawer-identity.mjs test/whatsapp-read-latest.mjs test/whatsapp-read-preparation.mjs test/whatsapp-chat-completion.mjs test/whatsapp-chat-restart.mjs test/chat-goal-lifecycle.mjs test/chat-completion.mjs test/chat-generic-dispatch.mjs test/completion-live-status-browser.mjs test/native-confirmation-visibility.mjs test/native-button-preparation.mjs test/read-continuation-browser.mjs test/iframe-preparation-focus.mjs test/contenteditable-literal.mjs test/native-chat-empty-editor.mjs test/native-chat-trusted-clear.mjs test/chat-steering-ui.mjs test/assistant-message-visibility-ui.mjs test/page-monitor-ui.mjs test/page-monitor-dialog.mjs test/page-monitor-dialog-native.mjs test/page-monitor-inline-post.mjs test/bluesky-composer-context.mjs test/page-monitor-anchored.mjs test/page-monitor-upload-owner.mjs test/page-monitor-native-preparation.mjs test/native-preparation-cold.mjs test/page-monitor-navigation.mjs test/page-monitor-main-owner.mjs test/page-monitor-collateral-controls.mjs test/accessibility-tree-scoped-budget.mjs test/ax-reference-origin.mjs test/page-monitor-decorative-hit.mjs test/social-ui-navigation.mjs test/dropdown-menu-inventory.mjs test/page-monitor-passive-progress.mjs test/media-download-binding.mjs test/media-download-capture-integration.mjs test/social-media-downloader-reinjection.mjs test/upload-file-local-state-guidance.mjs",
"test:shard:browser-b1": "node test/message-recipient-upload.mjs && node --test test/message-recipient-whatsapp.mjs && node test/quality/integration-mv3.mjs --full-form && node test/quality/literal-calculate-midrun-mv3.mjs && node test/quality/telegram-cold-preparation-mv3.mjs && node test/quality/native-chat-mv3.mjs && node test/quality/native-chat-presubmit-mv3.mjs && node test/quality/cloud-bot-memory-mv3.mjs",
"test:shard:browser-b2": "node test/quality/whatsapp-contact-matrix.mjs && node test/quality/pdf-detection-mv3.mjs && node test/quality/download-upload-mv3.mjs && node test/quality/downloaded-file-saved-bytes-mv3.mjs && node test/quality/file-upload-accept-mv3.mjs && node test/quality/bulk-resume-mv3.mjs",
"test:shard:browser-c": "node test/quality/literal-frame-mv3.mjs && node test/quality/mastodon-publication-mv3.mjs && node test/quality/mastodon-deletion-mv3.mjs && node test/quality/anchored-action-mv3.mjs && node test/quality/inline-social-composer-mv3.mjs && node test/quality/bluesky-composer-mv3.mjs && node test/quality/dropdown-menu-matrix.mjs && node test/quality/mastodon-deletion-navigation-mv3.mjs && node test/quality/semantic-navigation-matrix.mjs && node test/quality/workflow-replay-mv3.mjs && node test/quality/generative-media-mv3.mjs && node test/quality/cdp-worker-restart-mv3.mjs",
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 95c1baf02..6ca5bd0a6 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -37749,7 +37749,15 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
if (coordinates.block) return coordinates.block;
args = coordinates.args;
let prepared;
- try { prepared = await memory.prepare(tabId, name, args || {}, { signal: linked.signal }); }
+ if (name === 'click' && args?.index != null) {
+ const index = typeof args.index === 'number' || (typeof args.index === 'string' && /^\d+$/.test(args.index)) ? Number(args.index) : NaN;
+ if (!Number.isSafeInteger(index) || index < 0) return { success: false, botMemory: true, noDispatch: true, dispatched: false, error: 'Choose a valid observed element index.' };
+ args = { ...args, index };
+ }
+ const detected = executionContext?.detectedSubmitAction || executionContext?.detectedSubmit;
+ const detectedSubmit = detected?.isSubmit === true;
+ const consequentialSubmit = detectedSubmit && classifySubmitRisk(detected, detected.url).risk !== SubmitRisk.LOW_RISK_SEARCH;
+ try { prepared = await memory.prepare(tabId, name, args || {}, { signal: linked.signal, detectedSubmit, consequentialSubmit }); }
catch (error) {
this._throwIfAborted(linked.signal);
return { success: false, botMemory: true, noDispatch: true, dispatched: false, error: error.message };
diff --git a/src/chrome/src/agent/bot-memory.js b/src/chrome/src/agent/bot-memory.js
index 0fbf1595d..cf69846d1 100644
--- a/src/chrome/src/agent/bot-memory.js
+++ b/src/chrome/src/agent/bot-memory.js
@@ -22,7 +22,7 @@ const OBSERVATION_TOOLS = new Set([
// and recipient checks still run before the guarded inner dispatch.
const DELEGATED_GUARD_TOOLS = new Set(['chat_send']);
const SECRET_KEY = /password|passwd|secret|token|authorization|cookie|credential|api.?key|otp|pin|binary|image|base64/i;
-const SAFE_ARGUMENTS = new Set(['ref_id', 'selector', 'text', 'key', 'keys', 'url', 'urlFilter', 'method', 'submit', 'clear', 'checked', 'x', 'y', 'tool_id', 'name', 'action', 'intent']);
+const SAFE_ARGUMENTS = new Set(['ref_id', 'selector', 'text', 'index', 'exact', 'matchIndex', 'textMatch', 'key', 'keys', 'url', 'urlFilter', 'method', 'submit', 'clear', 'checked', 'x', 'y', 'tool_id', 'name', 'action', 'intent']);
const compact = (value, max = 512) => String(value ?? '').slice(0, max);
const randomId = () => globalThis.crypto.randomUUID();
@@ -177,7 +177,9 @@ export function observeMemoryPageInPage(args = {}) {
return text;
};
let element = null;
- if (typeof args.ref_id === 'string' && typeof window.__wb_ax_lookup === 'function') {
+ if (args.toolName === 'click' && typeof window.__wb_resolve_click_target_for_submit_probe === 'function') {
+ try { element = window.__wb_resolve_click_target_for_submit_probe(args); } catch { /* unavailable or ambiguous target */ }
+ } else if (typeof args.ref_id === 'string' && typeof window.__wb_ax_lookup === 'function') {
try { element = window.__wb_ax_lookup(args.ref_id); } catch { /* stale ref */ }
} else if (typeof args.selector === 'string') {
try {
@@ -192,10 +194,19 @@ export function observeMemoryPageInPage(args = {}) {
const label = clip(safeText(element?.getAttribute?.('aria-label') || element?.innerText || element?.getAttribute?.('title') || ''), 512);
const form = element?.form || element?.closest?.('form');
let scope = form || element;
- for (let depth = 0; scope && depth < 4 && !scope.querySelector?.('textarea,[contenteditable=true],input:not([type=hidden])'); depth++) scope = scope.parentElement;
+ const draftSelector = 'textarea,[contenteditable=true],input:not([type=hidden]):not([type=submit]):not([type=button]):not([type=reset]):not([type=file]):not([type=checkbox]):not([type=radio])';
+ let draftFields = [];
+ for (let depth = 0; scope && depth < 10 && scope !== document.body && scope !== document.documentElement; depth++) {
+ const fields = [...(scope.matches?.(draftSelector) ? [scope] : []), ...Array.from(scope.querySelectorAll?.(draftSelector) || [])]
+ .filter(field => !sensitive(metadata(field)));
+ // A form binds its fields. A form-less composer needs one nearest editor;
+ // a shared page container with multiple editors cannot identify this draft.
+ if (form || fields.length === 1) { draftFields = fields; break; }
+ if (fields.length > 1) break;
+ scope = scope.parentElement || scope.getRootNode?.()?.host;
+ }
const drafts = [];
- for (const field of Array.from(scope?.querySelectorAll?.('textarea,[contenteditable=true],input:not([type=hidden])') || []).slice(0, 12)) {
- if (sensitive(metadata(field))) continue;
+ for (const field of draftFields.slice(0, 12)) {
const value = clip(field.isContentEditable ? field.innerText : field.value, 2048);
if (value) drafts.push(value);
}
@@ -366,7 +377,7 @@ export class BotMemorySession {
return ['about:blank', 'about:newtab', 'about:home', 'chrome://newtab/',
'chrome://new-tab-page/', 'edge://newtab/', 'chrome-search://local-ntp/local-ntp.html'].includes(current);
}
- async prepare(tabId, name, args, { signal = null } = {}) {
+ async prepare(tabId, name, args, { signal = null, consequentialSubmit = false, detectedSubmit = false } = {}) {
if (this.pending.size && (name === 'done_json' || (name === 'done' && !['partial', 'failed'].includes(args?.outcome)))) {
return { allowed: false, reason: 'A dispatched action still has an uncertain outcome. Read its result and use bot_memory_verify before claiming success, or finish with an honest partial/failed outcome.' };
}
@@ -379,6 +390,10 @@ export class BotMemorySession {
if (await this.canBootstrapNavigation(tabId, name, args)) return { allowed: true, readOnly: true, bootstrapNavigation: true };
const observation = await this.observation(tabId, name, args);
if (!observation.url || !observation.targets.length) return { allowed: false, reason: 'Observe the target page before acting.' };
+ if ((name === 'click' || consequentialSubmit) && observation.elementFacts?.found !== true) {
+ return { allowed: false, reason: 'The action target could not be resolved. Read the page and choose an observed control.' };
+ }
+ if (detectedSubmit) observation.elementFacts = { ...observation.elementFacts, isSubmit: true, consequentialSubmit: consequentialSubmit === true };
const requestId = randomId();
const safeArgs = sanitizeMemoryArguments(args, observation);
const response = await this.call('prepare', { requestId,
@@ -387,7 +402,9 @@ export class BotMemorySession {
elementFacts: observation.elementFacts },
}, { signal });
if (response.allowed !== true) return { allowed: false, reason: compact(response.reason || 'This action is already completed or cannot be safely identified.') };
- if (response.readOnly === true) return { allowed: true, readOnly: true };
+ if (response.readOnly === true) return consequentialSubmit
+ ? { allowed: false, reason: 'A consequential submission requires a matching action reservation.' }
+ : { allowed: true, readOnly: true };
const reservation = response.reservation;
if (!reservation || typeof reservation.attemptId !== 'string' || reservation.requestId !== requestId
|| !this.config.policy.actions.some(action => action.key === reservation.actionKey)
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 9772b12ea..9dd4250f7 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -754,12 +754,92 @@
}
window.__wb_resolve_click_target_for_submit_probe = function resolveClickTargetForSubmitProbe(params = {}) {
- if (params?.index == null) return null;
- const index = Number(params.index);
- if (!Number.isInteger(index) || index < 0) return null;
- return queryInteractiveForToolIndex()[index] || null;
+ return resolveObservedClickTarget(params);
};
+ // Pure counterpart of clickElement's dispatch resolution. Observation must
+ // never scroll, focus, select an option or dispatch while identifying a target.
+ function resolveObservedClickTarget(params = {}) {
+ if (params.selector && /:contains\(|:has-text\(/.test(params.selector)) return null;
+ if (params.text) {
+ if (typeof params.text !== 'string') return null;
+ const needle = params.text.toLowerCase();
+ const scope = _findTopmostModal() || document;
+ const modes = params.textMatch ? [params.textMatch] : ['exact', 'prefix', 'contains'];
+ if (modes.some(mode => !['exact', 'prefix', 'contains'].includes(mode))) return null;
+ const candidates = _clickTextCandidates(scope);
+ let matches = [];
+ let usedMode = modes[0];
+ for (const mode of modes) {
+ usedMode = mode;
+ matches = candidates.filter(({ txt }) => mode === 'exact' ? txt === needle
+ : mode === 'prefix' ? txt.startsWith(needle)
+ : needle.length >= 3 ? txt.includes(needle)
+ : txt === needle || txt.startsWith(needle + ' ') || txt.endsWith(' ' + needle) || txt.includes(' ' + needle + ' '));
+ if (matches.length) break;
+ }
+ const labelTarget = label => {
+ let target = label.htmlFor ? document.getElementById(label.htmlFor) : null;
+ target ||= label.querySelector('input,textarea,select');
+ if (!target && label.nextElementSibling) {
+ const sibling = label.nextElementSibling;
+ target = /^(INPUT|TEXTAREA|SELECT)$/.test(sibling.tagName) ? sibling : sibling.querySelector('input,textarea,select');
+ }
+ return target;
+ };
+ if (!matches.length) {
+ const labelMap = new Map();
+ for (const label of scope.querySelectorAll('label')) {
+ const text = (label.innerText || '').trim().toLowerCase();
+ if (!text) continue;
+ const target = (label.htmlFor ? document.getElementById(label.htmlFor) : null) || label.querySelector('input,textarea,select');
+ if (target) labelMap.set(text, target);
+ }
+ for (const [text, target] of labelMap) if (needle === text || text.startsWith(needle) || text.includes(needle)) return target;
+ return null;
+ }
+ if (matches.length > 1) {
+ const owners = new Map();
+ for (const match of matches) {
+ const owner = _resolveInteractiveAncestor(match.e) || match.e;
+ if (!owners.has(owner) || (match.primary === true && owners.get(owner).primary !== true)) owners.set(owner, match);
+ }
+ matches = [...owners.values()];
+ }
+ if (matches.length > 1) {
+ const preferred = _preferredPrimaryTextClickMatch(matches);
+ if (preferred) matches = [preferred];
+ }
+ if (matches.length > 1) {
+ const interactive = matches.filter(({ e }) => _isInteractive(e));
+ if (interactive.length === 1) matches = interactive;
+ else {
+ const eligible = (interactive.length > 1 ? interactive : matches).filter(({ e }) => _isEligibleTextClickTarget(e));
+ if (eligible.length === 1) matches = eligible;
+ }
+ }
+ if (matches.length !== 1) return null;
+ const selected = matches[0].e;
+ const resolved = selected.tagName === 'LABEL' ? labelTarget(selected) || selected : selected;
+ // Dispatch may choose a select option instead of a non-exact text
+ // target. Until that local selection is separately bound, do not
+ // authorize the earlier button as though it were the clicked control.
+ if (resolved instanceof HTMLSelectElement || usedMode !== 'exact') {
+ const selects = resolved instanceof HTMLSelectElement ? [resolved] : scope.querySelectorAll('select');
+ for (const select of selects) for (const option of select.options) {
+ if ([option.textContent, option.label, option.value].some(value => String(value || '').trim().toLowerCase() === needle.trim())) return null;
+ }
+ }
+ return _resolveInteractiveAncestor(resolved) || resolved;
+ }
+ if (params.selector) return safeIndexedQuerySelector(params.selector, params.matchIndex).element;
+ if (params.index != null) {
+ const index = typeof params.index === 'number' || (typeof params.index === 'string' && /^\d+$/.test(params.index)) ? Number(params.index) : NaN;
+ return Number.isSafeInteger(index) && index >= 0 ? queryInteractiveForToolIndex()[index] || null : null;
+ }
+ return Number.isFinite(params.x) && Number.isFinite(params.y) ? _shadowAwareElementFromPoint(params.x, params.y) : null;
+ }
+
/**
* Get a simplified DOM snapshot for the agent.
*/
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index a4f90d217..c06b52b05 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -34494,7 +34494,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (coordinates.block) return coordinates.block;
args = coordinates.args;
let prepared;
- try { prepared = await memory.prepare(tabId, name, args || {}, { signal: linked.signal }); }
+ if (name === 'click' && args?.index != null) {
+ const index = typeof args.index === 'number' || (typeof args.index === 'string' && /^\d+$/.test(args.index)) ? Number(args.index) : NaN;
+ if (!Number.isSafeInteger(index) || index < 0) return { success: false, botMemory: true, noDispatch: true, dispatched: false, error: 'Choose a valid observed element index.' };
+ args = { ...args, index };
+ }
+ const detected = executionContext?.detectedSubmitAction || executionContext?.detectedSubmit;
+ const detectedSubmit = detected?.isSubmit === true;
+ const consequentialSubmit = detectedSubmit && classifySubmitRisk(detected, detected.url).risk !== SubmitRisk.LOW_RISK_SEARCH;
+ try { prepared = await memory.prepare(tabId, name, args || {}, { signal: linked.signal, detectedSubmit, consequentialSubmit }); }
catch (error) {
this._throwIfAborted(linked.signal);
return { success: false, botMemory: true, noDispatch: true, dispatched: false, error: error.message };
diff --git a/src/firefox/src/agent/bot-memory.js b/src/firefox/src/agent/bot-memory.js
index 0fbf1595d..cf69846d1 100644
--- a/src/firefox/src/agent/bot-memory.js
+++ b/src/firefox/src/agent/bot-memory.js
@@ -22,7 +22,7 @@ const OBSERVATION_TOOLS = new Set([
// and recipient checks still run before the guarded inner dispatch.
const DELEGATED_GUARD_TOOLS = new Set(['chat_send']);
const SECRET_KEY = /password|passwd|secret|token|authorization|cookie|credential|api.?key|otp|pin|binary|image|base64/i;
-const SAFE_ARGUMENTS = new Set(['ref_id', 'selector', 'text', 'key', 'keys', 'url', 'urlFilter', 'method', 'submit', 'clear', 'checked', 'x', 'y', 'tool_id', 'name', 'action', 'intent']);
+const SAFE_ARGUMENTS = new Set(['ref_id', 'selector', 'text', 'index', 'exact', 'matchIndex', 'textMatch', 'key', 'keys', 'url', 'urlFilter', 'method', 'submit', 'clear', 'checked', 'x', 'y', 'tool_id', 'name', 'action', 'intent']);
const compact = (value, max = 512) => String(value ?? '').slice(0, max);
const randomId = () => globalThis.crypto.randomUUID();
@@ -177,7 +177,9 @@ export function observeMemoryPageInPage(args = {}) {
return text;
};
let element = null;
- if (typeof args.ref_id === 'string' && typeof window.__wb_ax_lookup === 'function') {
+ if (args.toolName === 'click' && typeof window.__wb_resolve_click_target_for_submit_probe === 'function') {
+ try { element = window.__wb_resolve_click_target_for_submit_probe(args); } catch { /* unavailable or ambiguous target */ }
+ } else if (typeof args.ref_id === 'string' && typeof window.__wb_ax_lookup === 'function') {
try { element = window.__wb_ax_lookup(args.ref_id); } catch { /* stale ref */ }
} else if (typeof args.selector === 'string') {
try {
@@ -192,10 +194,19 @@ export function observeMemoryPageInPage(args = {}) {
const label = clip(safeText(element?.getAttribute?.('aria-label') || element?.innerText || element?.getAttribute?.('title') || ''), 512);
const form = element?.form || element?.closest?.('form');
let scope = form || element;
- for (let depth = 0; scope && depth < 4 && !scope.querySelector?.('textarea,[contenteditable=true],input:not([type=hidden])'); depth++) scope = scope.parentElement;
+ const draftSelector = 'textarea,[contenteditable=true],input:not([type=hidden]):not([type=submit]):not([type=button]):not([type=reset]):not([type=file]):not([type=checkbox]):not([type=radio])';
+ let draftFields = [];
+ for (let depth = 0; scope && depth < 10 && scope !== document.body && scope !== document.documentElement; depth++) {
+ const fields = [...(scope.matches?.(draftSelector) ? [scope] : []), ...Array.from(scope.querySelectorAll?.(draftSelector) || [])]
+ .filter(field => !sensitive(metadata(field)));
+ // A form binds its fields. A form-less composer needs one nearest editor;
+ // a shared page container with multiple editors cannot identify this draft.
+ if (form || fields.length === 1) { draftFields = fields; break; }
+ if (fields.length > 1) break;
+ scope = scope.parentElement || scope.getRootNode?.()?.host;
+ }
const drafts = [];
- for (const field of Array.from(scope?.querySelectorAll?.('textarea,[contenteditable=true],input:not([type=hidden])') || []).slice(0, 12)) {
- if (sensitive(metadata(field))) continue;
+ for (const field of draftFields.slice(0, 12)) {
const value = clip(field.isContentEditable ? field.innerText : field.value, 2048);
if (value) drafts.push(value);
}
@@ -366,7 +377,7 @@ export class BotMemorySession {
return ['about:blank', 'about:newtab', 'about:home', 'chrome://newtab/',
'chrome://new-tab-page/', 'edge://newtab/', 'chrome-search://local-ntp/local-ntp.html'].includes(current);
}
- async prepare(tabId, name, args, { signal = null } = {}) {
+ async prepare(tabId, name, args, { signal = null, consequentialSubmit = false, detectedSubmit = false } = {}) {
if (this.pending.size && (name === 'done_json' || (name === 'done' && !['partial', 'failed'].includes(args?.outcome)))) {
return { allowed: false, reason: 'A dispatched action still has an uncertain outcome. Read its result and use bot_memory_verify before claiming success, or finish with an honest partial/failed outcome.' };
}
@@ -379,6 +390,10 @@ export class BotMemorySession {
if (await this.canBootstrapNavigation(tabId, name, args)) return { allowed: true, readOnly: true, bootstrapNavigation: true };
const observation = await this.observation(tabId, name, args);
if (!observation.url || !observation.targets.length) return { allowed: false, reason: 'Observe the target page before acting.' };
+ if ((name === 'click' || consequentialSubmit) && observation.elementFacts?.found !== true) {
+ return { allowed: false, reason: 'The action target could not be resolved. Read the page and choose an observed control.' };
+ }
+ if (detectedSubmit) observation.elementFacts = { ...observation.elementFacts, isSubmit: true, consequentialSubmit: consequentialSubmit === true };
const requestId = randomId();
const safeArgs = sanitizeMemoryArguments(args, observation);
const response = await this.call('prepare', { requestId,
@@ -387,7 +402,9 @@ export class BotMemorySession {
elementFacts: observation.elementFacts },
}, { signal });
if (response.allowed !== true) return { allowed: false, reason: compact(response.reason || 'This action is already completed or cannot be safely identified.') };
- if (response.readOnly === true) return { allowed: true, readOnly: true };
+ if (response.readOnly === true) return consequentialSubmit
+ ? { allowed: false, reason: 'A consequential submission requires a matching action reservation.' }
+ : { allowed: true, readOnly: true };
const reservation = response.reservation;
if (!reservation || typeof reservation.attemptId !== 'string' || reservation.requestId !== requestId
|| !this.config.policy.actions.some(action => action.key === reservation.actionKey)
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index 1d84ed23f..410e193ce 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -1258,12 +1258,92 @@
}
window.__wb_resolve_click_target_for_submit_probe = function resolveClickTargetForSubmitProbe(params = {}) {
- if (params?.index == null) return null;
- const index = Number(params.index);
- if (!Number.isInteger(index) || index < 0) return null;
- return queryInteractiveForToolIndex()[index] || null;
+ return resolveObservedClickTarget(params);
};
+ // Pure counterpart of clickElement's dispatch resolution. Observation must
+ // never scroll, focus, select an option or dispatch while identifying a target.
+ function resolveObservedClickTarget(params = {}) {
+ if (params.selector && /:contains\(|:has-text\(/.test(params.selector)) return null;
+ if (params.text) {
+ if (typeof params.text !== 'string') return null;
+ const needle = params.text.toLowerCase();
+ const scope = _findTopmostModal() || document;
+ const modes = params.textMatch ? [params.textMatch] : ['exact', 'prefix', 'contains'];
+ if (modes.some(mode => !['exact', 'prefix', 'contains'].includes(mode))) return null;
+ const candidates = _clickTextCandidates(scope);
+ let matches = [];
+ let usedMode = modes[0];
+ for (const mode of modes) {
+ usedMode = mode;
+ matches = candidates.filter(({ txt }) => mode === 'exact' ? txt === needle
+ : mode === 'prefix' ? txt.startsWith(needle)
+ : needle.length >= 3 ? txt.includes(needle)
+ : txt === needle || txt.startsWith(needle + ' ') || txt.endsWith(' ' + needle) || txt.includes(' ' + needle + ' '));
+ if (matches.length) break;
+ }
+ const labelTarget = label => {
+ let target = label.htmlFor ? document.getElementById(label.htmlFor) : null;
+ target ||= label.querySelector('input,textarea,select');
+ if (!target && label.nextElementSibling) {
+ const sibling = label.nextElementSibling;
+ target = /^(INPUT|TEXTAREA|SELECT)$/.test(sibling.tagName) ? sibling : sibling.querySelector('input,textarea,select');
+ }
+ return target;
+ };
+ if (!matches.length) {
+ const labelMap = new Map();
+ for (const label of scope.querySelectorAll('label')) {
+ const text = (label.innerText || '').trim().toLowerCase();
+ if (!text) continue;
+ const target = (label.htmlFor ? document.getElementById(label.htmlFor) : null) || label.querySelector('input,textarea,select');
+ if (target) labelMap.set(text, target);
+ }
+ for (const [text, target] of labelMap) if (needle === text || text.startsWith(needle) || text.includes(needle)) return target;
+ return null;
+ }
+ if (matches.length > 1) {
+ const owners = new Map();
+ for (const match of matches) {
+ const owner = _resolveInteractiveAncestor(match.e) || match.e;
+ if (!owners.has(owner) || (match.primary === true && owners.get(owner).primary !== true)) owners.set(owner, match);
+ }
+ matches = [...owners.values()];
+ }
+ if (matches.length > 1) {
+ const preferred = _preferredPrimaryTextClickMatch(matches);
+ if (preferred) matches = [preferred];
+ }
+ if (matches.length > 1) {
+ const interactive = matches.filter(({ e }) => _isInteractive(e));
+ if (interactive.length === 1) matches = interactive;
+ else {
+ const eligible = (interactive.length > 1 ? interactive : matches).filter(({ e }) => _isEligibleTextClickTarget(e));
+ if (eligible.length === 1) matches = eligible;
+ }
+ }
+ if (matches.length !== 1) return null;
+ const selected = matches[0].e;
+ const resolved = selected.tagName === 'LABEL' ? labelTarget(selected) || selected : selected;
+ // Dispatch may choose a select option instead of a non-exact text
+ // target. Until that local selection is separately bound, do not
+ // authorize the earlier button as though it were the clicked control.
+ if (resolved instanceof HTMLSelectElement || usedMode !== 'exact') {
+ const selects = resolved instanceof HTMLSelectElement ? [resolved] : scope.querySelectorAll('select');
+ for (const select of selects) for (const option of select.options) {
+ if ([option.textContent, option.label, option.value].some(value => String(value || '').trim().toLowerCase() === needle.trim())) return null;
+ }
+ }
+ return _resolveInteractiveAncestor(resolved) || resolved;
+ }
+ if (params.selector) return safeIndexedQuerySelector(params.selector, params.matchIndex).element;
+ if (params.index != null) {
+ const index = typeof params.index === 'number' || (typeof params.index === 'string' && /^\d+$/.test(params.index)) ? Number(params.index) : NaN;
+ return Number.isSafeInteger(index) && index >= 0 ? queryInteractiveForToolIndex()[index] || null : null;
+ }
+ return Number.isFinite(params.x) && Number.isFinite(params.y) ? _shadowAwareElementFromPoint(params.x, params.y) : null;
+ }
+
function getInteractiveElementsFull() {
return queryInteractiveFull().map((c, i) => {
const el = c.el;
diff --git a/test/bot-memory-click-dom.mjs b/test/bot-memory-click-dom.mjs
new file mode 100644
index 000000000..0d2b129bd
--- /dev/null
+++ b/test/bot-memory-click-dom.mjs
@@ -0,0 +1,84 @@
+import assert from 'node:assert/strict';
+import { createServer } from 'node:http';
+import { readFile } from 'node:fs/promises';
+import { chromium, firefox } from 'playwright';
+
+const html = `<!doctype html><style>button,input,textarea,[contenteditable]{margin:8px} dialog{position:fixed;top:40px;width:400px;height:200px}</style>
+<input value="Unrelated search draft" aria-label="Unrelated search">
+<section id="composer" data-record-id="7"><div contenteditable="true" aria-label="Comment">Exact observed developer comment.</div><input type="password" value="fixture-password">
+<div><div><div><div><div><div><button id="post" type="button"><span>Post</span></button></div></div></div></div></div></div></section>
+<section id="other"><button id="menu">Open menu</button></section>
+<script>window.count=0;window.clicked=null;document.addEventListener('click',e=>{const button=e.target.closest('button');if(button){window.clicked=button.id;if(button.id==='post')window.count++;}});</script>`;
+const server = createServer((_request, response) => { response.writeHead(200, { 'content-type': 'text/html' }); response.end(html); });
+await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+const origin = `http://127.0.0.1:${server.address().port}`;
+let browser;
+try {
+ for (const [name, engine] of [['chrome', chromium], ['firefox', firefox]]) {
+ browser = await engine.launch({ headless: true });
+ const page = await browser.newPage();
+ await page.context().addInitScript(() => {
+ window.qaListeners = []; window.qaRun = 'click-memory-fixture';
+ window.chrome = window.browser = { runtime: { getURL: value => value, onMessage: { addListener: listener => qaListeners.push(listener), removeListener() {} }, async sendMessage(message) { return message.action === 'get_page_monitor_state' ? { active: true, runToken: qaRun, documentToken: message.documentToken } : { accepted: true }; } } };
+ window.qaMessage = (action, params = {}) => new Promise(resolve => { for (const listener of qaListeners) { let answered = false; const result = listener({ target: 'content', action, params: { runToken: qaRun, ...params } }, {}, value => { answered = true; resolve(value); }); if (answered || result === true) break; } });
+ });
+ for (const file of ['rich-text-toolbar-heuristic.js', 'accessibility-tree.js', 'content.js']) await page.context().addInitScript({ content: await readFile(new URL(`../src/${name}/src/content/${file}`, import.meta.url), 'utf8') });
+ await page.goto(origin + '/item/7');
+ const { observeMemoryPageInPage, BotMemorySession } = await import(`../src/${name}/src/agent/bot-memory.js`);
+ const { Agent } = await import(`../src/${name}/src/agent/agent.js`);
+ globalThis.chrome = globalThis.browser = { storage: { local: { get: async () => ({}), set: async () => {} }, session: { get: async () => ({}), set: async () => {} } }, runtime: { getURL: value => value } };
+ const read = args => page.evaluate(observeMemoryPageInPage, { toolName: 'click', ...args });
+ const indexed = await page.evaluate(() => {
+ for (let index = 0; index < 20; index++) if (__wb_resolve_click_target_for_submit_probe({ index })?.id === 'post') return index;
+ throw Error('Fixture post index not found');
+ });
+ for (const args of [{ text: 'Post', textMatch: 'exact' }, { index: indexed }, { selector: '#post', matchIndex: 0 }]) {
+ const observation = await read(args);
+ assert.equal(observation.elementFacts.found, true);
+ assert.equal(observation.elementFacts.label, 'Post');
+ assert.equal(observation.elementFacts.draftText, 'Exact observed developer comment.', 'Deep form-less sibling composer is captured');
+ assert.ok(!observation.elementFacts.draftText.includes('Unrelated search'));
+ assert.ok(!JSON.stringify(observation).includes('fixture-password'));
+ assert.equal(await page.evaluate(() => window.count), 0, 'Observation does not click or focus');
+ }
+ const menu = await read({ text: 'Open menu' });
+ assert.equal(menu.elementFacts.draftText, '', 'A button without its own editor does not collect unrelated page inputs');
+ const selector = await read({ selector: 'button', matchIndex: 1 });
+ assert.equal(selector.elementFacts.label, 'Open menu');
+ const hidden = await page.evaluate(() => { const button=document.createElement('button');button.id='other-post';button.textContent='Post';document.querySelector('#other').append(button);return true; });
+ assert.equal(hidden, true);
+ assert.equal((await read({ text: 'Post', textMatch: 'exact' })).elementFacts.found, false, 'Multiple distinct Post controls remain ambiguous');
+ await page.locator('#other-post').evaluate(node => node.remove());
+ await page.evaluate(() => { const dialog=document.createElement('dialog');dialog.open=true;dialog.innerHTML='<button id="modal-post">Post</button>';document.body.append(dialog); });
+ assert.equal(await page.evaluate(() => __wb_resolve_click_target_for_submit_probe({ text: 'Post' })?.id), 'modal-post', 'Text matching is scoped to the current dialog');
+ await page.locator('dialog').evaluate(node => node.remove());
+ await page.evaluate(() => { const box=document.createElement('section');box.id='labels-fixture';box.innerHTML='<label style="display:none" for="first-label">Hidden fallback</label><input id="first-label"><label style="display:none" for="last-label">Hidden fallback</label><input id="last-label"><label style="display:none">Hidden adjacent</label><input id="adjacent-label">';document.body.append(box); });
+ assert.equal(await page.evaluate(() => __wb_resolve_click_target_for_submit_probe({text:'Hidden fallback'})?.id),'last-label','Duplicate fallback labels keep the dispatcher Map last-wins order');
+ assert.equal(await page.evaluate(() => !!__wb_resolve_click_target_for_submit_probe({text:'Hidden adjacent'})),false,'No-match fallback does not invent an adjacent label target');
+ await page.locator('#labels-fixture').evaluate(node=>node.remove());
+ await page.evaluate(()=>{const box=document.createElement('section');box.id='select-fixture';box.innerHTML='<button id="prefix-save">Save settings</button><select><option>Other</option><option>Save</option></select>';document.body.append(box);});
+ assert.equal(await page.evaluate(()=>!!__wb_resolve_click_target_for_submit_probe({text:'Save'})),false,'Exact option rescue cannot authorize the earlier prefix button');
+ assert.equal(await page.evaluate(()=>document.querySelector('#select-fixture select').value),'Other','Memory observation never changes a selected option');
+ await page.locator('#select-fixture').evaluate(node=>node.remove());
+
+ const target = { kind: 'url', url: origin + '/item/7' }, calls = [];
+ const session = new BotMemorySession({ version: 1, policyVersion: 1, botId: 'fixture', runId: 'fixture-run', callbackUrl: origin + '/api/internal/bot-memory/fixture-run', callbackToken: 'fixture-only-capability-'.repeat(3), policy: { schemaVersion: 1, actions: [{ key: 'post', effect: 'write' }] }, context: {} }, {
+ observe: async (_tab, args) => { const value = await page.evaluate(observeMemoryPageInPage, args); value.targets=[target]; return value; },
+ fetchImpl: async (_url, options) => { const value=JSON.parse(options.body);calls.push(value); if(value.operation==='prepare'){assert.equal(value.args.observation.elementFacts.draftText,'Exact observed developer comment.');return Response.json({allowed:true,reservation:{attemptId:'attempt-7',requestId:value.args.requestId,actionKey:'post',target}});}if(value.operation==='dispatch')return Response.json({attempt:{id:'attempt-7',status:'dispatched'}});if(value.operation==='verify')return Response.json({verified:true,status:'verified'});return Response.json({status:value.args.status}); },
+ });
+ const agent = new Agent({ getActive: () => ({ model: 'fixture' }) });agent._botMemorySessions.set(7,session);
+ agent._executeToolWithLifecycle = async (_tab, tool, args) => {
+ if(tool==='click'){assert.deepEqual(calls.map(call=>call.operation),['prepare','dispatch']);return page.evaluate(args=>qaMessage('click',args),args);}
+ return {success:true,text:'Exact observed developer comment. Posted on item 7.'};
+ };
+ const action = await agent.executeTool(7,'click',{index:String(indexed)},null,{detectedSubmitAction:{isSubmit:true,method:'POST',url:target.url,action:'/comments'}});
+ assert.equal(action.success,true,JSON.stringify(action));
+ assert.equal(await page.evaluate(()=>window.count),1,'The real content dispatcher clicks the same observed Post control');
+ assert.equal(action.memoryAttemptId,'attempt-7');
+ await agent.executeTool(7,'read_page',{});
+ assert.deepEqual(calls.map(call=>call.operation),['prepare','dispatch','verify']);
+ await session.close();
+ console.log(`${name}: click-by-index/text/selector shares observed target, rejects ambiguity, and records before local fixture submission`);
+ await browser.close();browser=null;
+ }
+} finally { if(browser)await browser.close();await new Promise(resolve=>server.close(resolve)); }
diff --git a/test/bot-memory-click-guard.mjs b/test/bot-memory-click-guard.mjs
new file mode 100644
index 000000000..fd0e0b20f
--- /dev/null
+++ b/test/bot-memory-click-guard.mjs
@@ -0,0 +1,66 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+globalThis.chrome = globalThis.browser = { storage: { local: { get: async () => ({}), set: async () => {} }, session: { get: async () => ({}), set: async () => {} } }, runtime: { getURL: value => value } };
+const target = { kind: 'url', url: 'https://example.test/item/7' };
+const config = { version: 1, policyVersion: 1, botId: 'bot-1', runId: 'run-1', callbackUrl: 'https://app.test/api/internal/bot-memory/run-1', callbackToken: 'test-capability-only-'.repeat(3), policy: { schemaVersion: 1, actions: [{ key: 'post', effect: 'write' }] }, context: {} };
+const post = { isSubmit: true, method: 'POST', url: target.url, action: '/comments', fields: [{ name: 'comment', type: 'textarea' }] };
+const search = { isSubmit: true, method: 'GET', url: target.url, action: '/search', fields: [{ name: 'q', type: 'search' }] };
+for (const browser of ['chrome', 'firefox']) {
+ const { Agent } = await import(`../src/${browser}/src/agent/agent.js`);
+ const { BotMemorySession, sanitizeMemoryArguments } = await import(`../src/${browser}/src/agent/bot-memory.js`);
+ function harness({ readOnly = false, found = true } = {}) {
+ const calls = [], actions = [];
+ const session = new BotMemorySession(config, { observe: async () => ({ url: target.url, title: 'Item 7', targets: [target], text: 'Item 7', elementFacts: { found, tag: 'BUTTON', label: 'Post', draftText: 'Exact observed draft' } }), fetchImpl: async (_url, options) => {
+ const value = JSON.parse(options.body); calls.push(value);
+ if (value.operation === 'prepare') return Response.json(readOnly ? { allowed: true, readOnly: true } : { allowed: true, reservation: { attemptId: 'attempt-7', requestId: value.args.requestId, actionKey: 'post', target } });
+ if (value.operation === 'dispatch') return Response.json({ attempt: { id: 'attempt-7', status: 'dispatched' } });
+ return Response.json({ status: value.args.status });
+ } });
+ const agent = new Agent({ getActive: () => ({ model: 'test' }) }); agent._botMemorySessions.set(7, session);
+ agent._executeToolWithLifecycle = async (_tab, name, args) => { actions.push({ name, args }); return { success: true }; };
+ return { session, agent, calls, actions };
+ }
+ test(`${browser}: indexed clicks canonicalize before both observation and dispatch and retain matching arguments`, async () => {
+ const h = harness();
+ assert.equal((await h.agent.executeTool(7, 'click', { index: '001', exact: true, matchIndex: 2, textMatch: 'exact' })).success, true);
+ assert.deepEqual(h.calls.map(call => call.operation), ['prepare', 'dispatch']);
+ assert.deepEqual(h.calls[0].args.tool.arguments, { index: 1, exact: true, matchIndex: 2, textMatch: 'exact' });
+ assert.equal(h.actions[0].args.index, 1);
+ assert.equal(h.calls[0].args.observation.elementFacts.draftText, 'Exact observed draft');
+ for (const index of [-1, 1.5, '1e0', true, '']) assert.equal((await h.agent.executeTool(7, 'click', { index })).noDispatch, true);
+ assert.equal(h.actions.length, 1);
+ await h.session.close();
+ });
+ test(`${browser}: both trusted caller context names make consequential read classifications fail before dispatch`, async () => {
+ for (const contextName of ['detectedSubmitAction', 'detectedSubmit']) {
+ const h = harness({ readOnly: true });
+ const result = await h.agent.executeTool(7, 'click', { text: 'Post' }, null, { [contextName]: post });
+ assert.equal(result.noDispatch, true); assert.deepEqual(h.actions, []);
+ assert.deepEqual(h.calls.map(call => call.operation), ['prepare']);
+ assert.equal(h.calls[0].args.observation.elementFacts.isSubmit, true);
+ assert.equal(h.calls[0].args.observation.elementFacts.consequentialSubmit, true);
+ assert.equal(JSON.stringify(h.calls).includes('fields'), false, 'Only trusted booleans are forwarded, never raw submit context');
+ await h.session.close();
+ }
+ });
+ test(`${browser}: resolved search submit remains read while unresolvable consequential target is blocked`, async () => {
+ const h = harness({ readOnly: true });
+ assert.equal((await h.agent.executeTool(7, 'click', { text: 'Search' }, null, { detectedSubmitAction: search })).success, true);
+ assert.equal(h.calls[0].args.observation.elementFacts.isSubmit, true);
+ assert.equal(h.calls[0].args.observation.elementFacts.consequentialSubmit, false);
+ assert.deepEqual(h.calls.map(call => call.operation), ['prepare']);
+ assert.equal(h.actions.length, 1); await h.session.close();
+ const missing = harness({ found: false });
+ assert.equal((await missing.agent.executeTool(7, 'click', { text: 'Post' }, null, { detectedSubmitAction: post })).noDispatch, true);
+ assert.deepEqual(missing.actions, []); assert.deepEqual(missing.calls, []); await missing.session.close();
+ });
+ test(`${browser}: model arguments cannot forge trusted submit facts and normal drafts remain read`, async () => {
+ const h = harness({ readOnly: true });
+ assert.equal((await h.agent.executeTool(7, 'type_text', { text: 'Only draft', isSubmit: true, consequentialSubmit: true, detectedSubmitAction: post })).success, true);
+ assert.deepEqual(h.calls[0].args.tool.arguments, { text: 'Only draft' });
+ assert.equal(h.calls[0].args.observation.elementFacts.isSubmit, undefined);
+ assert.equal(h.calls[0].args.observation.elementFacts.consequentialSubmit, undefined);
+ assert.equal(h.actions.length, 1); await h.session.close();
+ assert.deepEqual(sanitizeMemoryArguments({ index: 3, exact: true, matchIndex: 1, textMatch: 'prefix', callbackToken: 'fake', consequentialSubmit: true }), { index: 3, exact: true, matchIndex: 1, textMatch: 'prefix' });
+ });
+}