Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Ask cards: an approved repeat click releases its lock only when it is sent
Barack Sokullu5296f9847de4
diff --git a/docs/guards.md b/docs/guards.mdindex 084a715d5..14c6e4ae8 100644--- a/docs/guards.md+++ b/docs/guards.md@@ -95,7 +95,10 @@ When a guard cannot verify a consequential step and its `onCannotDecide` is- **After `once`.** The tool result carries a note for the model. A commit orpost made this way is marked `publishesUnverifiedContent: true` on its submitreceipt; the checks after the submit (GitHub's committed raw-file check, the- published-post check) stay the authority.+ published-post check) stay the authority. A `clickax-task-click-lock` answer+ covers one click that is actually sent: the target's lock is released only+ as that click is dispatched, so a correction, Stop or refusal before it+ keeps the lock, and the next click on the target asks again.| Guard | Asks when | The card shows ||---|---|---|diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 8be1c51fa..b9150fc47 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -15600,7 +15600,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d|| await this._workflowPreSubmitDispatchBlock(tabId, fnName, fnArgs, detectedSubmitAction, provider, callState.guardAdvisories,)- || this._taskClickLockAsk(tabId, fnName, fnArgs);+ || this._taskClickLockAsk(tabId, fnName, fnArgs, callState);let workflowPreSubmitBlock = await runWorkflowPreSubmitChecks();if (workflowPreSubmitBlock?.mutationConfirmation&& this.scheduledRunPolicies.get(tabId)?.requireConsequentialConfirmation !== false) {@@ -16402,6 +16402,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dguardAdvisories: callState.guardAdvisories,dispatchBinding: pipelineToolbarPreflight.probe?.dispatchBinding || null,...messageRecipientExecutionContext,+ taskClickLockRelease: callState.taskClickLockRelease || null,iframeTargetUnresolved: pipelineToolbarPreflight.iframeTargetUnresolved === true,richTextToolbarWrongTargetAdvised: pipelineToolbarPreflight.wrongTargetAdvised === true,imageGenConfig,@@ -19908,6 +19909,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dtraceStep = null,) {const dispatchState = upstreamDispatchState || { started: false };+ // The user's 'once' for a repeat of an unreconciled click releases this+ // target's locks only now, for this click, and they return if it is not sent.+ const approvedRepeat = this._releaseApprovedTaskClickLock(tabId, args, axScope,+ messageRecipientContext?.taskClickLockRelease);const timing = {startedAt: this._clickAxClockNow(),startedAtEpoch: Date.now(),@@ -19955,6 +19960,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}return result;} finally {+ if (approvedRepeat) this._restoreUnsentTaskClickLock(tabId, approvedRepeat, result, dispatchState);timing.clickPathMs = Math.max(0, this._clickAxClockNow() - timing.startedAt);timing.outcomeUnknown = result?.outcomeUnknown === true|| (!timing.syntheticDispatched && !timing.trustedInputEvents@@ -42644,19 +42650,61 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dthis._taskClickLocks?.get(tabId)?.keys.delete(key);}+ // The user's 'once' on the clickax-task-click-lock card reaches the click+ // as a grant for its exact task and document|ref (_taskClickLockAsk). Only+ // here, as that click is dispatched, are the target's task locks and its+ // per-run trusted attempts released; what was released is returned so+ // _restoreUnsentTaskClickLock can put it back if the click is not sent.+ _releaseApprovedTaskClickLock(tabId, args, axScope, grant) {+ const taskToken = this._taskTokens?.get(tabId);+ if (!grant || typeof taskToken !== 'string' || !taskToken || grant.taskToken !== taskToken+ || !axScope?.documentToken || typeof args?.ref_id !== 'string'+ || grant.target !== `${axScope.documentToken}|${args.ref_id}`) return null;+ const entry = this._taskClickLocks?.get(tabId);+ const locks = entry?.taskToken === taskToken ? entry.keys : null;+ const attempted = this._clickAxCdpFallbacks.get(tabId) || null;+ const release = { entry, attempted, locks: [], attempts: [] };+ for (const key of [`native-button-pending:${grant.target}`, grant.target]) {+ if (locks?.delete(key)) release.locks.push(key);+ if (attempted?.delete(key)) release.attempts.push(key);+ }+ for (const key of [...(attempted || [])]) {+ if (key.startsWith(`native-button:${grant.target}|`) && attempted.delete(key)) release.attempts.push(key);+ }+ return release;+ }++ // A click that was not sent (refused, vetoed, timed out or stopped before+ // dispatch) leaves the earlier click unreconciled and the approved repeat+ // undone, so the target is locked again; the answer is used up, and the+ // next click on it asks again. A click that may have been sent keeps the+ // release; its own trusted paths lock it again if its input is unresolved.+ _restoreUnsentTaskClickLock(tabId, release, result, dispatchState) {+ if (dispatchState?.started === true && result?.dispatched !== false && result?.noDispatch !== true) return;+ if (release.locks.length && this._taskClickLocks?.get(tabId) === release.entry+ && release.entry?.taskToken === this._taskTokens?.get(tabId)) {+ for (const key of release.locks) this._rememberTaskClickLock(tabId, key);+ }+ for (const key of release.attempts) release.attempted?.add(key);+ }+/*** clickax-task-click-lock (a hard class: a repeat of a click that may* already have taken effect) cannot decide: an earlier trusted click on this* exact document|ref may have landed and was never verified. Instead of* refusing for the rest of the task, WebBrain asks the user on its card,* which never times out and never goes ahead on its own (an unattended run- * is refused). Only the user's 'once' releases this target's locks, for- * this one click. This applies with Strict checks too: it replaces a dead- * end with the user's explicit decision. The lock in _dispatchClickAxImpl- * stays the final fence. Chrome only: Firefox sends no trusted click- * fallback, so it has no such lock.+ * is refused). The user's 'once' covers one click on this target that is+ * actually sent: it becomes this call's grant (`callState`), and+ * _dispatchClickAx releases the target's locks only as that click is+ * dispatched, and restores them if it is not sent. A correction, Stop or+ * any other refusal before dispatch leaves the locks in place, and the+ * next click on this target asks again. This applies with Strict checks+ * too: it replaces a dead end with the user's explicit decision. The lock+ * in _dispatchClickAxImpl stays the final fence. Chrome only: Firefox+ * sends no trusted click fallback, so it has no such lock.*/- _taskClickLockAsk(tabId, name, args = {}) {+ _taskClickLockAsk(tabId, name, args = {}, callState = null) {if (name !== 'click_ax' || typeof args?.ref_id !== 'string' || !args.ref_id) return null;const scope = this._lastAxScopes.get(tabId);const documentToken = scope?.documentToken;@@ -42666,16 +42714,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst keys = [`native-button-pending:${target}`, target];if (!keys.some(key => locks.has(key))|| this._guardOutcome('clickax-task-click-lock', { decided: false }) !== 'ask') return null;- const stepKey = this._sha256TextSync(JSON.stringify(['clickax-task-click-lock', this._taskTokens.get(tabId) || '', target]));- if (this._unverifiedStepApproval(tabId, 'clickax-task-click-lock', stepKey, { consume: true })) {- const attempted = this._clickAxCdpFallbacks.get(tabId);- for (const key of keys) {- this._releaseTaskClickLock(tabId, key);- attempted?.delete(key);- }- for (const key of [...(attempted || [])]) {- if (key.startsWith(`native-button:${target}|`)) attempted.delete(key);- }+ const taskToken = this._taskTokens.get(tabId) || '';+ // The checks can run again before dispatch; this call already holds the answer.+ if (callState?.taskClickLockRelease?.target === target && callState.taskClickLockRelease.taskToken === taskToken) return null;+ const stepKey = this._sha256TextSync(JSON.stringify(['clickax-task-click-lock', taskToken, target]));+ if (callState && this._unverifiedStepApproval(tabId, 'clickax-task-click-lock', stepKey, { consume: true })) {+ callState.taskClickLockRelease = { taskToken, target };return null;}return { success: false, dispatched: false, noDispatch: true, retryable: false, repeatBlocked: true,@@ -49871,7 +49915,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dnativeTelegramReadRepairOwner: dispatchContext.nativeTelegramReadRepairOwner,nativeTelegramReadRepairRef: dispatchContext.nativeTelegramReadRepairRef,nativeTelegramReadRepairBinding: dispatchContext.nativeTelegramReadRepairBinding,- nativeWhatsappReadRepairOwner: dispatchContext.nativeWhatsappReadRepairOwner },+ nativeWhatsappReadRepairOwner: dispatchContext.nativeWhatsappReadRepairOwner,+ taskClickLockRelease: dispatchContext.taskClickLockRelease || null },earlyCdpAbortSignal,earlyCdpDispatchState,dispatchContext.traceStep,diff --git a/test/guard-ask-cards.mjs b/test/guard-ask-cards.mjsindex 3e552b0e7..51efe7a17 100644--- a/test/guard-ask-cards.mjs+++ b/test/guard-ask-cards.mjs@@ -782,8 +782,10 @@ test('chrome: a click that may repeat an unverified earlier click asks on a hardagent._auditRichTextToolbarTarget = async () => {};agent._shouldAutoScreenshot = () => false;agent._refreshPageFeedbackForCall = async () => false;- agent._executeToolWithLifecycleInner = async (_tab, name, args) => {- dispatched.push({ name, args });+ // The click itself: _dispatchClickAx releases an approved target's locks around it.+ agent._dispatchClickAxImpl = async (_tab, args, _scope, _binding, _context, _deadline, _upstream, dispatchState) => {+ dispatched.push({ name: 'click_ax', args });+ dispatchState.started = true;return { success: true, dispatched: true, verified: true };};const run = async (id, ref, answer) => {@@ -825,6 +827,108 @@ test('chrome: a click that may repeat an unverified earlier click asks on a hard}});+// The user's 'once' covers one click that is actually sent. A correction+// that arrives just after the answer hands the turn to the corrected task+// before dispatch: no click, the target stays locked, and the next click on+// it asks again. The real click path runs (its lock fence included) with+// the page's content script stubbed.+test('chrome: an approved repeat click releases its lock only when it is sent', async () => {+ const { Agent } = builds.find(item => item.build === 'chrome');+ const provider = { name: 'lock test', model: 'test', promptTier: 'full', contextWindow: 128000,+ supportsTools: true, supportsVision: false, chat: async () => ({ content: '' }) };+ const agent = new Agent({ getActive: () => provider, getProvider: () => provider, getVisionProvider: async () => null });+ agent.clarifyTimeoutSec = 0;+ agent._persist = () => {};+ agent._persistNow = async () => ({ ok: true });+ const tab = nextTab++;+ agent._taskTokens.set(tab, `task-${tab}`);+ agent._planExecutionGuards.set(tab, { enabled: false, taskKey: `key-${tab}` });+ agent._lastAxScopes.set(tab, { documentToken: 'doc-1', pageUrl: 'https://shop.example/cart' });+ agent._currentUrl = async () => 'https://shop.example/cart';+ for (const ref of ['ref_buy', 'ref_pay']) {+ agent._rememberTaskClickLock(tab, `doc-1|${ref}`);+ agent._rememberTaskClickLock(tab, `native-button-pending:doc-1|${ref}`);+ }+ const tried = ['doc-1|ref_buy', 'native-button-pending:doc-1|ref_buy', 'native-button:doc-1|ref_buy|snapshot-1'];+ agent._clickAxCdpFallbacks.set(tab, new Set(tried));+ const locked = ref => ['doc-1|', 'native-button-pending:doc-1|'].map(prefix => agent._taskClickLockKeys(tab).has(prefix + ref));+ agent._skipPermissionGate = true;+ agent._ensureGateSetting = async () => true;+ for (const name of ['_chromeProtectedPageFailure', '_captchaMutationPreflight', '_messageRecipientGuardBlock',+ '_detectLikelySubmitAction', '_socialPublicationPreSubmitBlock']) agent[name] = async () => null;+ agent._adoptLiveSocialPublishWorkflow = async () => false;+ agent._captureFormValidationState = async () => [];+ agent._waitForFormValidationFailure = async () => null;+ agent._auditRichTextToolbarTarget = async () => {};+ agent._shouldAutoScreenshot = () => false;+ agent._refreshPageFeedbackForCall = async () => false;+ agent._isPdfTab = async () => false;+ agent._captureClickAxObservation = async (_tab, snapshot, sideEffectWatch, startedAt) => ({ startedAt, snapshot, sideEffectWatch });+ agent._maybeFallbackClickAxWithCdp = async (_tab, _args, response) => response;+ // The content script's click: what it reports for each target.+ const clicks = [];+ const content = { ref_buy: () => ({ success: true, dispatched: true, verified: true }),+ ref_pay: () => ({ success: false, dispatched: false, noDispatch: true, fallbackAttempted: false,+ error: 'The element is no longer in the page.' }) };+ const sendMessage = api.tabs.sendMessage;+ api.tabs.sendMessage = async (tabId, message, options) => {+ if (message?.action !== 'click_ax') return sendMessage(tabId, message, options);+ const report = content[message.params.ref_id]();+ if (report.dispatched) clicks.push(message.params.ref_id);+ return report;+ };+ // A correction sent just after the user's answer, before the click.+ let correctAfterAnswer = false;+ const remember = agent._rememberUnverifiedStepDecision.bind(agent);+ agent._rememberUnverifiedStepDecision = (...args) => {+ const entry = remember(...args);+ if (correctAfterAnswer && entry?.choice === 'once') {+ agent._steeringRuns.set(tab, { messages: [{ role: 'user', content: 'Check the cart total first.' }] });+ }+ return entry;+ };+ const cards = [];+ const run = async (id, ref) => {+ const messages = [];+ await agent._executeToolBatch(tab, [call(id, 'click_ax', { ref_id: ref })], messages, (type, data) => {+ if (type !== 'clarify') return;+ cards.push(data);+ setTimeout(() => agent.submitClarifyResponse(tab, data.clarifyId, 'once', 'option'), 20);+ }, provider, null, new Set(['click_ax', 'done']), 1, {});+ return resultFor(messages, id);+ };+ try {+ correctAfterAnswer = true;+ const corrected = await run('buy-1', 'ref_buy');+ correctAfterAnswer = false;+ agent._steeringRuns.delete(tab);+ assert.equal(cards.length, 1);+ assert.equal(cards[0].unverifiedStepConfirmation.hard, true);+ assert.equal(corrected.skipped, true, JSON.stringify(corrected));+ assert.match(corrected.error, /the user steered the current task/);+ assert.deepEqual(clicks, [], 'no click was sent');+ assert.deepEqual(locked('ref_buy'), [true, true], 'the target stays locked');+ assert.deepEqual([...agent._clickAxCdpFallbacks.get(tab)], tried, 'its trusted attempts stay recorded');+ // The answer was used up: the next click asks again, and 'once' sends it.+ const approved = await run('buy-2', 'ref_buy');+ assert.equal(cards.length, 2, 'the next attempt asks again');+ assert.equal(approved.success, true, JSON.stringify(approved));+ assert.deepEqual(clicks, ['ref_buy'], 'exactly one click');+ assert.deepEqual(locked('ref_buy'), [false, false], 'the lock is released by the click that was sent');+ assert.deepEqual([...agent._clickAxCdpFallbacks.get(tab)], []);+ // A click that the page refuses before dispatch locks its target again.+ const refused = await run('pay-1', 'ref_pay');+ assert.equal(cards.length, 3);+ assert.equal(refused.noDispatch, true, JSON.stringify(refused));+ assert.deepEqual(clicks, ['ref_buy']);+ assert.deepEqual(locked('ref_pay'), [true, true], 'an unsent click gets its lock back');+ await run('pay-2', 'ref_pay');+ assert.equal(cards.length, 4, 'and the next attempt asks again');+ } finally {+ api.tabs.sendMessage = sendMessage;+ }+});+// ββ side panel βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββlet chromiumBrowser;before(async () => { chromiumBrowser = await chromium.launch({ headless: true }); });