Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Ask cards: an approved repeat click releases its lock only when it is sent

Barack Sokullu5296f9847de4
diff --git a/docs/guards.md b/docs/guards.md
index 084a715d5..14c6e4ae8 100644
--- a/docs/guards.md
+++ b/docs/guards.md
@@ -95,7 +95,10 @@ When a guard cannot verify a consequential step and its `onCannotDecide` is
- **After `once`.** The tool result carries a note for the model. A commit or
post made this way is marked `publishesUnverifiedContent: true` on its submit
receipt; the checks after the submit (GitHub's committed raw-file check, the
- published-post check) stay the authority.
+ published-post check) stay the authority. A `clickax-task-click-lock` answer
+ covers one click that is actually sent: the target's lock is released only
+ as that click is dispatched, so a correction, Stop or refusal before it
+ keeps the lock, and the next click on the target asks again.
| Guard | Asks when | The card shows |
|---|---|---|
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 8be1c51fa..b9150fc47 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -15600,7 +15600,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
|| await this._workflowPreSubmitDispatchBlock(
tabId, fnName, fnArgs, detectedSubmitAction, provider, callState.guardAdvisories,
)
- || this._taskClickLockAsk(tabId, fnName, fnArgs);
+ || this._taskClickLockAsk(tabId, fnName, fnArgs, callState);
let workflowPreSubmitBlock = await runWorkflowPreSubmitChecks();
if (workflowPreSubmitBlock?.mutationConfirmation
&& this.scheduledRunPolicies.get(tabId)?.requireConsequentialConfirmation !== false) {
@@ -16402,6 +16402,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
guardAdvisories: callState.guardAdvisories,
dispatchBinding: pipelineToolbarPreflight.probe?.dispatchBinding || null,
...messageRecipientExecutionContext,
+ taskClickLockRelease: callState.taskClickLockRelease || null,
iframeTargetUnresolved: pipelineToolbarPreflight.iframeTargetUnresolved === true,
richTextToolbarWrongTargetAdvised: pipelineToolbarPreflight.wrongTargetAdvised === true,
imageGenConfig,
@@ -19908,6 +19909,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
traceStep = null,
) {
const dispatchState = upstreamDispatchState || { started: false };
+ // The user's 'once' for a repeat of an unreconciled click releases this
+ // target's locks only now, for this click, and they return if it is not sent.
+ const approvedRepeat = this._releaseApprovedTaskClickLock(tabId, args, axScope,
+ messageRecipientContext?.taskClickLockRelease);
const timing = {
startedAt: this._clickAxClockNow(),
startedAtEpoch: Date.now(),
@@ -19955,6 +19960,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
return result;
} finally {
+ if (approvedRepeat) this._restoreUnsentTaskClickLock(tabId, approvedRepeat, result, dispatchState);
timing.clickPathMs = Math.max(0, this._clickAxClockNow() - timing.startedAt);
timing.outcomeUnknown = result?.outcomeUnknown === true
|| (!timing.syntheticDispatched && !timing.trustedInputEvents
@@ -42644,19 +42650,61 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
this._taskClickLocks?.get(tabId)?.keys.delete(key);
}
+ // The user's 'once' on the clickax-task-click-lock card reaches the click
+ // as a grant for its exact task and document|ref (_taskClickLockAsk). Only
+ // here, as that click is dispatched, are the target's task locks and its
+ // per-run trusted attempts released; what was released is returned so
+ // _restoreUnsentTaskClickLock can put it back if the click is not sent.
+ _releaseApprovedTaskClickLock(tabId, args, axScope, grant) {
+ const taskToken = this._taskTokens?.get(tabId);
+ if (!grant || typeof taskToken !== 'string' || !taskToken || grant.taskToken !== taskToken
+ || !axScope?.documentToken || typeof args?.ref_id !== 'string'
+ || grant.target !== `${axScope.documentToken}|${args.ref_id}`) return null;
+ const entry = this._taskClickLocks?.get(tabId);
+ const locks = entry?.taskToken === taskToken ? entry.keys : null;
+ const attempted = this._clickAxCdpFallbacks.get(tabId) || null;
+ const release = { entry, attempted, locks: [], attempts: [] };
+ for (const key of [`native-button-pending:${grant.target}`, grant.target]) {
+ if (locks?.delete(key)) release.locks.push(key);
+ if (attempted?.delete(key)) release.attempts.push(key);
+ }
+ for (const key of [...(attempted || [])]) {
+ if (key.startsWith(`native-button:${grant.target}|`) && attempted.delete(key)) release.attempts.push(key);
+ }
+ return release;
+ }
+
+ // A click that was not sent (refused, vetoed, timed out or stopped before
+ // dispatch) leaves the earlier click unreconciled and the approved repeat
+ // undone, so the target is locked again; the answer is used up, and the
+ // next click on it asks again. A click that may have been sent keeps the
+ // release; its own trusted paths lock it again if its input is unresolved.
+ _restoreUnsentTaskClickLock(tabId, release, result, dispatchState) {
+ if (dispatchState?.started === true && result?.dispatched !== false && result?.noDispatch !== true) return;
+ if (release.locks.length && this._taskClickLocks?.get(tabId) === release.entry
+ && release.entry?.taskToken === this._taskTokens?.get(tabId)) {
+ for (const key of release.locks) this._rememberTaskClickLock(tabId, key);
+ }
+ for (const key of release.attempts) release.attempted?.add(key);
+ }
+
/**
* clickax-task-click-lock (a hard class: a repeat of a click that may
* already have taken effect) cannot decide: an earlier trusted click on this
* exact document|ref may have landed and was never verified. Instead of
* refusing for the rest of the task, WebBrain asks the user on its card,
* which never times out and never goes ahead on its own (an unattended run
- * is refused). Only the user's 'once' releases this target's locks, for
- * this one click. This applies with Strict checks too: it replaces a dead
- * end with the user's explicit decision. The lock in _dispatchClickAxImpl
- * stays the final fence. Chrome only: Firefox sends no trusted click
- * fallback, so it has no such lock.
+ * is refused). The user's 'once' covers one click on this target that is
+ * actually sent: it becomes this call's grant (`callState`), and
+ * _dispatchClickAx releases the target's locks only as that click is
+ * dispatched, and restores them if it is not sent. A correction, Stop or
+ * any other refusal before dispatch leaves the locks in place, and the
+ * next click on this target asks again. This applies with Strict checks
+ * too: it replaces a dead end with the user's explicit decision. The lock
+ * in _dispatchClickAxImpl stays the final fence. Chrome only: Firefox
+ * sends no trusted click fallback, so it has no such lock.
*/
- _taskClickLockAsk(tabId, name, args = {}) {
+ _taskClickLockAsk(tabId, name, args = {}, callState = null) {
if (name !== 'click_ax' || typeof args?.ref_id !== 'string' || !args.ref_id) return null;
const scope = this._lastAxScopes.get(tabId);
const documentToken = scope?.documentToken;
@@ -42666,16 +42714,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const keys = [`native-button-pending:${target}`, target];
if (!keys.some(key => locks.has(key))
|| this._guardOutcome('clickax-task-click-lock', { decided: false }) !== 'ask') return null;
- const stepKey = this._sha256TextSync(JSON.stringify(['clickax-task-click-lock', this._taskTokens.get(tabId) || '', target]));
- if (this._unverifiedStepApproval(tabId, 'clickax-task-click-lock', stepKey, { consume: true })) {
- const attempted = this._clickAxCdpFallbacks.get(tabId);
- for (const key of keys) {
- this._releaseTaskClickLock(tabId, key);
- attempted?.delete(key);
- }
- for (const key of [...(attempted || [])]) {
- if (key.startsWith(`native-button:${target}|`)) attempted.delete(key);
- }
+ const taskToken = this._taskTokens.get(tabId) || '';
+ // The checks can run again before dispatch; this call already holds the answer.
+ if (callState?.taskClickLockRelease?.target === target && callState.taskClickLockRelease.taskToken === taskToken) return null;
+ const stepKey = this._sha256TextSync(JSON.stringify(['clickax-task-click-lock', taskToken, target]));
+ if (callState && this._unverifiedStepApproval(tabId, 'clickax-task-click-lock', stepKey, { consume: true })) {
+ callState.taskClickLockRelease = { taskToken, target };
return null;
}
return { success: false, dispatched: false, noDispatch: true, retryable: false, repeatBlocked: true,
@@ -49871,7 +49915,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
nativeTelegramReadRepairOwner: dispatchContext.nativeTelegramReadRepairOwner,
nativeTelegramReadRepairRef: dispatchContext.nativeTelegramReadRepairRef,
nativeTelegramReadRepairBinding: dispatchContext.nativeTelegramReadRepairBinding,
- nativeWhatsappReadRepairOwner: dispatchContext.nativeWhatsappReadRepairOwner },
+ nativeWhatsappReadRepairOwner: dispatchContext.nativeWhatsappReadRepairOwner,
+ taskClickLockRelease: dispatchContext.taskClickLockRelease || null },
earlyCdpAbortSignal,
earlyCdpDispatchState,
dispatchContext.traceStep,
diff --git a/test/guard-ask-cards.mjs b/test/guard-ask-cards.mjs
index 3e552b0e7..51efe7a17 100644
--- a/test/guard-ask-cards.mjs
+++ b/test/guard-ask-cards.mjs
@@ -782,8 +782,10 @@ test('chrome: a click that may repeat an unverified earlier click asks on a hard
agent._auditRichTextToolbarTarget = async () => {};
agent._shouldAutoScreenshot = () => false;
agent._refreshPageFeedbackForCall = async () => false;
- agent._executeToolWithLifecycleInner = async (_tab, name, args) => {
- dispatched.push({ name, args });
+ // The click itself: _dispatchClickAx releases an approved target's locks around it.
+ agent._dispatchClickAxImpl = async (_tab, args, _scope, _binding, _context, _deadline, _upstream, dispatchState) => {
+ dispatched.push({ name: 'click_ax', args });
+ dispatchState.started = true;
return { success: true, dispatched: true, verified: true };
};
const run = async (id, ref, answer) => {
@@ -825,6 +827,108 @@ test('chrome: a click that may repeat an unverified earlier click asks on a hard
}
});
+// The user's 'once' covers one click that is actually sent. A correction
+// that arrives just after the answer hands the turn to the corrected task
+// before dispatch: no click, the target stays locked, and the next click on
+// it asks again. The real click path runs (its lock fence included) with
+// the page's content script stubbed.
+test('chrome: an approved repeat click releases its lock only when it is sent', async () => {
+ const { Agent } = builds.find(item => item.build === 'chrome');
+ const provider = { name: 'lock test', model: 'test', promptTier: 'full', contextWindow: 128000,
+ supportsTools: true, supportsVision: false, chat: async () => ({ content: '' }) };
+ const agent = new Agent({ getActive: () => provider, getProvider: () => provider, getVisionProvider: async () => null });
+ agent.clarifyTimeoutSec = 0;
+ agent._persist = () => {};
+ agent._persistNow = async () => ({ ok: true });
+ const tab = nextTab++;
+ agent._taskTokens.set(tab, `task-${tab}`);
+ agent._planExecutionGuards.set(tab, { enabled: false, taskKey: `key-${tab}` });
+ agent._lastAxScopes.set(tab, { documentToken: 'doc-1', pageUrl: 'https://shop.example/cart' });
+ agent._currentUrl = async () => 'https://shop.example/cart';
+ for (const ref of ['ref_buy', 'ref_pay']) {
+ agent._rememberTaskClickLock(tab, `doc-1|${ref}`);
+ agent._rememberTaskClickLock(tab, `native-button-pending:doc-1|${ref}`);
+ }
+ const tried = ['doc-1|ref_buy', 'native-button-pending:doc-1|ref_buy', 'native-button:doc-1|ref_buy|snapshot-1'];
+ agent._clickAxCdpFallbacks.set(tab, new Set(tried));
+ const locked = ref => ['doc-1|', 'native-button-pending:doc-1|'].map(prefix => agent._taskClickLockKeys(tab).has(prefix + ref));
+ agent._skipPermissionGate = true;
+ agent._ensureGateSetting = async () => true;
+ for (const name of ['_chromeProtectedPageFailure', '_captchaMutationPreflight', '_messageRecipientGuardBlock',
+ '_detectLikelySubmitAction', '_socialPublicationPreSubmitBlock']) agent[name] = async () => null;
+ agent._adoptLiveSocialPublishWorkflow = async () => false;
+ agent._captureFormValidationState = async () => [];
+ agent._waitForFormValidationFailure = async () => null;
+ agent._auditRichTextToolbarTarget = async () => {};
+ agent._shouldAutoScreenshot = () => false;
+ agent._refreshPageFeedbackForCall = async () => false;
+ agent._isPdfTab = async () => false;
+ agent._captureClickAxObservation = async (_tab, snapshot, sideEffectWatch, startedAt) => ({ startedAt, snapshot, sideEffectWatch });
+ agent._maybeFallbackClickAxWithCdp = async (_tab, _args, response) => response;
+ // The content script's click: what it reports for each target.
+ const clicks = [];
+ const content = { ref_buy: () => ({ success: true, dispatched: true, verified: true }),
+ ref_pay: () => ({ success: false, dispatched: false, noDispatch: true, fallbackAttempted: false,
+ error: 'The element is no longer in the page.' }) };
+ const sendMessage = api.tabs.sendMessage;
+ api.tabs.sendMessage = async (tabId, message, options) => {
+ if (message?.action !== 'click_ax') return sendMessage(tabId, message, options);
+ const report = content[message.params.ref_id]();
+ if (report.dispatched) clicks.push(message.params.ref_id);
+ return report;
+ };
+ // A correction sent just after the user's answer, before the click.
+ let correctAfterAnswer = false;
+ const remember = agent._rememberUnverifiedStepDecision.bind(agent);
+ agent._rememberUnverifiedStepDecision = (...args) => {
+ const entry = remember(...args);
+ if (correctAfterAnswer && entry?.choice === 'once') {
+ agent._steeringRuns.set(tab, { messages: [{ role: 'user', content: 'Check the cart total first.' }] });
+ }
+ return entry;
+ };
+ const cards = [];
+ const run = async (id, ref) => {
+ const messages = [];
+ await agent._executeToolBatch(tab, [call(id, 'click_ax', { ref_id: ref })], messages, (type, data) => {
+ if (type !== 'clarify') return;
+ cards.push(data);
+ setTimeout(() => agent.submitClarifyResponse(tab, data.clarifyId, 'once', 'option'), 20);
+ }, provider, null, new Set(['click_ax', 'done']), 1, {});
+ return resultFor(messages, id);
+ };
+ try {
+ correctAfterAnswer = true;
+ const corrected = await run('buy-1', 'ref_buy');
+ correctAfterAnswer = false;
+ agent._steeringRuns.delete(tab);
+ assert.equal(cards.length, 1);
+ assert.equal(cards[0].unverifiedStepConfirmation.hard, true);
+ assert.equal(corrected.skipped, true, JSON.stringify(corrected));
+ assert.match(corrected.error, /the user steered the current task/);
+ assert.deepEqual(clicks, [], 'no click was sent');
+ assert.deepEqual(locked('ref_buy'), [true, true], 'the target stays locked');
+ assert.deepEqual([...agent._clickAxCdpFallbacks.get(tab)], tried, 'its trusted attempts stay recorded');
+ // The answer was used up: the next click asks again, and 'once' sends it.
+ const approved = await run('buy-2', 'ref_buy');
+ assert.equal(cards.length, 2, 'the next attempt asks again');
+ assert.equal(approved.success, true, JSON.stringify(approved));
+ assert.deepEqual(clicks, ['ref_buy'], 'exactly one click');
+ assert.deepEqual(locked('ref_buy'), [false, false], 'the lock is released by the click that was sent');
+ assert.deepEqual([...agent._clickAxCdpFallbacks.get(tab)], []);
+ // A click that the page refuses before dispatch locks its target again.
+ const refused = await run('pay-1', 'ref_pay');
+ assert.equal(cards.length, 3);
+ assert.equal(refused.noDispatch, true, JSON.stringify(refused));
+ assert.deepEqual(clicks, ['ref_buy']);
+ assert.deepEqual(locked('ref_pay'), [true, true], 'an unsent click gets its lock back');
+ await run('pay-2', 'ref_pay');
+ assert.equal(cards.length, 4, 'and the next attempt asks again');
+ } finally {
+ api.tabs.sendMessage = sendMessage;
+ }
+});
+
// ── side panel ───────────────────────────────────────────────────────────
let chromiumBrowser;
before(async () => { chromiumBrowser = await chromium.launch({ headless: true }); });