Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Publication prefetch: a call stopped at a card or the permission gate records its timing

Barack Sokulluc9f2c6b9070d
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 40c4a09d8..183123ae7 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -15940,6 +15940,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
|| this._taskClickLockAsk(tabId, fnName, fnArgs, callState);
const predispatchConsequential = () => missingResponseOutcomeUnknown
|| detectedSubmitAction?.isSubmit === true || detectedSubmitAction?.publicationControl === true;
+ // A consequential call that a confirmation card or the permission gate
+ // stops still records where its time went, the wait on the card
+ // included.
+ const predispatchBlocked = stage => {
+ predispatch.lap(stage);
+ if (predispatchConsequential()) predispatch.emit(step, { tool: fnName, outcome: 'blocked' });
+ };
let workflowPreSubmitBlock = await runWorkflowPreSubmitChecks();
if (workflowPreSubmitBlock?.mutationConfirmation
&& this.scheduledRunPolicies.get(tabId)?.requireConsequentialConfirmation !== false) {
@@ -16152,6 +16159,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}),
});
onUpdate('warning', { message: canGroupSearch ? 'Low-risk search blocked until the user allows the grouped actions.' : 'Form submission blocked until the user confirms.' });
+ predispatchBlocked('submitConfirmationMs');
if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }
continue;
}
@@ -16214,6 +16222,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}),
});
onUpdate('warning', { message: 'Low-risk search blocked until the user allows the grouped actions.' });
+ predispatchBlocked('submitConfirmationMs');
if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }
continue;
}
@@ -16294,6 +16303,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}),
});
onUpdate('warning', { message: 'Low-risk search blocked until the user allows the grouped actions.' });
+ predispatchBlocked('permissionMs');
if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }
continue;
}
@@ -16363,6 +16373,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
error: `Cannot run ${fnName}: the target frame/host couldn't be identified, so it can't be permission-checked. Pass a urlFilter naming the iframe's domain (read it first with iframe_read / get_accessibility_tree) and retry.`,
}),
});
+ predispatchBlocked('permissionMs');
if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }
continue;
}
@@ -16376,6 +16387,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
error: `The user denied permission to ${CAPABILITY_LABEL[blocked.capability]} ${blocked.host}. Do NOT retry this action on that site. Continue with what you can without it, or ask the user how to proceed.`,
}),
});
+ predispatchBlocked('permissionMs');
if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }
continue;
}
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index c2763f931..1dec3a2b8 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -14239,6 +14239,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
));
const predispatchConsequential = () => missingResponseOutcomeUnknown
|| detectedSubmitAction?.isSubmit === true || detectedSubmitAction?.publicationControl === true;
+ // A consequential call that a confirmation card or the permission gate
+ // stops still records where its time went, the wait on the card
+ // included.
+ const predispatchBlocked = stage => {
+ predispatch.lap(stage);
+ if (predispatchConsequential()) predispatch.emit(step, { tool: fnName, outcome: 'blocked' });
+ };
let workflowPreSubmitBlock = await runWorkflowPreSubmitChecks();
if (workflowPreSubmitBlock?.mutationConfirmation
&& this.scheduledRunPolicies.get(tabId)?.requireConsequentialConfirmation !== false) {
@@ -14444,6 +14451,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}),
});
onUpdate('warning', { message: canGroupSearch ? 'Low-risk search blocked until the user allows the grouped actions.' : 'Form submission blocked until the user confirms.' });
+ predispatchBlocked('submitConfirmationMs');
if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }
continue;
}
@@ -14507,6 +14515,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}),
});
onUpdate('warning', { message: 'Low-risk search blocked until the user allows the grouped actions.' });
+ predispatchBlocked('submitConfirmationMs');
if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }
continue;
}
@@ -14587,6 +14596,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}),
});
onUpdate('warning', { message: 'Low-risk search blocked until the user allows the grouped actions.' });
+ predispatchBlocked('permissionMs');
if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }
continue;
}
@@ -14656,6 +14666,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
error: `Cannot run ${fnName}: the target frame/host couldn't be identified, so it can't be permission-checked. Pass a urlFilter naming the iframe's domain (read it first with iframe_read / get_accessibility_tree) and retry.`,
}),
});
+ predispatchBlocked('permissionMs');
if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }
continue;
}
@@ -14669,6 +14680,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
error: `The user denied permission to ${CAPABILITY_LABEL[blocked.capability]} ${blocked.host}. Do NOT retry this action on that site. Continue with what you can without it, or ask the user how to proceed.`,
}),
});
+ predispatchBlocked('permissionMs');
if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }
continue;
}
diff --git a/test/publication-prefetch.mjs b/test/publication-prefetch.mjs
index 252266888..7ed14a031 100644
--- a/test/publication-prefetch.mjs
+++ b/test/publication-prefetch.mjs
@@ -422,6 +422,57 @@ for (const browser of ['chrome', 'firefox']) {
assert.equal(gates.length, 3);
});
+ test(`${browser}: a consequential click a card or the permission gate stops still records predispatch_timing`, async () => {
+ const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
+ const run = async (name, args, { submitChoice = 'once', permissionChoice = 'deny' } = {}) => {
+ const f = setup();
+ const { agent, tabId, provider } = f;
+ assert.equal(await f.write(), 'authorized');
+ agent.currentRunId.set(tabId, 'prefetch-run');
+ agent._skipPermissionGate = false;
+ agent._ensureGateSetting = async () => false;
+ agent.permissions.hydrate = async () => {};
+ agent.permissions.check = () => ({ allowed: false, needsPrompt: true });
+ agent.permissions.record = async () => {};
+ agent._promptSubmitConfirmation = async () => { await wait(15); return submitChoice; };
+ agent._promptPermission = async () => { await wait(15); return permissionChoice; };
+ agent._recordProgressObservation = async () => null;
+ agent._autoRecordProgressAction = () => null;
+ agent._progressWarningForAction = () => '';
+ agent._captureFormValidationState = async () => [];
+ agent._waitForFormValidationFailure = async () => null;
+ agent._readCompletionResult = async () => null;
+ agent._messageRecipientGuardBlock = async () => null;
+ agent._captchaMutationPreflight = async () => null;
+ const dispatched = [];
+ agent.executeTool = async (_tabId, tool) => { dispatched.push(tool); return { success: true, dispatched: true }; };
+ const notes = [];
+ agent._recordPredispatchTiming = (_tabId, step, timing) => notes.push({ step, timing });
+ const messages = [];
+ await agent._executeToolBatch(tabId, [{ id: 'post', function: { name, arguments: JSON.stringify(args) } }],
+ messages, () => {}, provider, '', new Set([name]), 5);
+ return { dispatched, notes, result: String(messages.at(-1)?.content || '') };
+ };
+ // The Post click's own confirmation card, declined.
+ const declined = await run('click_ax', { ref_id: 'ref_9' }, { submitChoice: 'deny' });
+ // The permission gate after the card: the frame's host denied, and a
+ // frame whose host cannot be identified.
+ const denied = await run('iframe_click', { urlFilter: 'x.com', selector: '[data-testid="tweetButton"]' });
+ const unidentified = await run('iframe_click', { urlFilter: '', selector: '[data-testid="tweetButton"]' });
+ for (const [label, outcome, stage] of [[`${browser} card declined`, declined, 'submitConfirmationMs'],
+ [`${browser} permission denied`, denied, 'permissionMs'], [`${browser} host unidentified`, unidentified, 'permissionMs']]) {
+ assert.deepEqual(outcome.dispatched, [], `${label}: nothing was sent`);
+ assert.equal(outcome.notes.length, 1, `${label}: the blocked click records its timing (${outcome.result.slice(0, 160)})`);
+ const [{ step, timing }] = outcome.notes;
+ assert.equal(step, 5, label);
+ assert.equal(timing.outcome, 'blocked', label);
+ assert.equal(typeof timing[stage], 'number', `${label}: ${stage}`);
+ assert.equal(typeof timing.totalMs, 'number', label);
+ }
+ assert.ok(declined.notes[0].timing.submitConfirmationMs >= 10, 'the wait on the confirmation card is recorded');
+ assert.ok(denied.notes[0].timing.permissionMs >= 10, 'the wait on the permission card is recorded');
+ });
+
test(`${browser}: every predispatch stage the agent records survives the trace projection`, async () => {
const fs = await import('node:fs/promises');
const source = await fs.readFile(new URL(`../src/${browser}/src/agent/agent.js`, import.meta.url), 'utf8');