Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Publication prefetch: a call stopped at a card or the permission gate records its timing
Barack Sokulluc9f2c6b9070d
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 40c4a09d8..183123ae7 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -15940,6 +15940,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d|| this._taskClickLockAsk(tabId, fnName, fnArgs, callState);const predispatchConsequential = () => missingResponseOutcomeUnknown|| detectedSubmitAction?.isSubmit === true || detectedSubmitAction?.publicationControl === true;+ // A consequential call that a confirmation card or the permission gate+ // stops still records where its time went, the wait on the card+ // included.+ const predispatchBlocked = stage => {+ predispatch.lap(stage);+ if (predispatchConsequential()) predispatch.emit(step, { tool: fnName, outcome: 'blocked' });+ };let workflowPreSubmitBlock = await runWorkflowPreSubmitChecks();if (workflowPreSubmitBlock?.mutationConfirmation&& this.scheduledRunPolicies.get(tabId)?.requireConsequentialConfirmation !== false) {@@ -16152,6 +16159,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}),});onUpdate('warning', { message: canGroupSearch ? 'Low-risk search blocked until the user allows the grouped actions.' : 'Form submission blocked until the user confirms.' });+ predispatchBlocked('submitConfirmationMs');if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}@@ -16214,6 +16222,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}),});onUpdate('warning', { message: 'Low-risk search blocked until the user allows the grouped actions.' });+ predispatchBlocked('submitConfirmationMs');if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}@@ -16294,6 +16303,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}),});onUpdate('warning', { message: 'Low-risk search blocked until the user allows the grouped actions.' });+ predispatchBlocked('permissionMs');if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}@@ -16363,6 +16373,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout derror: `Cannot run ${fnName}: the target frame/host couldn't be identified, so it can't be permission-checked. Pass a urlFilter naming the iframe's domain (read it first with iframe_read / get_accessibility_tree) and retry.`,}),});+ predispatchBlocked('permissionMs');if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}@@ -16376,6 +16387,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout derror: `The user denied permission to ${CAPABILITY_LABEL[blocked.capability]} ${blocked.host}. Do NOT retry this action on that site. Continue with what you can without it, or ask the user how to proceed.`,}),});+ predispatchBlocked('permissionMs');if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex c2763f931..1dec3a2b8 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -14239,6 +14239,13 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d));const predispatchConsequential = () => missingResponseOutcomeUnknown|| detectedSubmitAction?.isSubmit === true || detectedSubmitAction?.publicationControl === true;+ // A consequential call that a confirmation card or the permission gate+ // stops still records where its time went, the wait on the card+ // included.+ const predispatchBlocked = stage => {+ predispatch.lap(stage);+ if (predispatchConsequential()) predispatch.emit(step, { tool: fnName, outcome: 'blocked' });+ };let workflowPreSubmitBlock = await runWorkflowPreSubmitChecks();if (workflowPreSubmitBlock?.mutationConfirmation&& this.scheduledRunPolicies.get(tabId)?.requireConsequentialConfirmation !== false) {@@ -14444,6 +14451,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}),});onUpdate('warning', { message: canGroupSearch ? 'Low-risk search blocked until the user allows the grouped actions.' : 'Form submission blocked until the user confirms.' });+ predispatchBlocked('submitConfirmationMs');if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}@@ -14507,6 +14515,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}),});onUpdate('warning', { message: 'Low-risk search blocked until the user allows the grouped actions.' });+ predispatchBlocked('submitConfirmationMs');if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}@@ -14587,6 +14596,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}),});onUpdate('warning', { message: 'Low-risk search blocked until the user allows the grouped actions.' });+ predispatchBlocked('permissionMs');if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}@@ -14656,6 +14666,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout derror: `Cannot run ${fnName}: the target frame/host couldn't be identified, so it can't be permission-checked. Pass a urlFilter naming the iframe's domain (read it first with iframe_read / get_accessibility_tree) and retry.`,}),});+ predispatchBlocked('permissionMs');if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}@@ -14669,6 +14680,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout derror: `The user denied permission to ${CAPABILITY_LABEL[blocked.capability]} ${blocked.host}. Do NOT retry this action on that site. Continue with what you can without it, or ask the user how to proceed.`,}),});+ predispatchBlocked('permissionMs');if (interruptFailedBrowserAction(toolIndex, fnName)) { navNotices.length = 0; break; }continue;}diff --git a/test/publication-prefetch.mjs b/test/publication-prefetch.mjsindex 252266888..7ed14a031 100644--- a/test/publication-prefetch.mjs+++ b/test/publication-prefetch.mjs@@ -422,6 +422,57 @@ for (const browser of ['chrome', 'firefox']) {assert.equal(gates.length, 3);});+ test(`${browser}: a consequential click a card or the permission gate stops still records predispatch_timing`, async () => {+ const wait = ms => new Promise(resolve => setTimeout(resolve, ms));+ const run = async (name, args, { submitChoice = 'once', permissionChoice = 'deny' } = {}) => {+ const f = setup();+ const { agent, tabId, provider } = f;+ assert.equal(await f.write(), 'authorized');+ agent.currentRunId.set(tabId, 'prefetch-run');+ agent._skipPermissionGate = false;+ agent._ensureGateSetting = async () => false;+ agent.permissions.hydrate = async () => {};+ agent.permissions.check = () => ({ allowed: false, needsPrompt: true });+ agent.permissions.record = async () => {};+ agent._promptSubmitConfirmation = async () => { await wait(15); return submitChoice; };+ agent._promptPermission = async () => { await wait(15); return permissionChoice; };+ agent._recordProgressObservation = async () => null;+ agent._autoRecordProgressAction = () => null;+ agent._progressWarningForAction = () => '';+ agent._captureFormValidationState = async () => [];+ agent._waitForFormValidationFailure = async () => null;+ agent._readCompletionResult = async () => null;+ agent._messageRecipientGuardBlock = async () => null;+ agent._captchaMutationPreflight = async () => null;+ const dispatched = [];+ agent.executeTool = async (_tabId, tool) => { dispatched.push(tool); return { success: true, dispatched: true }; };+ const notes = [];+ agent._recordPredispatchTiming = (_tabId, step, timing) => notes.push({ step, timing });+ const messages = [];+ await agent._executeToolBatch(tabId, [{ id: 'post', function: { name, arguments: JSON.stringify(args) } }],+ messages, () => {}, provider, '', new Set([name]), 5);+ return { dispatched, notes, result: String(messages.at(-1)?.content || '') };+ };+ // The Post click's own confirmation card, declined.+ const declined = await run('click_ax', { ref_id: 'ref_9' }, { submitChoice: 'deny' });+ // The permission gate after the card: the frame's host denied, and a+ // frame whose host cannot be identified.+ const denied = await run('iframe_click', { urlFilter: 'x.com', selector: '[data-testid="tweetButton"]' });+ const unidentified = await run('iframe_click', { urlFilter: '', selector: '[data-testid="tweetButton"]' });+ for (const [label, outcome, stage] of [[`${browser} card declined`, declined, 'submitConfirmationMs'],+ [`${browser} permission denied`, denied, 'permissionMs'], [`${browser} host unidentified`, unidentified, 'permissionMs']]) {+ assert.deepEqual(outcome.dispatched, [], `${label}: nothing was sent`);+ assert.equal(outcome.notes.length, 1, `${label}: the blocked click records its timing (${outcome.result.slice(0, 160)})`);+ const [{ step, timing }] = outcome.notes;+ assert.equal(step, 5, label);+ assert.equal(timing.outcome, 'blocked', label);+ assert.equal(typeof timing[stage], 'number', `${label}: ${stage}`);+ assert.equal(typeof timing.totalMs, 'number', label);+ }+ assert.ok(declined.notes[0].timing.submitConfirmationMs >= 10, 'the wait on the confirmation card is recorded');+ assert.ok(denied.notes[0].timing.permissionMs >= 10, 'the wait on the permission card is recorded');+ });+test(`${browser}: every predispatch stage the agent records survives the trace projection`, async () => {const fs = await import('node:fs/promises');const source = await fs.readFile(new URL(`../src/${browser}/src/agent/agent.js`, import.meta.url), 'utf8');