Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Publication prefetch: a call with no timing note leaves no timing record
Barack Sokullu544c908ab716
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 94df1077e..ca12a30d9 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -6230,6 +6230,7 @@ export class Agent extends LoopDetector {_releaseRunEntry(tabId) {this._settleUnverifiedStepScope(tabId);this._cancelSocialPublicationBackground(tabId);+ this._predispatchTimings?.delete(tabId);const feedbackRun = this._pageFeedbackRuns?.get(tabId);if (feedbackRun?.monitoringRetry && !feedbackRun.monitoringRetry.started) {this._restoreFailureRecoveryStart(tabId, feedbackRun.monitoringRetry.requestId, feedbackRun.onUpdate, feedbackRun.monitoringRetry);@@ -9433,6 +9434,10 @@ export class Agent extends LoopDetector {const add = (stage, ms) => {if (Number.isFinite(ms) && ms >= 0) stages[stage] = (stages[stage] || 0) + ms;};+ // The tab's open record goes once the call has its note or none is due.+ const discard = () => {+ if (this._predispatchTimings?.get(tabId) === stages) this._predispatchTimings.delete(tabId);+ };return {lap: stage => { const now = Date.now(); add(stage, now - last); last = now; },add,@@ -9441,9 +9446,10 @@ export class Agent extends LoopDetector {emit: (step, extra = {}) => {if (emitted) return;emitted = true;- if (this._predispatchTimings?.get(tabId) === stages) this._predispatchTimings.delete(tabId);+ discard();this._recordPredispatchTiming(tabId, step, { ...stages, totalMs: Date.now() - startedAt, ...extra });},+ discard,get emitted() { return emitted; },};}@@ -9469,7 +9475,9 @@ export class Agent extends LoopDetector {if (predispatch.emitted) return;let consequential = false;try { consequential = !!pending.consequential(); } catch {}- if (!consequential) return;+ // A read or other call that changes nothing gets no note; its open+ // record for the tab goes with it.+ if (!consequential) { predispatch.discard(); return; }predispatch.lap('stoppedMs');predispatch.emit(pending.step, { tool: pending.tool,outcome: callState.dispatchState?.started === true ? 'dispatched' : 'not_dispatched' });@@ -16064,7 +16072,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dtabId, fnName, fnArgs, detectedSubmitAction, provider, callState.guardAdvisories,))|| this._taskClickLockAsk(tabId, fnName, fnArgs, callState);- // From here the permission capabilities and the submit probe decide.+ // From here the permission capabilities and the submit probe decide:+ // missingResponseOutcomeUnknown (above) holds when the call needs a+ // permission capability or is a state-changing tool, so navigation,+ // typing, uploads and deletions stay consequential; the probe adds a+ // submit or a publication control that needs neither.predispatchConsequential = () => missingResponseOutcomeUnknown|| detectedSubmitAction?.isSubmit === true || detectedSubmitAction?.publicationControl === true;let workflowPreSubmitBlock = await runWorkflowPreSubmitChecks();@@ -35527,6 +35539,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dthis._verifiedTextReplacements.delete(tabId);// Task-scoped like the proofs it sits beside; it never guarded anything.this._unverifiedTextWrites?.delete(tabId);+ this._predispatchTimings?.delete(tabId);this._lastCdpClickIdent?.delete(tabId);this._lastClickProgress?.delete(tabId);this._clickAxCdpFallbacks?.delete(tabId);diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex b04274330..884f3e7e2 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -5776,6 +5776,7 @@ export class Agent extends LoopDetector {_releaseRunEntry(tabId) {this._settleUnverifiedStepScope(tabId);this._cancelSocialPublicationBackground(tabId);+ this._predispatchTimings?.delete(tabId);const feedbackRun = this._pageFeedbackRuns?.get(tabId);if (feedbackRun?.monitoringRetry && !feedbackRun.monitoringRetry.started) {this._restoreFailureRecoveryStart(tabId, feedbackRun.monitoringRetry.requestId, feedbackRun.onUpdate, feedbackRun.monitoringRetry);@@ -9146,6 +9147,10 @@ export class Agent extends LoopDetector {const add = (stage, ms) => {if (Number.isFinite(ms) && ms >= 0) stages[stage] = (stages[stage] || 0) + ms;};+ // The tab's open record goes once the call has its note or none is due.+ const discard = () => {+ if (this._predispatchTimings?.get(tabId) === stages) this._predispatchTimings.delete(tabId);+ };return {lap: stage => { const now = Date.now(); add(stage, now - last); last = now; },add,@@ -9154,9 +9159,10 @@ export class Agent extends LoopDetector {emit: (step, extra = {}) => {if (emitted) return;emitted = true;- if (this._predispatchTimings?.get(tabId) === stages) this._predispatchTimings.delete(tabId);+ discard();this._recordPredispatchTiming(tabId, step, { ...stages, totalMs: Date.now() - startedAt, ...extra });},+ discard,get emitted() { return emitted; },};}@@ -9182,7 +9188,9 @@ export class Agent extends LoopDetector {if (predispatch.emitted) return;let consequential = false;try { consequential = !!pending.consequential(); } catch {}- if (!consequential) return;+ // A read or other call that changes nothing gets no note; its open+ // record for the tab goes with it.+ if (!consequential) { predispatch.discard(); return; }predispatch.lap('stoppedMs');predispatch.emit(pending.step, { tool: pending.tool,outcome: callState.dispatchState?.started === true ? 'dispatched' : 'not_dispatched' });@@ -14362,7 +14370,11 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d|| await timedCheck('workflowDispatchMs', () => this._workflowPreSubmitDispatchBlock(tabId, fnName, fnArgs, detectedSubmitAction, provider, callState.guardAdvisories,));- // From here the permission capabilities and the submit probe decide.+ // From here the permission capabilities and the submit probe decide:+ // missingResponseOutcomeUnknown (above) holds when the call needs a+ // permission capability or is a state-changing tool, so navigation,+ // typing, uploads and deletions stay consequential; the probe adds a+ // submit or a publication control that needs neither.predispatchConsequential = () => missingResponseOutcomeUnknown|| detectedSubmitAction?.isSubmit === true || detectedSubmitAction?.publicationControl === true;let workflowPreSubmitBlock = await runWorkflowPreSubmitChecks();@@ -33664,6 +33676,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dthis._verifiedTextReplacements.delete(tabId);// Task-scoped like the proofs it sits beside; it never guarded anything.this._unverifiedTextWrites?.delete(tabId);+ this._predispatchTimings?.delete(tabId);this.progressPageScopes.delete(tabId);this.progressSessions.delete(tabId);this.progressExpectedItems.delete(tabId);diff --git a/test/publication-prefetch.mjs b/test/publication-prefetch.mjsindex 728526a1d..8b46e3f43 100644--- a/test/publication-prefetch.mjs+++ b/test/publication-prefetch.mjs@@ -1224,6 +1224,58 @@ for (const browser of ['chrome', 'firefox']) {assert.equal(JSON.stringify(projected).includes('PRIVATE'), false);assert(Object.values(projected.extra).every(value => typeof value === 'number' || ['click_ax', 'dispatched'].includes(value)));});++ test(`${browser}: navigation and typing record predispatch_timing too, and a read leaves no timing record behind`, async () => {+ const run = async (name, args) => {+ const provider = { model: 'fake-checker', chat: async () => ({ content: '{}' }) };+ const agent = new Agent({ getActive: () => provider });+ const tabId = 812;+ agent._persist = () => {};+ agent._currentUrl = async () => 'https://example.com/search';+ agent.conversations.set(tabId, [{ role: 'system', content: 'system' }, { role: 'user', content: 'Search the site for hello' }]);+ agent._startPlanExecutionGuard(tabId, 'act', { requestKind: 'execute', requiresStateChange: true });+ agent.currentRunId.set(tabId, 'timing-run');+ agent._skipPermissionGate = true;+ agent._ensureGateSetting = async () => true;+ agent._recordProgressObservation = async () => null;+ agent._autoRecordProgressAction = () => null;+ agent._progressWarningForAction = () => '';+ agent._captureFormValidationState = async () => [];+ agent._waitForFormValidationFailure = async () => null;+ agent._readCompletionResult = async () => null;+ agent._messageRecipientGuardBlock = async () => null;+ agent._captchaMutationPreflight = async () => null;+ agent._detectLikelySubmitAction = async () => ({ isSubmit: false });+ const dispatched = [];+ agent.executeTool = async (_tabId, tool) => { dispatched.push(tool); return { success: true, dispatched: true }; };+ const notes = [];+ agent._recordPredispatchTiming = (_tabId, step, timing) => notes.push({ step, timing });+ await agent._executeToolBatch(tabId, [{ id: 'call', function: { name, arguments: JSON.stringify(args) } }],+ [], () => {}, provider, '', new Set([name]), 7);+ return { agent, tabId, dispatched, notes };+ };+ // None of these is a submit; the permission capability each needs makes+ // it consequential.+ for (const [name, args] of [['navigate', { url: 'https://example.com/next' }],+ ['type_text', { selector: '#q', text: 'hello' }], ['set_field', { ref_id: 'ref_3', text: 'hello' }]]) {+ const { agent, tabId, dispatched, notes } = await run(name, args);+ assert.deepEqual(dispatched, [name], name);+ assert.equal(notes.length, 1, `${name}: its timing is recorded`);+ assert.equal(notes[0].timing.tool, name);+ assert.equal(notes[0].timing.outcome, 'dispatched');+ assert.equal(agent._predispatchTimings?.has(tabId) || false, false, `${name}: no record is left open`);+ }+ // A read gets no note, and its record for the tab goes when its batch ends.+ const read = await run('get_accessibility_tree', {});+ assert.deepEqual(read.dispatched, ['get_accessibility_tree']);+ assert.equal(read.notes.length, 0);+ assert.equal(read.agent._predispatchTimings?.has(read.tabId) || false, false, 'the read left its timing record behind');+ // The end of a run drops whatever record is still open.+ read.agent._predispatchTimer(read.tabId);+ assert.equal(read.agent._predispatchTimings.has(read.tabId), true);+ read.agent._releaseRunEntry(read.tabId);+ assert.equal(read.agent._predispatchTimings.has(read.tabId), false, 'the end of the run dropped the open record');+ });}// The prefetch probe, in a real page: it must find the composer's own Post