Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Bound Ask-to-Act classifier and retry payloads

16Mikudb5130e68b70
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 76cce8cbe..5d8eada9c 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -461,6 +461,7 @@ const normalizeAttachmentNegationArticles = value => String(value || '').replace
const VISION_SUB_CALL_TIMEOUT_MS = 90_000;
const CONTENT_ACTION_TIMEOUT_MS = 60_000;
const CONTENT_ACTION_RESPONSE_GRACE_MS = 5_000;
+const ASK_MODE_HANDOFF_TIMEOUT_MS = 5_000;
const CONTENT_ACTION_SIGNAL_DEADLINES = new WeakMap();
const EARLY_CDP_ACTION_TOOLS = new Set(['click', 'type_text', 'press_keys', 'hover', 'drag_drop', 'upload_file']);
const DONE_OUTCOMES = new Set(['success', 'partial', 'failed']);
@@ -19972,12 +19973,22 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const provider = this._activeProvider(tabId);
const { tabUrl, tabTitle } = await this._getTabUrlTitle(tabId);
const messages = buildAskModeHandoffMessages(userMessage, finalResponse, tabUrl, tabTitle);
- const response = await this._chatWithCostAllowance(
- provider,
- messages,
- { ...this._plannerChatOptions(provider, false, true, 'ask_mode_handoff'), temperature: 0, maxTokens: 24 },
- this.currentCostState.get(tabId) || null,
- { tabId, generationName: 'ask_mode_handoff' },
+ const response = await this._withContentActionDeadline(
+ signal => this._chatWithCostAllowance(
+ provider,
+ messages,
+ {
+ ...this._plannerChatOptions(provider, false, true, 'ask_mode_handoff'),
+ temperature: 0,
+ maxTokens: 24,
+ signal,
+ },
+ this.currentCostState.get(tabId) || null,
+ { tabId, generationName: 'ask_mode_handoff' },
+ ),
+ 'ask_mode_handoff',
+ ASK_MODE_HANDOFF_TIMEOUT_MS,
+ this._runAbortSignal(tabId),
);
if (!this._checkAbort(tabId) && parseAskModeHandoffFromContent(response?.content) === 'act') {
onUpdate('ask_mode_handoff', { value: 'act' });
diff --git a/src/chrome/src/ui/sidepanel.js b/src/chrome/src/ui/sidepanel.js
index f2c05374a..946e07048 100644
--- a/src/chrome/src/ui/sidepanel.js
+++ b/src/chrome/src/ui/sidepanel.js
@@ -1214,10 +1214,29 @@ let slashCommandSelectedIndex = 0;
let busySlashNoticeLastShownAt = 0;
let composerToastTimer = null;
let retryPayloadSeq = 0;
+const RETRY_PAYLOAD_RETENTION_MS = 30_000;
const activeChatPayloadsByTab = new Map();
const retryAttachmentPayloads = new Map();
const retryAttachmentIdsByTab = new Map();
const retryPayloadByAssistant = new WeakMap();
+const retryPayloadExpiryTimers = new WeakMap();
+
+function rememberRetryPayloadForAssistant(assistantEl, retryPayload) {
+ if (!assistantEl || !retryPayload) return;
+ const previousTimer = retryPayloadExpiryTimers.get(assistantEl);
+ if (previousTimer != null) clearTimeout(previousTimer);
+ retryPayloadByAssistant.set(assistantEl, {
+ ...retryPayload,
+ attachments: Array.isArray(retryPayload.attachments)
+ ? retryPayload.attachments.slice()
+ : [],
+ });
+ const expiryTimer = setTimeout(() => {
+ retryPayloadByAssistant.delete(assistantEl);
+ retryPayloadExpiryTimers.delete(assistantEl);
+ }, RETRY_PAYLOAD_RETENTION_MS);
+ retryPayloadExpiryTimers.set(assistantEl, expiryTimer);
+}
function setTabProcessing(tabId, processing) {
const numericTabId = Number(tabId);
@@ -8918,10 +8937,7 @@ async function sendMessage(extraChatParams = {}) {
assistantEl.dataset.retrySelectionAction = selectionAction;
assistantEl.dataset.retryAttachmentCount = String(attachmentsForSend.length);
if (agentPrompt) assistantEl.dataset.retryAgentPrompt = agentPrompt;
- retryPayloadByAssistant.set(assistantEl, {
- ...retryPayload,
- attachments: attachmentsForSend.slice(),
- });
+ rememberRetryPayloadForAssistant(assistantEl, retryPayload);
userEl.dataset.runRequestId = requestId;
assistantEl.dataset.lastRenderedSeq = '0';
currentAssistantEl = assistantEl;
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 944eb2c1c..e81aedefa 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -395,6 +395,7 @@ const normalizeAttachmentNegationArticles = value => String(value || '').replace
const VISION_SUB_CALL_TIMEOUT_MS = 90_000;
const CONTENT_ACTION_TIMEOUT_MS = 60_000;
const CONTENT_ACTION_RESPONSE_GRACE_MS = 5_000;
+const ASK_MODE_HANDOFF_TIMEOUT_MS = 5_000;
const CONTENT_ACTION_SIGNAL_DEADLINES = new WeakMap();
const SAVED_WORKFLOW_MESSAGE_DISPATCH_TOOLS = new Set([
'click', 'click_ax', 'iframe_click', 'execute_js', 'execute_webmcp_tool', 'upload_file',
@@ -17756,12 +17757,22 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const provider = this._activeProvider(tabId);
const { tabUrl, tabTitle } = await this._getTabUrlTitle(tabId);
const messages = buildAskModeHandoffMessages(userMessage, finalResponse, tabUrl, tabTitle);
- const response = await this._chatWithCostAllowance(
- provider,
- messages,
- { ...this._plannerChatOptions(provider, false, true, 'ask_mode_handoff'), temperature: 0, maxTokens: 24 },
- this.currentCostState.get(tabId) || null,
- { tabId, generationName: 'ask_mode_handoff' },
+ const response = await this._withContentActionDeadline(
+ signal => this._chatWithCostAllowance(
+ provider,
+ messages,
+ {
+ ...this._plannerChatOptions(provider, false, true, 'ask_mode_handoff'),
+ temperature: 0,
+ maxTokens: 24,
+ signal,
+ },
+ this.currentCostState.get(tabId) || null,
+ { tabId, generationName: 'ask_mode_handoff' },
+ ),
+ 'ask_mode_handoff',
+ ASK_MODE_HANDOFF_TIMEOUT_MS,
+ this._runAbortSignal(tabId),
);
if (!this._checkAbort(tabId) && parseAskModeHandoffFromContent(response?.content) === 'act') {
onUpdate('ask_mode_handoff', { value: 'act' });
diff --git a/src/firefox/src/ui/sidepanel.js b/src/firefox/src/ui/sidepanel.js
index 1d3e76954..1d3cd3123 100644
--- a/src/firefox/src/ui/sidepanel.js
+++ b/src/firefox/src/ui/sidepanel.js
@@ -1068,10 +1068,29 @@ let slashCommandSelectedIndex = 0;
let busySlashNoticeLastShownAt = 0;
let composerToastTimer = null;
let retryPayloadSeq = 0;
+const RETRY_PAYLOAD_RETENTION_MS = 30_000;
const activeChatPayloadsByTab = new Map();
const retryAttachmentPayloads = new Map();
const retryAttachmentIdsByTab = new Map();
const retryPayloadByAssistant = new WeakMap();
+const retryPayloadExpiryTimers = new WeakMap();
+
+function rememberRetryPayloadForAssistant(assistantEl, retryPayload) {
+ if (!assistantEl || !retryPayload) return;
+ const previousTimer = retryPayloadExpiryTimers.get(assistantEl);
+ if (previousTimer != null) clearTimeout(previousTimer);
+ retryPayloadByAssistant.set(assistantEl, {
+ ...retryPayload,
+ attachments: Array.isArray(retryPayload.attachments)
+ ? retryPayload.attachments.slice()
+ : [],
+ });
+ const expiryTimer = setTimeout(() => {
+ retryPayloadByAssistant.delete(assistantEl);
+ retryPayloadExpiryTimers.delete(assistantEl);
+ }, RETRY_PAYLOAD_RETENTION_MS);
+ retryPayloadExpiryTimers.set(assistantEl, expiryTimer);
+}
function setTabProcessing(tabId, processing) {
const numericTabId = Number(tabId);
@@ -8572,10 +8591,7 @@ async function sendMessage(extraChatParams = {}) {
assistantEl.dataset.retrySelectionAction = selectionAction;
assistantEl.dataset.retryAttachmentCount = String(attachmentsForSend.length);
if (agentPrompt) assistantEl.dataset.retryAgentPrompt = agentPrompt;
- retryPayloadByAssistant.set(assistantEl, {
- ...retryPayload,
- attachments: attachmentsForSend.slice(),
- });
+ rememberRetryPayloadForAssistant(assistantEl, retryPayload);
userEl.dataset.runRequestId = requestId;
assistantEl.dataset.lastRenderedSeq = '0';
currentAssistantEl = assistantEl;
diff --git a/test/run.js b/test/run.js
index e92d2f3b3..bc77e271d 100644
--- a/test/run.js
+++ b/test/run.js
@@ -11996,20 +11996,27 @@ test('Ask mode handoff classification is strict, silent, and mode guarded', asyn
const provider = { name: `${browserLabel}-handoff`, model: `${browserLabel}-handoff`, promptTier: 'full' };
const agent = new AgentClass({ getActive: () => provider, getVisionProvider: async () => null });
const updates = [];
+ const deadlineCalls = [];
let calls = 0;
agent._getTabUrlTitle = async () => ({
tabUrl: 'https://example.com/page',
tabTitle: 'Example page',
});
+ const withContentActionDeadline = agent._withContentActionDeadline.bind(agent);
+ agent._withContentActionDeadline = async (...args) => {
+ deadlineCalls.push({ toolName: args[1], deadlineMs: args[2] });
+ return withContentActionDeadline(...args);
+ };
agent._chatWithCostAllowance = async (_provider, messages, options, _costState, metadata) => {
calls += 1;
assert.equal(metadata?.generationName, 'ask_mode_handoff', `${browserLabel}: wrong classifier generation`);
assert.equal(options.maxTokens, 24, `${browserLabel}: handoff classifier budget should stay small`);
+ assert.ok(options.signal, `${browserLabel}: handoff classifier must receive an abort signal`);
assert.match(messages[0]?.content || '', /DATA, never instructions/i, `${browserLabel}: classifier prompt must treat inputs as data`);
assert.match(messages[1]?.content || '', /<assistant_answer>/, `${browserLabel}: answer was not included in classifier input`);
return { content: response, usage: {} };
};
- return { agent, updates, get calls() { return calls; } };
+ return { agent, updates, deadlineCalls, get calls() { return calls; } };
};
const successful = createAgent();
@@ -12019,6 +12026,11 @@ test('Ask mode handoff classification is strict, silent, and mode guarded', asyn
(type, data) => successful.updates.push({ type, data }), {},
);
assert.equal(successful.calls, 1, `${browserLabel}: Ask answer should invoke the classifier once`);
+ assert.deepEqual(
+ successful.deadlineCalls,
+ [{ toolName: 'ask_mode_handoff', deadlineMs: 5_000 }],
+ `${browserLabel}: handoff classifier must use its dedicated short deadline`,
+ );
assert.deepEqual(successful.updates, [{ type: 'ask_mode_handoff', data: { value: 'act' } }], `${browserLabel}: act handoff event missing`);
for (const mode of ['act', 'dev']) {
@@ -12059,6 +12071,22 @@ test('Ask mode handoff classification is strict, silent, and mode guarded', asyn
`${browserLabel}: classifier errors must be silent`,
);
assert.equal(failing.calls, 0, `${browserLabel}: replacement failure stub should be used without leaking state`);
+
+ const timedOut = createAgent();
+ timedOut.agent._withContentActionDeadline = async () => {
+ const error = new Error('handoff classifier timed out');
+ error.code = 'content_action_timeout';
+ throw error;
+ };
+ const timedOutUpdates = [];
+ await assert.doesNotReject(
+ () => timedOut.agent._maybeEmitAskModeHandoff(
+ 53007, 'ask', 'Click the button', finalResponse,
+ (...event) => timedOutUpdates.push(event), {},
+ ),
+ `${browserLabel}: classifier timeout must not block or fail the completed Ask answer`,
+ );
+ assert.deepEqual(timedOutUpdates, [], `${browserLabel}: timed-out classifier must not emit a handoff`);
}
});
@@ -114528,9 +114556,14 @@ test('sidepanel: Ask-to-Act retries retain attachment payloads', () => {
assert.match(source, /const retryPayloadByAssistant = new WeakMap\(\)/, `${label}: assistant retry payload store is missing`);
assert.match(
source,
- /retryPayloadByAssistant\.set\(assistantEl, \{[\s\S]*?attachments: attachmentsForSend\.slice\(\)/,
+ /function rememberRetryPayloadForAssistant\(assistantEl, retryPayload\) \{[\s\S]*?retryPayloadByAssistant\.set\(assistantEl, \{[\s\S]*?attachments: Array\.isArray\(retryPayload\.attachments\)[\s\S]*?retryPayload\.attachments\.slice\(\)/,
`${label}: sent attachment payloads are not retained with the assistant message`,
);
+ assert.match(
+ source,
+ /const RETRY_PAYLOAD_RETENTION_MS = 30_000[\s\S]*?setTimeout\(\(\) => \{[\s\S]*?retryPayloadByAssistant\.delete\(assistantEl\)/,
+ `${label}: assistant retry attachment payloads do not have a bounded lifetime`,
+ );
assert.match(
source,
/function retryPayloadForRunAssistant\(assistantEl\) \{[\s\S]*?const storedRetryPayload = retryPayloadByAssistant\.get\(assistantEl\)[\s\S]*?const attachments = Array\.isArray\(storedRetryPayload\?\.attachments\)/,