Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Bound Ask-to-Act classifier and retry payloads
16Mikudb5130e68b70
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 76cce8cbe..5d8eada9c 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -461,6 +461,7 @@ const normalizeAttachmentNegationArticles = value => String(value || '').replaceconst VISION_SUB_CALL_TIMEOUT_MS = 90_000;const CONTENT_ACTION_TIMEOUT_MS = 60_000;const CONTENT_ACTION_RESPONSE_GRACE_MS = 5_000;+const ASK_MODE_HANDOFF_TIMEOUT_MS = 5_000;const CONTENT_ACTION_SIGNAL_DEADLINES = new WeakMap();const EARLY_CDP_ACTION_TOOLS = new Set(['click', 'type_text', 'press_keys', 'hover', 'drag_drop', 'upload_file']);const DONE_OUTCOMES = new Set(['success', 'partial', 'failed']);@@ -19972,12 +19973,22 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst provider = this._activeProvider(tabId);const { tabUrl, tabTitle } = await this._getTabUrlTitle(tabId);const messages = buildAskModeHandoffMessages(userMessage, finalResponse, tabUrl, tabTitle);- const response = await this._chatWithCostAllowance(- provider,- messages,- { ...this._plannerChatOptions(provider, false, true, 'ask_mode_handoff'), temperature: 0, maxTokens: 24 },- this.currentCostState.get(tabId) || null,- { tabId, generationName: 'ask_mode_handoff' },+ const response = await this._withContentActionDeadline(+ signal => this._chatWithCostAllowance(+ provider,+ messages,+ {+ ...this._plannerChatOptions(provider, false, true, 'ask_mode_handoff'),+ temperature: 0,+ maxTokens: 24,+ signal,+ },+ this.currentCostState.get(tabId) || null,+ { tabId, generationName: 'ask_mode_handoff' },+ ),+ 'ask_mode_handoff',+ ASK_MODE_HANDOFF_TIMEOUT_MS,+ this._runAbortSignal(tabId),);if (!this._checkAbort(tabId) && parseAskModeHandoffFromContent(response?.content) === 'act') {onUpdate('ask_mode_handoff', { value: 'act' });diff --git a/src/chrome/src/ui/sidepanel.js b/src/chrome/src/ui/sidepanel.jsindex f2c05374a..946e07048 100644--- a/src/chrome/src/ui/sidepanel.js+++ b/src/chrome/src/ui/sidepanel.js@@ -1214,10 +1214,29 @@ let slashCommandSelectedIndex = 0;let busySlashNoticeLastShownAt = 0;let composerToastTimer = null;let retryPayloadSeq = 0;+const RETRY_PAYLOAD_RETENTION_MS = 30_000;const activeChatPayloadsByTab = new Map();const retryAttachmentPayloads = new Map();const retryAttachmentIdsByTab = new Map();const retryPayloadByAssistant = new WeakMap();+const retryPayloadExpiryTimers = new WeakMap();++function rememberRetryPayloadForAssistant(assistantEl, retryPayload) {+ if (!assistantEl || !retryPayload) return;+ const previousTimer = retryPayloadExpiryTimers.get(assistantEl);+ if (previousTimer != null) clearTimeout(previousTimer);+ retryPayloadByAssistant.set(assistantEl, {+ ...retryPayload,+ attachments: Array.isArray(retryPayload.attachments)+ ? retryPayload.attachments.slice()+ : [],+ });+ const expiryTimer = setTimeout(() => {+ retryPayloadByAssistant.delete(assistantEl);+ retryPayloadExpiryTimers.delete(assistantEl);+ }, RETRY_PAYLOAD_RETENTION_MS);+ retryPayloadExpiryTimers.set(assistantEl, expiryTimer);+}function setTabProcessing(tabId, processing) {const numericTabId = Number(tabId);@@ -8918,10 +8937,7 @@ async function sendMessage(extraChatParams = {}) {assistantEl.dataset.retrySelectionAction = selectionAction;assistantEl.dataset.retryAttachmentCount = String(attachmentsForSend.length);if (agentPrompt) assistantEl.dataset.retryAgentPrompt = agentPrompt;- retryPayloadByAssistant.set(assistantEl, {- ...retryPayload,- attachments: attachmentsForSend.slice(),- });+ rememberRetryPayloadForAssistant(assistantEl, retryPayload);userEl.dataset.runRequestId = requestId;assistantEl.dataset.lastRenderedSeq = '0';currentAssistantEl = assistantEl;diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 944eb2c1c..e81aedefa 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -395,6 +395,7 @@ const normalizeAttachmentNegationArticles = value => String(value || '').replaceconst VISION_SUB_CALL_TIMEOUT_MS = 90_000;const CONTENT_ACTION_TIMEOUT_MS = 60_000;const CONTENT_ACTION_RESPONSE_GRACE_MS = 5_000;+const ASK_MODE_HANDOFF_TIMEOUT_MS = 5_000;const CONTENT_ACTION_SIGNAL_DEADLINES = new WeakMap();const SAVED_WORKFLOW_MESSAGE_DISPATCH_TOOLS = new Set(['click', 'click_ax', 'iframe_click', 'execute_js', 'execute_webmcp_tool', 'upload_file',@@ -17756,12 +17757,22 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst provider = this._activeProvider(tabId);const { tabUrl, tabTitle } = await this._getTabUrlTitle(tabId);const messages = buildAskModeHandoffMessages(userMessage, finalResponse, tabUrl, tabTitle);- const response = await this._chatWithCostAllowance(- provider,- messages,- { ...this._plannerChatOptions(provider, false, true, 'ask_mode_handoff'), temperature: 0, maxTokens: 24 },- this.currentCostState.get(tabId) || null,- { tabId, generationName: 'ask_mode_handoff' },+ const response = await this._withContentActionDeadline(+ signal => this._chatWithCostAllowance(+ provider,+ messages,+ {+ ...this._plannerChatOptions(provider, false, true, 'ask_mode_handoff'),+ temperature: 0,+ maxTokens: 24,+ signal,+ },+ this.currentCostState.get(tabId) || null,+ { tabId, generationName: 'ask_mode_handoff' },+ ),+ 'ask_mode_handoff',+ ASK_MODE_HANDOFF_TIMEOUT_MS,+ this._runAbortSignal(tabId),);if (!this._checkAbort(tabId) && parseAskModeHandoffFromContent(response?.content) === 'act') {onUpdate('ask_mode_handoff', { value: 'act' });diff --git a/src/firefox/src/ui/sidepanel.js b/src/firefox/src/ui/sidepanel.jsindex 1d3e76954..1d3cd3123 100644--- a/src/firefox/src/ui/sidepanel.js+++ b/src/firefox/src/ui/sidepanel.js@@ -1068,10 +1068,29 @@ let slashCommandSelectedIndex = 0;let busySlashNoticeLastShownAt = 0;let composerToastTimer = null;let retryPayloadSeq = 0;+const RETRY_PAYLOAD_RETENTION_MS = 30_000;const activeChatPayloadsByTab = new Map();const retryAttachmentPayloads = new Map();const retryAttachmentIdsByTab = new Map();const retryPayloadByAssistant = new WeakMap();+const retryPayloadExpiryTimers = new WeakMap();++function rememberRetryPayloadForAssistant(assistantEl, retryPayload) {+ if (!assistantEl || !retryPayload) return;+ const previousTimer = retryPayloadExpiryTimers.get(assistantEl);+ if (previousTimer != null) clearTimeout(previousTimer);+ retryPayloadByAssistant.set(assistantEl, {+ ...retryPayload,+ attachments: Array.isArray(retryPayload.attachments)+ ? retryPayload.attachments.slice()+ : [],+ });+ const expiryTimer = setTimeout(() => {+ retryPayloadByAssistant.delete(assistantEl);+ retryPayloadExpiryTimers.delete(assistantEl);+ }, RETRY_PAYLOAD_RETENTION_MS);+ retryPayloadExpiryTimers.set(assistantEl, expiryTimer);+}function setTabProcessing(tabId, processing) {const numericTabId = Number(tabId);@@ -8572,10 +8591,7 @@ async function sendMessage(extraChatParams = {}) {assistantEl.dataset.retrySelectionAction = selectionAction;assistantEl.dataset.retryAttachmentCount = String(attachmentsForSend.length);if (agentPrompt) assistantEl.dataset.retryAgentPrompt = agentPrompt;- retryPayloadByAssistant.set(assistantEl, {- ...retryPayload,- attachments: attachmentsForSend.slice(),- });+ rememberRetryPayloadForAssistant(assistantEl, retryPayload);userEl.dataset.runRequestId = requestId;assistantEl.dataset.lastRenderedSeq = '0';currentAssistantEl = assistantEl;diff --git a/test/run.js b/test/run.jsindex e92d2f3b3..bc77e271d 100644--- a/test/run.js+++ b/test/run.js@@ -11996,20 +11996,27 @@ test('Ask mode handoff classification is strict, silent, and mode guarded', asynconst provider = { name: `${browserLabel}-handoff`, model: `${browserLabel}-handoff`, promptTier: 'full' };const agent = new AgentClass({ getActive: () => provider, getVisionProvider: async () => null });const updates = [];+ const deadlineCalls = [];let calls = 0;agent._getTabUrlTitle = async () => ({tabUrl: 'https://example.com/page',tabTitle: 'Example page',});+ const withContentActionDeadline = agent._withContentActionDeadline.bind(agent);+ agent._withContentActionDeadline = async (...args) => {+ deadlineCalls.push({ toolName: args[1], deadlineMs: args[2] });+ return withContentActionDeadline(...args);+ };agent._chatWithCostAllowance = async (_provider, messages, options, _costState, metadata) => {calls += 1;assert.equal(metadata?.generationName, 'ask_mode_handoff', `${browserLabel}: wrong classifier generation`);assert.equal(options.maxTokens, 24, `${browserLabel}: handoff classifier budget should stay small`);+ assert.ok(options.signal, `${browserLabel}: handoff classifier must receive an abort signal`);assert.match(messages[0]?.content || '', /DATA, never instructions/i, `${browserLabel}: classifier prompt must treat inputs as data`);assert.match(messages[1]?.content || '', /<assistant_answer>/, `${browserLabel}: answer was not included in classifier input`);return { content: response, usage: {} };};- return { agent, updates, get calls() { return calls; } };+ return { agent, updates, deadlineCalls, get calls() { return calls; } };};const successful = createAgent();@@ -12019,6 +12026,11 @@ test('Ask mode handoff classification is strict, silent, and mode guarded', asyn(type, data) => successful.updates.push({ type, data }), {},);assert.equal(successful.calls, 1, `${browserLabel}: Ask answer should invoke the classifier once`);+ assert.deepEqual(+ successful.deadlineCalls,+ [{ toolName: 'ask_mode_handoff', deadlineMs: 5_000 }],+ `${browserLabel}: handoff classifier must use its dedicated short deadline`,+ );assert.deepEqual(successful.updates, [{ type: 'ask_mode_handoff', data: { value: 'act' } }], `${browserLabel}: act handoff event missing`);for (const mode of ['act', 'dev']) {@@ -12059,6 +12071,22 @@ test('Ask mode handoff classification is strict, silent, and mode guarded', asyn`${browserLabel}: classifier errors must be silent`,);assert.equal(failing.calls, 0, `${browserLabel}: replacement failure stub should be used without leaking state`);++ const timedOut = createAgent();+ timedOut.agent._withContentActionDeadline = async () => {+ const error = new Error('handoff classifier timed out');+ error.code = 'content_action_timeout';+ throw error;+ };+ const timedOutUpdates = [];+ await assert.doesNotReject(+ () => timedOut.agent._maybeEmitAskModeHandoff(+ 53007, 'ask', 'Click the button', finalResponse,+ (...event) => timedOutUpdates.push(event), {},+ ),+ `${browserLabel}: classifier timeout must not block or fail the completed Ask answer`,+ );+ assert.deepEqual(timedOutUpdates, [], `${browserLabel}: timed-out classifier must not emit a handoff`);}});@@ -114528,9 +114556,14 @@ test('sidepanel: Ask-to-Act retries retain attachment payloads', () => {assert.match(source, /const retryPayloadByAssistant = new WeakMap\(\)/, `${label}: assistant retry payload store is missing`);assert.match(source,- /retryPayloadByAssistant\.set\(assistantEl, \{[\s\S]*?attachments: attachmentsForSend\.slice\(\)/,+ /function rememberRetryPayloadForAssistant\(assistantEl, retryPayload\) \{[\s\S]*?retryPayloadByAssistant\.set\(assistantEl, \{[\s\S]*?attachments: Array\.isArray\(retryPayload\.attachments\)[\s\S]*?retryPayload\.attachments\.slice\(\)/,`${label}: sent attachment payloads are not retained with the assistant message`,);+ assert.match(+ source,+ /const RETRY_PAYLOAD_RETENTION_MS = 30_000[\s\S]*?setTimeout\(\(\) => \{[\s\S]*?retryPayloadByAssistant\.delete\(assistantEl\)/,+ `${label}: assistant retry attachment payloads do not have a bounded lifetime`,+ );assert.match(source,/function retryPayloadForRunAssistant\(assistantEl\) \{[\s\S]*?const storedRetryPayload = retryPayloadByAssistant\.get\(assistantEl\)[\s\S]*?const attachments = Array\.isArray\(storedRetryPayload\?\.attachments\)/,