Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Fix Ask-to-Act retry payload handling
16Mikub18782719e13
diff --git a/src/chrome/src/agent/planner.js b/src/chrome/src/agent/planner.jsindex 3844398a4..d7a5b5efd 100644--- a/src/chrome/src/agent/planner.js+++ b/src/chrome/src/agent/planner.js@@ -821,6 +821,9 @@ export function buildAskModeHandoffMessages(userMessage, assistantAnswer, pageUrexport function parseAskModeHandoffFromContent(content) {const obj = extractFirstJsonObject(content);+ if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return null;+ const keys = Object.keys(obj);+ if (keys.length !== 1 || keys[0] !== 'mode_handoff') return null;const value = obj?.mode_handoff;return value === 'act' || value === 'none' ? value : null;}diff --git a/src/chrome/src/ui/sidepanel.js b/src/chrome/src/ui/sidepanel.jsindex 60e799b5c..f2c05374a 100644--- a/src/chrome/src/ui/sidepanel.js+++ b/src/chrome/src/ui/sidepanel.js@@ -1217,6 +1217,7 @@ let retryPayloadSeq = 0;const activeChatPayloadsByTab = new Map();const retryAttachmentPayloads = new Map();const retryAttachmentIdsByTab = new Map();+const retryPayloadByAssistant = new WeakMap();function setTabProcessing(tabId, processing) {const numericTabId = Number(tabId);@@ -2136,10 +2137,10 @@ function releaseRetryAttachmentPayload(retryId) {function releaseRetryAttachmentsInTree(root) {if (!root) return;- if (root.matches?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id]')) {+ if (root.matches?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id], .ask-act-handoff-btn[data-retry-id]')) {releaseRetryAttachmentPayload(root.dataset.retryId);}- root.querySelectorAll?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id]').forEach((btn) => {+ root.querySelectorAll?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id], .ask-act-handoff-btn[data-retry-id]').forEach((btn) => {releaseRetryAttachmentPayload(btn.dataset.retryId);});}@@ -6661,12 +6662,16 @@ function retryPayloadForRunAssistant(assistantEl) {const userEl = userMessageForRunAssistant(assistantEl);const displayText = String(userEl ? getComposerHistoryTextFromMessage(userEl) : '').trim();if (!displayText) return null;+ const storedRetryPayload = retryPayloadByAssistant.get(assistantEl);const internalPrompt = String(assistantEl?.dataset.retryAgentPrompt || '').trim();const text = internalPrompt || displayText;const sourceGrounding = normalizeSelectionSourceGrounding(assistantEl?.dataset.retrySourceGrounding) || null;const selectionAction = sourceGrounding? normalizeSelectionAction(assistantEl?.dataset.retrySelectionAction): '';+ const attachments = Array.isArray(storedRetryPayload?.attachments)+ ? storedRetryPayload.attachments.slice()+ : [];return {text,displayText,@@ -6677,8 +6682,11 @@ function retryPayloadForRunAssistant(assistantEl) {foreground: assistantEl?.dataset.retryForeground === 'true',...(sourceGrounding ? { sourceGrounding } : {}),...(selectionAction ? { selectionAction } : {}),- attachments: [],- attachmentCount: Number(assistantEl?.dataset.retryAttachmentCount || 0) || 0,+ attachments,+ attachmentCount: Math.max(+ Number(assistantEl?.dataset.retryAttachmentCount || 0) || 0,+ attachments.length,+ ),};}@@ -8910,6 +8918,10 @@ async function sendMessage(extraChatParams = {}) {assistantEl.dataset.retrySelectionAction = selectionAction;assistantEl.dataset.retryAttachmentCount = String(attachmentsForSend.length);if (agentPrompt) assistantEl.dataset.retryAgentPrompt = agentPrompt;+ retryPayloadByAssistant.set(assistantEl, {+ ...retryPayload,+ attachments: attachmentsForSend.slice(),+ });userEl.dataset.runRequestId = requestId;assistantEl.dataset.lastRenderedSeq = '0';currentAssistantEl = assistantEl;@@ -11340,7 +11352,16 @@ function renderAskActHandoffButton(assistantEl, tabId, requestId) {const content = assistantEl.querySelector('.message-content');if (!content || content.querySelector('.ask-act-handoff-btn')) return;- const retryPayload = { ...retryPayloadForRunAssistant(assistantEl), mode: 'act' };+ const baseRetryPayload = activeRetryPayloadForRequest(tabId, requestId)+ || retryPayloadForRunAssistant(assistantEl);+ if (!baseRetryPayload) return;+ const retryPayload = {+ ...baseRetryPayload,+ mode: 'act',+ attachments: Array.isArray(baseRetryPayload.attachments)+ ? baseRetryPayload.attachments.slice()+ : [],+ };const btn = document.createElement('button');btn.type = 'button';btn.className = 'ask-act-handoff-btn';diff --git a/src/firefox/src/agent/planner.js b/src/firefox/src/agent/planner.jsindex 3844398a4..d7a5b5efd 100644--- a/src/firefox/src/agent/planner.js+++ b/src/firefox/src/agent/planner.js@@ -821,6 +821,9 @@ export function buildAskModeHandoffMessages(userMessage, assistantAnswer, pageUrexport function parseAskModeHandoffFromContent(content) {const obj = extractFirstJsonObject(content);+ if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return null;+ const keys = Object.keys(obj);+ if (keys.length !== 1 || keys[0] !== 'mode_handoff') return null;const value = obj?.mode_handoff;return value === 'act' || value === 'none' ? value : null;}diff --git a/src/firefox/src/ui/sidepanel.js b/src/firefox/src/ui/sidepanel.jsindex b7b16cbab..1d3e76954 100644--- a/src/firefox/src/ui/sidepanel.js+++ b/src/firefox/src/ui/sidepanel.js@@ -1071,6 +1071,7 @@ let retryPayloadSeq = 0;const activeChatPayloadsByTab = new Map();const retryAttachmentPayloads = new Map();const retryAttachmentIdsByTab = new Map();+const retryPayloadByAssistant = new WeakMap();function setTabProcessing(tabId, processing) {const numericTabId = Number(tabId);@@ -2368,10 +2369,10 @@ function releaseRetryAttachmentPayload(retryId) {function releaseRetryAttachmentsInTree(root) {if (!root) return;- if (root.matches?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id]')) {+ if (root.matches?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id], .ask-act-handoff-btn[data-retry-id]')) {releaseRetryAttachmentPayload(root.dataset.retryId);}- root.querySelectorAll?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id]').forEach((btn) => {+ root.querySelectorAll?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id], .ask-act-handoff-btn[data-retry-id]').forEach((btn) => {releaseRetryAttachmentPayload(btn.dataset.retryId);});}@@ -6495,12 +6496,16 @@ function retryPayloadForRunAssistant(assistantEl) {const userEl = userMessageForRunAssistant(assistantEl);const displayText = String(userEl ? getComposerHistoryTextFromMessage(userEl) : '').trim();if (!displayText) return null;+ const storedRetryPayload = retryPayloadByAssistant.get(assistantEl);const internalPrompt = String(assistantEl?.dataset.retryAgentPrompt || '').trim();const text = internalPrompt || displayText;const sourceGrounding = normalizeSelectionSourceGrounding(assistantEl?.dataset.retrySourceGrounding) || null;const selectionAction = sourceGrounding? normalizeSelectionAction(assistantEl?.dataset.retrySelectionAction): '';+ const attachments = Array.isArray(storedRetryPayload?.attachments)+ ? storedRetryPayload.attachments.slice()+ : [];return {text,displayText,@@ -6511,8 +6516,11 @@ function retryPayloadForRunAssistant(assistantEl) {foreground: assistantEl?.dataset.retryForeground === 'true',...(sourceGrounding ? { sourceGrounding } : {}),...(selectionAction ? { selectionAction } : {}),- attachments: [],- attachmentCount: Number(assistantEl?.dataset.retryAttachmentCount || 0) || 0,+ attachments,+ attachmentCount: Math.max(+ Number(assistantEl?.dataset.retryAttachmentCount || 0) || 0,+ attachments.length,+ ),};}@@ -8564,6 +8572,10 @@ async function sendMessage(extraChatParams = {}) {assistantEl.dataset.retrySelectionAction = selectionAction;assistantEl.dataset.retryAttachmentCount = String(attachmentsForSend.length);if (agentPrompt) assistantEl.dataset.retryAgentPrompt = agentPrompt;+ retryPayloadByAssistant.set(assistantEl, {+ ...retryPayload,+ attachments: attachmentsForSend.slice(),+ });userEl.dataset.runRequestId = requestId;assistantEl.dataset.lastRenderedSeq = '0';currentAssistantEl = assistantEl;@@ -10942,7 +10954,16 @@ function renderAskActHandoffButton(assistantEl, tabId, requestId) {const content = assistantEl.querySelector('.message-content');if (!content || content.querySelector('.ask-act-handoff-btn')) return;- const retryPayload = { ...retryPayloadForRunAssistant(assistantEl), mode: 'act' };+ const baseRetryPayload = activeRetryPayloadForRequest(tabId, requestId)+ || retryPayloadForRunAssistant(assistantEl);+ if (!baseRetryPayload) return;+ const retryPayload = {+ ...baseRetryPayload,+ mode: 'act',+ attachments: Array.isArray(baseRetryPayload.attachments)+ ? baseRetryPayload.attachments.slice()+ : [],+ };const btn = document.createElement('button');btn.type = 'button';btn.className = 'ask-act-handoff-btn';diff --git a/test/run.js b/test/run.jsindex eaa3b5467..e92d2f3b3 100644--- a/test/run.js+++ b/test/run.js@@ -11978,11 +11978,15 @@ test('Ask and managed cloud classify communication read scope across languages',});test('Ask mode handoff classification is strict, silent, and mode guarded', async () => {- assert.equal(parseAskModeHandoffFromContent('{"mode_handoff":"act"}'), 'act');- assert.equal(parseAskModeHandoffFromContent('prefix {"mode_handoff":"none"} suffix'), 'none');- assert.equal(parseAskModeHandoffFromContent('{"mode_handoff":"maybe"}'), null);- assert.equal(parseAskModeHandoffFromContent('not JSON'), null);- assert.equal(parseAskModeHandoffFromContent('{"mode_handoff":"act","extra":true}'), 'act');+ for (const parse of [parseAskModeHandoffFromContent, parseAskModeHandoffFromContentFx]) {+ assert.equal(parse('{"mode_handoff":"act"}'), 'act');+ assert.equal(parse('prefix {"mode_handoff":"none"} suffix'), 'none');+ assert.equal(parse('{"mode_handoff":"maybe"}'), null);+ assert.equal(parse('{"mode_handoff":42}'), null);+ assert.equal(parse('not JSON'), null);+ assert.equal(parse('{"mode_handoff":"act","extra":true}'), null);+ assert.equal(parse('{"extra":true,"mode_handoff":"none"}'), null);+ }for (const [browserLabel, AgentClass, selectionGrounding] of [['chrome', AgentCh, SELECTION_ONLY_SOURCE_GROUNDING_CH],@@ -46090,12 +46094,14 @@ test('sidepanel restored suggested-action retries preserve hidden prompts', () ='normalizeSelectionSourceGrounding','normalizeSelectionAction','agentMode',+ 'retryPayloadByAssistant',`${panel.slice(retryStart, retryEnd + 2)}\n${panel.slice(userStart, userEnd + 2)}\nreturn retryPayloadForRunAssistant;`,)(() => visibleText,value => String(value || '').trim(),value => String(value || '').trim(),'ask',+ new WeakMap(),);const internalPrompt = 'Read the complete active thread, follow every continuation, then summarize it.';@@ -114514,6 +114520,35 @@ test('sidepanel: pending attachments are tab-scoped and send-gated while loading}});+test('sidepanel: Ask-to-Act retries retain attachment payloads', () => {+ for (const [label, source] of [+ ['chrome', sidepanelSources[0]],+ ['firefox', sidepanelSources[1]],+ ]) {+ assert.match(source, /const retryPayloadByAssistant = new WeakMap\(\)/, `${label}: assistant retry payload store is missing`);+ assert.match(+ source,+ /retryPayloadByAssistant\.set\(assistantEl, \{[\s\S]*?attachments: attachmentsForSend\.slice\(\)/,+ `${label}: sent attachment payloads are not retained with the assistant message`,+ );+ assert.match(+ source,+ /function retryPayloadForRunAssistant\(assistantEl\) \{[\s\S]*?const storedRetryPayload = retryPayloadByAssistant\.get\(assistantEl\)[\s\S]*?const attachments = Array\.isArray\(storedRetryPayload\?\.attachments\)/,+ `${label}: assistant fallback retries do not restore their attachment payloads`,+ );+ assert.match(+ source,+ /const baseRetryPayload = activeRetryPayloadForRequest\(tabId, requestId\)[\s\S]*?\|\| retryPayloadForRunAssistant\(assistantEl\)/,+ `${label}: Ask-to-Act handoff does not prefer the active attachment-aware retry payload`,+ );+ assert.match(+ source,+ /function releaseRetryAttachmentsInTree\(root\) \{[\s\S]*?ask-act-handoff-btn\[data-retry-id\]/,+ `${label}: handoff retry attachments are not released with the message tree`,+ );+ }+});+test('chrome sidepanel resets mic icon after speech recognition ends', () => {const source = fs.readFileSync(path.join(ROOT, 'src/chrome/src/ui/sidepanel.js'), 'utf8');assert.match(source, /function setMicIdleIcon\(\) \{[\s\S]*?micBtn\.innerHTML = `<svg width="18"/, 'chrome: idle mic icon helper missing');