Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fix Ask-to-Act retry payload handling

16Mikub18782719e13
diff --git a/src/chrome/src/agent/planner.js b/src/chrome/src/agent/planner.js
index 3844398a4..d7a5b5efd 100644
--- a/src/chrome/src/agent/planner.js
+++ b/src/chrome/src/agent/planner.js
@@ -821,6 +821,9 @@ export function buildAskModeHandoffMessages(userMessage, assistantAnswer, pageUr
export function parseAskModeHandoffFromContent(content) {
const obj = extractFirstJsonObject(content);
+ if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return null;
+ const keys = Object.keys(obj);
+ if (keys.length !== 1 || keys[0] !== 'mode_handoff') return null;
const value = obj?.mode_handoff;
return value === 'act' || value === 'none' ? value : null;
}
diff --git a/src/chrome/src/ui/sidepanel.js b/src/chrome/src/ui/sidepanel.js
index 60e799b5c..f2c05374a 100644
--- a/src/chrome/src/ui/sidepanel.js
+++ b/src/chrome/src/ui/sidepanel.js
@@ -1217,6 +1217,7 @@ let retryPayloadSeq = 0;
const activeChatPayloadsByTab = new Map();
const retryAttachmentPayloads = new Map();
const retryAttachmentIdsByTab = new Map();
+const retryPayloadByAssistant = new WeakMap();
function setTabProcessing(tabId, processing) {
const numericTabId = Number(tabId);
@@ -2136,10 +2137,10 @@ function releaseRetryAttachmentPayload(retryId) {
function releaseRetryAttachmentsInTree(root) {
if (!root) return;
- if (root.matches?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id]')) {
+ if (root.matches?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id], .ask-act-handoff-btn[data-retry-id]')) {
releaseRetryAttachmentPayload(root.dataset.retryId);
}
- root.querySelectorAll?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id]').forEach((btn) => {
+ root.querySelectorAll?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id], .ask-act-handoff-btn[data-retry-id]').forEach((btn) => {
releaseRetryAttachmentPayload(btn.dataset.retryId);
});
}
@@ -6661,12 +6662,16 @@ function retryPayloadForRunAssistant(assistantEl) {
const userEl = userMessageForRunAssistant(assistantEl);
const displayText = String(userEl ? getComposerHistoryTextFromMessage(userEl) : '').trim();
if (!displayText) return null;
+ const storedRetryPayload = retryPayloadByAssistant.get(assistantEl);
const internalPrompt = String(assistantEl?.dataset.retryAgentPrompt || '').trim();
const text = internalPrompt || displayText;
const sourceGrounding = normalizeSelectionSourceGrounding(assistantEl?.dataset.retrySourceGrounding) || null;
const selectionAction = sourceGrounding
? normalizeSelectionAction(assistantEl?.dataset.retrySelectionAction)
: '';
+ const attachments = Array.isArray(storedRetryPayload?.attachments)
+ ? storedRetryPayload.attachments.slice()
+ : [];
return {
text,
displayText,
@@ -6677,8 +6682,11 @@ function retryPayloadForRunAssistant(assistantEl) {
foreground: assistantEl?.dataset.retryForeground === 'true',
...(sourceGrounding ? { sourceGrounding } : {}),
...(selectionAction ? { selectionAction } : {}),
- attachments: [],
- attachmentCount: Number(assistantEl?.dataset.retryAttachmentCount || 0) || 0,
+ attachments,
+ attachmentCount: Math.max(
+ Number(assistantEl?.dataset.retryAttachmentCount || 0) || 0,
+ attachments.length,
+ ),
};
}
@@ -8910,6 +8918,10 @@ async function sendMessage(extraChatParams = {}) {
assistantEl.dataset.retrySelectionAction = selectionAction;
assistantEl.dataset.retryAttachmentCount = String(attachmentsForSend.length);
if (agentPrompt) assistantEl.dataset.retryAgentPrompt = agentPrompt;
+ retryPayloadByAssistant.set(assistantEl, {
+ ...retryPayload,
+ attachments: attachmentsForSend.slice(),
+ });
userEl.dataset.runRequestId = requestId;
assistantEl.dataset.lastRenderedSeq = '0';
currentAssistantEl = assistantEl;
@@ -11340,7 +11352,16 @@ function renderAskActHandoffButton(assistantEl, tabId, requestId) {
const content = assistantEl.querySelector('.message-content');
if (!content || content.querySelector('.ask-act-handoff-btn')) return;
- const retryPayload = { ...retryPayloadForRunAssistant(assistantEl), mode: 'act' };
+ const baseRetryPayload = activeRetryPayloadForRequest(tabId, requestId)
+ || retryPayloadForRunAssistant(assistantEl);
+ if (!baseRetryPayload) return;
+ const retryPayload = {
+ ...baseRetryPayload,
+ mode: 'act',
+ attachments: Array.isArray(baseRetryPayload.attachments)
+ ? baseRetryPayload.attachments.slice()
+ : [],
+ };
const btn = document.createElement('button');
btn.type = 'button';
btn.className = 'ask-act-handoff-btn';
diff --git a/src/firefox/src/agent/planner.js b/src/firefox/src/agent/planner.js
index 3844398a4..d7a5b5efd 100644
--- a/src/firefox/src/agent/planner.js
+++ b/src/firefox/src/agent/planner.js
@@ -821,6 +821,9 @@ export function buildAskModeHandoffMessages(userMessage, assistantAnswer, pageUr
export function parseAskModeHandoffFromContent(content) {
const obj = extractFirstJsonObject(content);
+ if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return null;
+ const keys = Object.keys(obj);
+ if (keys.length !== 1 || keys[0] !== 'mode_handoff') return null;
const value = obj?.mode_handoff;
return value === 'act' || value === 'none' ? value : null;
}
diff --git a/src/firefox/src/ui/sidepanel.js b/src/firefox/src/ui/sidepanel.js
index b7b16cbab..1d3e76954 100644
--- a/src/firefox/src/ui/sidepanel.js
+++ b/src/firefox/src/ui/sidepanel.js
@@ -1071,6 +1071,7 @@ let retryPayloadSeq = 0;
const activeChatPayloadsByTab = new Map();
const retryAttachmentPayloads = new Map();
const retryAttachmentIdsByTab = new Map();
+const retryPayloadByAssistant = new WeakMap();
function setTabProcessing(tabId, processing) {
const numericTabId = Number(tabId);
@@ -2368,10 +2369,10 @@ function releaseRetryAttachmentPayload(retryId) {
function releaseRetryAttachmentsInTree(root) {
if (!root) return;
- if (root.matches?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id]')) {
+ if (root.matches?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id], .ask-act-handoff-btn[data-retry-id]')) {
releaseRetryAttachmentPayload(root.dataset.retryId);
}
- root.querySelectorAll?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id]').forEach((btn) => {
+ root.querySelectorAll?.('.error-retry-btn[data-retry-id], .cost-allowance-retry-btn[data-retry-id], .planner-request-failure-retry-btn[data-retry-id], .plan-review-retry[data-retry-id], .ask-act-handoff-btn[data-retry-id]').forEach((btn) => {
releaseRetryAttachmentPayload(btn.dataset.retryId);
});
}
@@ -6495,12 +6496,16 @@ function retryPayloadForRunAssistant(assistantEl) {
const userEl = userMessageForRunAssistant(assistantEl);
const displayText = String(userEl ? getComposerHistoryTextFromMessage(userEl) : '').trim();
if (!displayText) return null;
+ const storedRetryPayload = retryPayloadByAssistant.get(assistantEl);
const internalPrompt = String(assistantEl?.dataset.retryAgentPrompt || '').trim();
const text = internalPrompt || displayText;
const sourceGrounding = normalizeSelectionSourceGrounding(assistantEl?.dataset.retrySourceGrounding) || null;
const selectionAction = sourceGrounding
? normalizeSelectionAction(assistantEl?.dataset.retrySelectionAction)
: '';
+ const attachments = Array.isArray(storedRetryPayload?.attachments)
+ ? storedRetryPayload.attachments.slice()
+ : [];
return {
text,
displayText,
@@ -6511,8 +6516,11 @@ function retryPayloadForRunAssistant(assistantEl) {
foreground: assistantEl?.dataset.retryForeground === 'true',
...(sourceGrounding ? { sourceGrounding } : {}),
...(selectionAction ? { selectionAction } : {}),
- attachments: [],
- attachmentCount: Number(assistantEl?.dataset.retryAttachmentCount || 0) || 0,
+ attachments,
+ attachmentCount: Math.max(
+ Number(assistantEl?.dataset.retryAttachmentCount || 0) || 0,
+ attachments.length,
+ ),
};
}
@@ -8564,6 +8572,10 @@ async function sendMessage(extraChatParams = {}) {
assistantEl.dataset.retrySelectionAction = selectionAction;
assistantEl.dataset.retryAttachmentCount = String(attachmentsForSend.length);
if (agentPrompt) assistantEl.dataset.retryAgentPrompt = agentPrompt;
+ retryPayloadByAssistant.set(assistantEl, {
+ ...retryPayload,
+ attachments: attachmentsForSend.slice(),
+ });
userEl.dataset.runRequestId = requestId;
assistantEl.dataset.lastRenderedSeq = '0';
currentAssistantEl = assistantEl;
@@ -10942,7 +10954,16 @@ function renderAskActHandoffButton(assistantEl, tabId, requestId) {
const content = assistantEl.querySelector('.message-content');
if (!content || content.querySelector('.ask-act-handoff-btn')) return;
- const retryPayload = { ...retryPayloadForRunAssistant(assistantEl), mode: 'act' };
+ const baseRetryPayload = activeRetryPayloadForRequest(tabId, requestId)
+ || retryPayloadForRunAssistant(assistantEl);
+ if (!baseRetryPayload) return;
+ const retryPayload = {
+ ...baseRetryPayload,
+ mode: 'act',
+ attachments: Array.isArray(baseRetryPayload.attachments)
+ ? baseRetryPayload.attachments.slice()
+ : [],
+ };
const btn = document.createElement('button');
btn.type = 'button';
btn.className = 'ask-act-handoff-btn';
diff --git a/test/run.js b/test/run.js
index eaa3b5467..e92d2f3b3 100644
--- a/test/run.js
+++ b/test/run.js
@@ -11978,11 +11978,15 @@ test('Ask and managed cloud classify communication read scope across languages',
});
test('Ask mode handoff classification is strict, silent, and mode guarded', async () => {
- assert.equal(parseAskModeHandoffFromContent('{"mode_handoff":"act"}'), 'act');
- assert.equal(parseAskModeHandoffFromContent('prefix {"mode_handoff":"none"} suffix'), 'none');
- assert.equal(parseAskModeHandoffFromContent('{"mode_handoff":"maybe"}'), null);
- assert.equal(parseAskModeHandoffFromContent('not JSON'), null);
- assert.equal(parseAskModeHandoffFromContent('{"mode_handoff":"act","extra":true}'), 'act');
+ for (const parse of [parseAskModeHandoffFromContent, parseAskModeHandoffFromContentFx]) {
+ assert.equal(parse('{"mode_handoff":"act"}'), 'act');
+ assert.equal(parse('prefix {"mode_handoff":"none"} suffix'), 'none');
+ assert.equal(parse('{"mode_handoff":"maybe"}'), null);
+ assert.equal(parse('{"mode_handoff":42}'), null);
+ assert.equal(parse('not JSON'), null);
+ assert.equal(parse('{"mode_handoff":"act","extra":true}'), null);
+ assert.equal(parse('{"extra":true,"mode_handoff":"none"}'), null);
+ }
for (const [browserLabel, AgentClass, selectionGrounding] of [
['chrome', AgentCh, SELECTION_ONLY_SOURCE_GROUNDING_CH],
@@ -46090,12 +46094,14 @@ test('sidepanel restored suggested-action retries preserve hidden prompts', () =
'normalizeSelectionSourceGrounding',
'normalizeSelectionAction',
'agentMode',
+ 'retryPayloadByAssistant',
`${panel.slice(retryStart, retryEnd + 2)}\n${panel.slice(userStart, userEnd + 2)}\nreturn retryPayloadForRunAssistant;`,
)(
() => visibleText,
value => String(value || '').trim(),
value => String(value || '').trim(),
'ask',
+ new WeakMap(),
);
const internalPrompt = 'Read the complete active thread, follow every continuation, then summarize it.';
@@ -114514,6 +114520,35 @@ test('sidepanel: pending attachments are tab-scoped and send-gated while loading
}
});
+test('sidepanel: Ask-to-Act retries retain attachment payloads', () => {
+ for (const [label, source] of [
+ ['chrome', sidepanelSources[0]],
+ ['firefox', sidepanelSources[1]],
+ ]) {
+ assert.match(source, /const retryPayloadByAssistant = new WeakMap\(\)/, `${label}: assistant retry payload store is missing`);
+ assert.match(
+ source,
+ /retryPayloadByAssistant\.set\(assistantEl, \{[\s\S]*?attachments: attachmentsForSend\.slice\(\)/,
+ `${label}: sent attachment payloads are not retained with the assistant message`,
+ );
+ assert.match(
+ source,
+ /function retryPayloadForRunAssistant\(assistantEl\) \{[\s\S]*?const storedRetryPayload = retryPayloadByAssistant\.get\(assistantEl\)[\s\S]*?const attachments = Array\.isArray\(storedRetryPayload\?\.attachments\)/,
+ `${label}: assistant fallback retries do not restore their attachment payloads`,
+ );
+ assert.match(
+ source,
+ /const baseRetryPayload = activeRetryPayloadForRequest\(tabId, requestId\)[\s\S]*?\|\| retryPayloadForRunAssistant\(assistantEl\)/,
+ `${label}: Ask-to-Act handoff does not prefer the active attachment-aware retry payload`,
+ );
+ assert.match(
+ source,
+ /function releaseRetryAttachmentsInTree\(root\) \{[\s\S]*?ask-act-handoff-btn\[data-retry-id\]/,
+ `${label}: handoff retry attachments are not released with the message tree`,
+ );
+ }
+});
+
test('chrome sidepanel resets mic icon after speech recognition ends', () => {
const source = fs.readFileSync(path.join(ROOT, 'src/chrome/src/ui/sidepanel.js'), 'utf8');
assert.match(source, /function setMicIdleIcon\(\) \{[\s\S]*?micBtn\.innerHTML = `<svg width="18"/, 'chrome: idle mic icon helper missing');