Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Skip superseded planner recommendations and allow form-free image previews
Emre Sokullu827371b3b951
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex d40e567de..1949d1c29 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -43293,14 +43293,15 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai|| gateOutcome.responseLanguageApprovedPlanOverride === true,trustedContinuation: runOptions?.trustedContinuation === true,});- let initialResponseRevalidated = false;+ // The initial planner can authorize steering before response-only handling.+ let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0;if (gateOutcome.responseOnly === true) {const responseOutcome = await this._completeInitialResponseOnlyTurn(tabId, messages, onUpdate, mode, provider, costState, runId,runOptions, gateOutcome, enriched, sourceBoundPriorMessages,);gateOutcome = responseOutcome.gate;- initialResponseRevalidated = responseOutcome.revalidated;+ initialTaskRevalidated ||= responseOutcome.revalidated;if (!gateOutcome.proceed) {_traceStatus = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled';return (finalResponse = gateOutcome.message || 'More information is required.');@@ -43323,7 +43324,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai}}if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId);- if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);+ if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) {await this._ensureProgressSessionForCurrentTask(tabId, {@@ -43529,7 +43530,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaithis._appendStandaloneUngroundedPolicy(enriched, standaloneGroundingGap);}- const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(+ const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas,);if (recommendedFirstTool?.action === 'return') {@@ -44650,14 +44651,15 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai|| gateOutcome.responseLanguageApprovedPlanOverride === true,trustedContinuation: runOptions?.trustedContinuation === true,});- let initialResponseRevalidated = false;+ // The initial planner can authorize steering before response-only handling.+ let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0;if (gateOutcome.responseOnly === true) {const responseOutcome = await this._completeInitialResponseOnlyTurn(tabId, messages, onUpdate, mode, provider, costState, runId,runOptions, gateOutcome, enriched, sourceBoundPriorMessages,);gateOutcome = responseOutcome.gate;- initialResponseRevalidated = responseOutcome.revalidated;+ initialTaskRevalidated ||= responseOutcome.revalidated;if (!gateOutcome.proceed) {const status = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled';return finish(gateOutcome.message || 'More information is required.', status);@@ -44675,7 +44677,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai}}if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId);- if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);+ if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);let standaloneGroundingGap = this._standaloneOfflineGroundingGap(localWikipediaRag, runOptions);if (standaloneGroundingGap) {@@ -44739,7 +44741,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai// currentStreamRequestMessages is declared at the top of this function so// pre-loop exits (e.g. response-only turns) can capture their request.- const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(+ const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas,);if (recommendedFirstTool?.action === 'return') {diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex e9fe24e0b..ec4d911a9 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -5875,7 +5875,7 @@};if (!button || !painted(button) || button.disabled || button.matches(':disabled')|| button.getAttribute('aria-disabled') === 'true'- || String(button.getAttribute('type') || '').toLowerCase() !== 'button'+ || (button.hasAttribute('type') && String(button.getAttribute('type')).toLowerCase() !== 'button')|| button.form || button.hasAttribute('form') || button.hasAttribute('download')|| /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '')|| _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false;diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 9704135f8..fdb610644 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -36257,14 +36257,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d|| gateOutcome.responseLanguageApprovedPlanOverride === true,trustedContinuation: runOptions?.trustedContinuation === true,});- let initialResponseRevalidated = false;+ // The initial planner can authorize steering before response-only handling.+ let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0;if (gateOutcome.responseOnly === true) {const responseOutcome = await this._completeInitialResponseOnlyTurn(tabId, messages, onUpdate, mode, provider, costState, runId,runOptions, gateOutcome, enriched, sourceBoundPriorMessages,);gateOutcome = responseOutcome.gate;- initialResponseRevalidated = responseOutcome.revalidated;+ initialTaskRevalidated ||= responseOutcome.revalidated;if (!gateOutcome.proceed) {_traceStatus = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled';return (finalResponse = gateOutcome.message || 'More information is required.');@@ -36287,7 +36288,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}}if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId);- if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);+ if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) {await this._ensureProgressSessionForCurrentTask(tabId, {@@ -36482,7 +36483,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d);}- const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(+ const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas,);if (recommendedFirstTool?.action === 'return') {@@ -37445,14 +37446,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d|| gateOutcome.responseLanguageApprovedPlanOverride === true,trustedContinuation: runOptions?.trustedContinuation === true,});- let initialResponseRevalidated = false;+ // The initial planner can authorize steering before response-only handling.+ let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0;if (gateOutcome.responseOnly === true) {const responseOutcome = await this._completeInitialResponseOnlyTurn(tabId, messages, onUpdate, mode, provider, costState, runId,runOptions, gateOutcome, enriched, sourceBoundPriorMessages,);gateOutcome = responseOutcome.gate;- initialResponseRevalidated = responseOutcome.revalidated;+ initialTaskRevalidated ||= responseOutcome.revalidated;if (!gateOutcome.proceed) {const status = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled';return finish(gateOutcome.message || 'More information is required.', status);@@ -37470,7 +37472,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d}}if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId);- if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);+ if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) {await this._ensureProgressSessionForCurrentTask(tabId, {@@ -37521,7 +37523,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// currentStreamRequestMessages is declared at the top of this function so// pre-loop exits (e.g. response-only turns) can capture their request.- const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(+ const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas,);if (recommendedFirstTool?.action === 'return') {diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex 38a0e2216..f20c87713 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -4963,7 +4963,7 @@};if (!button || !painted(button) || button.disabled || button.matches(':disabled')|| button.getAttribute('aria-disabled') === 'true'- || String(button.getAttribute('type') || '').toLowerCase() !== 'button'+ || (button.hasAttribute('type') && String(button.getAttribute('type')).toLowerCase() !== 'button')|| button.form || button.hasAttribute('form') || button.hasAttribute('download')|| /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '')|| _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false;diff --git a/test/chat-steering.mjs b/test/chat-steering.mjsindex c5b8046f0..fc546dce2 100644--- a/test/chat-steering.mjs+++ b/test/chat-steering.mjs@@ -117,6 +117,49 @@ for (const build of ['chrome', 'firefox']) {for (const streaming of [false, true]) {for (const mode of ['act', 'dev']) {+ for (const [id, tool] of [['summarize-page', 'read_page'], ['download-media', 'screenshot']]) {+ test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} planner steering skips the original recommended ${tool}`, async () => {+ const entered = deferred(), release = deferred(), plans = [], dispatched = [], recommendedAttempts = [];+ let modelCalls = 0;+ const next = async () => ++modelCalls === 1+ ? { toolCalls: [{ id: 'dashboard', function: { name: 'navigate', arguments: '{"url":"https://example.com/dashboard"}' } }] }+ : { toolCalls: [{ id: 'finished', function: { name: 'done', arguments: '{"summary":"Dashboard opened","outcome":"success"}' } }] };+ const agent = actSetup(Agent, {+ supportsVision: true,+ chat: next,+ async *chatStream() { yield { type: 'tool_call', content: (await next()).toolCalls }; yield { type: 'done' }; },+ }, async (_tab, enriched) => {+ plans.push(enriched.content);+ if (plans.length === 1) { entered.resolve(); await release.promise; }+ return { proceed: true, requestKind: 'execute', requiresStateChange: plans.length > 1, requiresSubmission: false };+ });+ agent.executeTool = async (_tab, name, args) => {+ if (name === 'done') {+ assert.equal(agent._planOnlyTerminalDecision(tabId, args.summary, { viaDone: true, outcome: args.outcome }), null);+ return { done: true, summary: args.summary, outcome: args.outcome };+ }+ dispatched.push(name); return { success: true, url: args.url };+ };+ const runOptions = { ...options, recommendedAction: { id, tool, autoExecute: true } };+ assert.ok(agent._recommendedActionFirstTool(runOptions), 'Fixture must activate the real recommended first tool');+ const firstTool = agent._maybeExecuteRecommendedActionFirstTool.bind(agent);+ agent._maybeExecuteRecommendedActionFirstTool = async (...args) => {+ recommendedAttempts.push(args[1]?.recommendedAction?.tool);+ return firstTool(...args);+ };+ const run = streaming+ ? agent.processMessageStream(tabId, 'Inspect this page', () => {}, mode, runOptions)+ : agent.processMessage(tabId, 'Inspect this page', () => {}, mode, [], runOptions);+ await Promise.race([entered.promise, run.then(result => assert.fail(`Early completion: ${result}`))]);+ assert.equal(steer(agent, 'Cancel the page inspection; open my dashboard instead').accepted, true);+ release.resolve();+ assert.equal(await run, 'Dashboard opened');+ assert.equal(plans.length, 2);+ assert.match(plans[1], /Cancel the page inspection; open my dashboard instead/);+ assert.deepEqual(recommendedAttempts, [], 'Steering must suppress the stale recommendation before any preparation or dispatch');+ assert.deepEqual(dispatched, ['navigate'], 'The original recommended read/screenshot must never dispatch');+ });+ }for (const revisedKind of ['respond', 'execute']) {test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} initial response-only steering switches to ${revisedKind}`, async () => {const entered = deferred(), release = deferred(), updates = [], persisted = [], dispatched = [];diff --git a/test/fixtures/message-recipient-navigation.mjs b/test/fixtures/message-recipient-navigation.mjsindex a48a8edd5..972b4d327 100644--- a/test/fixtures/message-recipient-navigation.mjs+++ b/test/fixtures/message-recipient-navigation.mjs@@ -187,6 +187,37 @@ export function registerMessageRecipientNavigationFixtures({assert.equal(await page.evaluate(() => window.sentMessages), 0);});+ register(`${kind}: form-free LinkedIn previews need no explicit button type`, async page => {+ for (const tag of ['button', 'div']) {+ const { guard } = await setupMessageHistory(page);+ await page.evaluate(tag => {+ const original = document.querySelector('#preview');+ original.removeAttribute('type');+ if (tag === 'div') {+ const control = document.createElement('div');+ for (const attribute of original.attributes) control.setAttribute(attribute.name, attribute.value);+ control.setAttribute('role', 'button'); control.tabIndex = 0;+ control.append(...original.childNodes); original.replaceWith(control);+ control.onclick = () => { document.querySelector('#viewer').hidden = false; };+ }+ }, tag);+ const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxChars: 20000 });+ const ref = /button "Click or press enter to display in the image preview" \[([^\]]+)\]/.exec(tree.pageContent)?.[1];+ assert.ok(ref, tree.pageContent);+ const point = await page.locator('#preview').evaluate(el => {+ const r = el.getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2 };+ });+ for (const [tool, args] of [['click', { selector: '#preview' }], ['click', { text: 'Click or press enter to display in the image preview' }],+ ['click_ax', { ref_id: ref }], ['click', point]]) {+ assert.equal(await guard(tool, args), null, `${tag}/${tool}`);+ assert.equal((await call(page, tool, args)).success, true);+ assert.equal(await page.locator('#viewer').isVisible(), true);+ await page.locator('#close-viewer').click();+ }+ assert.equal(await page.evaluate(() => window.sentMessages), 0);+ }+ });+register(`${kind}: LinkedIn preview lookalikes and hidden or occluded images stay protected`, async page => {const { guard } = await setupMessageHistory(page);for (const change of ['no-image', 'composer', 'hidden', 'transparent', 'ancestor-transparent', 'occluded', 'disabled', 'commit-label', 'commit-text',@@ -204,7 +235,7 @@ export function registerMessageRecipientNavigationFixtures({if(change==='commit-text'){button.title='View image';button.append('Send');}if(change==='localized-send')button.setAttribute('aria-label','Gönder');if(change==='submit')button.type='submit';- if(change==='external-form')button.setAttribute('form','composer');+ if(change==='external-form'){button.removeAttribute('type');button.setAttribute('form','composer');}if(change==='outside-history')document.body.append(button);if(change==='small-icon')button.querySelector('img').style='width:20px;height:20px';if(change==='menu')button.setAttribute('aria-haspopup','menu');