Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Skip superseded planner recommendations and allow form-free image previews

Emre Sokullu827371b3b951
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index d40e567de..1949d1c29 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -43293,14 +43293,15 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
|| gateOutcome.responseLanguageApprovedPlanOverride === true,
trustedContinuation: runOptions?.trustedContinuation === true,
});
- let initialResponseRevalidated = false;
+ // The initial planner can authorize steering before response-only handling.
+ let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0;
if (gateOutcome.responseOnly === true) {
const responseOutcome = await this._completeInitialResponseOnlyTurn(
tabId, messages, onUpdate, mode, provider, costState, runId,
runOptions, gateOutcome, enriched, sourceBoundPriorMessages,
);
gateOutcome = responseOutcome.gate;
- initialResponseRevalidated = responseOutcome.revalidated;
+ initialTaskRevalidated ||= responseOutcome.revalidated;
if (!gateOutcome.proceed) {
_traceStatus = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled';
return (finalResponse = gateOutcome.message || 'More information is required.');
@@ -43323,7 +43324,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
}
}
if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId);
- if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);
+ if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);
if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) {
await this._ensureProgressSessionForCurrentTask(tabId, {
@@ -43529,7 +43530,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
this._appendStandaloneUngroundedPolicy(enriched, standaloneGroundingGap);
}
- const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(
+ const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(
tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas,
);
if (recommendedFirstTool?.action === 'return') {
@@ -44650,14 +44651,15 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
|| gateOutcome.responseLanguageApprovedPlanOverride === true,
trustedContinuation: runOptions?.trustedContinuation === true,
});
- let initialResponseRevalidated = false;
+ // The initial planner can authorize steering before response-only handling.
+ let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0;
if (gateOutcome.responseOnly === true) {
const responseOutcome = await this._completeInitialResponseOnlyTurn(
tabId, messages, onUpdate, mode, provider, costState, runId,
runOptions, gateOutcome, enriched, sourceBoundPriorMessages,
);
gateOutcome = responseOutcome.gate;
- initialResponseRevalidated = responseOutcome.revalidated;
+ initialTaskRevalidated ||= responseOutcome.revalidated;
if (!gateOutcome.proceed) {
const status = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled';
return finish(gateOutcome.message || 'More information is required.', status);
@@ -44675,7 +44677,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
}
}
if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId);
- if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);
+ if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);
let standaloneGroundingGap = this._standaloneOfflineGroundingGap(localWikipediaRag, runOptions);
if (standaloneGroundingGap) {
@@ -44739,7 +44741,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
// currentStreamRequestMessages is declared at the top of this function so
// pre-loop exits (e.g. response-only turns) can capture their request.
- const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(
+ const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(
tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas,
);
if (recommendedFirstTool?.action === 'return') {
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index e9fe24e0b..ec4d911a9 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -5875,7 +5875,7 @@
};
if (!button || !painted(button) || button.disabled || button.matches(':disabled')
|| button.getAttribute('aria-disabled') === 'true'
- || String(button.getAttribute('type') || '').toLowerCase() !== 'button'
+ || (button.hasAttribute('type') && String(button.getAttribute('type')).toLowerCase() !== 'button')
|| button.form || button.hasAttribute('form') || button.hasAttribute('download')
|| /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '')
|| _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false;
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 9704135f8..fdb610644 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -36257,14 +36257,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
|| gateOutcome.responseLanguageApprovedPlanOverride === true,
trustedContinuation: runOptions?.trustedContinuation === true,
});
- let initialResponseRevalidated = false;
+ // The initial planner can authorize steering before response-only handling.
+ let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0;
if (gateOutcome.responseOnly === true) {
const responseOutcome = await this._completeInitialResponseOnlyTurn(
tabId, messages, onUpdate, mode, provider, costState, runId,
runOptions, gateOutcome, enriched, sourceBoundPriorMessages,
);
gateOutcome = responseOutcome.gate;
- initialResponseRevalidated = responseOutcome.revalidated;
+ initialTaskRevalidated ||= responseOutcome.revalidated;
if (!gateOutcome.proceed) {
_traceStatus = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled';
return (finalResponse = gateOutcome.message || 'More information is required.');
@@ -36287,7 +36288,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
}
if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId);
- if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);
+ if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);
if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) {
await this._ensureProgressSessionForCurrentTask(tabId, {
@@ -36482,7 +36483,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
);
}
- const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(
+ const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(
tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas,
);
if (recommendedFirstTool?.action === 'return') {
@@ -37445,14 +37446,15 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
|| gateOutcome.responseLanguageApprovedPlanOverride === true,
trustedContinuation: runOptions?.trustedContinuation === true,
});
- let initialResponseRevalidated = false;
+ // The initial planner can authorize steering before response-only handling.
+ let initialTaskRevalidated = (this._steeringRuns.get(tabId)?.authorizedRevision || 0) > 0;
if (gateOutcome.responseOnly === true) {
const responseOutcome = await this._completeInitialResponseOnlyTurn(
tabId, messages, onUpdate, mode, provider, costState, runId,
runOptions, gateOutcome, enriched, sourceBoundPriorMessages,
);
gateOutcome = responseOutcome.gate;
- initialResponseRevalidated = responseOutcome.revalidated;
+ initialTaskRevalidated ||= responseOutcome.revalidated;
if (!gateOutcome.proceed) {
const status = gateOutcome.reason === 'plan_only' ? 'plan_only_output' : gateOutcome.reason || 'cancelled';
return finish(gateOutcome.message || 'More information is required.', status);
@@ -37470,7 +37472,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
}
}
if (this._consumeSelectionGroundingRestoration(tabId, enriched)) this._persist(tabId);
- if (!initialResponseRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);
+ if (!initialTaskRevalidated) this._startPlanExecutionGuard(tabId, mode, gateOutcome, runOptions);
if (this._isActionMode(mode) && !selectionOnly && !standaloneChatRun) {
await this._ensureProgressSessionForCurrentTask(tabId, {
@@ -37521,7 +37523,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// currentStreamRequestMessages is declared at the top of this function so
// pre-loop exits (e.g. response-only turns) can capture their request.
- const recommendedFirstTool = initialResponseRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(
+ const recommendedFirstTool = initialTaskRevalidated ? null : await this._maybeExecuteRecommendedActionFirstTool(
tabId, runOptions, messages, onUpdate, provider, allowedToolNames, toolSchemas,
);
if (recommendedFirstTool?.action === 'return') {
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index 38a0e2216..f20c87713 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -4963,7 +4963,7 @@
};
if (!button || !painted(button) || button.disabled || button.matches(':disabled')
|| button.getAttribute('aria-disabled') === 'true'
- || String(button.getAttribute('type') || '').toLowerCase() !== 'button'
+ || (button.hasAttribute('type') && String(button.getAttribute('type')).toLowerCase() !== 'button')
|| button.form || button.hasAttribute('form') || button.hasAttribute('download')
|| /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '')
|| _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false;
diff --git a/test/chat-steering.mjs b/test/chat-steering.mjs
index c5b8046f0..fc546dce2 100644
--- a/test/chat-steering.mjs
+++ b/test/chat-steering.mjs
@@ -117,6 +117,49 @@ for (const build of ['chrome', 'firefox']) {
for (const streaming of [false, true]) {
for (const mode of ['act', 'dev']) {
+ for (const [id, tool] of [['summarize-page', 'read_page'], ['download-media', 'screenshot']]) {
+ test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} planner steering skips the original recommended ${tool}`, async () => {
+ const entered = deferred(), release = deferred(), plans = [], dispatched = [], recommendedAttempts = [];
+ let modelCalls = 0;
+ const next = async () => ++modelCalls === 1
+ ? { toolCalls: [{ id: 'dashboard', function: { name: 'navigate', arguments: '{"url":"https://example.com/dashboard"}' } }] }
+ : { toolCalls: [{ id: 'finished', function: { name: 'done', arguments: '{"summary":"Dashboard opened","outcome":"success"}' } }] };
+ const agent = actSetup(Agent, {
+ supportsVision: true,
+ chat: next,
+ async *chatStream() { yield { type: 'tool_call', content: (await next()).toolCalls }; yield { type: 'done' }; },
+ }, async (_tab, enriched) => {
+ plans.push(enriched.content);
+ if (plans.length === 1) { entered.resolve(); await release.promise; }
+ return { proceed: true, requestKind: 'execute', requiresStateChange: plans.length > 1, requiresSubmission: false };
+ });
+ agent.executeTool = async (_tab, name, args) => {
+ if (name === 'done') {
+ assert.equal(agent._planOnlyTerminalDecision(tabId, args.summary, { viaDone: true, outcome: args.outcome }), null);
+ return { done: true, summary: args.summary, outcome: args.outcome };
+ }
+ dispatched.push(name); return { success: true, url: args.url };
+ };
+ const runOptions = { ...options, recommendedAction: { id, tool, autoExecute: true } };
+ assert.ok(agent._recommendedActionFirstTool(runOptions), 'Fixture must activate the real recommended first tool');
+ const firstTool = agent._maybeExecuteRecommendedActionFirstTool.bind(agent);
+ agent._maybeExecuteRecommendedActionFirstTool = async (...args) => {
+ recommendedAttempts.push(args[1]?.recommendedAction?.tool);
+ return firstTool(...args);
+ };
+ const run = streaming
+ ? agent.processMessageStream(tabId, 'Inspect this page', () => {}, mode, runOptions)
+ : agent.processMessage(tabId, 'Inspect this page', () => {}, mode, [], runOptions);
+ await Promise.race([entered.promise, run.then(result => assert.fail(`Early completion: ${result}`))]);
+ assert.equal(steer(agent, 'Cancel the page inspection; open my dashboard instead').accepted, true);
+ release.resolve();
+ assert.equal(await run, 'Dashboard opened');
+ assert.equal(plans.length, 2);
+ assert.match(plans[1], /Cancel the page inspection; open my dashboard instead/);
+ assert.deepEqual(recommendedAttempts, [], 'Steering must suppress the stale recommendation before any preparation or dispatch');
+ assert.deepEqual(dispatched, ['navigate'], 'The original recommended read/screenshot must never dispatch');
+ });
+ }
for (const revisedKind of ['respond', 'execute']) {
test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} initial response-only steering switches to ${revisedKind}`, async () => {
const entered = deferred(), release = deferred(), updates = [], persisted = [], dispatched = [];
diff --git a/test/fixtures/message-recipient-navigation.mjs b/test/fixtures/message-recipient-navigation.mjs
index a48a8edd5..972b4d327 100644
--- a/test/fixtures/message-recipient-navigation.mjs
+++ b/test/fixtures/message-recipient-navigation.mjs
@@ -187,6 +187,37 @@ export function registerMessageRecipientNavigationFixtures({
assert.equal(await page.evaluate(() => window.sentMessages), 0);
});
+ register(`${kind}: form-free LinkedIn previews need no explicit button type`, async page => {
+ for (const tag of ['button', 'div']) {
+ const { guard } = await setupMessageHistory(page);
+ await page.evaluate(tag => {
+ const original = document.querySelector('#preview');
+ original.removeAttribute('type');
+ if (tag === 'div') {
+ const control = document.createElement('div');
+ for (const attribute of original.attributes) control.setAttribute(attribute.name, attribute.value);
+ control.setAttribute('role', 'button'); control.tabIndex = 0;
+ control.append(...original.childNodes); original.replaceWith(control);
+ control.onclick = () => { document.querySelector('#viewer').hidden = false; };
+ }
+ }, tag);
+ const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxChars: 20000 });
+ const ref = /button "Click or press enter to display in the image preview" \[([^\]]+)\]/.exec(tree.pageContent)?.[1];
+ assert.ok(ref, tree.pageContent);
+ const point = await page.locator('#preview').evaluate(el => {
+ const r = el.getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2 };
+ });
+ for (const [tool, args] of [['click', { selector: '#preview' }], ['click', { text: 'Click or press enter to display in the image preview' }],
+ ['click_ax', { ref_id: ref }], ['click', point]]) {
+ assert.equal(await guard(tool, args), null, `${tag}/${tool}`);
+ assert.equal((await call(page, tool, args)).success, true);
+ assert.equal(await page.locator('#viewer').isVisible(), true);
+ await page.locator('#close-viewer').click();
+ }
+ assert.equal(await page.evaluate(() => window.sentMessages), 0);
+ }
+ });
+
register(`${kind}: LinkedIn preview lookalikes and hidden or occluded images stay protected`, async page => {
const { guard } = await setupMessageHistory(page);
for (const change of ['no-image', 'composer', 'hidden', 'transparent', 'ancestor-transparent', 'occluded', 'disabled', 'commit-label', 'commit-text',
@@ -204,7 +235,7 @@ export function registerMessageRecipientNavigationFixtures({
if(change==='commit-text'){button.title='View image';button.append('Send');}
if(change==='localized-send')button.setAttribute('aria-label','Gönder');
if(change==='submit')button.type='submit';
- if(change==='external-form')button.setAttribute('form','composer');
+ if(change==='external-form'){button.removeAttribute('type');button.setAttribute('form','composer');}
if(change==='outside-history')document.body.append(button);
if(change==='small-icon')button.querySelector('img').style='width:20px;height:20px';
if(change==='menu')button.setAttribute('aria-haspopup','menu');