Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Consume final-step steering before terminating a run

Emre Sokullu990bf57a93aa
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 1674a9b9d..6cc04ce0c 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -6096,6 +6096,11 @@ export class Agent extends LoopDetector {
return !!this._steeringRuns.get(tabId)?.messages.length;
}
+ _hasUnvalidatedSteering(tabId) {
+ const run = this._steeringRuns.get(tabId);
+ return !!run && (run.messages.length > 0 || run.revision > run.authorizedRevision);
+ }
+
_applyPendingSteering(tabId, messages, onUpdate) {
const run = this._steeringRuns.get(tabId);
if (!run?.messages.length || this._checkAbort(tabId)) return false;
@@ -9141,7 +9146,12 @@ export class Agent extends LoopDetector {
const toolCounts = new Map();
let lastAssistantText = '';
let lastToolCall = null;
- for (const m of messages) {
+ const binding = this._activeTaskBinding(messages);
+ // Earlier attempts remain in history, but this handoff describes the
+ // latest authorized revision rather than quoting its superseded answer.
+ const summaryMessages = binding.updates?.some(update => update.kind === 'steering')
+ ? messages.slice(binding.index + 1) : messages;
+ for (const m of summaryMessages) {
if (m.role === 'assistant') {
if (Array.isArray(m.tool_calls)) {
for (const tc of m.tool_calls) {
@@ -43584,7 +43594,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
// consider this run. Failures/cancels are still excluded via _traceStatus.
shareHadProviderCompletion = true;
- while (steps < this.maxSteps) {
+ while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) {
// Each turn's raw provider text belongs to that turn only: reset so a
// non-terminal final-text pass can never pair stale output with a later
// tool-batch result in the shared record.
@@ -43602,6 +43612,8 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
tabId, messages, onUpdate, mode, provider, costState, runId, runOptions,
);
if (steeringOutcome?.changed) {
+ // Replace the superseded final turn with one freshly authorized turn.
+ if (steps >= this.maxSteps) steps--;
forceCompletionVerificationTurn = false;
allowCompletionFailureTurn = false;
forceCompletionDoneAfterVerification = false;
@@ -43874,7 +43886,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
break;
}
- if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) {
+ if (this._applyPendingSteering(tabId, messages, onUpdate)) {
onUpdate('text', { content: '', replace: true });
continue;
}
@@ -43978,7 +43990,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
const batchResult = await this._executeToolBatch(
tabId, result.toolCalls, messages, onUpdate, provider, assistantToolContent, allowedToolNames, steps, runOptions, toolSchemas
);
- if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps
+ if (['continue', 'return'].includes(batchResult.action) && !batchResult.status
&& this._applyPendingSteering(tabId, messages, onUpdate)) {
onUpdate('text', { content: '', replace: true });
continue;
@@ -44789,7 +44801,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
// Past local-only fast paths: the streaming loop below calls the provider.
shareHadProviderCompletion = true;
- while (steps < this.maxSteps) {
+ while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) {
// See the non-streaming loop: raw provider text must not survive into
// a later turn's shared record.
shareRawResponse = null;
@@ -44805,6 +44817,8 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
tabId, messages, onUpdate, mode, provider, costState, runId, runOptions,
);
if (steeringOutcome?.changed) {
+ // Replace the superseded final turn with one freshly authorized turn.
+ if (steps >= this.maxSteps) steps--;
pendingVisionFallbackMessages = null;
forceCompletionVerificationTurn = false;
allowCompletionFailureTurn = false;
@@ -45024,7 +45038,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
toolCalls: streamedToolCalls,
}));
- if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) {
+ if (this._applyPendingSteering(tabId, messages, onUpdate)) {
onUpdate('text', { content: '', replace: true });
continue;
}
@@ -45114,7 +45128,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai
const batchResult = await this._executeToolBatch(
tabId, toolCalls, messages, onUpdate, provider, fullText, allowedToolNames, steps, runOptions, toolSchemas
);
- if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps
+ if (['continue', 'return'].includes(batchResult.action) && !batchResult.status
&& this._applyPendingSteering(tabId, messages, onUpdate)) {
onUpdate('text', { content: '', replace: true });
continue;
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index c73c837a2..90b8e2366 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -5810,6 +5810,11 @@ export class Agent extends LoopDetector {
return !!this._steeringRuns.get(tabId)?.messages.length;
}
+ _hasUnvalidatedSteering(tabId) {
+ const run = this._steeringRuns.get(tabId);
+ return !!run && (run.messages.length > 0 || run.revision > run.authorizedRevision);
+ }
+
_applyPendingSteering(tabId, messages, onUpdate) {
const run = this._steeringRuns.get(tabId);
if (!run?.messages.length || this._checkAbort(tabId)) return false;
@@ -8973,7 +8978,12 @@ export class Agent extends LoopDetector {
const toolCounts = new Map();
let lastAssistantText = '';
let lastToolCall = null;
- for (const m of messages) {
+ const binding = this._activeTaskBinding(messages);
+ // Earlier attempts remain in history, but this handoff describes the
+ // latest authorized revision rather than quoting its superseded answer.
+ const summaryMessages = binding.updates?.some(update => update.kind === 'steering')
+ ? messages.slice(binding.index + 1) : messages;
+ for (const m of summaryMessages) {
if (m.role === 'assistant') {
if (Array.isArray(m.tool_calls)) {
for (const tc of m.tool_calls) {
@@ -36537,7 +36547,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// consider this run. Failures/cancels are still excluded via _traceStatus.
shareHadProviderCompletion = true;
- while (steps < this.maxSteps) {
+ while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) {
// Each turn's raw provider text belongs to that turn only: reset so a
// non-terminal final-text pass can never pair stale output with a later
// tool-batch result in the shared record.
@@ -36554,6 +36564,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
tabId, messages, onUpdate, mode, provider, costState, runId, runOptions,
);
if (steeringOutcome?.changed) {
+ // Replace the superseded final turn with one freshly authorized turn.
+ if (steps >= this.maxSteps) steps--;
forceCompletionVerificationTurn = false;
allowCompletionFailureTurn = false;
forceCompletionDoneAfterVerification = false;
@@ -36801,7 +36813,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
break;
}
- if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) {
+ if (this._applyPendingSteering(tabId, messages, onUpdate)) {
onUpdate('text', { content: '', replace: true });
continue;
}
@@ -36857,7 +36869,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const batchResult = await this._executeToolBatch(
tabId, result.toolCalls, messages, onUpdate, provider, assistantToolContent, allowedToolNames, steps, runOptions, toolSchemas
);
- if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps
+ if (['continue', 'return'].includes(batchResult.action) && !batchResult.status
&& this._applyPendingSteering(tabId, messages, onUpdate)) {
onUpdate('text', { content: '', replace: true });
continue;
@@ -37571,7 +37583,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// Past local-only fast paths: the streaming loop below calls the provider.
shareHadProviderCompletion = true;
- while (steps < this.maxSteps) {
+ while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) {
// See the non-streaming loop: raw provider text must not survive into
// a later turn's shared record.
shareRawResponse = null;
@@ -37587,6 +37599,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
tabId, messages, onUpdate, mode, provider, costState, runId, runOptions,
);
if (steeringOutcome?.changed) {
+ // Replace the superseded final turn with one freshly authorized turn.
+ if (steps >= this.maxSteps) steps--;
pendingVisionFallbackMessages = null;
forceCompletionVerificationTurn = false;
allowCompletionFailureTurn = false;
@@ -37802,7 +37816,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
toolCalls: streamedToolCalls,
}));
- if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) {
+ if (this._applyPendingSteering(tabId, messages, onUpdate)) {
onUpdate('text', { content: '', replace: true });
continue;
}
@@ -37845,7 +37859,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const batchResult = await this._executeToolBatch(
tabId, toolCalls, messages, onUpdate, provider, fullText, allowedToolNames, steps, runOptions, toolSchemas
);
- if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps
+ if (['continue', 'return'].includes(batchResult.action) && !batchResult.status
&& this._applyPendingSteering(tabId, messages, onUpdate)) {
onUpdate('text', { content: '', replace: true });
continue;
diff --git a/test/chat-steering.mjs b/test/chat-steering.mjs
index 17e0adc18..b3c6d249b 100644
--- a/test/chat-steering.mjs
+++ b/test/chat-steering.mjs
@@ -117,6 +117,59 @@ for (const build of ['chrome', 'firefox']) {
for (const streaming of [false, true]) {
for (const mode of ['act', 'dev']) {
+ for (const phase of ['model', 'tool']) {
+ for (const revisedKind of ['respond', 'execute']) {
+ test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} final ${phase} step revalidates steering to ${revisedKind}`, async () => {
+ const entered = deferred(), release = deferred(), plans = [], updates = [], dispatched = [];
+ let calls = 0;
+ const next = async () => {
+ if (++calls === 1) {
+ if (phase === 'model') { entered.resolve(); await release.promise; }
+ return { content: 'Obsolete final-step answer', toolCalls: [
+ { id: 'old-read', function: { name: 'get_accessibility_tree', arguments: '{}' } },
+ { id: 'old-action', function: { name: 'navigate', arguments: '{"url":"https://example.com/old"}' } },
+ ] };
+ }
+ return revisedKind === 'respond' ? { content: 'Revised answer' } : { toolCalls: [
+ { id: 'revised-action', function: { name: 'navigate', arguments: '{"url":"https://example.com/dashboard"}' } },
+ ] };
+ };
+ const agent = actSetup(Agent, {
+ chat: next,
+ async *chatStream() { const r = await next(); if (r.content) yield { type: 'text', content: r.content }; if (r.toolCalls) yield { type: 'tool_call', content: r.toolCalls.map((call, index) => ({ ...call, index })) }; yield { type: 'done' }; },
+ }, async (_tab, enriched) => {
+ plans.push(enriched.content);
+ return { proceed: true, requestKind: plans.length > 1 ? revisedKind : 'execute',
+ responseOnly: plans.length > 1 && revisedKind === 'respond', requiresStateChange: plans.length > 1 && revisedKind === 'execute' };
+ });
+ agent.maxSteps = 1;
+ agent.executeTool = async (_tab, name, args) => {
+ if (name === 'done') return { done: true, summary: args.summary, outcome: args.outcome };
+ dispatched.push(args.url || name);
+ if (name === 'get_accessibility_tree') { entered.resolve(); await release.promise; return { success: true, pageContent: 'Completed old observation' }; }
+ return { success: true, url: args.url };
+ };
+ const update = (type, data) => updates.push({ type, data });
+ const run = streaming ? agent.processMessageStream(tabId, 'Inspect the page', update, mode, options)
+ : agent.processMessage(tabId, 'Inspect the page', update, mode, [], options);
+ await Promise.race([entered.promise, run.then(r => assert.fail(`Early completion: ${r}`))]);
+ assert.equal(steer(agent, 'Use the corrected task instead').accepted, true); release.resolve();
+ const result = await run;
+ if (revisedKind === 'respond') assert.equal(result, 'Revised answer');
+ else {
+ // The replacement turn keeps the normal step cap and completion verification.
+ assert.match(result, /Step limit reached after 1 steps/);
+ assert.doesNotMatch(result, /Obsolete final-step answer|example\.com\/old/);
+ assert.equal(calls, 2);
+ }
+ assert.equal(plans.length, 2); assert.match(plans[1], /Use the corrected task instead/);
+ assert.deepEqual(dispatched, [...(phase === 'tool' ? ['get_accessibility_tree'] : []), ...(revisedKind === 'execute' ? ['https://example.com/dashboard'] : [])]);
+ assert.equal(updates.some(u => u.type === 'steering_queued'), false);
+ assert.equal(agent.conversations.get(tabId).some(m => m.role === 'assistant' && m.content === 'Obsolete final-step answer' && !m.tool_calls), false);
+ assertPairedTools(agent.conversations.get(tabId));
+ });
+ }
+ }
for (const [id, tool] of [['summarize-page', 'read_page'], ['download-media', 'screenshot']]) {
test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} planner steering skips the original recommended ${tool}`, async () => {
const entered = deferred(), release = deferred(), plans = [], dispatched = [], recommendedAttempts = [];
@@ -349,11 +402,12 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(agent.isRunning(tabId), false);
});
- test(`${build}: ${streaming ? 'stream' : 'chat'} returns unconsumed steering when the step budget ends`, async () => {
+ test(`${build}: ${streaming ? 'stream' : 'chat'} final Ask step consumes steering and replaces the obsolete answer`, async () => {
const entered = deferred(); const release = deferred();
+ let calls = 0;
const agent = setup(Agent, {
- chat: async () => { entered.resolve(); await release.promise; return { content: 'Finished' }; },
- async *chatStream() { entered.resolve(); await release.promise; yield { type: 'text', content: 'Finished' }; yield { type: 'done' }; },
+ chat: async () => { if (++calls === 1) { entered.resolve(); await release.promise; return { content: 'Finished' }; } return { content: 'Corrected answer' }; },
+ async *chatStream() { if (++calls === 1) { entered.resolve(); await release.promise; yield { type: 'text', content: 'Finished' }; } else yield { type: 'text', content: 'Corrected answer' }; yield { type: 'done' }; },
});
agent.maxSteps = 1;
const updates = [];
@@ -364,10 +418,11 @@ for (const build of ['chrome', 'firefox']) {
await Promise.race([entered.promise, run.then(result => assert.fail(`Early completion: ${result}`))]);
assert.equal(steer(agent, 'Follow-up').accepted, true);
release.resolve();
- await run;
- assert.deepEqual(updates.find(update => update.type === 'steering_queued').data.messages,
- [{ id: 'correction-1', text: 'Follow-up' }]);
- assert.equal(updates.filter(update => update.type === 'steering_applied').length, 0);
+ assert.equal(await run, 'Corrected answer');
+ assert.equal(calls, 2);
+ assert.equal(updates.some(update => update.type === 'steering_queued'), false);
+ assert.equal(updates.filter(update => update.type === 'steering_applied').length, 1);
+ assert.equal(agent.conversations.get(tabId).some(message => message.role === 'assistant' && message.content === 'Finished'), false);
assert.equal(agent._steeringRuns.size, 0);
});
}