Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Consume final-step steering before terminating a run
Emre Sokullu990bf57a93aa
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 1674a9b9d..6cc04ce0c 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -6096,6 +6096,11 @@ export class Agent extends LoopDetector {return !!this._steeringRuns.get(tabId)?.messages.length;}+ _hasUnvalidatedSteering(tabId) {+ const run = this._steeringRuns.get(tabId);+ return !!run && (run.messages.length > 0 || run.revision > run.authorizedRevision);+ }+_applyPendingSteering(tabId, messages, onUpdate) {const run = this._steeringRuns.get(tabId);if (!run?.messages.length || this._checkAbort(tabId)) return false;@@ -9141,7 +9146,12 @@ export class Agent extends LoopDetector {const toolCounts = new Map();let lastAssistantText = '';let lastToolCall = null;- for (const m of messages) {+ const binding = this._activeTaskBinding(messages);+ // Earlier attempts remain in history, but this handoff describes the+ // latest authorized revision rather than quoting its superseded answer.+ const summaryMessages = binding.updates?.some(update => update.kind === 'steering')+ ? messages.slice(binding.index + 1) : messages;+ for (const m of summaryMessages) {if (m.role === 'assistant') {if (Array.isArray(m.tool_calls)) {for (const tc of m.tool_calls) {@@ -43584,7 +43594,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai// consider this run. Failures/cancels are still excluded via _traceStatus.shareHadProviderCompletion = true;- while (steps < this.maxSteps) {+ while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) {// Each turn's raw provider text belongs to that turn only: reset so a// non-terminal final-text pass can never pair stale output with a later// tool-batch result in the shared record.@@ -43602,6 +43612,8 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaitabId, messages, onUpdate, mode, provider, costState, runId, runOptions,);if (steeringOutcome?.changed) {+ // Replace the superseded final turn with one freshly authorized turn.+ if (steps >= this.maxSteps) steps--;forceCompletionVerificationTurn = false;allowCompletionFailureTurn = false;forceCompletionDoneAfterVerification = false;@@ -43874,7 +43886,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaibreak;}- if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) {+ if (this._applyPendingSteering(tabId, messages, onUpdate)) {onUpdate('text', { content: '', replace: true });continue;}@@ -43978,7 +43990,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaiconst batchResult = await this._executeToolBatch(tabId, result.toolCalls, messages, onUpdate, provider, assistantToolContent, allowedToolNames, steps, runOptions, toolSchemas);- if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps+ if (['continue', 'return'].includes(batchResult.action) && !batchResult.status&& this._applyPendingSteering(tabId, messages, onUpdate)) {onUpdate('text', { content: '', replace: true });continue;@@ -44789,7 +44801,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agai// Past local-only fast paths: the streaming loop below calls the provider.shareHadProviderCompletion = true;- while (steps < this.maxSteps) {+ while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) {// See the non-streaming loop: raw provider text must not survive into// a later turn's shared record.shareRawResponse = null;@@ -44805,6 +44817,8 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaitabId, messages, onUpdate, mode, provider, costState, runId, runOptions,);if (steeringOutcome?.changed) {+ // Replace the superseded final turn with one freshly authorized turn.+ if (steps >= this.maxSteps) steps--;pendingVisionFallbackMessages = null;forceCompletionVerificationTurn = false;allowCompletionFailureTurn = false;@@ -45024,7 +45038,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaitoolCalls: streamedToolCalls,}));- if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) {+ if (this._applyPendingSteering(tabId, messages, onUpdate)) {onUpdate('text', { content: '', replace: true });continue;}@@ -45114,7 +45128,7 @@ If the user has already named or confirmed this exact recipient, do NOT ask agaiconst batchResult = await this._executeToolBatch(tabId, toolCalls, messages, onUpdate, provider, fullText, allowedToolNames, steps, runOptions, toolSchemas);- if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps+ if (['continue', 'return'].includes(batchResult.action) && !batchResult.status&& this._applyPendingSteering(tabId, messages, onUpdate)) {onUpdate('text', { content: '', replace: true });continue;diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex c73c837a2..90b8e2366 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -5810,6 +5810,11 @@ export class Agent extends LoopDetector {return !!this._steeringRuns.get(tabId)?.messages.length;}+ _hasUnvalidatedSteering(tabId) {+ const run = this._steeringRuns.get(tabId);+ return !!run && (run.messages.length > 0 || run.revision > run.authorizedRevision);+ }+_applyPendingSteering(tabId, messages, onUpdate) {const run = this._steeringRuns.get(tabId);if (!run?.messages.length || this._checkAbort(tabId)) return false;@@ -8973,7 +8978,12 @@ export class Agent extends LoopDetector {const toolCounts = new Map();let lastAssistantText = '';let lastToolCall = null;- for (const m of messages) {+ const binding = this._activeTaskBinding(messages);+ // Earlier attempts remain in history, but this handoff describes the+ // latest authorized revision rather than quoting its superseded answer.+ const summaryMessages = binding.updates?.some(update => update.kind === 'steering')+ ? messages.slice(binding.index + 1) : messages;+ for (const m of summaryMessages) {if (m.role === 'assistant') {if (Array.isArray(m.tool_calls)) {for (const tc of m.tool_calls) {@@ -36537,7 +36547,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// consider this run. Failures/cancels are still excluded via _traceStatus.shareHadProviderCompletion = true;- while (steps < this.maxSteps) {+ while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) {// Each turn's raw provider text belongs to that turn only: reset so a// non-terminal final-text pass can never pair stale output with a later// tool-batch result in the shared record.@@ -36554,6 +36564,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dtabId, messages, onUpdate, mode, provider, costState, runId, runOptions,);if (steeringOutcome?.changed) {+ // Replace the superseded final turn with one freshly authorized turn.+ if (steps >= this.maxSteps) steps--;forceCompletionVerificationTurn = false;allowCompletionFailureTurn = false;forceCompletionDoneAfterVerification = false;@@ -36801,7 +36813,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dbreak;}- if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) {+ if (this._applyPendingSteering(tabId, messages, onUpdate)) {onUpdate('text', { content: '', replace: true });continue;}@@ -36857,7 +36869,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst batchResult = await this._executeToolBatch(tabId, result.toolCalls, messages, onUpdate, provider, assistantToolContent, allowedToolNames, steps, runOptions, toolSchemas);- if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps+ if (['continue', 'return'].includes(batchResult.action) && !batchResult.status&& this._applyPendingSteering(tabId, messages, onUpdate)) {onUpdate('text', { content: '', replace: true });continue;@@ -37571,7 +37583,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// Past local-only fast paths: the streaming loop below calls the provider.shareHadProviderCompletion = true;- while (steps < this.maxSteps) {+ while (steps < this.maxSteps || this._hasUnvalidatedSteering(tabId)) {// See the non-streaming loop: raw provider text must not survive into// a later turn's shared record.shareRawResponse = null;@@ -37587,6 +37599,8 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dtabId, messages, onUpdate, mode, provider, costState, runId, runOptions,);if (steeringOutcome?.changed) {+ // Replace the superseded final turn with one freshly authorized turn.+ if (steps >= this.maxSteps) steps--;pendingVisionFallbackMessages = null;forceCompletionVerificationTurn = false;allowCompletionFailureTurn = false;@@ -37802,7 +37816,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dtoolCalls: streamedToolCalls,}));- if (steps < this.maxSteps && this._applyPendingSteering(tabId, messages, onUpdate)) {+ if (this._applyPendingSteering(tabId, messages, onUpdate)) {onUpdate('text', { content: '', replace: true });continue;}@@ -37845,7 +37859,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst batchResult = await this._executeToolBatch(tabId, toolCalls, messages, onUpdate, provider, fullText, allowedToolNames, steps, runOptions, toolSchemas);- if (['continue', 'return'].includes(batchResult.action) && !batchResult.status && steps < this.maxSteps+ if (['continue', 'return'].includes(batchResult.action) && !batchResult.status&& this._applyPendingSteering(tabId, messages, onUpdate)) {onUpdate('text', { content: '', replace: true });continue;diff --git a/test/chat-steering.mjs b/test/chat-steering.mjsindex 17e0adc18..b3c6d249b 100644--- a/test/chat-steering.mjs+++ b/test/chat-steering.mjs@@ -117,6 +117,59 @@ for (const build of ['chrome', 'firefox']) {for (const streaming of [false, true]) {for (const mode of ['act', 'dev']) {+ for (const phase of ['model', 'tool']) {+ for (const revisedKind of ['respond', 'execute']) {+ test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} final ${phase} step revalidates steering to ${revisedKind}`, async () => {+ const entered = deferred(), release = deferred(), plans = [], updates = [], dispatched = [];+ let calls = 0;+ const next = async () => {+ if (++calls === 1) {+ if (phase === 'model') { entered.resolve(); await release.promise; }+ return { content: 'Obsolete final-step answer', toolCalls: [+ { id: 'old-read', function: { name: 'get_accessibility_tree', arguments: '{}' } },+ { id: 'old-action', function: { name: 'navigate', arguments: '{"url":"https://example.com/old"}' } },+ ] };+ }+ return revisedKind === 'respond' ? { content: 'Revised answer' } : { toolCalls: [+ { id: 'revised-action', function: { name: 'navigate', arguments: '{"url":"https://example.com/dashboard"}' } },+ ] };+ };+ const agent = actSetup(Agent, {+ chat: next,+ async *chatStream() { const r = await next(); if (r.content) yield { type: 'text', content: r.content }; if (r.toolCalls) yield { type: 'tool_call', content: r.toolCalls.map((call, index) => ({ ...call, index })) }; yield { type: 'done' }; },+ }, async (_tab, enriched) => {+ plans.push(enriched.content);+ return { proceed: true, requestKind: plans.length > 1 ? revisedKind : 'execute',+ responseOnly: plans.length > 1 && revisedKind === 'respond', requiresStateChange: plans.length > 1 && revisedKind === 'execute' };+ });+ agent.maxSteps = 1;+ agent.executeTool = async (_tab, name, args) => {+ if (name === 'done') return { done: true, summary: args.summary, outcome: args.outcome };+ dispatched.push(args.url || name);+ if (name === 'get_accessibility_tree') { entered.resolve(); await release.promise; return { success: true, pageContent: 'Completed old observation' }; }+ return { success: true, url: args.url };+ };+ const update = (type, data) => updates.push({ type, data });+ const run = streaming ? agent.processMessageStream(tabId, 'Inspect the page', update, mode, options)+ : agent.processMessage(tabId, 'Inspect the page', update, mode, [], options);+ await Promise.race([entered.promise, run.then(r => assert.fail(`Early completion: ${r}`))]);+ assert.equal(steer(agent, 'Use the corrected task instead').accepted, true); release.resolve();+ const result = await run;+ if (revisedKind === 'respond') assert.equal(result, 'Revised answer');+ else {+ // The replacement turn keeps the normal step cap and completion verification.+ assert.match(result, /Step limit reached after 1 steps/);+ assert.doesNotMatch(result, /Obsolete final-step answer|example\.com\/old/);+ assert.equal(calls, 2);+ }+ assert.equal(plans.length, 2); assert.match(plans[1], /Use the corrected task instead/);+ assert.deepEqual(dispatched, [...(phase === 'tool' ? ['get_accessibility_tree'] : []), ...(revisedKind === 'execute' ? ['https://example.com/dashboard'] : [])]);+ assert.equal(updates.some(u => u.type === 'steering_queued'), false);+ assert.equal(agent.conversations.get(tabId).some(m => m.role === 'assistant' && m.content === 'Obsolete final-step answer' && !m.tool_calls), false);+ assertPairedTools(agent.conversations.get(tabId));+ });+ }+ }for (const [id, tool] of [['summarize-page', 'read_page'], ['download-media', 'screenshot']]) {test(`${build}: ${mode} ${streaming ? 'stream' : 'chat'} planner steering skips the original recommended ${tool}`, async () => {const entered = deferred(), release = deferred(), plans = [], dispatched = [], recommendedAttempts = [];@@ -349,11 +402,12 @@ for (const build of ['chrome', 'firefox']) {assert.equal(agent.isRunning(tabId), false);});- test(`${build}: ${streaming ? 'stream' : 'chat'} returns unconsumed steering when the step budget ends`, async () => {+ test(`${build}: ${streaming ? 'stream' : 'chat'} final Ask step consumes steering and replaces the obsolete answer`, async () => {const entered = deferred(); const release = deferred();+ let calls = 0;const agent = setup(Agent, {- chat: async () => { entered.resolve(); await release.promise; return { content: 'Finished' }; },- async *chatStream() { entered.resolve(); await release.promise; yield { type: 'text', content: 'Finished' }; yield { type: 'done' }; },+ chat: async () => { if (++calls === 1) { entered.resolve(); await release.promise; return { content: 'Finished' }; } return { content: 'Corrected answer' }; },+ async *chatStream() { if (++calls === 1) { entered.resolve(); await release.promise; yield { type: 'text', content: 'Finished' }; } else yield { type: 'text', content: 'Corrected answer' }; yield { type: 'done' }; },});agent.maxSteps = 1;const updates = [];@@ -364,10 +418,11 @@ for (const build of ['chrome', 'firefox']) {await Promise.race([entered.promise, run.then(result => assert.fail(`Early completion: ${result}`))]);assert.equal(steer(agent, 'Follow-up').accepted, true);release.resolve();- await run;- assert.deepEqual(updates.find(update => update.type === 'steering_queued').data.messages,- [{ id: 'correction-1', text: 'Follow-up' }]);- assert.equal(updates.filter(update => update.type === 'steering_applied').length, 0);+ assert.equal(await run, 'Corrected answer');+ assert.equal(calls, 2);+ assert.equal(updates.some(update => update.type === 'steering_queued'), false);+ assert.equal(updates.filter(update => update.type === 'steering_applied').length, 1);+ assert.equal(agent.conversations.get(tabId).some(message => message.role === 'assistant' && message.content === 'Finished'), false);assert.equal(agent._steeringRuns.size, 0);});}