Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: a refused done on its own gets a screen update
Emre Sokullud6eba60fa44e
diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex 9f6166e46..c50f91a7f 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -866,7 +866,11 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {proto._visionObserveAfterBatchOnce = async function _visionObserveAfterBatchOnce(tabId, messages, onUpdate, { state, toolCalls = [], force = false } = {}) {const session = this._visionSessionFor(tabId, state);const names = toolCalls.map(call => call?._vision?.tool || call?.function?.name).filter(Boolean);- const observe = force || names.some(name => !VISION_NO_OBSERVE_TOOLS.has(name)) || session.forceKeyframe;+ // An accepted done ends the run before this point, so a batch with done+ // only gets here when done was refused: the model is told to look again,+ // and gets a screen to look at.+ const observe = force || session.forceKeyframe+ || names.some(name => !VISION_NO_OBSERVE_TOOLS.has(name) || name === 'done' || name === 'done_json');const zoomsOnly = !observe && session.pendingZooms?.length;if (zoomsOnly) {const zooms = this._visionTakePendingZooms(tabId);diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex 9f6166e46..c50f91a7f 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -866,7 +866,11 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {proto._visionObserveAfterBatchOnce = async function _visionObserveAfterBatchOnce(tabId, messages, onUpdate, { state, toolCalls = [], force = false } = {}) {const session = this._visionSessionFor(tabId, state);const names = toolCalls.map(call => call?._vision?.tool || call?.function?.name).filter(Boolean);- const observe = force || names.some(name => !VISION_NO_OBSERVE_TOOLS.has(name)) || session.forceKeyframe;+ // An accepted done ends the run before this point, so a batch with done+ // only gets here when done was refused: the model is told to look again,+ // and gets a screen to look at.+ const observe = force || session.forceKeyframe+ || names.some(name => !VISION_NO_OBSERVE_TOOLS.has(name) || name === 'done' || name === 'done_json');const zoomsOnly = !observe && session.pendingZooms?.length;if (zoomsOnly) {const zooms = this._visionTakePendingZooms(tabId);diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjsindex a7310b176..b6983041d 100644--- a/test/vision-mode-agent.mjs+++ b/test/vision-mode-agent.mjs@@ -673,6 +673,22 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))assert.equal(attempts, 1);});+ test(`${build}: a refused done on its own is followed by a screen update`, async () => {+ const ctx = runAgent(step => {+ // No outcome: refused once, with nothing else in the response.+ if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.' })] };+ if (step === 2) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };+ return { content: 'done' };+ });+ await runLoop(ctx.agent, nextTab++, false);+ assert.equal(ctx.requests.length, 2, 'the first done was refused');+ const messages = ctx.requests[1].messages;+ const refusal = messages.findLastIndex(message => message.role === 'tool');+ const update = messages.findLastIndex(message => message.webbrainVisionFrame);+ assert.ok(update > refusal, 'a screen update follows the refused done');+ assert.ok(['none', 'patch', 'key'].includes(messages[update].webbrainVisionFrame.kind), textOf(messages[update]));+ });+test(`${build}: a new run never shows or uses the previous run's frame when its first capture fails`, async () => {const ctx = runAgent(step => {if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };