Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Vision mode: a refused done on its own gets a screen update

Emre Sokullud6eba60fa44e
diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.js
index 9f6166e46..c50f91a7f 100644
--- a/src/chrome/src/agent/vision-runtime.js
+++ b/src/chrome/src/agent/vision-runtime.js
@@ -866,7 +866,11 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
proto._visionObserveAfterBatchOnce = async function _visionObserveAfterBatchOnce(tabId, messages, onUpdate, { state, toolCalls = [], force = false } = {}) {
const session = this._visionSessionFor(tabId, state);
const names = toolCalls.map(call => call?._vision?.tool || call?.function?.name).filter(Boolean);
- const observe = force || names.some(name => !VISION_NO_OBSERVE_TOOLS.has(name)) || session.forceKeyframe;
+ // An accepted done ends the run before this point, so a batch with done
+ // only gets here when done was refused: the model is told to look again,
+ // and gets a screen to look at.
+ const observe = force || session.forceKeyframe
+ || names.some(name => !VISION_NO_OBSERVE_TOOLS.has(name) || name === 'done' || name === 'done_json');
const zoomsOnly = !observe && session.pendingZooms?.length;
if (zoomsOnly) {
const zooms = this._visionTakePendingZooms(tabId);
diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.js
index 9f6166e46..c50f91a7f 100644
--- a/src/firefox/src/agent/vision-runtime.js
+++ b/src/firefox/src/agent/vision-runtime.js
@@ -866,7 +866,11 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
proto._visionObserveAfterBatchOnce = async function _visionObserveAfterBatchOnce(tabId, messages, onUpdate, { state, toolCalls = [], force = false } = {}) {
const session = this._visionSessionFor(tabId, state);
const names = toolCalls.map(call => call?._vision?.tool || call?.function?.name).filter(Boolean);
- const observe = force || names.some(name => !VISION_NO_OBSERVE_TOOLS.has(name)) || session.forceKeyframe;
+ // An accepted done ends the run before this point, so a batch with done
+ // only gets here when done was refused: the model is told to look again,
+ // and gets a screen to look at.
+ const observe = force || session.forceKeyframe
+ || names.some(name => !VISION_NO_OBSERVE_TOOLS.has(name) || name === 'done' || name === 'done_json');
const zoomsOnly = !observe && session.pendingZooms?.length;
if (zoomsOnly) {
const zooms = this._visionTakePendingZooms(tabId);
diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjs
index a7310b176..b6983041d 100644
--- a/test/vision-mode-agent.mjs
+++ b/test/vision-mode-agent.mjs
@@ -673,6 +673,22 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))
assert.equal(attempts, 1);
});
+ test(`${build}: a refused done on its own is followed by a screen update`, async () => {
+ const ctx = runAgent(step => {
+ // No outcome: refused once, with nothing else in the response.
+ if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.' })] };
+ if (step === 2) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };
+ return { content: 'done' };
+ });
+ await runLoop(ctx.agent, nextTab++, false);
+ assert.equal(ctx.requests.length, 2, 'the first done was refused');
+ const messages = ctx.requests[1].messages;
+ const refusal = messages.findLastIndex(message => message.role === 'tool');
+ const update = messages.findLastIndex(message => message.webbrainVisionFrame);
+ assert.ok(update > refusal, 'a screen update follows the refused done');
+ assert.ok(['none', 'patch', 'key'].includes(messages[update].webbrainVisionFrame.kind), textOf(messages[update]));
+ });
+
test(`${build}: a new run never shows or uses the previous run's frame when its first capture fails`, async () => {
const ctx = runAgent(step => {
if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };