Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Vision mode: Stop during the layout check before a batch answers its calls

Emre Sokullue3603a52241b
diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.js
index 2ea74f73f..87f8404ed 100644
--- a/src/chrome/src/agent/vision-runtime.js
+++ b/src/chrome/src/agent/vision-runtime.js
@@ -1160,7 +1160,9 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
let probe = null;
try { probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId)); }
catch (error) {
- if (isControlFlowError(error)) throw error;
+ // Stop: this runs before the batch can answer its calls on a throw;
+ // the batch's per-call abort check answers every one of them instead.
+ if (isControlFlowError(error) && error?.name !== 'AbortError') throw error;
return null;
}
if (!probe) return null;
diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.js
index 2ea74f73f..87f8404ed 100644
--- a/src/firefox/src/agent/vision-runtime.js
+++ b/src/firefox/src/agent/vision-runtime.js
@@ -1160,7 +1160,9 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
let probe = null;
try { probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId)); }
catch (error) {
- if (isControlFlowError(error)) throw error;
+ // Stop: this runs before the batch can answer its calls on a throw;
+ // the batch's per-call abort check answers every one of them instead.
+ if (isControlFlowError(error) && error?.name !== 'AbortError') throw error;
return null;
}
if (!probe) return null;
diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjs
index 3284b2ed4..babedb58e 100644
--- a/test/vision-mode-agent.mjs
+++ b/test/vision-mode-agent.mjs
@@ -767,6 +767,26 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))
assert.ok(runTwo.messages.some(message => /Screenshots from earlier vision steps are not shown/.test(textOf(message))));
});
+ test(`${build}: Stop while the layout before a batch is measured still answers every call of the batch`, async () => {
+ let tab;
+ const ctx = runAgent(step => (step === 1 ? { toolCalls: [call('click_at', { x: 300, y: 200 })] } : { content: 'done' }));
+ const probe = ctx.agent._visionPlatformProbe;
+ let probes = 0;
+ ctx.agent._visionPlatformProbe = async (...args) => {
+ // The second measurement is the layout check before the click's batch: Stop arrives while it hangs.
+ if (++probes === 2) { setTimeout(() => ctx.agent.abort(tab), 20); return new Promise(() => {}); }
+ return probe(...args);
+ };
+ tab = nextTab++;
+ await runLoop(ctx.agent, tab, false);
+ const conversation = ctx.agent.conversations.get(tab) || [];
+ const calls = conversation.filter(message => message.role === 'assistant').flatMap(message => message.tool_calls || []).map(item => item.id);
+ const answered = new Set(conversation.filter(message => message.role === 'tool').map(message => message.tool_call_id));
+ assert.ok(calls.length >= 1);
+ assert.deepEqual(calls.filter(id => !answered.has(id)), [], 'no tool call is left without a result');
+ assert.equal(ctx.dispatched.filter(entry => entry.name === 'click').length, 0, 'nothing was clicked');
+ });
+
test(`${build}: a new run never shows or uses the previous run's frame when its first capture fails`, async () => {
const ctx = runAgent(step => {
if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };