Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: Stop during the layout check before a batch answers its calls
Emre Sokullue3603a52241b
diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex 2ea74f73f..87f8404ed 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -1160,7 +1160,9 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {let probe = null;try { probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId)); }catch (error) {- if (isControlFlowError(error)) throw error;+ // Stop: this runs before the batch can answer its calls on a throw;+ // the batch's per-call abort check answers every one of them instead.+ if (isControlFlowError(error) && error?.name !== 'AbortError') throw error;return null;}if (!probe) return null;diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex 2ea74f73f..87f8404ed 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -1160,7 +1160,9 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {let probe = null;try { probe = await this._visionBounded(tabId, 'viewport measurement', () => this._visionPlatformProbe(tabId)); }catch (error) {- if (isControlFlowError(error)) throw error;+ // Stop: this runs before the batch can answer its calls on a throw;+ // the batch's per-call abort check answers every one of them instead.+ if (isControlFlowError(error) && error?.name !== 'AbortError') throw error;return null;}if (!probe) return null;diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjsindex 3284b2ed4..babedb58e 100644--- a/test/vision-mode-agent.mjs+++ b/test/vision-mode-agent.mjs@@ -767,6 +767,26 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))assert.ok(runTwo.messages.some(message => /Screenshots from earlier vision steps are not shown/.test(textOf(message))));});+ test(`${build}: Stop while the layout before a batch is measured still answers every call of the batch`, async () => {+ let tab;+ const ctx = runAgent(step => (step === 1 ? { toolCalls: [call('click_at', { x: 300, y: 200 })] } : { content: 'done' }));+ const probe = ctx.agent._visionPlatformProbe;+ let probes = 0;+ ctx.agent._visionPlatformProbe = async (...args) => {+ // The second measurement is the layout check before the click's batch: Stop arrives while it hangs.+ if (++probes === 2) { setTimeout(() => ctx.agent.abort(tab), 20); return new Promise(() => {}); }+ return probe(...args);+ };+ tab = nextTab++;+ await runLoop(ctx.agent, tab, false);+ const conversation = ctx.agent.conversations.get(tab) || [];+ const calls = conversation.filter(message => message.role === 'assistant').flatMap(message => message.tool_calls || []).map(item => item.id);+ const answered = new Set(conversation.filter(message => message.role === 'tool').map(message => message.tool_call_id));+ assert.ok(calls.length >= 1);+ assert.deepEqual(calls.filter(id => !answered.has(id)), [], 'no tool call is left without a result');+ assert.equal(ctx.dispatched.filter(entry => entry.name === 'click').length, 0, 'nothing was clicked');+ });+test(`${build}: a new run never shows or uses the previous run's frame when its first capture fails`, async () => {const ctx = runAgent(step => {if (step === 1) return { toolCalls: [call('done', { summary: 'Looked.', outcome: 'success' })] };