Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: tell the model how to attach a file
Claude59e491e67eaa
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex b9a6daa68..815c4f811 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -22,7 +22,7 @@ import { SOCIAL_PLATFORMS, socialPublicationApiPlatform, normalizePublicationConimport { mastodonPublicationIntentMessages, normalizeMastodonPublicationIntent, captureNativeMastodonPublication, mastodonPublicationBinding, mastodonPublicationReceipt, mastodonPublicationNavigation } from './mastodon-publication.js';import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, RETIRED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_BATCHING, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_LEAN_NOTE, SYSTEM_PROMPT_GENERATIVE_MEDIA, SYSTEM_PROMPT_DEV_APPENDIX, SYSTEM_PROMPT_WEBMCP_ASK, SYSTEM_PROMPT_WEBMCP_ACT, getVisionActPrompt } from './tools.js';import { installVisionRuntime } from './vision-runtime.js';-import { VISION_CAPTCHA_MANUAL_NOTE, VISION_CAPTCHA_UNAPPLIED_NOTE, VISION_NATIVE_TOOLS, VISION_SITE_NOTES_PREFACE, visionCaptchaSolverNote, visionFilePickerBlockedResult, visionUniversalPreamble } from './vision-tools.js';+import { VISION_CAPTCHA_MANUAL_NOTE, VISION_CAPTCHA_UNAPPLIED_NOTE, VISION_NATIVE_TOOLS, visionCaptchaSolverNote, visionFilePickerBlockedResult, visionSiteNotesPreface, visionUniversalPreamble } from './vision-tools.js';import { visionDomProbeExpression } from './vision-dom-probe.js';import { normalizeVisionModeSetting } from './vision-mode.js';import { rejectedCompletionRecovery } from './completion-recovery.js';@@ -12978,7 +12978,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst heading = adapter.category === 'finance'? `[Site guidance for ${adapter.name} β FINANCE / HIGH-STAKES]`: `[Site guidance for ${adapter.name}]`;- const visionNote = this._visionRunState(tabId).active ? `${VISION_SITE_NOTES_PREFACE}\n` : '';+ const visionNote = this._visionRunState(tabId).active ? `${visionSiteNotesPreface(adapter.notes)}\n` : '';contextLine += `${heading}\n${visionNote}${adapter.notes.trim()}\n\n`;}if (adapter) this._queueAdapterMatchTrace(tabId, adapter, !!shouldInjectAdapter);@@ -13717,7 +13717,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst heading = adapter.category === 'finance'? `[Site context changed β now on ${adapter.name} β FINANCE / HIGH-STAKES. Apply these rules from now on:]`: `[Site context changed β now on ${adapter.name}. Apply these notes from now on:]`;- const visionNote = this._visionRunState(tabId).active ? `${VISION_SITE_NOTES_PREFACE}\n` : '';+ const visionNote = this._visionRunState(tabId).active ? `${visionSiteNotesPreface(adapter.notes)}\n` : '';messages.push({role: 'user',content: `${heading}\n${visionNote}${adapter.notes.trim()}`,diff --git a/src/chrome/src/agent/vision-tools.js b/src/chrome/src/agent/vision-tools.jsindex 65f580aca..a33310955 100644--- a/src/chrome/src/agent/vision-tools.js+++ b/src/chrome/src/agent/vision-tools.js@@ -299,6 +299,11 @@ export function buildVisionActPrompt({const scheduling = tier === 'compact'? '- You cannot schedule, sleep or check back later. If something must wait for an external event, call done({summary, outcome:"partial"}) with the current state.': '- Future work only through schedule_resume (pause this task) or schedule_task (a task the user explicitly asked to schedule); claim it is scheduled only after the tool succeeds.';+ // upload_file is not a vision tool: clicking the page's upload control is+ // what hands it over (vision-runtime.js _visionNoteUploadHandoff).+ const uploadSource = tier === 'compact'+ ? 'a file the user attached to their message'+ : 'a file the user attached, or one downloaded in this conversation';return `You are WebBrain, an AI browser agent running in Act mode with VISION CONTROL. You see the browser tab only through screenshots and act with mouse and keyboard at coordinates. There is no accessibility tree, no element refs and no CSS selectors in this mode.OPERATING ENVIRONMENT:@@ -325,6 +330,7 @@ TYPING AND FORMS:- After submitting, CONFIRM success on screen (toast, the new item, a detail page). An item that existed before is not yours.- When a dialog is open, finish or dismiss it before acting behind it.- CAPTCHAs: never dismiss or resubmit a verification challenge; stop and ask the user unless a CAPTCHA solver note says otherwise.+- File uploads: to attach ${uploadSource}, click the page's own attach or upload control once with click_at. WebBrain blocks the OS file dialog, which screenshots cannot show, and gives you upload_file for the next steps; screenshots come back after the upload. Only if WebBrain then says the upload is unavailable, finish the rest and tell the user.READING:- Screenshots are the primary view. Use read_page when you need exact long text (an article, a full post) and zoom for small print.@@ -377,6 +383,20 @@ export function visionCaptchaSolverNote(coverage) {export const VISION_SITE_NOTES_PREFACE = '(Written for WebBrain\'s element tools: click_ax, set_field, type_ax, ref ids, selectors and the accessibility tree are not available in this run. Apply the facts about the site with click_at, type_text, press_keys and scroll_at on what the screenshots show.)';+// Site notes send uploads to upload_file on a hidden file input and forbid+// clicking the upload control, because that opens the OS file dialog. A vision+// run gets upload_file only through that very click (vision-runtime.js+// _visionNoteUploadHandoff), so obeying the notes would leave it no way to+// attach a file.+export const VISION_SITE_NOTES_UPLOAD_CLAUSE = '(Uploads in this run: where these notes say to use upload_file and not to click the upload control, click that control once with click_at instead. WebBrain blocks the OS file dialog and then gives you upload_file for the next steps.)';++/** The line put before a site's notes in a vision run. */+export function visionSiteNotesPreface(notes = '') {+ return /\bupload_file\b/.test(String(notes || ''))+ ? `${VISION_SITE_NOTES_PREFACE}\n${VISION_SITE_NOTES_UPLOAD_CLAUSE}`+ : VISION_SITE_NOTES_PREFACE;+}+/*** A vision click the native file-chooser guard stopped. The guard's own* guidance points to upload_file, which the vision tools do not include.diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex a83e4a13c..1f5b210a6 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -21,7 +21,7 @@ import { SOCIAL_PLATFORMS, socialPublicationApiPlatform, normalizePublicationConimport { mastodonPublicationIntentMessages, normalizeMastodonPublicationIntent, captureNativeMastodonPublication, mastodonPublicationBinding, mastodonPublicationReceipt, mastodonPublicationNavigation } from './mastodon-publication.js';import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, RETIRED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_BATCHING, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_LEAN_NOTE, SYSTEM_PROMPT_GENERATIVE_MEDIA, SYSTEM_PROMPT_DEV_APPENDIX, getVisionActPrompt } from './tools.js';import { installVisionRuntime } from './vision-runtime.js';-import { VISION_CAPTCHA_MANUAL_NOTE, VISION_CAPTCHA_UNAPPLIED_NOTE, VISION_NATIVE_TOOLS, VISION_SITE_NOTES_PREFACE, visionCaptchaSolverNote, visionFilePickerBlockedResult, visionUniversalPreamble } from './vision-tools.js';+import { VISION_CAPTCHA_MANUAL_NOTE, VISION_CAPTCHA_UNAPPLIED_NOTE, VISION_NATIVE_TOOLS, visionCaptchaSolverNote, visionFilePickerBlockedResult, visionSiteNotesPreface, visionUniversalPreamble } from './vision-tools.js';import { visionDomProbeExpression } from './vision-dom-probe.js';import { normalizeVisionModeSetting } from './vision-mode.js';import { rejectedCompletionRecovery } from './completion-recovery.js';@@ -12700,7 +12700,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst heading = adapter.category === 'finance'? `[Site context changed β now on ${adapter.name} β FINANCE / HIGH-STAKES. Apply these rules from now on:]`: `[Site context changed β now on ${adapter.name}. Apply these notes from now on:]`;- const visionNote = this._visionRunState(tabId).active ? `${VISION_SITE_NOTES_PREFACE}\n` : '';+ const visionNote = this._visionRunState(tabId).active ? `${visionSiteNotesPreface(adapter.notes)}\n` : '';messages.push({role: 'user',content: `${heading}\n${visionNote}${adapter.notes.trim()}`,@@ -19615,7 +19615,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout dconst heading = adapter.category === 'finance'? `[Site guidance for ${adapter.name} β FINANCE / HIGH-STAKES]`: `[Site guidance for ${adapter.name}]`;- const visionNote = this._visionRunState(tabId).active ? `${VISION_SITE_NOTES_PREFACE}\n` : '';+ const visionNote = this._visionRunState(tabId).active ? `${visionSiteNotesPreface(adapter.notes)}\n` : '';contextLine += `${heading}\n${visionNote}${adapter.notes.trim()}\n\n`;}if (adapter) this._queueAdapterMatchTrace(tabId, adapter, !!shouldInjectAdapter);diff --git a/src/firefox/src/agent/vision-tools.js b/src/firefox/src/agent/vision-tools.jsindex 65f580aca..a33310955 100644--- a/src/firefox/src/agent/vision-tools.js+++ b/src/firefox/src/agent/vision-tools.js@@ -299,6 +299,11 @@ export function buildVisionActPrompt({const scheduling = tier === 'compact'? '- You cannot schedule, sleep or check back later. If something must wait for an external event, call done({summary, outcome:"partial"}) with the current state.': '- Future work only through schedule_resume (pause this task) or schedule_task (a task the user explicitly asked to schedule); claim it is scheduled only after the tool succeeds.';+ // upload_file is not a vision tool: clicking the page's upload control is+ // what hands it over (vision-runtime.js _visionNoteUploadHandoff).+ const uploadSource = tier === 'compact'+ ? 'a file the user attached to their message'+ : 'a file the user attached, or one downloaded in this conversation';return `You are WebBrain, an AI browser agent running in Act mode with VISION CONTROL. You see the browser tab only through screenshots and act with mouse and keyboard at coordinates. There is no accessibility tree, no element refs and no CSS selectors in this mode.OPERATING ENVIRONMENT:@@ -325,6 +330,7 @@ TYPING AND FORMS:- After submitting, CONFIRM success on screen (toast, the new item, a detail page). An item that existed before is not yours.- When a dialog is open, finish or dismiss it before acting behind it.- CAPTCHAs: never dismiss or resubmit a verification challenge; stop and ask the user unless a CAPTCHA solver note says otherwise.+- File uploads: to attach ${uploadSource}, click the page's own attach or upload control once with click_at. WebBrain blocks the OS file dialog, which screenshots cannot show, and gives you upload_file for the next steps; screenshots come back after the upload. Only if WebBrain then says the upload is unavailable, finish the rest and tell the user.READING:- Screenshots are the primary view. Use read_page when you need exact long text (an article, a full post) and zoom for small print.@@ -377,6 +383,20 @@ export function visionCaptchaSolverNote(coverage) {export const VISION_SITE_NOTES_PREFACE = '(Written for WebBrain\'s element tools: click_ax, set_field, type_ax, ref ids, selectors and the accessibility tree are not available in this run. Apply the facts about the site with click_at, type_text, press_keys and scroll_at on what the screenshots show.)';+// Site notes send uploads to upload_file on a hidden file input and forbid+// clicking the upload control, because that opens the OS file dialog. A vision+// run gets upload_file only through that very click (vision-runtime.js+// _visionNoteUploadHandoff), so obeying the notes would leave it no way to+// attach a file.+export const VISION_SITE_NOTES_UPLOAD_CLAUSE = '(Uploads in this run: where these notes say to use upload_file and not to click the upload control, click that control once with click_at instead. WebBrain blocks the OS file dialog and then gives you upload_file for the next steps.)';++/** The line put before a site's notes in a vision run. */+export function visionSiteNotesPreface(notes = '') {+ return /\bupload_file\b/.test(String(notes || ''))+ ? `${VISION_SITE_NOTES_PREFACE}\n${VISION_SITE_NOTES_UPLOAD_CLAUSE}`+ : VISION_SITE_NOTES_PREFACE;+}+/*** A vision click the native file-chooser guard stopped. The guard's own* guidance points to upload_file, which the vision tools do not include.diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjsindex 79e1b45de..cd4d4781e 100644--- a/test/vision-mode-agent.mjs+++ b/test/vision-mode-agent.mjs@@ -1001,7 +1001,43 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))}for (const file of [`../src/${build}/src/agent/agent.js`]) {const source = (await import('node:fs')).readFileSync(new URL(file, import.meta.url), 'utf8');- assert.match(source, /const visionNote = this\._visionRunState\(tabId\)\.active \? `\$\{VISION_SITE_NOTES_PREFACE\}\\n` : '';\s*contextLine \+= `\$\{heading\}\\n\$\{visionNote\}/);+ assert.match(source, /const visionNote = this\._visionRunState\(tabId\)\.active \? `\$\{visionSiteNotesPreface\(adapter\.notes\)\}\\n` : '';\s*contextLine \+= `\$\{heading\}\\n\$\{visionNote\}/);+ }+ });++ test(`${build}: site notes that route uploads to upload_file tell a vision run how to get it`, async () => {+ // The notes forbid clicking the upload control, but in a vision run that+ // click is what hands upload_file over (_visionNoteUploadHandoff).+ const { agent } = runAgent(() => ({ content: 'done' }));+ const tab = nextTab++;+ agent.conversationModes.set(tab, 'act');+ const api = globalThis.chrome;+ const get = api.tabs.get;+ let url = 'https://x.com/compose/post';+ api.tabs.get = async id => ({ id, url, title: 'Post', status: 'complete' });+ const reinject = (...args) => Agent.prototype._maybeReinjectAdapter.apply(agent, args);+ try {+ agent.lastSeenAdapter.set(tab, null);+ const x = [];+ assert.equal(await reinject(tab, x), true);+ assert.match(x[0].content, /never click that button/, 'fixture: the X notes forbid the click');+ assert.match(x[0].content, /\(Uploads in this run: where these notes say to use upload_file and not to click the upload control, click that control once with click_at instead\./);+ // Notes without uploads keep only the tool line.+ url = 'https://mastodon.social/home';+ agent.lastSeenAdapter.set(tab, null);+ const mastodon = [];+ assert.equal(await reinject(tab, mastodon), true);+ assert.match(mastodon[0].content, /Written for WebBrain's element tools/);+ assert.doesNotMatch(mastodon[0].content, /Uploads in this run/);+ // The element-tool path keeps the notes as written.+ url = 'https://x.com/compose/post';+ agent.visionMode = 'off';+ agent.lastSeenAdapter.set(tab, null);+ const legacy = [];+ assert.equal(await reinject(tab, legacy), true);+ assert.doesNotMatch(legacy[0].content, /Uploads in this run|Written for WebBrain's element tools/);+ } finally {+ api.tabs.get = get;}});diff --git a/test/vision-mode.mjs b/test/vision-mode.mjsindex 2ba8e52cb..d033ebf87 100644--- a/test/vision-mode.mjs+++ b/test/vision-mode.mjs@@ -437,6 +437,18 @@ for (const build of ['chrome', 'firefox']) {assert.match(tools.getVisionActPrompt({ coordinates: 'normalized' }), /0\.\.1000/);assert.match(tools.getVisionActPrompt({ tier: 'compact' }), /cannot schedule/);});++ test(`${build}: the vision Act prompt says how a file gets attached without the element tools`, () => {+ // upload_file is not a vision tool; clicking the upload control hands it over.+ for (const tier of ['compact', 'mid', 'full']) {+ const prompt = tools.getVisionActPrompt({ coordinates: 'pixel', tier });+ assert.match(prompt, /File uploads: to attach [^\n]*click the page's own attach or upload control once with click_at\. WebBrain blocks the OS file dialog[^\n]*gives you upload_file for the next steps/);+ assert.match(prompt, /Only if WebBrain then says the upload is unavailable, finish the rest and tell the user\./);+ }+ // Compact upload_file attaches only the user's own attachments.+ assert.match(tools.getVisionActPrompt({ tier: 'compact' }), /to attach a file the user attached to their message,/);+ assert.match(tools.getVisionActPrompt({ tier: 'full' }), /to attach a file the user attached, or one downloaded in this conversation,/);+ });}for (const build of ['chrome', 'firefox']) {