Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: grid-aligned patches, OpenAI base URL matched by host
Emre Sokullub92ab67f30f1
diff --git a/docs/vision-mode.md b/docs/vision-mode.mdindex 9ad001eec..98da32532 100644--- a/docs/vision-mode.md+++ b/docs/vision-mode.md@@ -84,7 +84,7 @@ Screenshot redaction maps element boxes into the captured visual-viewport box (pAfter a batch WebBrain waits until two consecutive captures match (bounded, ~1 s; a micro change between them, a few 16 px cells such as a blinking caret, a clock or a spinner, counts as a match, `isMicroScreenChange`, since it never stops), then compares the new frame with what the model has seen (`vision-diff.js`): 16 px cells, a per-pixel threshold that ignores encoder noise and caret blinks, nearby changes merged, at most 4 padded regions. `planFrameUpdate` sends:- **nothing but a one-line note** when nothing changed: "no visible change" is also the clearest signal that a click missed;-- **patches** for local changes (typing, a menu, a toast);+- **patches** for local changes (typing, a menu, a toast). For a provider with an image grid (28 px for Qwen2/2.5-VL, UI-TARS, GLM-4.xV, InternVL and local servers; 32 px for Qwen3-VL) each patch is grown to multiples of the grid, like frames and zooms, so the server never resizes it and its pixels map 1:1 to the frame;- **a new frame** when more than 45% changed, the patch chain is 6 long, patches would cost β₯ 70% of a frame, or the URL, viewport size or DPR changed.The "seen" buffer copies only patched regions, so changes below the threshold accumulate until they are sent instead of drifting forever.diff --git a/src/chrome/src/agent/vision-budget.js b/src/chrome/src/agent/vision-budget.jsindex 277a9ac29..76aa741e9 100644--- a/src/chrome/src/agent/vision-budget.js+++ b/src/chrome/src/agent/vision-budget.js@@ -79,6 +79,18 @@ function lower(value) {return String(value || '').trim().toLowerCase();}+// The host of a configured base URL, compared exactly: a host name that only+// appears somewhere in the URL (a path, a longer domain) is not that host.+function urlHost(value) {+ const text = String(value || '').trim();+ if (!text) return '';+ try {+ return new URL(/^[a-z][a-z0-9+.-]*:\/\//i.test(text) ? text : `https://${text}`).hostname.toLowerCase();+ } catch {+ return '';+ }+}+/*** Pick the vision profile for a provider object (or its config). Model ids* decide first because routers (OpenRouter, Bedrock, Vertex) serve several@@ -99,7 +111,7 @@ export function detectVisionProfile(provider = {}) {}if (/qwen|ui-tars|uitars|holo1|opencua/.test(model)) return VISION_PROFILES.qwen;if (/(^|\/)(gpt-|o\d|chatgpt|computer-use)/.test(model) || name === 'openai' || name === 'azure-openai'- || baseUrl.includes('api.openai.com')) {+ || urlHost(baseUrl) === 'api.openai.com') {return VISION_PROFILES.openai;}if (LOCAL_PROVIDER_NAMES.has(name) || category === 'local') return VISION_PROFILES.local;diff --git a/src/chrome/src/agent/vision-diff.js b/src/chrome/src/agent/vision-diff.jsindex 9b90fc4a3..30d880fb8 100644--- a/src/chrome/src/agent/vision-diff.js+++ b/src/chrome/src/agent/vision-diff.js@@ -42,6 +42,11 @@ export const VISION_UPDATE_DEFAULTS = Object.freeze({// Smallest patch side (image px). Tiny slivers are hard to read, so they// are grown to at least this size around their centre.minPatchSide: 48,+ // The provider's image grid (vision-budget.js profile.align). Patch sides+ // are grown to multiples of it, like frames and zooms, so a server that+ // resizes images to its grid (Qwen2/2.5-VL, UI-TARS: 28 px) receives the+ // patch unchanged and its pixels map 1:1 to the frame.+ align: 1,});function clamp(value, lo, hi) {@@ -232,6 +237,37 @@ export function ensureMinimumSize(region, minSide, width, height) {return { ...region, x, y, width: rw, height: rh };}+/**+ * Grow a region so both sides are multiples of `align` (at least two grid+ * cells), centred on the original and kept inside the frame. A frame planned+ * for the profile is itself a multiple of `align`, so the result covers the+ * original region whenever that fits.+ */+export function alignRegion(region, align, width, height) {+ const step = Math.max(1, Math.floor(Number(align) || 1));+ if (step <= 1) return region;+ const fit = (size, bound) => {+ const aligned = Math.max(step * 2, Math.ceil(size / step) * step);+ return aligned <= bound ? aligned : (Math.floor(bound / step) * step || bound);+ };+ const w = fit(region.width, width);+ const h = fit(region.height, height);+ const x = clamp(Math.round(region.x + region.width / 2 - w / 2), 0, Math.max(0, width - w));+ const y = clamp(Math.round(region.y + region.height / 2 - h / 2), 0, Math.max(0, height - h));+ return { ...region, x, y, width: w, height: h };+}++// Aligned regions can touch or overlap again: merge, re-align, until stable.+function alignRegions(regions, align, width, height) {+ let list = regions.map(region => alignRegion(region, align, width, height));+ for (let pass = 0; pass < 8; pass++) {+ const merged = mergeOverlappingRegions(list);+ if (merged.length === list.length) return list;+ list = merged.map(region => alignRegion(region, align, width, height));+ }+ return list;+}+/*** Decide how to show the model the screen after an action.*@@ -255,9 +291,10 @@ export function planFrameUpdate(diff, {if (!diff.regions.length) return { kind: 'none', reason: 'no_visible_change' };if (patchChainLength >= opts.maxPatchChain) return { kind: 'keyframe', reason: 'patch_chain_limit' };if (diff.regionAreaRatio > opts.maxPatchAreaRatio) return { kind: 'keyframe', reason: 'large_change' };- const regions = mergeOverlappingRegions(+ let regions = mergeOverlappingRegions(diff.regions.map(region => ensureMinimumSize(region, opts.minPatchSide, width, height)),);+ if (opts.align > 1) regions = alignRegions(regions, opts.align, width, height);const patchTokens = regions.reduce((sum, region) => sum + estimateTokens(region.width, region.height), 0);const frameTokens = estimateTokens(width, height);if (frameTokens > 0 && patchTokens >= frameTokens * opts.maxPatchTokenRatio) {diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex aa1721798..ac93a391e 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -528,6 +528,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {height: frame.size.height,patchChainLength: session.patchChainLength,estimateTokens: (w, h) => estimateVisionTokens(w, h, frame.profile),+ align: frame.profile.align,});if (plan.kind === 'keyframe') {await this._visionPushKeyframe(tabId, messages, onUpdate, { state, frame, reason: plan.reason === 'large_change' ? 'large change' : plan.reason.replace(/_/g, ' ') });diff --git a/src/firefox/src/agent/vision-budget.js b/src/firefox/src/agent/vision-budget.jsindex 277a9ac29..76aa741e9 100644--- a/src/firefox/src/agent/vision-budget.js+++ b/src/firefox/src/agent/vision-budget.js@@ -79,6 +79,18 @@ function lower(value) {return String(value || '').trim().toLowerCase();}+// The host of a configured base URL, compared exactly: a host name that only+// appears somewhere in the URL (a path, a longer domain) is not that host.+function urlHost(value) {+ const text = String(value || '').trim();+ if (!text) return '';+ try {+ return new URL(/^[a-z][a-z0-9+.-]*:\/\//i.test(text) ? text : `https://${text}`).hostname.toLowerCase();+ } catch {+ return '';+ }+}+/*** Pick the vision profile for a provider object (or its config). Model ids* decide first because routers (OpenRouter, Bedrock, Vertex) serve several@@ -99,7 +111,7 @@ export function detectVisionProfile(provider = {}) {}if (/qwen|ui-tars|uitars|holo1|opencua/.test(model)) return VISION_PROFILES.qwen;if (/(^|\/)(gpt-|o\d|chatgpt|computer-use)/.test(model) || name === 'openai' || name === 'azure-openai'- || baseUrl.includes('api.openai.com')) {+ || urlHost(baseUrl) === 'api.openai.com') {return VISION_PROFILES.openai;}if (LOCAL_PROVIDER_NAMES.has(name) || category === 'local') return VISION_PROFILES.local;diff --git a/src/firefox/src/agent/vision-diff.js b/src/firefox/src/agent/vision-diff.jsindex 9b90fc4a3..30d880fb8 100644--- a/src/firefox/src/agent/vision-diff.js+++ b/src/firefox/src/agent/vision-diff.js@@ -42,6 +42,11 @@ export const VISION_UPDATE_DEFAULTS = Object.freeze({// Smallest patch side (image px). Tiny slivers are hard to read, so they// are grown to at least this size around their centre.minPatchSide: 48,+ // The provider's image grid (vision-budget.js profile.align). Patch sides+ // are grown to multiples of it, like frames and zooms, so a server that+ // resizes images to its grid (Qwen2/2.5-VL, UI-TARS: 28 px) receives the+ // patch unchanged and its pixels map 1:1 to the frame.+ align: 1,});function clamp(value, lo, hi) {@@ -232,6 +237,37 @@ export function ensureMinimumSize(region, minSide, width, height) {return { ...region, x, y, width: rw, height: rh };}+/**+ * Grow a region so both sides are multiples of `align` (at least two grid+ * cells), centred on the original and kept inside the frame. A frame planned+ * for the profile is itself a multiple of `align`, so the result covers the+ * original region whenever that fits.+ */+export function alignRegion(region, align, width, height) {+ const step = Math.max(1, Math.floor(Number(align) || 1));+ if (step <= 1) return region;+ const fit = (size, bound) => {+ const aligned = Math.max(step * 2, Math.ceil(size / step) * step);+ return aligned <= bound ? aligned : (Math.floor(bound / step) * step || bound);+ };+ const w = fit(region.width, width);+ const h = fit(region.height, height);+ const x = clamp(Math.round(region.x + region.width / 2 - w / 2), 0, Math.max(0, width - w));+ const y = clamp(Math.round(region.y + region.height / 2 - h / 2), 0, Math.max(0, height - h));+ return { ...region, x, y, width: w, height: h };+}++// Aligned regions can touch or overlap again: merge, re-align, until stable.+function alignRegions(regions, align, width, height) {+ let list = regions.map(region => alignRegion(region, align, width, height));+ for (let pass = 0; pass < 8; pass++) {+ const merged = mergeOverlappingRegions(list);+ if (merged.length === list.length) return list;+ list = merged.map(region => alignRegion(region, align, width, height));+ }+ return list;+}+/*** Decide how to show the model the screen after an action.*@@ -255,9 +291,10 @@ export function planFrameUpdate(diff, {if (!diff.regions.length) return { kind: 'none', reason: 'no_visible_change' };if (patchChainLength >= opts.maxPatchChain) return { kind: 'keyframe', reason: 'patch_chain_limit' };if (diff.regionAreaRatio > opts.maxPatchAreaRatio) return { kind: 'keyframe', reason: 'large_change' };- const regions = mergeOverlappingRegions(+ let regions = mergeOverlappingRegions(diff.regions.map(region => ensureMinimumSize(region, opts.minPatchSide, width, height)),);+ if (opts.align > 1) regions = alignRegions(regions, opts.align, width, height);const patchTokens = regions.reduce((sum, region) => sum + estimateTokens(region.width, region.height), 0);const frameTokens = estimateTokens(width, height);if (frameTokens > 0 && patchTokens >= frameTokens * opts.maxPatchTokenRatio) {diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex aa1721798..ac93a391e 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -528,6 +528,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {height: frame.size.height,patchChainLength: session.patchChainLength,estimateTokens: (w, h) => estimateVisionTokens(w, h, frame.profile),+ align: frame.profile.align,});if (plan.kind === 'keyframe') {await this._visionPushKeyframe(tabId, messages, onUpdate, { state, frame, reason: plan.reason === 'large_change' ? 'large change' : plan.reason.replace(/_/g, ' ') });diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjsindex efd5b3227..269fac880 100644--- a/test/vision-mode-agent.mjs+++ b/test/vision-mode-agent.mjs@@ -271,6 +271,37 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))assert.match(textOf(zoomMessage), /ZOOM z\d+ of f1 area x 100β300, y 100β200/);});+ test(`${label}: a Qwen2.5-VL run gets patches on its 28 px grid, and points in them map 1:1`, async () => {+ const QWEN = { name: 'openrouter', model: 'qwen/qwen2.5-vl-72b-instruct', config: { category: 'cloud', type: 'openrouter', model: 'qwen/qwen2.5-vl-72b-instruct' } };+ let patch = null;+ const { agent, requests, dispatched } = runAgent((step, messages) => {+ if (step === 1) return { toolCalls: [call('click_at', { x: 300, y: 200 })] };+ if (step === 2) {+ const note = messages.map(textOf).findLast(text => /UNTRUSTED SCREEN UPDATE/.test(text)) || '';+ const match = /(p\d+) = area x (\d+)β(\d+), y (\d+)β(\d+) of f\d+ \((\d+)x(\d+) px/.exec(note);+ assert.ok(match, note);+ patch = { id: match[1], x: Number(match[2]), y: Number(match[4]), width: Number(match[6]), height: Number(match[7]) };+ return { toolCalls: [call('click_at', { x: Math.round(patch.width / 2), y: Math.round(patch.height / 2), image: patch.id })] };+ }+ if (step === 3) return { toolCalls: [call('done', { summary: 'Clicked twice.', outcome: 'success' })] };+ return { content: 'done' };+ }, { provider: QWEN, onDispatch: (name, _args, scene) => {+ if (name === 'click' && scene.rects.length === 1) scene.rects.push({ x: 600, y: 420, w: 50, h: 30, color: [200, 40, 40] });+ } });+ const tab = nextTab++;+ await runLoop(agent, tab, streaming);+ // 1440x900 CSS -> a 1260x784 frame (multiples of 28) for this profile.+ assert.match(textOf(visionMessages(requests[0].messages)[0]), /1260x784 image/);+ assert.ok(patch, 'a patch was sent');+ assert.equal(patch.width % 28, 0, JSON.stringify(patch));+ assert.equal(patch.height % 28, 0, JSON.stringify(patch));+ const clicks = dispatched.filter(entry => entry.name === 'click');+ assert.equal(clicks.length, 2, JSON.stringify(dispatched.map(entry => entry.name)));+ const expected = { x: (patch.x + Math.round(patch.width / 2)) * 1440 / 1260, y: (patch.y + Math.round(patch.height / 2)) * 900 / 784 };+ assert.ok(Math.abs(clicks[1].args.x - expected.x) <= 1 && Math.abs(clicks[1].args.y - expected.y) <= 1,+ `patch pixels map 1:1 to the frame: ${JSON.stringify({ got: clicks[1].args, expected })}`);+ });+test(`${label}: vision mode off, or a model without images, keeps the accessibility-tree path`, async () => {for (const options of [{ visionMode: 'off' }, { provider: { ...VISION, supportsVision: false } }]) {const { agent, requests } = runAgent(step => (step === 1diff --git a/test/vision-mode.mjs b/test/vision-mode.mjsindex 43f96a017..4aebb732d 100644--- a/test/vision-mode.mjs+++ b/test/vision-mode.mjs@@ -49,6 +49,15 @@ test('profiles: model ids pick the provider family before router names', () => {assert.equal(pick('llamacpp', 'gemma-3-12b-it'), 'local');assert.equal(pick('ollama', 'llava:13b'), 'local');assert.equal(pick('custom', 'pixtral-large', { category: 'cloud' }), 'default');+ // An OpenAI base URL is recognized by its host, never by a substring of the URL.+ assert.equal(pick('custom', 'my-model', { baseUrl: 'https://api.openai.com/v1' }), 'openai');+ assert.equal(pick('custom', 'my-model', { baseUrl: 'api.openai.com/v1' }), 'openai');+ for (const baseUrl of ['https://proxy.example.test/api.openai.com/v1', 'https://api.openai.com.example.test/v1', 'https://example.test/?u=api.openai.com']) {+ assert.equal(pick('custom', 'my-model', { baseUrl, category: 'cloud' }), 'default', baseUrl);+ }+ for (const build of ['chrome', 'firefox']) {+ assert.doesNotMatch(read(`src/${build}/src/agent/vision-budget.js`), /\.includes\(['"]api\.openai\.com/);+ }assert.equal(budget.VISION_PROFILES.gemini.coordinates, 'normalized');assert.equal(budget.VISION_PROFILES.qwen3.coordinates, 'normalized');assert.equal(budget.VISION_PROFILES.anthropic.coordinates, 'pixel');@@ -188,6 +197,33 @@ test('frame update planning: none, patch, keyframe', () => {assert.ok(sliver.regions[0].width >= 48 && sliver.regions[0].height >= 48, JSON.stringify(sliver.regions));});+test('frame update planning: patches follow the provider\'s image grid', () => {+ // A Qwen2.5-VL frame for a 1440x900 viewport is 1260x784 (multiples of 28).+ const width = 1260, height = 784;+ const opts = { width, height, align: 28, estimateTokens: (w, h) => Math.ceil(w / 28) * Math.ceil(h / 28) + 2 };+ const changes = [{ x: 517, y: 358, width: 64, height: 48 }, { x: 1200, y: 740, width: 60, height: 44 }, { x: 0, y: 0, width: 50, height: 30 }];+ for (const change of changes) {+ const plan = diff.planFrameUpdate({ comparable: true, regions: [change], regionAreaRatio: 0.004 }, opts);+ assert.equal(plan.kind, 'patch');+ for (const region of plan.regions) {+ assert.equal(region.width % 28, 0, JSON.stringify(region));+ assert.equal(region.height % 28, 0, JSON.stringify(region));+ assert.ok(region.width >= 56 && region.height >= 56, JSON.stringify(region));+ assert.ok(region.x >= 0 && region.y >= 0 && region.x + region.width <= width && region.y + region.height <= height, JSON.stringify(region));+ assert.ok(region.x <= change.x && region.y <= change.y && region.x + region.width >= change.x + change.width+ && region.y + region.height >= change.y + change.height, `the patch covers the change: ${JSON.stringify({ region, change })}`);+ }+ }+ // Two changes whose aligned patches would overlap become one aligned patch.+ const close = diff.planFrameUpdate({ comparable: true, regions: [{ x: 100, y: 100, width: 50, height: 40 }, { x: 160, y: 110, width: 40, height: 40 }], regionAreaRatio: 0.004 }, opts);+ assert.equal(close.regions.length, 1, JSON.stringify(close.regions));+ assert.equal(close.regions[0].width % 28, 0);+ assert.equal(close.regions[0].height % 28, 0);+ // Profiles without a grid keep the exact padded region.+ const plain = diff.planFrameUpdate({ comparable: true, regions: [{ x: 517, y: 358, width: 64, height: 48 }], regionAreaRatio: 0.004 }, { ...opts, align: 1 });+ assert.deepEqual(plain.regions.map(({ x, y, width: w, height: h }) => ({ x, y, w, h })), [{ x: 517, y: 358, w: 64, h: 48 }]);+});+test('copyRegions updates only the patched area of the seen buffer', () => {const seen = frame(64, 64, [0, 0, 0]);const next = frame(64, 64, [255, 255, 255]);