Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Vision mode: grid-aligned patches, OpenAI base URL matched by host

Emre Sokullub92ab67f30f1
diff --git a/docs/vision-mode.md b/docs/vision-mode.md
index 9ad001eec..98da32532 100644
--- a/docs/vision-mode.md
+++ b/docs/vision-mode.md
@@ -84,7 +84,7 @@ Screenshot redaction maps element boxes into the captured visual-viewport box (p
After a batch WebBrain waits until two consecutive captures match (bounded, ~1 s; a micro change between them, a few 16 px cells such as a blinking caret, a clock or a spinner, counts as a match, `isMicroScreenChange`, since it never stops), then compares the new frame with what the model has seen (`vision-diff.js`): 16 px cells, a per-pixel threshold that ignores encoder noise and caret blinks, nearby changes merged, at most 4 padded regions. `planFrameUpdate` sends:
- **nothing but a one-line note** when nothing changed: "no visible change" is also the clearest signal that a click missed;
-- **patches** for local changes (typing, a menu, a toast);
+- **patches** for local changes (typing, a menu, a toast). For a provider with an image grid (28 px for Qwen2/2.5-VL, UI-TARS, GLM-4.xV, InternVL and local servers; 32 px for Qwen3-VL) each patch is grown to multiples of the grid, like frames and zooms, so the server never resizes it and its pixels map 1:1 to the frame;
- **a new frame** when more than 45% changed, the patch chain is 6 long, patches would cost β‰₯ 70% of a frame, or the URL, viewport size or DPR changed.
The "seen" buffer copies only patched regions, so changes below the threshold accumulate until they are sent instead of drifting forever.
diff --git a/src/chrome/src/agent/vision-budget.js b/src/chrome/src/agent/vision-budget.js
index 277a9ac29..76aa741e9 100644
--- a/src/chrome/src/agent/vision-budget.js
+++ b/src/chrome/src/agent/vision-budget.js
@@ -79,6 +79,18 @@ function lower(value) {
return String(value || '').trim().toLowerCase();
}
+// The host of a configured base URL, compared exactly: a host name that only
+// appears somewhere in the URL (a path, a longer domain) is not that host.
+function urlHost(value) {
+ const text = String(value || '').trim();
+ if (!text) return '';
+ try {
+ return new URL(/^[a-z][a-z0-9+.-]*:\/\//i.test(text) ? text : `https://${text}`).hostname.toLowerCase();
+ } catch {
+ return '';
+ }
+}
+
/**
* Pick the vision profile for a provider object (or its config). Model ids
* decide first because routers (OpenRouter, Bedrock, Vertex) serve several
@@ -99,7 +111,7 @@ export function detectVisionProfile(provider = {}) {
}
if (/qwen|ui-tars|uitars|holo1|opencua/.test(model)) return VISION_PROFILES.qwen;
if (/(^|\/)(gpt-|o\d|chatgpt|computer-use)/.test(model) || name === 'openai' || name === 'azure-openai'
- || baseUrl.includes('api.openai.com')) {
+ || urlHost(baseUrl) === 'api.openai.com') {
return VISION_PROFILES.openai;
}
if (LOCAL_PROVIDER_NAMES.has(name) || category === 'local') return VISION_PROFILES.local;
diff --git a/src/chrome/src/agent/vision-diff.js b/src/chrome/src/agent/vision-diff.js
index 9b90fc4a3..30d880fb8 100644
--- a/src/chrome/src/agent/vision-diff.js
+++ b/src/chrome/src/agent/vision-diff.js
@@ -42,6 +42,11 @@ export const VISION_UPDATE_DEFAULTS = Object.freeze({
// Smallest patch side (image px). Tiny slivers are hard to read, so they
// are grown to at least this size around their centre.
minPatchSide: 48,
+ // The provider's image grid (vision-budget.js profile.align). Patch sides
+ // are grown to multiples of it, like frames and zooms, so a server that
+ // resizes images to its grid (Qwen2/2.5-VL, UI-TARS: 28 px) receives the
+ // patch unchanged and its pixels map 1:1 to the frame.
+ align: 1,
});
function clamp(value, lo, hi) {
@@ -232,6 +237,37 @@ export function ensureMinimumSize(region, minSide, width, height) {
return { ...region, x, y, width: rw, height: rh };
}
+/**
+ * Grow a region so both sides are multiples of `align` (at least two grid
+ * cells), centred on the original and kept inside the frame. A frame planned
+ * for the profile is itself a multiple of `align`, so the result covers the
+ * original region whenever that fits.
+ */
+export function alignRegion(region, align, width, height) {
+ const step = Math.max(1, Math.floor(Number(align) || 1));
+ if (step <= 1) return region;
+ const fit = (size, bound) => {
+ const aligned = Math.max(step * 2, Math.ceil(size / step) * step);
+ return aligned <= bound ? aligned : (Math.floor(bound / step) * step || bound);
+ };
+ const w = fit(region.width, width);
+ const h = fit(region.height, height);
+ const x = clamp(Math.round(region.x + region.width / 2 - w / 2), 0, Math.max(0, width - w));
+ const y = clamp(Math.round(region.y + region.height / 2 - h / 2), 0, Math.max(0, height - h));
+ return { ...region, x, y, width: w, height: h };
+}
+
+// Aligned regions can touch or overlap again: merge, re-align, until stable.
+function alignRegions(regions, align, width, height) {
+ let list = regions.map(region => alignRegion(region, align, width, height));
+ for (let pass = 0; pass < 8; pass++) {
+ const merged = mergeOverlappingRegions(list);
+ if (merged.length === list.length) return list;
+ list = merged.map(region => alignRegion(region, align, width, height));
+ }
+ return list;
+}
+
/**
* Decide how to show the model the screen after an action.
*
@@ -255,9 +291,10 @@ export function planFrameUpdate(diff, {
if (!diff.regions.length) return { kind: 'none', reason: 'no_visible_change' };
if (patchChainLength >= opts.maxPatchChain) return { kind: 'keyframe', reason: 'patch_chain_limit' };
if (diff.regionAreaRatio > opts.maxPatchAreaRatio) return { kind: 'keyframe', reason: 'large_change' };
- const regions = mergeOverlappingRegions(
+ let regions = mergeOverlappingRegions(
diff.regions.map(region => ensureMinimumSize(region, opts.minPatchSide, width, height)),
);
+ if (opts.align > 1) regions = alignRegions(regions, opts.align, width, height);
const patchTokens = regions.reduce((sum, region) => sum + estimateTokens(region.width, region.height), 0);
const frameTokens = estimateTokens(width, height);
if (frameTokens > 0 && patchTokens >= frameTokens * opts.maxPatchTokenRatio) {
diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.js
index aa1721798..ac93a391e 100644
--- a/src/chrome/src/agent/vision-runtime.js
+++ b/src/chrome/src/agent/vision-runtime.js
@@ -528,6 +528,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
height: frame.size.height,
patchChainLength: session.patchChainLength,
estimateTokens: (w, h) => estimateVisionTokens(w, h, frame.profile),
+ align: frame.profile.align,
});
if (plan.kind === 'keyframe') {
await this._visionPushKeyframe(tabId, messages, onUpdate, { state, frame, reason: plan.reason === 'large_change' ? 'large change' : plan.reason.replace(/_/g, ' ') });
diff --git a/src/firefox/src/agent/vision-budget.js b/src/firefox/src/agent/vision-budget.js
index 277a9ac29..76aa741e9 100644
--- a/src/firefox/src/agent/vision-budget.js
+++ b/src/firefox/src/agent/vision-budget.js
@@ -79,6 +79,18 @@ function lower(value) {
return String(value || '').trim().toLowerCase();
}
+// The host of a configured base URL, compared exactly: a host name that only
+// appears somewhere in the URL (a path, a longer domain) is not that host.
+function urlHost(value) {
+ const text = String(value || '').trim();
+ if (!text) return '';
+ try {
+ return new URL(/^[a-z][a-z0-9+.-]*:\/\//i.test(text) ? text : `https://${text}`).hostname.toLowerCase();
+ } catch {
+ return '';
+ }
+}
+
/**
* Pick the vision profile for a provider object (or its config). Model ids
* decide first because routers (OpenRouter, Bedrock, Vertex) serve several
@@ -99,7 +111,7 @@ export function detectVisionProfile(provider = {}) {
}
if (/qwen|ui-tars|uitars|holo1|opencua/.test(model)) return VISION_PROFILES.qwen;
if (/(^|\/)(gpt-|o\d|chatgpt|computer-use)/.test(model) || name === 'openai' || name === 'azure-openai'
- || baseUrl.includes('api.openai.com')) {
+ || urlHost(baseUrl) === 'api.openai.com') {
return VISION_PROFILES.openai;
}
if (LOCAL_PROVIDER_NAMES.has(name) || category === 'local') return VISION_PROFILES.local;
diff --git a/src/firefox/src/agent/vision-diff.js b/src/firefox/src/agent/vision-diff.js
index 9b90fc4a3..30d880fb8 100644
--- a/src/firefox/src/agent/vision-diff.js
+++ b/src/firefox/src/agent/vision-diff.js
@@ -42,6 +42,11 @@ export const VISION_UPDATE_DEFAULTS = Object.freeze({
// Smallest patch side (image px). Tiny slivers are hard to read, so they
// are grown to at least this size around their centre.
minPatchSide: 48,
+ // The provider's image grid (vision-budget.js profile.align). Patch sides
+ // are grown to multiples of it, like frames and zooms, so a server that
+ // resizes images to its grid (Qwen2/2.5-VL, UI-TARS: 28 px) receives the
+ // patch unchanged and its pixels map 1:1 to the frame.
+ align: 1,
});
function clamp(value, lo, hi) {
@@ -232,6 +237,37 @@ export function ensureMinimumSize(region, minSide, width, height) {
return { ...region, x, y, width: rw, height: rh };
}
+/**
+ * Grow a region so both sides are multiples of `align` (at least two grid
+ * cells), centred on the original and kept inside the frame. A frame planned
+ * for the profile is itself a multiple of `align`, so the result covers the
+ * original region whenever that fits.
+ */
+export function alignRegion(region, align, width, height) {
+ const step = Math.max(1, Math.floor(Number(align) || 1));
+ if (step <= 1) return region;
+ const fit = (size, bound) => {
+ const aligned = Math.max(step * 2, Math.ceil(size / step) * step);
+ return aligned <= bound ? aligned : (Math.floor(bound / step) * step || bound);
+ };
+ const w = fit(region.width, width);
+ const h = fit(region.height, height);
+ const x = clamp(Math.round(region.x + region.width / 2 - w / 2), 0, Math.max(0, width - w));
+ const y = clamp(Math.round(region.y + region.height / 2 - h / 2), 0, Math.max(0, height - h));
+ return { ...region, x, y, width: w, height: h };
+}
+
+// Aligned regions can touch or overlap again: merge, re-align, until stable.
+function alignRegions(regions, align, width, height) {
+ let list = regions.map(region => alignRegion(region, align, width, height));
+ for (let pass = 0; pass < 8; pass++) {
+ const merged = mergeOverlappingRegions(list);
+ if (merged.length === list.length) return list;
+ list = merged.map(region => alignRegion(region, align, width, height));
+ }
+ return list;
+}
+
/**
* Decide how to show the model the screen after an action.
*
@@ -255,9 +291,10 @@ export function planFrameUpdate(diff, {
if (!diff.regions.length) return { kind: 'none', reason: 'no_visible_change' };
if (patchChainLength >= opts.maxPatchChain) return { kind: 'keyframe', reason: 'patch_chain_limit' };
if (diff.regionAreaRatio > opts.maxPatchAreaRatio) return { kind: 'keyframe', reason: 'large_change' };
- const regions = mergeOverlappingRegions(
+ let regions = mergeOverlappingRegions(
diff.regions.map(region => ensureMinimumSize(region, opts.minPatchSide, width, height)),
);
+ if (opts.align > 1) regions = alignRegions(regions, opts.align, width, height);
const patchTokens = regions.reduce((sum, region) => sum + estimateTokens(region.width, region.height), 0);
const frameTokens = estimateTokens(width, height);
if (frameTokens > 0 && patchTokens >= frameTokens * opts.maxPatchTokenRatio) {
diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.js
index aa1721798..ac93a391e 100644
--- a/src/firefox/src/agent/vision-runtime.js
+++ b/src/firefox/src/agent/vision-runtime.js
@@ -528,6 +528,7 @@ export function installVisionRuntime(AgentClass, { trace = null } = {}) {
height: frame.size.height,
patchChainLength: session.patchChainLength,
estimateTokens: (w, h) => estimateVisionTokens(w, h, frame.profile),
+ align: frame.profile.align,
});
if (plan.kind === 'keyframe') {
await this._visionPushKeyframe(tabId, messages, onUpdate, { state, frame, reason: plan.reason === 'large_change' ? 'large change' : plan.reason.replace(/_/g, ' ') });
diff --git a/test/vision-mode-agent.mjs b/test/vision-mode-agent.mjs
index efd5b3227..269fac880 100644
--- a/test/vision-mode-agent.mjs
+++ b/test/vision-mode-agent.mjs
@@ -271,6 +271,37 @@ for (const build of (process.env.VISION_BUILDS || 'chrome,firefox').split(','))
assert.match(textOf(zoomMessage), /ZOOM z\d+ of f1 area x 100–300, y 100–200/);
});
+ test(`${label}: a Qwen2.5-VL run gets patches on its 28 px grid, and points in them map 1:1`, async () => {
+ const QWEN = { name: 'openrouter', model: 'qwen/qwen2.5-vl-72b-instruct', config: { category: 'cloud', type: 'openrouter', model: 'qwen/qwen2.5-vl-72b-instruct' } };
+ let patch = null;
+ const { agent, requests, dispatched } = runAgent((step, messages) => {
+ if (step === 1) return { toolCalls: [call('click_at', { x: 300, y: 200 })] };
+ if (step === 2) {
+ const note = messages.map(textOf).findLast(text => /UNTRUSTED SCREEN UPDATE/.test(text)) || '';
+ const match = /(p\d+) = area x (\d+)–(\d+), y (\d+)–(\d+) of f\d+ \((\d+)x(\d+) px/.exec(note);
+ assert.ok(match, note);
+ patch = { id: match[1], x: Number(match[2]), y: Number(match[4]), width: Number(match[6]), height: Number(match[7]) };
+ return { toolCalls: [call('click_at', { x: Math.round(patch.width / 2), y: Math.round(patch.height / 2), image: patch.id })] };
+ }
+ if (step === 3) return { toolCalls: [call('done', { summary: 'Clicked twice.', outcome: 'success' })] };
+ return { content: 'done' };
+ }, { provider: QWEN, onDispatch: (name, _args, scene) => {
+ if (name === 'click' && scene.rects.length === 1) scene.rects.push({ x: 600, y: 420, w: 50, h: 30, color: [200, 40, 40] });
+ } });
+ const tab = nextTab++;
+ await runLoop(agent, tab, streaming);
+ // 1440x900 CSS -> a 1260x784 frame (multiples of 28) for this profile.
+ assert.match(textOf(visionMessages(requests[0].messages)[0]), /1260x784 image/);
+ assert.ok(patch, 'a patch was sent');
+ assert.equal(patch.width % 28, 0, JSON.stringify(patch));
+ assert.equal(patch.height % 28, 0, JSON.stringify(patch));
+ const clicks = dispatched.filter(entry => entry.name === 'click');
+ assert.equal(clicks.length, 2, JSON.stringify(dispatched.map(entry => entry.name)));
+ const expected = { x: (patch.x + Math.round(patch.width / 2)) * 1440 / 1260, y: (patch.y + Math.round(patch.height / 2)) * 900 / 784 };
+ assert.ok(Math.abs(clicks[1].args.x - expected.x) <= 1 && Math.abs(clicks[1].args.y - expected.y) <= 1,
+ `patch pixels map 1:1 to the frame: ${JSON.stringify({ got: clicks[1].args, expected })}`);
+ });
+
test(`${label}: vision mode off, or a model without images, keeps the accessibility-tree path`, async () => {
for (const options of [{ visionMode: 'off' }, { provider: { ...VISION, supportsVision: false } }]) {
const { agent, requests } = runAgent(step => (step === 1
diff --git a/test/vision-mode.mjs b/test/vision-mode.mjs
index 43f96a017..4aebb732d 100644
--- a/test/vision-mode.mjs
+++ b/test/vision-mode.mjs
@@ -49,6 +49,15 @@ test('profiles: model ids pick the provider family before router names', () => {
assert.equal(pick('llamacpp', 'gemma-3-12b-it'), 'local');
assert.equal(pick('ollama', 'llava:13b'), 'local');
assert.equal(pick('custom', 'pixtral-large', { category: 'cloud' }), 'default');
+ // An OpenAI base URL is recognized by its host, never by a substring of the URL.
+ assert.equal(pick('custom', 'my-model', { baseUrl: 'https://api.openai.com/v1' }), 'openai');
+ assert.equal(pick('custom', 'my-model', { baseUrl: 'api.openai.com/v1' }), 'openai');
+ for (const baseUrl of ['https://proxy.example.test/api.openai.com/v1', 'https://api.openai.com.example.test/v1', 'https://example.test/?u=api.openai.com']) {
+ assert.equal(pick('custom', 'my-model', { baseUrl, category: 'cloud' }), 'default', baseUrl);
+ }
+ for (const build of ['chrome', 'firefox']) {
+ assert.doesNotMatch(read(`src/${build}/src/agent/vision-budget.js`), /\.includes\(['"]api\.openai\.com/);
+ }
assert.equal(budget.VISION_PROFILES.gemini.coordinates, 'normalized');
assert.equal(budget.VISION_PROFILES.qwen3.coordinates, 'normalized');
assert.equal(budget.VISION_PROFILES.anthropic.coordinates, 'pixel');
@@ -188,6 +197,33 @@ test('frame update planning: none, patch, keyframe', () => {
assert.ok(sliver.regions[0].width >= 48 && sliver.regions[0].height >= 48, JSON.stringify(sliver.regions));
});
+test('frame update planning: patches follow the provider\'s image grid', () => {
+ // A Qwen2.5-VL frame for a 1440x900 viewport is 1260x784 (multiples of 28).
+ const width = 1260, height = 784;
+ const opts = { width, height, align: 28, estimateTokens: (w, h) => Math.ceil(w / 28) * Math.ceil(h / 28) + 2 };
+ const changes = [{ x: 517, y: 358, width: 64, height: 48 }, { x: 1200, y: 740, width: 60, height: 44 }, { x: 0, y: 0, width: 50, height: 30 }];
+ for (const change of changes) {
+ const plan = diff.planFrameUpdate({ comparable: true, regions: [change], regionAreaRatio: 0.004 }, opts);
+ assert.equal(plan.kind, 'patch');
+ for (const region of plan.regions) {
+ assert.equal(region.width % 28, 0, JSON.stringify(region));
+ assert.equal(region.height % 28, 0, JSON.stringify(region));
+ assert.ok(region.width >= 56 && region.height >= 56, JSON.stringify(region));
+ assert.ok(region.x >= 0 && region.y >= 0 && region.x + region.width <= width && region.y + region.height <= height, JSON.stringify(region));
+ assert.ok(region.x <= change.x && region.y <= change.y && region.x + region.width >= change.x + change.width
+ && region.y + region.height >= change.y + change.height, `the patch covers the change: ${JSON.stringify({ region, change })}`);
+ }
+ }
+ // Two changes whose aligned patches would overlap become one aligned patch.
+ const close = diff.planFrameUpdate({ comparable: true, regions: [{ x: 100, y: 100, width: 50, height: 40 }, { x: 160, y: 110, width: 40, height: 40 }], regionAreaRatio: 0.004 }, opts);
+ assert.equal(close.regions.length, 1, JSON.stringify(close.regions));
+ assert.equal(close.regions[0].width % 28, 0);
+ assert.equal(close.regions[0].height % 28, 0);
+ // Profiles without a grid keep the exact padded region.
+ const plain = diff.planFrameUpdate({ comparable: true, regions: [{ x: 517, y: 358, width: 64, height: 48 }], regionAreaRatio: 0.004 }, { ...opts, align: 1 });
+ assert.deepEqual(plain.regions.map(({ x, y, width: w, height: h }) => ({ x, y, w, h })), [{ x: 517, y: 358, w: 64, h: 48 }]);
+});
+
test('copyRegions updates only the patched area of the seen buffer', () => {
const seen = frame(64, 64, [0, 0, 0]);
const next = frame(64, 64, [255, 255, 255]);