Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Vision mode: keep at most four zooms of the current frame
Emre Sokullu1068929fcac1
diff --git a/docs/vision-mode.md b/docs/vision-mode.mdindex 4244e7068..e484688f6 100644--- a/docs/vision-mode.md+++ b/docs/vision-mode.md@@ -100,7 +100,7 @@ After a batch WebBrain waits until two consecutive captures match (bounded, ~1 sThe "seen" buffer copies only patched regions, so changes below the threshold accumulate until they are sent instead of drifting forever.-Only the current frame, its patches and zooms stay in the model request (`pruneVisionImages`); older images become a short placeholder. Screen updates are app-owned user turns (`webbrainAppOwned`), never mistaken for a new request.+Only the current frame, its patches and its newest zooms stay in the model request (`pruneVisionImages`); older images become a short placeholder. At most four zooms of the current frame stay, and stay usable (`VISION_MAX_LIVE_ZOOMS`; a point on an older one is refused as `stale_image` with a note to zoom again), and while the kept images exceed about 8 MB of data URL the oldest zooms leave first. Patches are never dropped, since they are the current screen; a long patch chain becomes a new frame instead. Screen updates are app-owned user turns (`webbrainAppOwned`), never mistaken for a new request.A capture, encoding or input failure never ends the run: a screen update that cannot be prepared becomes a one-line note and the next update is a full frame; a vision tool that fails returns a tool error (an unknown outcome once input may have gone out).diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.jsindex c50f91a7f..bfba455c5 100644--- a/src/chrome/src/agent/vision-runtime.js+++ b/src/chrome/src/agent/vision-runtime.js@@ -15,7 +15,7 @@// older images become short placeholders (_visionPruneImages).import { resolveVisionMode, normalizeVisionModeSetting } from './vision-mode.js';-import { VisionSession } from './vision-session.js';+import { VisionSession, VISION_MAX_LIVE_ZOOMS } from './vision-session.js';import { diffFrames, planFrameUpdate, pointTouchesRegions } from './vision-diff.js';import {VISION_PROFILES, planVisionImageSize, planZoomImageSize, estimateVisionTokens, visionEncodingFor,@@ -52,6 +52,11 @@ const STABILITY_MARGIN_PX = 12;const SETTLE_INITIAL_DELAY_MS = 120;const SETTLE_POLL_MS = 220;const SETTLE_MAX_EXTRA_CAPTURES = 3;+// Images kept after the current frame (its patches and zooms), as data-URL+// characters: past this, the oldest zooms leave the request first. Patches+// are never dropped (they are the current screen); a long patch chain turns+// into a new frame instead (vision-diff.js planFrameUpdate).+const RETAINED_IMAGE_CHARS_BUDGET = 8_000_000;// An upload handoff (_visionNoteUploadHandoff) lasts this many model// decisions without a successful upload_file, and a run hands off at most// this many times: a page whose upload never succeeds goes back to vision@@ -230,7 +235,46 @@ export function pruneVisionImages(messages) {});return changed ? { ...message, content } : message;});- return { messages: out, latestEpoch };+ return { messages: dropOldZooms(out, boundary), latestEpoch };+}++/**+ * After the current frame, only the newest VISION_MAX_LIVE_ZOOMS zooms keep+ * their images (as only they stay usable, VisionSession.registerZoom), and+ * older zooms go first while the kept images exceed+ * RETAINED_IMAGE_CHARS_BUDGET. A tagged message's image blocks are its ids,+ * in order.+ */+function dropOldZooms(messages, boundary) {+ const slots = [];+ let chars = 0;+ messages.forEach((message, index) => {+ const tag = visionFrameTag(message);+ if (index < boundary || !tag || !Array.isArray(message.content) || !Array.isArray(tag.ids)) return;+ const blocks = message.content.map((block, at) => (isImageBlock(block) ? at : -1)).filter(at => at >= 0);+ blocks.forEach((at, order) => {+ const block = message.content[at];+ const size = String(block.image_url?.url || block.image_url || block.source?.data || '').length;+ chars += size;+ if (blocks.length === tag.ids.length && /^z\d+$/.test(tag.ids[order])) slots.push({ index, at, id: tag.ids[order], size });+ });+ });+ const drop = new Set();+ const excess = slots.length - VISION_MAX_LIVE_ZOOMS;+ for (const slot of slots.slice(0, Math.max(0, excess))) { drop.add(slot); chars -= slot.size; }+ for (const slot of slots) {+ if (chars <= RETAINED_IMAGE_CHARS_BUDGET) break;+ if (!drop.has(slot)) { drop.add(slot); chars -= slot.size; }+ }+ if (!drop.size) return messages;+ const out = messages.slice();+ for (const slot of drop) {+ const message = out[slot.index];+ const content = message.content.slice();+ content[slot.at] = { type: 'text', text: `[zoom ${slot.id} omitted — older than the zooms kept; zoom again if needed]` };+ out[slot.index] = { ...message, content };+ }+ return out;}export function installVisionRuntime(AgentClass, { trace = null } = {}) {diff --git a/src/chrome/src/agent/vision-session.js b/src/chrome/src/agent/vision-session.jsindex 919b9e682..819f536d7 100644--- a/src/chrome/src/agent/vision-session.js+++ b/src/chrome/src/agent/vision-session.js@@ -35,6 +35,10 @@ export function normalizeImageId(value) {return IMAGE_ID_RE.test(id) ? id : '';}+// Zooms of the current frame kept usable and in the request; older ones are+// retired (VisionSession.registerZoom, vision-runtime.js pruneVisionImages).+export const VISION_MAX_LIVE_ZOOMS = 4;+export class VisionSession {/*** `startEpoch` / `startCounter` continue the numbering of an earlier@@ -203,6 +207,13 @@ export class VisionSession {scaleY: region.height / Math.max(1, height),};this.images.set(id, image);+ // Only the newest zooms stay usable, as only they stay in the model's+ // request (vision-runtime.js pruneVisionImages): an older one is retired.+ const zooms = [...this.images.values()].filter(item => item.kind === 'zoom');+ for (const old of zooms.slice(0, Math.max(0, zooms.length - VISION_MAX_LIVE_ZOOMS))) {+ this.images.delete(old.id);+ this.retired.set(old.id, { epoch: old.epoch, replacedBy: null, zoomLimit: true });+ }return image;}@@ -266,7 +277,9 @@ export class VisionSession {const lookup = this._imageFor(image);if (!lookup.image) {if (lookup.retired) {- return { ok: false, code: 'stale_image', error: `Image ${lookup.id} is from an earlier screen. The current full frame is ${this.keyframe.id}; read the point from it or from a later patch.` };+ return { ok: false, code: 'stale_image', error: lookup.retired.zoomLimit+ ? `Zoom ${lookup.id} is no longer shown: only the newest ${VISION_MAX_LIVE_ZOOMS} zooms are kept. Read the point from the current frame ${this.keyframe.id}, or zoom again.`+ : `Image ${lookup.id} is from an earlier screen. The current full frame is ${this.keyframe.id}; read the point from it or from a later patch.` };}return { ok: false, code: 'unknown_image', error: `Unknown image "${lookup.id}". Name an image exactly as labeled (valid now: ${[...this.images.keys()].join(', ')}), or omit image for the current screen.` };}diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.jsindex c50f91a7f..bfba455c5 100644--- a/src/firefox/src/agent/vision-runtime.js+++ b/src/firefox/src/agent/vision-runtime.js@@ -15,7 +15,7 @@// older images become short placeholders (_visionPruneImages).import { resolveVisionMode, normalizeVisionModeSetting } from './vision-mode.js';-import { VisionSession } from './vision-session.js';+import { VisionSession, VISION_MAX_LIVE_ZOOMS } from './vision-session.js';import { diffFrames, planFrameUpdate, pointTouchesRegions } from './vision-diff.js';import {VISION_PROFILES, planVisionImageSize, planZoomImageSize, estimateVisionTokens, visionEncodingFor,@@ -52,6 +52,11 @@ const STABILITY_MARGIN_PX = 12;const SETTLE_INITIAL_DELAY_MS = 120;const SETTLE_POLL_MS = 220;const SETTLE_MAX_EXTRA_CAPTURES = 3;+// Images kept after the current frame (its patches and zooms), as data-URL+// characters: past this, the oldest zooms leave the request first. Patches+// are never dropped (they are the current screen); a long patch chain turns+// into a new frame instead (vision-diff.js planFrameUpdate).+const RETAINED_IMAGE_CHARS_BUDGET = 8_000_000;// An upload handoff (_visionNoteUploadHandoff) lasts this many model// decisions without a successful upload_file, and a run hands off at most// this many times: a page whose upload never succeeds goes back to vision@@ -230,7 +235,46 @@ export function pruneVisionImages(messages) {});return changed ? { ...message, content } : message;});- return { messages: out, latestEpoch };+ return { messages: dropOldZooms(out, boundary), latestEpoch };+}++/**+ * After the current frame, only the newest VISION_MAX_LIVE_ZOOMS zooms keep+ * their images (as only they stay usable, VisionSession.registerZoom), and+ * older zooms go first while the kept images exceed+ * RETAINED_IMAGE_CHARS_BUDGET. A tagged message's image blocks are its ids,+ * in order.+ */+function dropOldZooms(messages, boundary) {+ const slots = [];+ let chars = 0;+ messages.forEach((message, index) => {+ const tag = visionFrameTag(message);+ if (index < boundary || !tag || !Array.isArray(message.content) || !Array.isArray(tag.ids)) return;+ const blocks = message.content.map((block, at) => (isImageBlock(block) ? at : -1)).filter(at => at >= 0);+ blocks.forEach((at, order) => {+ const block = message.content[at];+ const size = String(block.image_url?.url || block.image_url || block.source?.data || '').length;+ chars += size;+ if (blocks.length === tag.ids.length && /^z\d+$/.test(tag.ids[order])) slots.push({ index, at, id: tag.ids[order], size });+ });+ });+ const drop = new Set();+ const excess = slots.length - VISION_MAX_LIVE_ZOOMS;+ for (const slot of slots.slice(0, Math.max(0, excess))) { drop.add(slot); chars -= slot.size; }+ for (const slot of slots) {+ if (chars <= RETAINED_IMAGE_CHARS_BUDGET) break;+ if (!drop.has(slot)) { drop.add(slot); chars -= slot.size; }+ }+ if (!drop.size) return messages;+ const out = messages.slice();+ for (const slot of drop) {+ const message = out[slot.index];+ const content = message.content.slice();+ content[slot.at] = { type: 'text', text: `[zoom ${slot.id} omitted — older than the zooms kept; zoom again if needed]` };+ out[slot.index] = { ...message, content };+ }+ return out;}export function installVisionRuntime(AgentClass, { trace = null } = {}) {diff --git a/src/firefox/src/agent/vision-session.js b/src/firefox/src/agent/vision-session.jsindex 919b9e682..819f536d7 100644--- a/src/firefox/src/agent/vision-session.js+++ b/src/firefox/src/agent/vision-session.js@@ -35,6 +35,10 @@ export function normalizeImageId(value) {return IMAGE_ID_RE.test(id) ? id : '';}+// Zooms of the current frame kept usable and in the request; older ones are+// retired (VisionSession.registerZoom, vision-runtime.js pruneVisionImages).+export const VISION_MAX_LIVE_ZOOMS = 4;+export class VisionSession {/*** `startEpoch` / `startCounter` continue the numbering of an earlier@@ -203,6 +207,13 @@ export class VisionSession {scaleY: region.height / Math.max(1, height),};this.images.set(id, image);+ // Only the newest zooms stay usable, as only they stay in the model's+ // request (vision-runtime.js pruneVisionImages): an older one is retired.+ const zooms = [...this.images.values()].filter(item => item.kind === 'zoom');+ for (const old of zooms.slice(0, Math.max(0, zooms.length - VISION_MAX_LIVE_ZOOMS))) {+ this.images.delete(old.id);+ this.retired.set(old.id, { epoch: old.epoch, replacedBy: null, zoomLimit: true });+ }return image;}@@ -266,7 +277,9 @@ export class VisionSession {const lookup = this._imageFor(image);if (!lookup.image) {if (lookup.retired) {- return { ok: false, code: 'stale_image', error: `Image ${lookup.id} is from an earlier screen. The current full frame is ${this.keyframe.id}; read the point from it or from a later patch.` };+ return { ok: false, code: 'stale_image', error: lookup.retired.zoomLimit+ ? `Zoom ${lookup.id} is no longer shown: only the newest ${VISION_MAX_LIVE_ZOOMS} zooms are kept. Read the point from the current frame ${this.keyframe.id}, or zoom again.`+ : `Image ${lookup.id} is from an earlier screen. The current full frame is ${this.keyframe.id}; read the point from it or from a later patch.` };}return { ok: false, code: 'unknown_image', error: `Unknown image "${lookup.id}". Name an image exactly as labeled (valid now: ${[...this.images.keys()].join(', ')}), or omit image for the current screen.` };}diff --git a/test/vision-mode.mjs b/test/vision-mode.mjsindex 39a5d9874..32f4a24c3 100644--- a/test/vision-mode.mjs+++ b/test/vision-mode.mjs@@ -494,6 +494,27 @@ for (const build of ['chrome', 'firefox']) {const probeModule = await import('../src/chrome/src/agent/vision-dom-probe.js');const runtimeModule = await import('../src/chrome/src/agent/vision-runtime.js');+test('pruning: after the current frame only the newest zooms keep their images, and oldest zooms go first past the size budget', () => {+ const image = (chars = 40) => ({ type: 'image_url', image_url: { url: `data:image/webp;base64,${'A'.repeat(chars)}` } });+ const frame = (kind, ids, images) => ({ role: 'user', content: [{ type: 'text', text: ids.join(',') }, ...images], webbrainVisionFrame: { epoch: 1, kind, ids } });+ const messages = [+ frame('key', ['f1'], [image()]),+ ...['z2', 'z3', 'z4', 'z5', 'z6', 'z7'].map(id => frame('zoom', [id], [image()])),+ frame('patch', ['p8', 'z9'], [image(), image()]),+ ];+ const kept = runtimeModule.pruneVisionImages(messages).messages;+ const shown = kept.flatMap(message => message.content.filter(block => block.type === 'image_url').length ? message.webbrainVisionFrame.ids : []);+ const omitted = kept.flatMap(message => message.content.filter(block => block.type === 'text' && /^\[zoom z\d+ omitted/.test(block.text)).map(block => /z\d+/.exec(block.text)[0]));+ assert.deepEqual(omitted, ['z2', 'z3', 'z4']);+ assert.deepEqual(shown, ['f1', 'z5', 'z6', 'z7', 'p8', 'z9']);+ // A huge zoom pushes the kept images past the budget: the oldest zoom goes, never the frame or a patch.+ const big = runtimeModule.pruneVisionImages([+ frame('key', ['f1'], [image()]), frame('zoom', ['z2'], [image(8_100_000)]), frame('zoom', ['z3'], [image()]), frame('patch', ['p4'], [image()]),+ ]).messages;+ assert.equal(big[1].content.some(block => block.type === 'image_url'), false);+ for (const index of [0, 2, 3]) assert.equal(big[index].content.some(block => block.type === 'image_url'), true, String(index));+});+test('session: numbering continues across sessions and a retired screen refuses points', () => {const session = new VisionSession({ startEpoch: 3, startCounter: 7 });const frame = session.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 1000, cssHeight: 500 });@@ -525,6 +546,17 @@ test('session: a failed update stays recorded until the model is shown the scree}});+test('session: only the newest four zooms stay usable', () => {+ const session = new VisionSession();+ session.registerKeyframe({ imageWidth: 100, imageHeight: 50, cssWidth: 100, cssHeight: 50 });+ const zooms = [1, 2, 3, 4, 5].map(() => session.registerZoom({ x: 0, y: 0, width: 10, height: 10 }, 20, 20));+ assert.equal(sessionModule.VISION_MAX_LIVE_ZOOMS, 4);+ const oldest = session.resolvePoint({ x: 5, y: 5, image: zooms[0].id });+ assert.equal(oldest.code, 'stale_image');+ assert.match(oldest.error, /no longer shown: only the newest 4 zooms are kept/);+ for (const zoom of zooms.slice(1)) assert.equal(session.resolvePoint({ x: 5, y: 5, image: zoom.id }).ok, true, zoom.id);+});+test('session: an image value that is not an id is refused, not read as the current screen', () => {const session = new VisionSession();session.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 2000, cssHeight: 1000 });