Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Vision mode: keep at most four zooms of the current frame

Emre Sokullu1068929fcac1
diff --git a/docs/vision-mode.md b/docs/vision-mode.md
index 4244e7068..e484688f6 100644
--- a/docs/vision-mode.md
+++ b/docs/vision-mode.md
@@ -100,7 +100,7 @@ After a batch WebBrain waits until two consecutive captures match (bounded, ~1 s
The "seen" buffer copies only patched regions, so changes below the threshold accumulate until they are sent instead of drifting forever.
-Only the current frame, its patches and zooms stay in the model request (`pruneVisionImages`); older images become a short placeholder. Screen updates are app-owned user turns (`webbrainAppOwned`), never mistaken for a new request.
+Only the current frame, its patches and its newest zooms stay in the model request (`pruneVisionImages`); older images become a short placeholder. At most four zooms of the current frame stay, and stay usable (`VISION_MAX_LIVE_ZOOMS`; a point on an older one is refused as `stale_image` with a note to zoom again), and while the kept images exceed about 8 MB of data URL the oldest zooms leave first. Patches are never dropped, since they are the current screen; a long patch chain becomes a new frame instead. Screen updates are app-owned user turns (`webbrainAppOwned`), never mistaken for a new request.
A capture, encoding or input failure never ends the run: a screen update that cannot be prepared becomes a one-line note and the next update is a full frame; a vision tool that fails returns a tool error (an unknown outcome once input may have gone out).
diff --git a/src/chrome/src/agent/vision-runtime.js b/src/chrome/src/agent/vision-runtime.js
index c50f91a7f..bfba455c5 100644
--- a/src/chrome/src/agent/vision-runtime.js
+++ b/src/chrome/src/agent/vision-runtime.js
@@ -15,7 +15,7 @@
// older images become short placeholders (_visionPruneImages).
import { resolveVisionMode, normalizeVisionModeSetting } from './vision-mode.js';
-import { VisionSession } from './vision-session.js';
+import { VisionSession, VISION_MAX_LIVE_ZOOMS } from './vision-session.js';
import { diffFrames, planFrameUpdate, pointTouchesRegions } from './vision-diff.js';
import {
VISION_PROFILES, planVisionImageSize, planZoomImageSize, estimateVisionTokens, visionEncodingFor,
@@ -52,6 +52,11 @@ const STABILITY_MARGIN_PX = 12;
const SETTLE_INITIAL_DELAY_MS = 120;
const SETTLE_POLL_MS = 220;
const SETTLE_MAX_EXTRA_CAPTURES = 3;
+// Images kept after the current frame (its patches and zooms), as data-URL
+// characters: past this, the oldest zooms leave the request first. Patches
+// are never dropped (they are the current screen); a long patch chain turns
+// into a new frame instead (vision-diff.js planFrameUpdate).
+const RETAINED_IMAGE_CHARS_BUDGET = 8_000_000;
// An upload handoff (_visionNoteUploadHandoff) lasts this many model
// decisions without a successful upload_file, and a run hands off at most
// this many times: a page whose upload never succeeds goes back to vision
@@ -230,7 +235,46 @@ export function pruneVisionImages(messages) {
});
return changed ? { ...message, content } : message;
});
- return { messages: out, latestEpoch };
+ return { messages: dropOldZooms(out, boundary), latestEpoch };
+}
+
+/**
+ * After the current frame, only the newest VISION_MAX_LIVE_ZOOMS zooms keep
+ * their images (as only they stay usable, VisionSession.registerZoom), and
+ * older zooms go first while the kept images exceed
+ * RETAINED_IMAGE_CHARS_BUDGET. A tagged message's image blocks are its ids,
+ * in order.
+ */
+function dropOldZooms(messages, boundary) {
+ const slots = [];
+ let chars = 0;
+ messages.forEach((message, index) => {
+ const tag = visionFrameTag(message);
+ if (index < boundary || !tag || !Array.isArray(message.content) || !Array.isArray(tag.ids)) return;
+ const blocks = message.content.map((block, at) => (isImageBlock(block) ? at : -1)).filter(at => at >= 0);
+ blocks.forEach((at, order) => {
+ const block = message.content[at];
+ const size = String(block.image_url?.url || block.image_url || block.source?.data || '').length;
+ chars += size;
+ if (blocks.length === tag.ids.length && /^z\d+$/.test(tag.ids[order])) slots.push({ index, at, id: tag.ids[order], size });
+ });
+ });
+ const drop = new Set();
+ const excess = slots.length - VISION_MAX_LIVE_ZOOMS;
+ for (const slot of slots.slice(0, Math.max(0, excess))) { drop.add(slot); chars -= slot.size; }
+ for (const slot of slots) {
+ if (chars <= RETAINED_IMAGE_CHARS_BUDGET) break;
+ if (!drop.has(slot)) { drop.add(slot); chars -= slot.size; }
+ }
+ if (!drop.size) return messages;
+ const out = messages.slice();
+ for (const slot of drop) {
+ const message = out[slot.index];
+ const content = message.content.slice();
+ content[slot.at] = { type: 'text', text: `[zoom ${slot.id} omitted — older than the zooms kept; zoom again if needed]` };
+ out[slot.index] = { ...message, content };
+ }
+ return out;
}
export function installVisionRuntime(AgentClass, { trace = null } = {}) {
diff --git a/src/chrome/src/agent/vision-session.js b/src/chrome/src/agent/vision-session.js
index 919b9e682..819f536d7 100644
--- a/src/chrome/src/agent/vision-session.js
+++ b/src/chrome/src/agent/vision-session.js
@@ -35,6 +35,10 @@ export function normalizeImageId(value) {
return IMAGE_ID_RE.test(id) ? id : '';
}
+// Zooms of the current frame kept usable and in the request; older ones are
+// retired (VisionSession.registerZoom, vision-runtime.js pruneVisionImages).
+export const VISION_MAX_LIVE_ZOOMS = 4;
+
export class VisionSession {
/**
* `startEpoch` / `startCounter` continue the numbering of an earlier
@@ -203,6 +207,13 @@ export class VisionSession {
scaleY: region.height / Math.max(1, height),
};
this.images.set(id, image);
+ // Only the newest zooms stay usable, as only they stay in the model's
+ // request (vision-runtime.js pruneVisionImages): an older one is retired.
+ const zooms = [...this.images.values()].filter(item => item.kind === 'zoom');
+ for (const old of zooms.slice(0, Math.max(0, zooms.length - VISION_MAX_LIVE_ZOOMS))) {
+ this.images.delete(old.id);
+ this.retired.set(old.id, { epoch: old.epoch, replacedBy: null, zoomLimit: true });
+ }
return image;
}
@@ -266,7 +277,9 @@ export class VisionSession {
const lookup = this._imageFor(image);
if (!lookup.image) {
if (lookup.retired) {
- return { ok: false, code: 'stale_image', error: `Image ${lookup.id} is from an earlier screen. The current full frame is ${this.keyframe.id}; read the point from it or from a later patch.` };
+ return { ok: false, code: 'stale_image', error: lookup.retired.zoomLimit
+ ? `Zoom ${lookup.id} is no longer shown: only the newest ${VISION_MAX_LIVE_ZOOMS} zooms are kept. Read the point from the current frame ${this.keyframe.id}, or zoom again.`
+ : `Image ${lookup.id} is from an earlier screen. The current full frame is ${this.keyframe.id}; read the point from it or from a later patch.` };
}
return { ok: false, code: 'unknown_image', error: `Unknown image "${lookup.id}". Name an image exactly as labeled (valid now: ${[...this.images.keys()].join(', ')}), or omit image for the current screen.` };
}
diff --git a/src/firefox/src/agent/vision-runtime.js b/src/firefox/src/agent/vision-runtime.js
index c50f91a7f..bfba455c5 100644
--- a/src/firefox/src/agent/vision-runtime.js
+++ b/src/firefox/src/agent/vision-runtime.js
@@ -15,7 +15,7 @@
// older images become short placeholders (_visionPruneImages).
import { resolveVisionMode, normalizeVisionModeSetting } from './vision-mode.js';
-import { VisionSession } from './vision-session.js';
+import { VisionSession, VISION_MAX_LIVE_ZOOMS } from './vision-session.js';
import { diffFrames, planFrameUpdate, pointTouchesRegions } from './vision-diff.js';
import {
VISION_PROFILES, planVisionImageSize, planZoomImageSize, estimateVisionTokens, visionEncodingFor,
@@ -52,6 +52,11 @@ const STABILITY_MARGIN_PX = 12;
const SETTLE_INITIAL_DELAY_MS = 120;
const SETTLE_POLL_MS = 220;
const SETTLE_MAX_EXTRA_CAPTURES = 3;
+// Images kept after the current frame (its patches and zooms), as data-URL
+// characters: past this, the oldest zooms leave the request first. Patches
+// are never dropped (they are the current screen); a long patch chain turns
+// into a new frame instead (vision-diff.js planFrameUpdate).
+const RETAINED_IMAGE_CHARS_BUDGET = 8_000_000;
// An upload handoff (_visionNoteUploadHandoff) lasts this many model
// decisions without a successful upload_file, and a run hands off at most
// this many times: a page whose upload never succeeds goes back to vision
@@ -230,7 +235,46 @@ export function pruneVisionImages(messages) {
});
return changed ? { ...message, content } : message;
});
- return { messages: out, latestEpoch };
+ return { messages: dropOldZooms(out, boundary), latestEpoch };
+}
+
+/**
+ * After the current frame, only the newest VISION_MAX_LIVE_ZOOMS zooms keep
+ * their images (as only they stay usable, VisionSession.registerZoom), and
+ * older zooms go first while the kept images exceed
+ * RETAINED_IMAGE_CHARS_BUDGET. A tagged message's image blocks are its ids,
+ * in order.
+ */
+function dropOldZooms(messages, boundary) {
+ const slots = [];
+ let chars = 0;
+ messages.forEach((message, index) => {
+ const tag = visionFrameTag(message);
+ if (index < boundary || !tag || !Array.isArray(message.content) || !Array.isArray(tag.ids)) return;
+ const blocks = message.content.map((block, at) => (isImageBlock(block) ? at : -1)).filter(at => at >= 0);
+ blocks.forEach((at, order) => {
+ const block = message.content[at];
+ const size = String(block.image_url?.url || block.image_url || block.source?.data || '').length;
+ chars += size;
+ if (blocks.length === tag.ids.length && /^z\d+$/.test(tag.ids[order])) slots.push({ index, at, id: tag.ids[order], size });
+ });
+ });
+ const drop = new Set();
+ const excess = slots.length - VISION_MAX_LIVE_ZOOMS;
+ for (const slot of slots.slice(0, Math.max(0, excess))) { drop.add(slot); chars -= slot.size; }
+ for (const slot of slots) {
+ if (chars <= RETAINED_IMAGE_CHARS_BUDGET) break;
+ if (!drop.has(slot)) { drop.add(slot); chars -= slot.size; }
+ }
+ if (!drop.size) return messages;
+ const out = messages.slice();
+ for (const slot of drop) {
+ const message = out[slot.index];
+ const content = message.content.slice();
+ content[slot.at] = { type: 'text', text: `[zoom ${slot.id} omitted — older than the zooms kept; zoom again if needed]` };
+ out[slot.index] = { ...message, content };
+ }
+ return out;
}
export function installVisionRuntime(AgentClass, { trace = null } = {}) {
diff --git a/src/firefox/src/agent/vision-session.js b/src/firefox/src/agent/vision-session.js
index 919b9e682..819f536d7 100644
--- a/src/firefox/src/agent/vision-session.js
+++ b/src/firefox/src/agent/vision-session.js
@@ -35,6 +35,10 @@ export function normalizeImageId(value) {
return IMAGE_ID_RE.test(id) ? id : '';
}
+// Zooms of the current frame kept usable and in the request; older ones are
+// retired (VisionSession.registerZoom, vision-runtime.js pruneVisionImages).
+export const VISION_MAX_LIVE_ZOOMS = 4;
+
export class VisionSession {
/**
* `startEpoch` / `startCounter` continue the numbering of an earlier
@@ -203,6 +207,13 @@ export class VisionSession {
scaleY: region.height / Math.max(1, height),
};
this.images.set(id, image);
+ // Only the newest zooms stay usable, as only they stay in the model's
+ // request (vision-runtime.js pruneVisionImages): an older one is retired.
+ const zooms = [...this.images.values()].filter(item => item.kind === 'zoom');
+ for (const old of zooms.slice(0, Math.max(0, zooms.length - VISION_MAX_LIVE_ZOOMS))) {
+ this.images.delete(old.id);
+ this.retired.set(old.id, { epoch: old.epoch, replacedBy: null, zoomLimit: true });
+ }
return image;
}
@@ -266,7 +277,9 @@ export class VisionSession {
const lookup = this._imageFor(image);
if (!lookup.image) {
if (lookup.retired) {
- return { ok: false, code: 'stale_image', error: `Image ${lookup.id} is from an earlier screen. The current full frame is ${this.keyframe.id}; read the point from it or from a later patch.` };
+ return { ok: false, code: 'stale_image', error: lookup.retired.zoomLimit
+ ? `Zoom ${lookup.id} is no longer shown: only the newest ${VISION_MAX_LIVE_ZOOMS} zooms are kept. Read the point from the current frame ${this.keyframe.id}, or zoom again.`
+ : `Image ${lookup.id} is from an earlier screen. The current full frame is ${this.keyframe.id}; read the point from it or from a later patch.` };
}
return { ok: false, code: 'unknown_image', error: `Unknown image "${lookup.id}". Name an image exactly as labeled (valid now: ${[...this.images.keys()].join(', ')}), or omit image for the current screen.` };
}
diff --git a/test/vision-mode.mjs b/test/vision-mode.mjs
index 39a5d9874..32f4a24c3 100644
--- a/test/vision-mode.mjs
+++ b/test/vision-mode.mjs
@@ -494,6 +494,27 @@ for (const build of ['chrome', 'firefox']) {
const probeModule = await import('../src/chrome/src/agent/vision-dom-probe.js');
const runtimeModule = await import('../src/chrome/src/agent/vision-runtime.js');
+test('pruning: after the current frame only the newest zooms keep their images, and oldest zooms go first past the size budget', () => {
+ const image = (chars = 40) => ({ type: 'image_url', image_url: { url: `data:image/webp;base64,${'A'.repeat(chars)}` } });
+ const frame = (kind, ids, images) => ({ role: 'user', content: [{ type: 'text', text: ids.join(',') }, ...images], webbrainVisionFrame: { epoch: 1, kind, ids } });
+ const messages = [
+ frame('key', ['f1'], [image()]),
+ ...['z2', 'z3', 'z4', 'z5', 'z6', 'z7'].map(id => frame('zoom', [id], [image()])),
+ frame('patch', ['p8', 'z9'], [image(), image()]),
+ ];
+ const kept = runtimeModule.pruneVisionImages(messages).messages;
+ const shown = kept.flatMap(message => message.content.filter(block => block.type === 'image_url').length ? message.webbrainVisionFrame.ids : []);
+ const omitted = kept.flatMap(message => message.content.filter(block => block.type === 'text' && /^\[zoom z\d+ omitted/.test(block.text)).map(block => /z\d+/.exec(block.text)[0]));
+ assert.deepEqual(omitted, ['z2', 'z3', 'z4']);
+ assert.deepEqual(shown, ['f1', 'z5', 'z6', 'z7', 'p8', 'z9']);
+ // A huge zoom pushes the kept images past the budget: the oldest zoom goes, never the frame or a patch.
+ const big = runtimeModule.pruneVisionImages([
+ frame('key', ['f1'], [image()]), frame('zoom', ['z2'], [image(8_100_000)]), frame('zoom', ['z3'], [image()]), frame('patch', ['p4'], [image()]),
+ ]).messages;
+ assert.equal(big[1].content.some(block => block.type === 'image_url'), false);
+ for (const index of [0, 2, 3]) assert.equal(big[index].content.some(block => block.type === 'image_url'), true, String(index));
+});
+
test('session: numbering continues across sessions and a retired screen refuses points', () => {
const session = new VisionSession({ startEpoch: 3, startCounter: 7 });
const frame = session.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 1000, cssHeight: 500 });
@@ -525,6 +546,17 @@ test('session: a failed update stays recorded until the model is shown the scree
}
});
+test('session: only the newest four zooms stay usable', () => {
+ const session = new VisionSession();
+ session.registerKeyframe({ imageWidth: 100, imageHeight: 50, cssWidth: 100, cssHeight: 50 });
+ const zooms = [1, 2, 3, 4, 5].map(() => session.registerZoom({ x: 0, y: 0, width: 10, height: 10 }, 20, 20));
+ assert.equal(sessionModule.VISION_MAX_LIVE_ZOOMS, 4);
+ const oldest = session.resolvePoint({ x: 5, y: 5, image: zooms[0].id });
+ assert.equal(oldest.code, 'stale_image');
+ assert.match(oldest.error, /no longer shown: only the newest 4 zooms are kept/);
+ for (const zoom of zooms.slice(1)) assert.equal(session.resolvePoint({ x: 5, y: 5, image: zoom.id }).ok, true, zoom.id);
+});
+
test('session: an image value that is not an id is refused, not read as the current screen', () => {
const session = new VisionSession();
session.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 2000, cssHeight: 1000 });