Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Traces keep vision notes as vision notes

Emre Sokullue2abdca79d4a
diff --git a/src/chrome/src/trace/privacy.js b/src/chrome/src/trace/privacy.js
index 2ce6f2154..cb7159ed8 100644
--- a/src/chrome/src/trace/privacy.js
+++ b/src/chrome/src/trace/privacy.js
@@ -166,6 +166,58 @@ function projectClickAxTiming(data) {
};
}
+// Vision mode's notes (agent/vision-runtime.js _visionTraceNote). Metadata
+// traces keep their codes from closed lists and their bounded numbers, never
+// an error text, a region or a page size string: a capture failure must read
+// as one, not as the generic note stage 'model'.
+const VISION_NOTES = new Set([
+ 'vision_frame', 'vision_frame_error', 'vision_tool_error', 'vision_upload_handoff',
+ 'vision_redaction_skipped', 'vision_capture_retry', 'vision_layout_changed',
+]);
+const VISION_NOTE_ENUMS = {
+ kind: ['keyframe', 'patch', 'none', 'zoom'],
+ stage: ['start', 'observe', 'encode', 'resume', 'armed', 'ended'],
+ reason: ['start of turn', 'requested', 'page changed', 'layout changed', 'first frame', 'vision resumed',
+ 'forced', 'not comparable', 'no visible change', 'patch chain limit', 'large change', 'patch cost', 'local change',
+ 'page_changed', 'scrolled', 'uploaded', 'navigated', 'expired'],
+ code: ['decode_failed', 'scan_unavailable', 'pixelate_failed', 'error', 'transient'],
+ context: ['keyframe', 'patch', 'zoom'],
+ tool: ['take_screenshot', 'zoom', 'scroll_at', 'hover_at', 'drag_at', 'wait', 'click_at'],
+ mimeType: ['image/webp', 'image/jpeg', 'image/png'],
+};
+const VISION_NOTE_NUMBERS = {
+ width: 20_000, height: 20_000, cssWidth: 20_000, cssHeight: 20_000, captureWidth: 40_000, captureHeight: 40_000,
+ devicePixelRatio: 8, bytes: 50_000_000, estimatedTokens: 1_000_000, frameTokens: 1_000_000, changedRatio: 1,
+ count: 100, attempt: 100,
+};
+const VISION_IMAGE_ID = /^[fpz]\d{1,7}$/;
+
+function projectVisionNote(data) {
+ const source = data?.extra && typeof data.extra === 'object' && !Array.isArray(data.extra) ? data.extra : {};
+ const extra = {};
+ for (const [key, allowed] of Object.entries(VISION_NOTE_ENUMS)) {
+ if (allowed.includes(source[key])) extra[key] = source[key];
+ }
+ for (const [key, max] of Object.entries(VISION_NOTE_NUMBERS)) {
+ const value = source[key];
+ if (typeof value === 'number' && Number.isFinite(value) && value >= 0) extra[key] = Math.min(value, max);
+ }
+ for (const key of ['id', 'keyframe', 'frame']) {
+ if (typeof source[key] === 'string' && VISION_IMAGE_ID.test(source[key])) extra[key] = source[key];
+ }
+ if (Array.isArray(source.ids)) {
+ const ids = source.ids.filter(id => typeof id === 'string' && VISION_IMAGE_ID.test(id)).slice(0, 16);
+ if (ids.length) extra.ids = ids;
+ }
+ if (typeof source.dispatched === 'boolean') extra.dispatched = source.dispatched;
+ if (typeof source.profile === 'string' && /^[a-z0-9_-]{1,32}$/i.test(source.profile)) extra.profile = source.profile;
+ return {
+ step: Number.isFinite(data?.step) ? data.step : 0,
+ note: data.note,
+ ...(Object.keys(extra).length ? { extra } : {}),
+ };
+}
+
function pick(source, fields) {
if (!source || typeof source !== 'object' || Array.isArray(source)) return {};
const output = {};
@@ -326,6 +378,7 @@ export function projectTraceEventData(kind, data, { includeContent = false } = {
}
if (kind === 'error') return pick(source, ['step', 'phase', 'code']);
if (kind === 'streaming') return pick(source, STREAMING_METADATA_FIELDS);
+ if (kind === 'note' && VISION_NOTES.has(source.note)) return projectVisionNote(source);
if (kind === 'note') {
const projected = pick(source, ['step', 'note']);
const extra = projectNoteExtra(source.extra);
diff --git a/src/firefox/src/trace/privacy.js b/src/firefox/src/trace/privacy.js
index 2ce6f2154..cb7159ed8 100644
--- a/src/firefox/src/trace/privacy.js
+++ b/src/firefox/src/trace/privacy.js
@@ -166,6 +166,58 @@ function projectClickAxTiming(data) {
};
}
+// Vision mode's notes (agent/vision-runtime.js _visionTraceNote). Metadata
+// traces keep their codes from closed lists and their bounded numbers, never
+// an error text, a region or a page size string: a capture failure must read
+// as one, not as the generic note stage 'model'.
+const VISION_NOTES = new Set([
+ 'vision_frame', 'vision_frame_error', 'vision_tool_error', 'vision_upload_handoff',
+ 'vision_redaction_skipped', 'vision_capture_retry', 'vision_layout_changed',
+]);
+const VISION_NOTE_ENUMS = {
+ kind: ['keyframe', 'patch', 'none', 'zoom'],
+ stage: ['start', 'observe', 'encode', 'resume', 'armed', 'ended'],
+ reason: ['start of turn', 'requested', 'page changed', 'layout changed', 'first frame', 'vision resumed',
+ 'forced', 'not comparable', 'no visible change', 'patch chain limit', 'large change', 'patch cost', 'local change',
+ 'page_changed', 'scrolled', 'uploaded', 'navigated', 'expired'],
+ code: ['decode_failed', 'scan_unavailable', 'pixelate_failed', 'error', 'transient'],
+ context: ['keyframe', 'patch', 'zoom'],
+ tool: ['take_screenshot', 'zoom', 'scroll_at', 'hover_at', 'drag_at', 'wait', 'click_at'],
+ mimeType: ['image/webp', 'image/jpeg', 'image/png'],
+};
+const VISION_NOTE_NUMBERS = {
+ width: 20_000, height: 20_000, cssWidth: 20_000, cssHeight: 20_000, captureWidth: 40_000, captureHeight: 40_000,
+ devicePixelRatio: 8, bytes: 50_000_000, estimatedTokens: 1_000_000, frameTokens: 1_000_000, changedRatio: 1,
+ count: 100, attempt: 100,
+};
+const VISION_IMAGE_ID = /^[fpz]\d{1,7}$/;
+
+function projectVisionNote(data) {
+ const source = data?.extra && typeof data.extra === 'object' && !Array.isArray(data.extra) ? data.extra : {};
+ const extra = {};
+ for (const [key, allowed] of Object.entries(VISION_NOTE_ENUMS)) {
+ if (allowed.includes(source[key])) extra[key] = source[key];
+ }
+ for (const [key, max] of Object.entries(VISION_NOTE_NUMBERS)) {
+ const value = source[key];
+ if (typeof value === 'number' && Number.isFinite(value) && value >= 0) extra[key] = Math.min(value, max);
+ }
+ for (const key of ['id', 'keyframe', 'frame']) {
+ if (typeof source[key] === 'string' && VISION_IMAGE_ID.test(source[key])) extra[key] = source[key];
+ }
+ if (Array.isArray(source.ids)) {
+ const ids = source.ids.filter(id => typeof id === 'string' && VISION_IMAGE_ID.test(id)).slice(0, 16);
+ if (ids.length) extra.ids = ids;
+ }
+ if (typeof source.dispatched === 'boolean') extra.dispatched = source.dispatched;
+ if (typeof source.profile === 'string' && /^[a-z0-9_-]{1,32}$/i.test(source.profile)) extra.profile = source.profile;
+ return {
+ step: Number.isFinite(data?.step) ? data.step : 0,
+ note: data.note,
+ ...(Object.keys(extra).length ? { extra } : {}),
+ };
+}
+
function pick(source, fields) {
if (!source || typeof source !== 'object' || Array.isArray(source)) return {};
const output = {};
@@ -326,6 +378,7 @@ export function projectTraceEventData(kind, data, { includeContent = false } = {
}
if (kind === 'error') return pick(source, ['step', 'phase', 'code']);
if (kind === 'streaming') return pick(source, STREAMING_METADATA_FIELDS);
+ if (kind === 'note' && VISION_NOTES.has(source.note)) return projectVisionNote(source);
if (kind === 'note') {
const projected = pick(source, ['step', 'note']);
const extra = projectNoteExtra(source.extra);
diff --git a/test/vision-mode.mjs b/test/vision-mode.mjs
index 32f4a24c3..2ba8e52cb 100644
--- a/test/vision-mode.mjs
+++ b/test/vision-mode.mjs
@@ -557,6 +557,25 @@ test('session: only the newest four zooms stay usable', () => {
for (const zoom of zooms.slice(1)) assert.equal(session.resolvePoint({ x: 5, y: 5, image: zoom.id }).ok, true, zoom.id);
});
+test('traces: metadata traces keep vision notes as vision notes, without error texts', async () => {
+ const { projectTraceEventData } = await import('../src/chrome/src/trace/privacy.js');
+ assert.equal(read('src/chrome/src/trace/privacy.js'), read('src/firefox/src/trace/privacy.js'));
+ const failed = projectTraceEventData('note', { step: 3, note: 'vision_frame_error', extra: { stage: 'observe', error: 'Cannot read https://bank.example/account?id=42' } });
+ assert.deepEqual(failed, { step: 3, note: 'vision_frame_error', extra: { stage: 'observe' } });
+ const frame = projectTraceEventData('note', { step: 1, note: 'vision_frame', extra: {
+ kind: 'keyframe', id: 'f4', reason: 'vision resumed', width: 1356, height: 847, devicePixelRatio: 2, bytes: 81234,
+ mimeType: 'image/webp', profile: 'anthropic', regions: [{ x: 1, y: 2, width: 3, height: 4 }], url: 'https://bank.example/',
+ } });
+ assert.deepEqual(frame.extra, { kind: 'keyframe', reason: 'vision resumed', width: 1356, height: 847, devicePixelRatio: 2, bytes: 81234, mimeType: 'image/webp', id: 'f4', profile: 'anthropic' });
+ const skipped = projectTraceEventData('note', { step: 2, note: 'vision_redaction_skipped', extra: { code: 'scan_unavailable', context: 'zoom' } });
+ assert.deepEqual(skipped.extra, { code: 'scan_unavailable', context: 'zoom' });
+ // Values outside the closed lists are dropped, not passed through.
+ assert.equal(projectTraceEventData('note', { step: 2, note: 'vision_frame', extra: { kind: 'secret text', reason: 'typed: hunter2' } }).extra, undefined);
+ // The opt-in content tier keeps the note as recorded.
+ const raw = { step: 3, note: 'vision_frame_error', extra: { stage: 'observe', error: 'details' } };
+ assert.deepEqual(projectTraceEventData('note', raw, { includeContent: true }), raw);
+});
+
test('session: an image value that is not an id is refused, not read as the current screen', () => {
const session = new VisionSession();
session.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 2000, cssHeight: 1000 });