Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Traces keep vision notes as vision notes
Emre Sokullue2abdca79d4a
diff --git a/src/chrome/src/trace/privacy.js b/src/chrome/src/trace/privacy.jsindex 2ce6f2154..cb7159ed8 100644--- a/src/chrome/src/trace/privacy.js+++ b/src/chrome/src/trace/privacy.js@@ -166,6 +166,58 @@ function projectClickAxTiming(data) {};}+// Vision mode's notes (agent/vision-runtime.js _visionTraceNote). Metadata+// traces keep their codes from closed lists and their bounded numbers, never+// an error text, a region or a page size string: a capture failure must read+// as one, not as the generic note stage 'model'.+const VISION_NOTES = new Set([+ 'vision_frame', 'vision_frame_error', 'vision_tool_error', 'vision_upload_handoff',+ 'vision_redaction_skipped', 'vision_capture_retry', 'vision_layout_changed',+]);+const VISION_NOTE_ENUMS = {+ kind: ['keyframe', 'patch', 'none', 'zoom'],+ stage: ['start', 'observe', 'encode', 'resume', 'armed', 'ended'],+ reason: ['start of turn', 'requested', 'page changed', 'layout changed', 'first frame', 'vision resumed',+ 'forced', 'not comparable', 'no visible change', 'patch chain limit', 'large change', 'patch cost', 'local change',+ 'page_changed', 'scrolled', 'uploaded', 'navigated', 'expired'],+ code: ['decode_failed', 'scan_unavailable', 'pixelate_failed', 'error', 'transient'],+ context: ['keyframe', 'patch', 'zoom'],+ tool: ['take_screenshot', 'zoom', 'scroll_at', 'hover_at', 'drag_at', 'wait', 'click_at'],+ mimeType: ['image/webp', 'image/jpeg', 'image/png'],+};+const VISION_NOTE_NUMBERS = {+ width: 20_000, height: 20_000, cssWidth: 20_000, cssHeight: 20_000, captureWidth: 40_000, captureHeight: 40_000,+ devicePixelRatio: 8, bytes: 50_000_000, estimatedTokens: 1_000_000, frameTokens: 1_000_000, changedRatio: 1,+ count: 100, attempt: 100,+};+const VISION_IMAGE_ID = /^[fpz]\d{1,7}$/;++function projectVisionNote(data) {+ const source = data?.extra && typeof data.extra === 'object' && !Array.isArray(data.extra) ? data.extra : {};+ const extra = {};+ for (const [key, allowed] of Object.entries(VISION_NOTE_ENUMS)) {+ if (allowed.includes(source[key])) extra[key] = source[key];+ }+ for (const [key, max] of Object.entries(VISION_NOTE_NUMBERS)) {+ const value = source[key];+ if (typeof value === 'number' && Number.isFinite(value) && value >= 0) extra[key] = Math.min(value, max);+ }+ for (const key of ['id', 'keyframe', 'frame']) {+ if (typeof source[key] === 'string' && VISION_IMAGE_ID.test(source[key])) extra[key] = source[key];+ }+ if (Array.isArray(source.ids)) {+ const ids = source.ids.filter(id => typeof id === 'string' && VISION_IMAGE_ID.test(id)).slice(0, 16);+ if (ids.length) extra.ids = ids;+ }+ if (typeof source.dispatched === 'boolean') extra.dispatched = source.dispatched;+ if (typeof source.profile === 'string' && /^[a-z0-9_-]{1,32}$/i.test(source.profile)) extra.profile = source.profile;+ return {+ step: Number.isFinite(data?.step) ? data.step : 0,+ note: data.note,+ ...(Object.keys(extra).length ? { extra } : {}),+ };+}+function pick(source, fields) {if (!source || typeof source !== 'object' || Array.isArray(source)) return {};const output = {};@@ -326,6 +378,7 @@ export function projectTraceEventData(kind, data, { includeContent = false } = {}if (kind === 'error') return pick(source, ['step', 'phase', 'code']);if (kind === 'streaming') return pick(source, STREAMING_METADATA_FIELDS);+ if (kind === 'note' && VISION_NOTES.has(source.note)) return projectVisionNote(source);if (kind === 'note') {const projected = pick(source, ['step', 'note']);const extra = projectNoteExtra(source.extra);diff --git a/src/firefox/src/trace/privacy.js b/src/firefox/src/trace/privacy.jsindex 2ce6f2154..cb7159ed8 100644--- a/src/firefox/src/trace/privacy.js+++ b/src/firefox/src/trace/privacy.js@@ -166,6 +166,58 @@ function projectClickAxTiming(data) {};}+// Vision mode's notes (agent/vision-runtime.js _visionTraceNote). Metadata+// traces keep their codes from closed lists and their bounded numbers, never+// an error text, a region or a page size string: a capture failure must read+// as one, not as the generic note stage 'model'.+const VISION_NOTES = new Set([+ 'vision_frame', 'vision_frame_error', 'vision_tool_error', 'vision_upload_handoff',+ 'vision_redaction_skipped', 'vision_capture_retry', 'vision_layout_changed',+]);+const VISION_NOTE_ENUMS = {+ kind: ['keyframe', 'patch', 'none', 'zoom'],+ stage: ['start', 'observe', 'encode', 'resume', 'armed', 'ended'],+ reason: ['start of turn', 'requested', 'page changed', 'layout changed', 'first frame', 'vision resumed',+ 'forced', 'not comparable', 'no visible change', 'patch chain limit', 'large change', 'patch cost', 'local change',+ 'page_changed', 'scrolled', 'uploaded', 'navigated', 'expired'],+ code: ['decode_failed', 'scan_unavailable', 'pixelate_failed', 'error', 'transient'],+ context: ['keyframe', 'patch', 'zoom'],+ tool: ['take_screenshot', 'zoom', 'scroll_at', 'hover_at', 'drag_at', 'wait', 'click_at'],+ mimeType: ['image/webp', 'image/jpeg', 'image/png'],+};+const VISION_NOTE_NUMBERS = {+ width: 20_000, height: 20_000, cssWidth: 20_000, cssHeight: 20_000, captureWidth: 40_000, captureHeight: 40_000,+ devicePixelRatio: 8, bytes: 50_000_000, estimatedTokens: 1_000_000, frameTokens: 1_000_000, changedRatio: 1,+ count: 100, attempt: 100,+};+const VISION_IMAGE_ID = /^[fpz]\d{1,7}$/;++function projectVisionNote(data) {+ const source = data?.extra && typeof data.extra === 'object' && !Array.isArray(data.extra) ? data.extra : {};+ const extra = {};+ for (const [key, allowed] of Object.entries(VISION_NOTE_ENUMS)) {+ if (allowed.includes(source[key])) extra[key] = source[key];+ }+ for (const [key, max] of Object.entries(VISION_NOTE_NUMBERS)) {+ const value = source[key];+ if (typeof value === 'number' && Number.isFinite(value) && value >= 0) extra[key] = Math.min(value, max);+ }+ for (const key of ['id', 'keyframe', 'frame']) {+ if (typeof source[key] === 'string' && VISION_IMAGE_ID.test(source[key])) extra[key] = source[key];+ }+ if (Array.isArray(source.ids)) {+ const ids = source.ids.filter(id => typeof id === 'string' && VISION_IMAGE_ID.test(id)).slice(0, 16);+ if (ids.length) extra.ids = ids;+ }+ if (typeof source.dispatched === 'boolean') extra.dispatched = source.dispatched;+ if (typeof source.profile === 'string' && /^[a-z0-9_-]{1,32}$/i.test(source.profile)) extra.profile = source.profile;+ return {+ step: Number.isFinite(data?.step) ? data.step : 0,+ note: data.note,+ ...(Object.keys(extra).length ? { extra } : {}),+ };+}+function pick(source, fields) {if (!source || typeof source !== 'object' || Array.isArray(source)) return {};const output = {};@@ -326,6 +378,7 @@ export function projectTraceEventData(kind, data, { includeContent = false } = {}if (kind === 'error') return pick(source, ['step', 'phase', 'code']);if (kind === 'streaming') return pick(source, STREAMING_METADATA_FIELDS);+ if (kind === 'note' && VISION_NOTES.has(source.note)) return projectVisionNote(source);if (kind === 'note') {const projected = pick(source, ['step', 'note']);const extra = projectNoteExtra(source.extra);diff --git a/test/vision-mode.mjs b/test/vision-mode.mjsindex 32f4a24c3..2ba8e52cb 100644--- a/test/vision-mode.mjs+++ b/test/vision-mode.mjs@@ -557,6 +557,25 @@ test('session: only the newest four zooms stay usable', () => {for (const zoom of zooms.slice(1)) assert.equal(session.resolvePoint({ x: 5, y: 5, image: zoom.id }).ok, true, zoom.id);});+test('traces: metadata traces keep vision notes as vision notes, without error texts', async () => {+ const { projectTraceEventData } = await import('../src/chrome/src/trace/privacy.js');+ assert.equal(read('src/chrome/src/trace/privacy.js'), read('src/firefox/src/trace/privacy.js'));+ const failed = projectTraceEventData('note', { step: 3, note: 'vision_frame_error', extra: { stage: 'observe', error: 'Cannot read https://bank.example/account?id=42' } });+ assert.deepEqual(failed, { step: 3, note: 'vision_frame_error', extra: { stage: 'observe' } });+ const frame = projectTraceEventData('note', { step: 1, note: 'vision_frame', extra: {+ kind: 'keyframe', id: 'f4', reason: 'vision resumed', width: 1356, height: 847, devicePixelRatio: 2, bytes: 81234,+ mimeType: 'image/webp', profile: 'anthropic', regions: [{ x: 1, y: 2, width: 3, height: 4 }], url: 'https://bank.example/',+ } });+ assert.deepEqual(frame.extra, { kind: 'keyframe', reason: 'vision resumed', width: 1356, height: 847, devicePixelRatio: 2, bytes: 81234, mimeType: 'image/webp', id: 'f4', profile: 'anthropic' });+ const skipped = projectTraceEventData('note', { step: 2, note: 'vision_redaction_skipped', extra: { code: 'scan_unavailable', context: 'zoom' } });+ assert.deepEqual(skipped.extra, { code: 'scan_unavailable', context: 'zoom' });+ // Values outside the closed lists are dropped, not passed through.+ assert.equal(projectTraceEventData('note', { step: 2, note: 'vision_frame', extra: { kind: 'secret text', reason: 'typed: hunter2' } }).extra, undefined);+ // The opt-in content tier keeps the note as recorded.+ const raw = { step: 3, note: 'vision_frame_error', extra: { stage: 'observe', error: 'details' } };+ assert.deepEqual(projectTraceEventData('note', raw, { includeContent: true }), raw);+});+test('session: an image value that is not an id is refused, not read as the current screen', () => {const session = new VisionSession();session.registerKeyframe({ imageWidth: 1000, imageHeight: 500, cssWidth: 2000, cssHeight: 1000 });