Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Preserve screenshot descriptions in UI delivery and replay
Codex release-state test49ea3c43b0a4
diff --git a/src/chrome/src/run-ui-journal.js b/src/chrome/src/run-ui-journal.jsindex 0fbb47708..829f633bb 100644--- a/src/chrome/src/run-ui-journal.js+++ b/src/chrome/src/run-ui-journal.js@@ -78,6 +78,19 @@ function compactRunUiToolResult(result, name) {compacted.textTruncated = true;}}+ if (['auto_screenshot', 'staged_screenshot'].includes(name)) {+ // Preserve the vision output for live details and replay, without retaining+ // image bytes or allowing a single capture to exhaust the journal budget.+ if (typeof source.description === 'string') {+ compacted.description = source.description.slice(0, 12000);+ if (source.descriptionTruncated === true || source.description.length > 12000) {+ compacted.descriptionTruncated = true;+ }+ }+ for (const key of ['method', 'model', 'visionRoute']) {+ if (typeof source[key] === 'string') compacted[key] = source[key].slice(0, 300);+ }+ }if (source.truncated === true) compacted.truncated = true;if (source.hasMore === true) compacted.hasMore = true;if (source.notice) compacted.notice = String(source.notice).slice(0, 300);diff --git a/src/chrome/src/ui/sidepanel.js b/src/chrome/src/ui/sidepanel.jsindex 454ee0fa2..73e46fae9 100644--- a/src/chrome/src/ui/sidepanel.js+++ b/src/chrome/src/ui/sidepanel.js@@ -12142,7 +12142,7 @@ function markLastStepDone(toolName, result) {if (details && details.classList.contains('step-details')) {const resultDiv = document.createElement('div');resultDiv.className = 'detail-result';- resultDiv.innerHTML = `<div class="detail-label">${escapeHtml(t('sp.step.result_label'))}</div>${escapeHtml(truncate(JSON.stringify(result), 1200))}`;+ resultDiv.innerHTML = `<div class="detail-label">${escapeHtml(t('sp.step.result_label'))}</div>${escapeHtml((['auto_screenshot', 'staged_screenshot'].includes(toolName) && typeof result?.description === 'string' ? JSON.stringify(result, null, 2) : truncate(JSON.stringify(result), 1200)))}`;const expandAll = details.querySelector?.('.step-expand-all') || null;if (expandAll && typeof details.insertBefore === 'function') details.insertBefore(resultDiv, expandAll);else details.appendChild(resultDiv);@@ -12238,7 +12238,7 @@ function appendVerboseToolResult(name, result) {const resultEl = document.createElement('div');resultEl.className = 'tool-result';- resultEl.textContent = truncate(JSON.stringify(result), 200);+ resultEl.textContent = (['auto_screenshot', 'staged_screenshot'].includes(name) && typeof result?.description === 'string' ? JSON.stringify(result, null, 2) : truncate(JSON.stringify(result), 200));lastTool.appendChild(resultEl);if (name === 'done') {diff --git a/src/firefox/src/run-ui-journal.js b/src/firefox/src/run-ui-journal.jsindex 0fbb47708..829f633bb 100644--- a/src/firefox/src/run-ui-journal.js+++ b/src/firefox/src/run-ui-journal.js@@ -78,6 +78,19 @@ function compactRunUiToolResult(result, name) {compacted.textTruncated = true;}}+ if (['auto_screenshot', 'staged_screenshot'].includes(name)) {+ // Preserve the vision output for live details and replay, without retaining+ // image bytes or allowing a single capture to exhaust the journal budget.+ if (typeof source.description === 'string') {+ compacted.description = source.description.slice(0, 12000);+ if (source.descriptionTruncated === true || source.description.length > 12000) {+ compacted.descriptionTruncated = true;+ }+ }+ for (const key of ['method', 'model', 'visionRoute']) {+ if (typeof source[key] === 'string') compacted[key] = source[key].slice(0, 300);+ }+ }if (source.truncated === true) compacted.truncated = true;if (source.hasMore === true) compacted.hasMore = true;if (source.notice) compacted.notice = String(source.notice).slice(0, 300);diff --git a/src/firefox/src/ui/sidepanel.js b/src/firefox/src/ui/sidepanel.jsindex 2372806df..491975819 100644--- a/src/firefox/src/ui/sidepanel.js+++ b/src/firefox/src/ui/sidepanel.js@@ -11786,7 +11786,7 @@ function markLastStepDone(toolName, result) {if (details && details.classList.contains('step-details')) {const resultDiv = document.createElement('div');resultDiv.className = 'detail-result';- resultDiv.innerHTML = `<div class="detail-label">${escapeHtml(t('sp.step.result_label'))}</div>${escapeHtml(truncate(JSON.stringify(result), 1200))}`;+ resultDiv.innerHTML = `<div class="detail-label">${escapeHtml(t('sp.step.result_label'))}</div>${escapeHtml((['auto_screenshot', 'staged_screenshot'].includes(toolName) && typeof result?.description === 'string' ? JSON.stringify(result, null, 2) : truncate(JSON.stringify(result), 1200)))}`;const expandAll = details.querySelector?.('.step-expand-all') || null;if (expandAll && typeof details.insertBefore === 'function') details.insertBefore(resultDiv, expandAll);else details.appendChild(resultDiv);@@ -11882,7 +11882,7 @@ function appendVerboseToolResult(name, result) {const resultEl = document.createElement('div');resultEl.className = 'tool-result';- resultEl.textContent = truncate(JSON.stringify(result), 200);+ resultEl.textContent = (['auto_screenshot', 'staged_screenshot'].includes(name) && typeof result?.description === 'string' ? JSON.stringify(result, null, 2) : truncate(JSON.stringify(result), 200));lastTool.appendChild(resultEl);if (name === 'done') {diff --git a/test/screenshot-description-ui.mjs b/test/screenshot-description-ui.mjsnew file mode 100644index 000000000..ce894be06--- /dev/null+++ b/test/screenshot-description-ui.mjs@@ -0,0 +1,55 @@+import test from 'node:test';+import assert from 'node:assert/strict';+import { readFile } from 'node:fs/promises';+import { chromium, firefox } from 'playwright';++for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {+ const { RunUiJournal, compactRunUiData, compactRunUiSnapshotForPersist } = await import(`../src/${build}/src/run-ui-journal.js`);+ for (const name of ['auto_screenshot', 'staged_screenshot']) {+ test(`${build}: ${name} description survives delivery and persistence`, async () => {+ const description = 'Visible page: a search box and navigation. '.repeat(70) + 'END OF DESCRIPTION';+ const journal = new RunUiJournal();+ journal.begin(7, 'capture');+ const event = journal.record(7, 'capture', 'tool_result', { name, result: {+ success: true, description, method: 'vision_describe', model: 'fixture-vision', visionRoute: 'dedicated', dataUrl: 'data:image/png;base64,PRIVATE',+ } });+ assert.equal(event.data.result.description, description);+ assert.equal(event.data.result.dataUrl, undefined);+ const restored = new RunUiJournal();+ restored.restore(7, JSON.parse(JSON.stringify(compactRunUiSnapshotForPersist(journal.get(7), { tight: true }))));+ assert.deepEqual(restored.get(7).events[0].data, event.data);+ const source = await readFile(`src/${build}/src/ui/sidepanel.js`, 'utf8');+ const compact = source.slice(source.indexOf('function markLastStepDone('), source.indexOf('\nfunction markLastStepFailed('));+ const verbose = source.slice(source.indexOf('function appendVerboseToolResult('), source.indexOf('\nfunction ', source.indexOf('function appendVerboseToolResult(') + 1));+ if (process.env.WEBBRAIN_SCREENSHOT_UI !== '1') return;+ const browser = await engine.launch();+ try {+ const page = await browser.newPage();+ for (const mode of ['compact', 'verbose']) {+ const text = await page.evaluate(({ compact, verbose, name, result, mode }) => {+ document.body.innerHTML = '<div class="message-content"><div class="step-item active"><span class="step-icon"></span></div><div class="step-details"></div><div class="tool-call" data-awaiting-result="true"><div class="tool-call-name"></div></div></div>';+ const currentAssistantEl = document.body;+ document.querySelector('.tool-call').dataset.toolName = name;+ const findLastActiveCompactStep = () => document.querySelector('.step-item.active');+ const isTerminalDoneTool = () => false;+ const restoreStepFriendlyLabel = () => {};+ const t = key => key;+ const escapeHtml = text => { const el = document.createElement('span'); el.textContent = text; return el.innerHTML; };+ const truncate = (text, length) => text.slice(0, length);+ eval(`${compact}\n${verbose}\n${mode === 'compact' ? 'markLastStepDone' : 'appendVerboseToolResult'}(name, result);`);+ return document.querySelector(mode === 'compact' ? '.detail-result' : '.tool-result').textContent;+ }, { compact, verbose, name, result: event.data.result, mode });+ assert.ok(text.includes('END OF DESCRIPTION'), `${mode} must show the description beyond the generic cutoff`);+ assert.ok(text.includes('fixture-vision'));+ }+ } finally { await browser.close(); }+ });+ }+ test(`${build}: screenshot descriptions stay bounded through repeated compaction`, () => {+ const data = { name: 'auto_screenshot', result: { success: true, description: 'x'.repeat(13000) } };+ const once = compactRunUiData('tool_result', data);+ assert.equal(once.result.description.length, 12000);+ assert.equal(once.result.descriptionTruncated, true);+ assert.deepEqual(compactRunUiData('tool_result', once), once);+ });+}