Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Preserve screenshot descriptions in UI delivery and replay

Codex release-state test49ea3c43b0a4
diff --git a/src/chrome/src/run-ui-journal.js b/src/chrome/src/run-ui-journal.js
index 0fbb47708..829f633bb 100644
--- a/src/chrome/src/run-ui-journal.js
+++ b/src/chrome/src/run-ui-journal.js
@@ -78,6 +78,19 @@ function compactRunUiToolResult(result, name) {
compacted.textTruncated = true;
}
}
+ if (['auto_screenshot', 'staged_screenshot'].includes(name)) {
+ // Preserve the vision output for live details and replay, without retaining
+ // image bytes or allowing a single capture to exhaust the journal budget.
+ if (typeof source.description === 'string') {
+ compacted.description = source.description.slice(0, 12000);
+ if (source.descriptionTruncated === true || source.description.length > 12000) {
+ compacted.descriptionTruncated = true;
+ }
+ }
+ for (const key of ['method', 'model', 'visionRoute']) {
+ if (typeof source[key] === 'string') compacted[key] = source[key].slice(0, 300);
+ }
+ }
if (source.truncated === true) compacted.truncated = true;
if (source.hasMore === true) compacted.hasMore = true;
if (source.notice) compacted.notice = String(source.notice).slice(0, 300);
diff --git a/src/chrome/src/ui/sidepanel.js b/src/chrome/src/ui/sidepanel.js
index 454ee0fa2..73e46fae9 100644
--- a/src/chrome/src/ui/sidepanel.js
+++ b/src/chrome/src/ui/sidepanel.js
@@ -12142,7 +12142,7 @@ function markLastStepDone(toolName, result) {
if (details && details.classList.contains('step-details')) {
const resultDiv = document.createElement('div');
resultDiv.className = 'detail-result';
- resultDiv.innerHTML = `<div class="detail-label">${escapeHtml(t('sp.step.result_label'))}</div>${escapeHtml(truncate(JSON.stringify(result), 1200))}`;
+ resultDiv.innerHTML = `<div class="detail-label">${escapeHtml(t('sp.step.result_label'))}</div>${escapeHtml((['auto_screenshot', 'staged_screenshot'].includes(toolName) && typeof result?.description === 'string' ? JSON.stringify(result, null, 2) : truncate(JSON.stringify(result), 1200)))}`;
const expandAll = details.querySelector?.('.step-expand-all') || null;
if (expandAll && typeof details.insertBefore === 'function') details.insertBefore(resultDiv, expandAll);
else details.appendChild(resultDiv);
@@ -12238,7 +12238,7 @@ function appendVerboseToolResult(name, result) {
const resultEl = document.createElement('div');
resultEl.className = 'tool-result';
- resultEl.textContent = truncate(JSON.stringify(result), 200);
+ resultEl.textContent = (['auto_screenshot', 'staged_screenshot'].includes(name) && typeof result?.description === 'string' ? JSON.stringify(result, null, 2) : truncate(JSON.stringify(result), 200));
lastTool.appendChild(resultEl);
if (name === 'done') {
diff --git a/src/firefox/src/run-ui-journal.js b/src/firefox/src/run-ui-journal.js
index 0fbb47708..829f633bb 100644
--- a/src/firefox/src/run-ui-journal.js
+++ b/src/firefox/src/run-ui-journal.js
@@ -78,6 +78,19 @@ function compactRunUiToolResult(result, name) {
compacted.textTruncated = true;
}
}
+ if (['auto_screenshot', 'staged_screenshot'].includes(name)) {
+ // Preserve the vision output for live details and replay, without retaining
+ // image bytes or allowing a single capture to exhaust the journal budget.
+ if (typeof source.description === 'string') {
+ compacted.description = source.description.slice(0, 12000);
+ if (source.descriptionTruncated === true || source.description.length > 12000) {
+ compacted.descriptionTruncated = true;
+ }
+ }
+ for (const key of ['method', 'model', 'visionRoute']) {
+ if (typeof source[key] === 'string') compacted[key] = source[key].slice(0, 300);
+ }
+ }
if (source.truncated === true) compacted.truncated = true;
if (source.hasMore === true) compacted.hasMore = true;
if (source.notice) compacted.notice = String(source.notice).slice(0, 300);
diff --git a/src/firefox/src/ui/sidepanel.js b/src/firefox/src/ui/sidepanel.js
index 2372806df..491975819 100644
--- a/src/firefox/src/ui/sidepanel.js
+++ b/src/firefox/src/ui/sidepanel.js
@@ -11786,7 +11786,7 @@ function markLastStepDone(toolName, result) {
if (details && details.classList.contains('step-details')) {
const resultDiv = document.createElement('div');
resultDiv.className = 'detail-result';
- resultDiv.innerHTML = `<div class="detail-label">${escapeHtml(t('sp.step.result_label'))}</div>${escapeHtml(truncate(JSON.stringify(result), 1200))}`;
+ resultDiv.innerHTML = `<div class="detail-label">${escapeHtml(t('sp.step.result_label'))}</div>${escapeHtml((['auto_screenshot', 'staged_screenshot'].includes(toolName) && typeof result?.description === 'string' ? JSON.stringify(result, null, 2) : truncate(JSON.stringify(result), 1200)))}`;
const expandAll = details.querySelector?.('.step-expand-all') || null;
if (expandAll && typeof details.insertBefore === 'function') details.insertBefore(resultDiv, expandAll);
else details.appendChild(resultDiv);
@@ -11882,7 +11882,7 @@ function appendVerboseToolResult(name, result) {
const resultEl = document.createElement('div');
resultEl.className = 'tool-result';
- resultEl.textContent = truncate(JSON.stringify(result), 200);
+ resultEl.textContent = (['auto_screenshot', 'staged_screenshot'].includes(name) && typeof result?.description === 'string' ? JSON.stringify(result, null, 2) : truncate(JSON.stringify(result), 200));
lastTool.appendChild(resultEl);
if (name === 'done') {
diff --git a/test/screenshot-description-ui.mjs b/test/screenshot-description-ui.mjs
new file mode 100644
index 000000000..ce894be06
--- /dev/null
+++ b/test/screenshot-description-ui.mjs
@@ -0,0 +1,55 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { chromium, firefox } from 'playwright';
+
+for (const [build, engine] of [['chrome', chromium], ['firefox', firefox]]) {
+ const { RunUiJournal, compactRunUiData, compactRunUiSnapshotForPersist } = await import(`../src/${build}/src/run-ui-journal.js`);
+ for (const name of ['auto_screenshot', 'staged_screenshot']) {
+ test(`${build}: ${name} description survives delivery and persistence`, async () => {
+ const description = 'Visible page: a search box and navigation. '.repeat(70) + 'END OF DESCRIPTION';
+ const journal = new RunUiJournal();
+ journal.begin(7, 'capture');
+ const event = journal.record(7, 'capture', 'tool_result', { name, result: {
+ success: true, description, method: 'vision_describe', model: 'fixture-vision', visionRoute: 'dedicated', dataUrl: 'data:image/png;base64,PRIVATE',
+ } });
+ assert.equal(event.data.result.description, description);
+ assert.equal(event.data.result.dataUrl, undefined);
+ const restored = new RunUiJournal();
+ restored.restore(7, JSON.parse(JSON.stringify(compactRunUiSnapshotForPersist(journal.get(7), { tight: true }))));
+ assert.deepEqual(restored.get(7).events[0].data, event.data);
+ const source = await readFile(`src/${build}/src/ui/sidepanel.js`, 'utf8');
+ const compact = source.slice(source.indexOf('function markLastStepDone('), source.indexOf('\nfunction markLastStepFailed('));
+ const verbose = source.slice(source.indexOf('function appendVerboseToolResult('), source.indexOf('\nfunction ', source.indexOf('function appendVerboseToolResult(') + 1));
+ if (process.env.WEBBRAIN_SCREENSHOT_UI !== '1') return;
+ const browser = await engine.launch();
+ try {
+ const page = await browser.newPage();
+ for (const mode of ['compact', 'verbose']) {
+ const text = await page.evaluate(({ compact, verbose, name, result, mode }) => {
+ document.body.innerHTML = '<div class="message-content"><div class="step-item active"><span class="step-icon"></span></div><div class="step-details"></div><div class="tool-call" data-awaiting-result="true"><div class="tool-call-name"></div></div></div>';
+ const currentAssistantEl = document.body;
+ document.querySelector('.tool-call').dataset.toolName = name;
+ const findLastActiveCompactStep = () => document.querySelector('.step-item.active');
+ const isTerminalDoneTool = () => false;
+ const restoreStepFriendlyLabel = () => {};
+ const t = key => key;
+ const escapeHtml = text => { const el = document.createElement('span'); el.textContent = text; return el.innerHTML; };
+ const truncate = (text, length) => text.slice(0, length);
+ eval(`${compact}\n${verbose}\n${mode === 'compact' ? 'markLastStepDone' : 'appendVerboseToolResult'}(name, result);`);
+ return document.querySelector(mode === 'compact' ? '.detail-result' : '.tool-result').textContent;
+ }, { compact, verbose, name, result: event.data.result, mode });
+ assert.ok(text.includes('END OF DESCRIPTION'), `${mode} must show the description beyond the generic cutoff`);
+ assert.ok(text.includes('fixture-vision'));
+ }
+ } finally { await browser.close(); }
+ });
+ }
+ test(`${build}: screenshot descriptions stay bounded through repeated compaction`, () => {
+ const data = { name: 'auto_screenshot', result: { success: true, description: 'x'.repeat(13000) } };
+ const once = compactRunUiData('tool_result', data);
+ assert.equal(once.result.description.length, 12000);
+ assert.equal(once.result.descriptionTruncated, true);
+ assert.deepEqual(compactRunUiData('tool_result', once), once);
+ });
+}