Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

Avoid replaying historical inline images (#281)

Seth Karten9e0e6961dae8
diff --git a/packages/coding-agent/src/core/export-html/tool-renderer.ts b/packages/coding-agent/src/core/export-html/tool-renderer.ts
index ee4fcb865d..4f6d569efc 100644
--- a/packages/coding-agent/src/core/export-html/tool-renderer.ts
+++ b/packages/coding-agent/src/core/export-html/tool-renderer.ts
@@ -91,6 +91,7 @@ export function createToolHtmlRenderer(deps: ToolHtmlRendererDeps): ToolHtmlRend
isPartial,
expanded,
showImages: false,
+ includeImageDimensions: true,
isError,
};
};
diff --git a/packages/coding-agent/src/core/extensions/types.ts b/packages/coding-agent/src/core/extensions/types.ts
index 180a76b4d6..81d6c34f4a 100644
--- a/packages/coding-agent/src/core/extensions/types.ts
+++ b/packages/coding-agent/src/core/extensions/types.ts
@@ -409,6 +409,8 @@ export interface ToolRenderContext<TState = any, TArgs = any> {
expanded: boolean;
/** Whether inline images are currently shown in the TUI. */
showImages: boolean;
+ /** Whether image fallback labels may parse dimensions from base64 data. */
+ includeImageDimensions: boolean;
/** Whether the current result is an error. */
isError: boolean;
}
diff --git a/packages/coding-agent/src/core/tools/bash.ts b/packages/coding-agent/src/core/tools/bash.ts
index 0e17eb3d10..c4d6fc906c 100644
--- a/packages/coding-agent/src/core/tools/bash.ts
+++ b/packages/coding-agent/src/core/tools/bash.ts
@@ -203,13 +203,14 @@ function rebuildBashResultRenderComponent(
},
options: ToolRenderResultOptions,
showImages: boolean,
+ includeImageDimensions: boolean,
startedAt: number | undefined,
endedAt: number | undefined,
): void {
const state = component.state;
component.clear();
- const output = getTextOutput(result as any, showImages).trim();
+ const output = getTextOutput(result as any, showImages, { includeImageDimensions }).trim();
if (output) {
const styledOutput = output
@@ -436,6 +437,7 @@ export function createBashToolDefinition(
result as any,
options,
context.showImages,
+ context.includeImageDimensions,
state.startedAt,
state.endedAt,
);
diff --git a/packages/coding-agent/src/core/tools/render-utils.ts b/packages/coding-agent/src/core/tools/render-utils.ts
index 8cdfc475f6..5307527fe3 100644
--- a/packages/coding-agent/src/core/tools/render-utils.ts
+++ b/packages/coding-agent/src/core/tools/render-utils.ts
@@ -27,9 +27,15 @@ export function normalizeDisplayText(text: string): string {
return text.replace(/\r/g, "");
}
+export interface TextOutputOptions {
+ /** Whether image fallbacks should parse image dimensions from base64 data. */
+ includeImageDimensions?: boolean;
+}
+
export function getTextOutput(
result: { content: Array<{ type: string; text?: string; data?: string; mimeType?: string }> } | undefined,
showImages: boolean,
+ options: TextOutputOptions = {},
): string {
if (!result) return "";
@@ -39,12 +45,15 @@ export function getTextOutput(
let output = textBlocks.map((c) => sanitizeBinaryOutput(stripAnsi(c.text || "")).replace(/\r/g, "")).join("\n");
const caps = getCapabilities();
+ const includeImageDimensions = options.includeImageDimensions ?? true;
if (imageBlocks.length > 0 && (!caps.images || !showImages)) {
const imageIndicators = imageBlocks
.map((img) => {
const mimeType = img.mimeType ?? "image/unknown";
const dims =
- img.data && img.mimeType ? (getImageDimensions(img.data, img.mimeType) ?? undefined) : undefined;
+ includeImageDimensions && img.data && img.mimeType
+ ? (getImageDimensions(img.data, img.mimeType) ?? undefined)
+ : undefined;
return imageFallback(mimeType, dims);
})
.join("\n");
diff --git a/packages/coding-agent/src/modes/interactive/components/tool-execution.ts b/packages/coding-agent/src/modes/interactive/components/tool-execution.ts
index 2447e5bbd7..902ca95082 100644
--- a/packages/coding-agent/src/modes/interactive/components/tool-execution.ts
+++ b/packages/coding-agent/src/modes/interactive/components/tool-execution.ts
@@ -13,6 +13,12 @@ import { ToolPanel } from "./tool-panel.js";
export interface ToolExecutionOptions {
showImages?: boolean;
imageWidthCells?: number;
+ /**
+ * Whether this component may emit inline terminal image escape sequences.
+ * Historical session replay disables this so loading image-heavy sessions does
+ * not re-send every image in the scrollback.
+ */
+ allowInlineImages?: boolean;
}
export interface ToolExecutionRendererDefinition {
@@ -41,6 +47,7 @@ export class ToolExecutionComponent extends Container {
private args: any;
private expanded = false;
private showImages: boolean;
+ private allowInlineImages: boolean;
private imageWidthCells: number;
private isPartial = true;
private toolDefinition?: ToolExecutionDefinition;
@@ -73,6 +80,7 @@ export class ToolExecutionComponent extends Container {
this.toolDefinition = toolDefinition;
this.builtInToolDefinition = createAllToolDefinitions(cwd)[toolName as ToolName];
this.showImages = options.showImages ?? true;
+ this.allowInlineImages = options.allowInlineImages ?? true;
this.imageWidthCells = options.imageWidthCells ?? 60;
this.ui = ui;
this.cwd = cwd;
@@ -136,7 +144,12 @@ export class ToolExecutionComponent extends Container {
return this.toolName === "ipython" && !this.toolDefinition?.renderCall && !this.toolDefinition?.renderResult;
}
+ private shouldRenderInlineImages(): boolean {
+ return this.showImages && this.allowInlineImages;
+ }
+
private getRenderContext(lastComponent: Component | undefined): ToolRenderContext {
+ const renderInlineImages = this.shouldRenderInlineImages();
return {
args: this.args,
toolCallId: this.toolCallId,
@@ -151,7 +164,8 @@ export class ToolExecutionComponent extends Container {
argsComplete: this.argsComplete,
isPartial: this.isPartial,
expanded: this.expanded,
- showImages: this.showImages,
+ showImages: renderInlineImages,
+ includeImageDimensions: this.allowInlineImages,
isError: this.result?.isError ?? false,
};
}
@@ -200,6 +214,7 @@ export class ToolExecutionComponent extends Container {
}
private maybeConvertImagesForKitty(): void {
+ if (!this.allowInlineImages) return;
const caps = getCapabilities();
if (caps.images !== "kitty") return;
if (!this.result) return;
@@ -229,6 +244,17 @@ export class ToolExecutionComponent extends Container {
setShowImages(show: boolean): void {
this.showImages = show;
+ if (show) {
+ this.maybeConvertImagesForKitty();
+ }
+ this.updateDisplay();
+ }
+
+ setAllowInlineImages(allow: boolean): void {
+ this.allowInlineImages = allow;
+ if (allow) {
+ this.maybeConvertImagesForKitty();
+ }
this.updateDisplay();
}
@@ -270,6 +296,7 @@ export class ToolExecutionComponent extends Container {
}
private updateDisplay(): void {
+ const renderInlineImages = this.shouldRenderInlineImages();
let hasContent = false;
this.hideComponent = false;
if (this.hasRendererDefinition() && this.getRenderShell() === "self") {
@@ -285,7 +312,7 @@ export class ToolExecutionComponent extends Container {
expanded: this.expanded,
executionStarted: this.executionStarted,
argsComplete: this.argsComplete,
- showImages: this.showImages,
+ showImages: renderInlineImages,
cwd: this.cwd,
};
if (!this.ipythonCellComponent) {
@@ -328,7 +355,7 @@ export class ToolExecutionComponent extends Container {
const caps = getCapabilities();
for (let i = 0; i < imageBlocks.length; i++) {
const img = imageBlocks[i];
- if (caps.images && this.showImages && img.data && img.mimeType) {
+ if (caps.images && renderInlineImages && img.data && img.mimeType) {
const converted = this.convertedImages.get(i);
const imageData = converted?.data ?? img.data;
const imageMimeType = converted?.mimeType ?? img.mimeType;
@@ -435,7 +462,9 @@ export class ToolExecutionComponent extends Container {
}
private getTextOutput(): string {
- return getRenderedTextOutput(this.result, this.showImages);
+ return getRenderedTextOutput(this.result, this.shouldRenderInlineImages(), {
+ includeImageDimensions: this.allowInlineImages,
+ });
}
private formatToolExecution(): string {
diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts
index a8e06854d1..e7be4af83b 100644
--- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts
+++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts
@@ -4928,6 +4928,8 @@ export class InteractiveMode {
{
showImages: this.settingsManager.getShowImages(),
imageWidthCells: this.settingsManager.getImageWidthCells(),
+ // Do not replay historical inline image payloads on session load/rebuild.
+ allowInlineImages: false,
},
this.getCachedToolDefinition(content.name),
this.ui,
@@ -4969,6 +4971,8 @@ export class InteractiveMode {
}
for (const [toolCallId, component] of renderedPendingTools) {
+ // These tool calls have no historical result yet, so future updates are live output.
+ component.setAllowInlineImages(true);
this.pendingTools.set(toolCallId, component);
}
this.ui.requestRender();
diff --git a/packages/coding-agent/test/interactive-mode-status.test.ts b/packages/coding-agent/test/interactive-mode-status.test.ts
index cba191f954..d80c48eb78 100644
--- a/packages/coding-agent/test/interactive-mode-status.test.ts
+++ b/packages/coding-agent/test/interactive-mode-status.test.ts
@@ -4,6 +4,8 @@ import {
type AutocompleteProvider,
CombinedAutocompleteProvider,
Container,
+ resetCapabilitiesCache,
+ setCapabilities,
visibleWidth,
} from "@earendil-works/pi-tui";
import { beforeAll, describe, expect, test, vi } from "vitest";
@@ -169,6 +171,109 @@ describe("InteractiveMode.showStatus", () => {
});
});
+describe("InteractiveMode.renderSessionContext", () => {
+ beforeAll(() => {
+ initTheme("dark");
+ });
+
+ test("renders historical tool result images as fallbacks instead of replaying inline payloads", async () => {
+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+ try {
+ const chatContainer = new Container();
+ const fakeThis: any = {
+ pendingTools: new Map(),
+ toolOutputExpanded: false,
+ chatContainer,
+ footer: { invalidate: vi.fn() },
+ updateEditorBorderColor: vi.fn(),
+ resetPendingToolState: vi.fn(),
+ preloadToolDefinitions: vi.fn(async () => {}),
+ settingsManager: {
+ getShowImages: () => true,
+ getImageWidthCells: () => 60,
+ },
+ getCachedToolDefinition: () => undefined,
+ getCurrentCwd: () => process.cwd(),
+ getRetryAttempt: () => 0,
+ ui: { requestRender: vi.fn() },
+ addMessageToChat: vi.fn(() => {
+ chatContainer.addChild({ render: () => ["assistant"], invalidate: () => {} });
+ }),
+ };
+
+ await (InteractiveMode as any).prototype.renderSessionContext.call(fakeThis, {
+ messages: [
+ {
+ role: "assistant",
+ content: [{ type: "toolCall", name: "custom_tool", id: "tool-1", arguments: {} }],
+ },
+ {
+ role: "toolResult",
+ toolCallId: "tool-1",
+ content: [{ type: "image", data: "AAAA", mimeType: "image/png" }],
+ isError: false,
+ },
+ ],
+ thinkingLevel: "medium",
+ model: null,
+ });
+
+ const rendered = renderAll(chatContainer);
+ expect(rendered).not.toContain("\x1b_G");
+ expect(normalizeRenderedOutput(chatContainer)).toContain("[Image: [image/png]]");
+ } finally {
+ resetCapabilitiesCache();
+ }
+ });
+
+ test("keeps pending history tool calls eligible for live inline image updates", async () => {
+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+ try {
+ const chatContainer = new Container();
+ const pendingTools = new Map<string, ToolExecutionComponent>();
+ const fakeThis: any = {
+ pendingTools,
+ toolOutputExpanded: false,
+ chatContainer,
+ footer: { invalidate: vi.fn() },
+ updateEditorBorderColor: vi.fn(),
+ resetPendingToolState: vi.fn(),
+ preloadToolDefinitions: vi.fn(async () => {}),
+ settingsManager: {
+ getShowImages: () => true,
+ getImageWidthCells: () => 60,
+ },
+ getCachedToolDefinition: () => undefined,
+ getCurrentCwd: () => process.cwd(),
+ getRetryAttempt: () => 0,
+ ui: { requestRender: vi.fn() },
+ addMessageToChat: vi.fn(() => {
+ chatContainer.addChild({ render: () => ["assistant"], invalidate: () => {} });
+ }),
+ };
+
+ await (InteractiveMode as any).prototype.renderSessionContext.call(fakeThis, {
+ messages: [
+ {
+ role: "assistant",
+ content: [{ type: "toolCall", name: "custom_tool", id: "tool-1", arguments: {} }],
+ },
+ ],
+ thinkingLevel: "medium",
+ model: null,
+ });
+
+ const component = pendingTools.get("tool-1");
+ expect(component).toBeDefined();
+ component!.updateResult({ content: [{ type: "image", data: "AAAA", mimeType: "image/png" }], isError: false });
+
+ expect(component!.render(120).join("\n")).toContain("\x1b_G");
+ } finally {
+ resetCapabilitiesCache();
+ }
+ });
+});
+
type SubmitHandlerHarness = {
defaultEditor: { onSubmit?: (text: string) => Promise<void> };
editor: { setText: (text: string) => void; addToHistory?: (text: string) => void };
diff --git a/packages/coding-agent/test/tool-execution-component.test.ts b/packages/coding-agent/test/tool-execution-component.test.ts
index 42d0bf3247..ffdc89c4e2 100644
--- a/packages/coding-agent/test/tool-execution-component.test.ts
+++ b/packages/coding-agent/test/tool-execution-component.test.ts
@@ -1,4 +1,4 @@
-import { Text, type TUI } from "@earendil-works/pi-tui";
+import { resetCapabilitiesCache, setCapabilities, Text, type TUI } from "@earendil-works/pi-tui";
import stripAnsi from "strip-ansi";
import { Type } from "typebox";
import { beforeAll, describe, expect, test } from "vitest";
@@ -28,6 +28,14 @@ function createFakeTui(): TUI {
} as unknown as TUI;
}
+async function waitForCondition(condition: () => boolean): Promise<void> {
+ for (let i = 0; i < 50; i++) {
+ if (condition()) return;
+ await new Promise((resolve) => setTimeout(resolve, 10));
+ }
+ throw new Error("condition was not met");
+}
+
describe("ToolExecutionComponent parity", () => {
beforeAll(() => {
initTheme("dark");
@@ -65,6 +73,131 @@ describe("ToolExecutionComponent parity", () => {
expect(rendered).toContain("custom result");
});
+ test("renders inline images by default when terminal supports them", () => {
+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+ try {
+ const component = new ToolExecutionComponent(
+ "custom_tool",
+ "tool-image-default",
+ {},
+ { showImages: true },
+ undefined,
+ createFakeTui(),
+ process.cwd(),
+ );
+ component.updateResult({ content: [{ type: "image", data: "AAAA", mimeType: "image/png" }], isError: false });
+
+ expect(component.render(120).join("\n")).toContain("\x1b_G");
+ } finally {
+ resetCapabilitiesCache();
+ }
+ });
+
+ test("can suppress inline image escape sequences for session history", () => {
+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+ try {
+ const component = new ToolExecutionComponent(
+ "custom_tool",
+ "tool-image-history",
+ {},
+ { showImages: true, allowInlineImages: false },
+ undefined,
+ createFakeTui(),
+ process.cwd(),
+ );
+ component.updateResult({ content: [{ type: "image", data: "AAAA", mimeType: "image/png" }], isError: false });
+
+ let rendered = component.render(120).join("\n");
+ expect(rendered).not.toContain("\x1b_G");
+ expect(stripAnsi(rendered)).toContain("[Image: [image/png]]");
+
+ component.setShowImages(false);
+ component.setShowImages(true);
+ rendered = component.render(120).join("\n");
+ expect(rendered).not.toContain("\x1b_G");
+ } finally {
+ resetCapabilitiesCache();
+ }
+ });
+
+ test("suppressed history image fallbacks skip dimension parsing", () => {
+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+ try {
+ const onePixelPng =
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=";
+ const component = new ToolExecutionComponent(
+ "custom_tool",
+ "tool-image-history-dims",
+ {},
+ { showImages: true, allowInlineImages: false },
+ undefined,
+ createFakeTui(),
+ process.cwd(),
+ );
+ component.updateResult({
+ content: [{ type: "image", data: onePixelPng, mimeType: "image/png" }],
+ isError: false,
+ });
+
+ const rendered = stripAnsi(component.render(120).join("\n"));
+ expect(rendered).toContain("[Image: [image/png]]");
+ expect(rendered).not.toContain("1x1");
+ } finally {
+ resetCapabilitiesCache();
+ }
+ });
+
+ test("pending history components can be restored to live inline image rendering", () => {
+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+ try {
+ const component = new ToolExecutionComponent(
+ "custom_tool",
+ "tool-image-pending-history",
+ {},
+ { showImages: true, allowInlineImages: false },
+ undefined,
+ createFakeTui(),
+ process.cwd(),
+ );
+ component.updateResult({ content: [{ type: "image", data: "AAAA", mimeType: "image/png" }], isError: false });
+ expect(component.render(120).join("\n")).not.toContain("\x1b_G");
+
+ component.setAllowInlineImages(true);
+
+ expect(component.render(120).join("\n")).toContain("\x1b_G");
+ } finally {
+ resetCapabilitiesCache();
+ }
+ });
+
+ test("converts kitty images even if image display is toggled off when the result arrives", async () => {
+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });
+ try {
+ const tinyJpeg =
+ "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAACAAIDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/xAAVAQEBAAAAAAAAAAAAAAAAAAAGCf/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AD3VTB3/2Q==";
+ const component = new ToolExecutionComponent(
+ "custom_tool",
+ "tool-image-hidden-conversion",
+ {},
+ { showImages: false },
+ undefined,
+ createFakeTui(),
+ process.cwd(),
+ );
+ component.updateResult({
+ content: [{ type: "image", data: tinyJpeg, mimeType: "image/jpeg" }],
+ isError: false,
+ });
+
+ await waitForCondition(() => (component as any).convertedImages.size === 1);
+ component.setShowImages(true);
+
+ expect(component.render(120).join("\n")).toContain("\x1b_G");
+ } finally {
+ resetCapabilitiesCache();
+ }
+ });
+
test("uses built-in rendering for built-in overrides without custom renderers", () => {
const overrideDefinition: ToolDefinition = {
...createBaseToolDefinition("edit"),