A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
Avoid replaying historical inline images (#281)
Seth Karten9e0e6961dae8
diff --git a/packages/coding-agent/src/core/export-html/tool-renderer.ts b/packages/coding-agent/src/core/export-html/tool-renderer.tsindex ee4fcb865d..4f6d569efc 100644--- a/packages/coding-agent/src/core/export-html/tool-renderer.ts+++ b/packages/coding-agent/src/core/export-html/tool-renderer.ts@@ -91,6 +91,7 @@ export function createToolHtmlRenderer(deps: ToolHtmlRendererDeps): ToolHtmlRendisPartial,expanded,showImages: false,+ includeImageDimensions: true,isError,};};diff --git a/packages/coding-agent/src/core/extensions/types.ts b/packages/coding-agent/src/core/extensions/types.tsindex 180a76b4d6..81d6c34f4a 100644--- a/packages/coding-agent/src/core/extensions/types.ts+++ b/packages/coding-agent/src/core/extensions/types.ts@@ -409,6 +409,8 @@ export interface ToolRenderContext<TState = any, TArgs = any> {expanded: boolean;/** Whether inline images are currently shown in the TUI. */showImages: boolean;+ /** Whether image fallback labels may parse dimensions from base64 data. */+ includeImageDimensions: boolean;/** Whether the current result is an error. */isError: boolean;}diff --git a/packages/coding-agent/src/core/tools/bash.ts b/packages/coding-agent/src/core/tools/bash.tsindex 0e17eb3d10..c4d6fc906c 100644--- a/packages/coding-agent/src/core/tools/bash.ts+++ b/packages/coding-agent/src/core/tools/bash.ts@@ -203,13 +203,14 @@ function rebuildBashResultRenderComponent(},options: ToolRenderResultOptions,showImages: boolean,+ includeImageDimensions: boolean,startedAt: number | undefined,endedAt: number | undefined,): void {const state = component.state;component.clear();- const output = getTextOutput(result as any, showImages).trim();+ const output = getTextOutput(result as any, showImages, { includeImageDimensions }).trim();if (output) {const styledOutput = output@@ -436,6 +437,7 @@ export function createBashToolDefinition(result as any,options,context.showImages,+ context.includeImageDimensions,state.startedAt,state.endedAt,);diff --git a/packages/coding-agent/src/core/tools/render-utils.ts b/packages/coding-agent/src/core/tools/render-utils.tsindex 8cdfc475f6..5307527fe3 100644--- a/packages/coding-agent/src/core/tools/render-utils.ts+++ b/packages/coding-agent/src/core/tools/render-utils.ts@@ -27,9 +27,15 @@ export function normalizeDisplayText(text: string): string {return text.replace(/\r/g, "");}+export interface TextOutputOptions {+ /** Whether image fallbacks should parse image dimensions from base64 data. */+ includeImageDimensions?: boolean;+}+export function getTextOutput(result: { content: Array<{ type: string; text?: string; data?: string; mimeType?: string }> } | undefined,showImages: boolean,+ options: TextOutputOptions = {},): string {if (!result) return "";@@ -39,12 +45,15 @@ export function getTextOutput(let output = textBlocks.map((c) => sanitizeBinaryOutput(stripAnsi(c.text || "")).replace(/\r/g, "")).join("\n");const caps = getCapabilities();+ const includeImageDimensions = options.includeImageDimensions ?? true;if (imageBlocks.length > 0 && (!caps.images || !showImages)) {const imageIndicators = imageBlocks.map((img) => {const mimeType = img.mimeType ?? "image/unknown";const dims =- img.data && img.mimeType ? (getImageDimensions(img.data, img.mimeType) ?? undefined) : undefined;+ includeImageDimensions && img.data && img.mimeType+ ? (getImageDimensions(img.data, img.mimeType) ?? undefined)+ : undefined;return imageFallback(mimeType, dims);}).join("\n");diff --git a/packages/coding-agent/src/modes/interactive/components/tool-execution.ts b/packages/coding-agent/src/modes/interactive/components/tool-execution.tsindex 2447e5bbd7..902ca95082 100644--- a/packages/coding-agent/src/modes/interactive/components/tool-execution.ts+++ b/packages/coding-agent/src/modes/interactive/components/tool-execution.ts@@ -13,6 +13,12 @@ import { ToolPanel } from "./tool-panel.js";export interface ToolExecutionOptions {showImages?: boolean;imageWidthCells?: number;+ /**+ * Whether this component may emit inline terminal image escape sequences.+ * Historical session replay disables this so loading image-heavy sessions does+ * not re-send every image in the scrollback.+ */+ allowInlineImages?: boolean;}export interface ToolExecutionRendererDefinition {@@ -41,6 +47,7 @@ export class ToolExecutionComponent extends Container {private args: any;private expanded = false;private showImages: boolean;+ private allowInlineImages: boolean;private imageWidthCells: number;private isPartial = true;private toolDefinition?: ToolExecutionDefinition;@@ -73,6 +80,7 @@ export class ToolExecutionComponent extends Container {this.toolDefinition = toolDefinition;this.builtInToolDefinition = createAllToolDefinitions(cwd)[toolName as ToolName];this.showImages = options.showImages ?? true;+ this.allowInlineImages = options.allowInlineImages ?? true;this.imageWidthCells = options.imageWidthCells ?? 60;this.ui = ui;this.cwd = cwd;@@ -136,7 +144,12 @@ export class ToolExecutionComponent extends Container {return this.toolName === "ipython" && !this.toolDefinition?.renderCall && !this.toolDefinition?.renderResult;}+ private shouldRenderInlineImages(): boolean {+ return this.showImages && this.allowInlineImages;+ }+private getRenderContext(lastComponent: Component | undefined): ToolRenderContext {+ const renderInlineImages = this.shouldRenderInlineImages();return {args: this.args,toolCallId: this.toolCallId,@@ -151,7 +164,8 @@ export class ToolExecutionComponent extends Container {argsComplete: this.argsComplete,isPartial: this.isPartial,expanded: this.expanded,- showImages: this.showImages,+ showImages: renderInlineImages,+ includeImageDimensions: this.allowInlineImages,isError: this.result?.isError ?? false,};}@@ -200,6 +214,7 @@ export class ToolExecutionComponent extends Container {}private maybeConvertImagesForKitty(): void {+ if (!this.allowInlineImages) return;const caps = getCapabilities();if (caps.images !== "kitty") return;if (!this.result) return;@@ -229,6 +244,17 @@ export class ToolExecutionComponent extends Container {setShowImages(show: boolean): void {this.showImages = show;+ if (show) {+ this.maybeConvertImagesForKitty();+ }+ this.updateDisplay();+ }++ setAllowInlineImages(allow: boolean): void {+ this.allowInlineImages = allow;+ if (allow) {+ this.maybeConvertImagesForKitty();+ }this.updateDisplay();}@@ -270,6 +296,7 @@ export class ToolExecutionComponent extends Container {}private updateDisplay(): void {+ const renderInlineImages = this.shouldRenderInlineImages();let hasContent = false;this.hideComponent = false;if (this.hasRendererDefinition() && this.getRenderShell() === "self") {@@ -285,7 +312,7 @@ export class ToolExecutionComponent extends Container {expanded: this.expanded,executionStarted: this.executionStarted,argsComplete: this.argsComplete,- showImages: this.showImages,+ showImages: renderInlineImages,cwd: this.cwd,};if (!this.ipythonCellComponent) {@@ -328,7 +355,7 @@ export class ToolExecutionComponent extends Container {const caps = getCapabilities();for (let i = 0; i < imageBlocks.length; i++) {const img = imageBlocks[i];- if (caps.images && this.showImages && img.data && img.mimeType) {+ if (caps.images && renderInlineImages && img.data && img.mimeType) {const converted = this.convertedImages.get(i);const imageData = converted?.data ?? img.data;const imageMimeType = converted?.mimeType ?? img.mimeType;@@ -435,7 +462,9 @@ export class ToolExecutionComponent extends Container {}private getTextOutput(): string {- return getRenderedTextOutput(this.result, this.showImages);+ return getRenderedTextOutput(this.result, this.shouldRenderInlineImages(), {+ includeImageDimensions: this.allowInlineImages,+ });}private formatToolExecution(): string {diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.tsindex a8e06854d1..e7be4af83b 100644--- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts+++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts@@ -4928,6 +4928,8 @@ export class InteractiveMode {{showImages: this.settingsManager.getShowImages(),imageWidthCells: this.settingsManager.getImageWidthCells(),+ // Do not replay historical inline image payloads on session load/rebuild.+ allowInlineImages: false,},this.getCachedToolDefinition(content.name),this.ui,@@ -4969,6 +4971,8 @@ export class InteractiveMode {}for (const [toolCallId, component] of renderedPendingTools) {+ // These tool calls have no historical result yet, so future updates are live output.+ component.setAllowInlineImages(true);this.pendingTools.set(toolCallId, component);}this.ui.requestRender();diff --git a/packages/coding-agent/test/interactive-mode-status.test.ts b/packages/coding-agent/test/interactive-mode-status.test.tsindex cba191f954..d80c48eb78 100644--- a/packages/coding-agent/test/interactive-mode-status.test.ts+++ b/packages/coding-agent/test/interactive-mode-status.test.ts@@ -4,6 +4,8 @@ import {type AutocompleteProvider,CombinedAutocompleteProvider,Container,+ resetCapabilitiesCache,+ setCapabilities,visibleWidth,} from "@earendil-works/pi-tui";import { beforeAll, describe, expect, test, vi } from "vitest";@@ -169,6 +171,109 @@ describe("InteractiveMode.showStatus", () => {});});+describe("InteractiveMode.renderSessionContext", () => {+ beforeAll(() => {+ initTheme("dark");+ });++ test("renders historical tool result images as fallbacks instead of replaying inline payloads", async () => {+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });+ try {+ const chatContainer = new Container();+ const fakeThis: any = {+ pendingTools: new Map(),+ toolOutputExpanded: false,+ chatContainer,+ footer: { invalidate: vi.fn() },+ updateEditorBorderColor: vi.fn(),+ resetPendingToolState: vi.fn(),+ preloadToolDefinitions: vi.fn(async () => {}),+ settingsManager: {+ getShowImages: () => true,+ getImageWidthCells: () => 60,+ },+ getCachedToolDefinition: () => undefined,+ getCurrentCwd: () => process.cwd(),+ getRetryAttempt: () => 0,+ ui: { requestRender: vi.fn() },+ addMessageToChat: vi.fn(() => {+ chatContainer.addChild({ render: () => ["assistant"], invalidate: () => {} });+ }),+ };++ await (InteractiveMode as any).prototype.renderSessionContext.call(fakeThis, {+ messages: [+ {+ role: "assistant",+ content: [{ type: "toolCall", name: "custom_tool", id: "tool-1", arguments: {} }],+ },+ {+ role: "toolResult",+ toolCallId: "tool-1",+ content: [{ type: "image", data: "AAAA", mimeType: "image/png" }],+ isError: false,+ },+ ],+ thinkingLevel: "medium",+ model: null,+ });++ const rendered = renderAll(chatContainer);+ expect(rendered).not.toContain("\x1b_G");+ expect(normalizeRenderedOutput(chatContainer)).toContain("[Image: [image/png]]");+ } finally {+ resetCapabilitiesCache();+ }+ });++ test("keeps pending history tool calls eligible for live inline image updates", async () => {+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });+ try {+ const chatContainer = new Container();+ const pendingTools = new Map<string, ToolExecutionComponent>();+ const fakeThis: any = {+ pendingTools,+ toolOutputExpanded: false,+ chatContainer,+ footer: { invalidate: vi.fn() },+ updateEditorBorderColor: vi.fn(),+ resetPendingToolState: vi.fn(),+ preloadToolDefinitions: vi.fn(async () => {}),+ settingsManager: {+ getShowImages: () => true,+ getImageWidthCells: () => 60,+ },+ getCachedToolDefinition: () => undefined,+ getCurrentCwd: () => process.cwd(),+ getRetryAttempt: () => 0,+ ui: { requestRender: vi.fn() },+ addMessageToChat: vi.fn(() => {+ chatContainer.addChild({ render: () => ["assistant"], invalidate: () => {} });+ }),+ };++ await (InteractiveMode as any).prototype.renderSessionContext.call(fakeThis, {+ messages: [+ {+ role: "assistant",+ content: [{ type: "toolCall", name: "custom_tool", id: "tool-1", arguments: {} }],+ },+ ],+ thinkingLevel: "medium",+ model: null,+ });++ const component = pendingTools.get("tool-1");+ expect(component).toBeDefined();+ component!.updateResult({ content: [{ type: "image", data: "AAAA", mimeType: "image/png" }], isError: false });++ expect(component!.render(120).join("\n")).toContain("\x1b_G");+ } finally {+ resetCapabilitiesCache();+ }+ });+});+type SubmitHandlerHarness = {defaultEditor: { onSubmit?: (text: string) => Promise<void> };editor: { setText: (text: string) => void; addToHistory?: (text: string) => void };diff --git a/packages/coding-agent/test/tool-execution-component.test.ts b/packages/coding-agent/test/tool-execution-component.test.tsindex 42d0bf3247..ffdc89c4e2 100644--- a/packages/coding-agent/test/tool-execution-component.test.ts+++ b/packages/coding-agent/test/tool-execution-component.test.ts@@ -1,4 +1,4 @@-import { Text, type TUI } from "@earendil-works/pi-tui";+import { resetCapabilitiesCache, setCapabilities, Text, type TUI } from "@earendil-works/pi-tui";import stripAnsi from "strip-ansi";import { Type } from "typebox";import { beforeAll, describe, expect, test } from "vitest";@@ -28,6 +28,14 @@ function createFakeTui(): TUI {} as unknown as TUI;}+async function waitForCondition(condition: () => boolean): Promise<void> {+ for (let i = 0; i < 50; i++) {+ if (condition()) return;+ await new Promise((resolve) => setTimeout(resolve, 10));+ }+ throw new Error("condition was not met");+}+describe("ToolExecutionComponent parity", () => {beforeAll(() => {initTheme("dark");@@ -65,6 +73,131 @@ describe("ToolExecutionComponent parity", () => {expect(rendered).toContain("custom result");});+ test("renders inline images by default when terminal supports them", () => {+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });+ try {+ const component = new ToolExecutionComponent(+ "custom_tool",+ "tool-image-default",+ {},+ { showImages: true },+ undefined,+ createFakeTui(),+ process.cwd(),+ );+ component.updateResult({ content: [{ type: "image", data: "AAAA", mimeType: "image/png" }], isError: false });++ expect(component.render(120).join("\n")).toContain("\x1b_G");+ } finally {+ resetCapabilitiesCache();+ }+ });++ test("can suppress inline image escape sequences for session history", () => {+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });+ try {+ const component = new ToolExecutionComponent(+ "custom_tool",+ "tool-image-history",+ {},+ { showImages: true, allowInlineImages: false },+ undefined,+ createFakeTui(),+ process.cwd(),+ );+ component.updateResult({ content: [{ type: "image", data: "AAAA", mimeType: "image/png" }], isError: false });++ let rendered = component.render(120).join("\n");+ expect(rendered).not.toContain("\x1b_G");+ expect(stripAnsi(rendered)).toContain("[Image: [image/png]]");++ component.setShowImages(false);+ component.setShowImages(true);+ rendered = component.render(120).join("\n");+ expect(rendered).not.toContain("\x1b_G");+ } finally {+ resetCapabilitiesCache();+ }+ });++ test("suppressed history image fallbacks skip dimension parsing", () => {+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });+ try {+ const onePixelPng =+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=";+ const component = new ToolExecutionComponent(+ "custom_tool",+ "tool-image-history-dims",+ {},+ { showImages: true, allowInlineImages: false },+ undefined,+ createFakeTui(),+ process.cwd(),+ );+ component.updateResult({+ content: [{ type: "image", data: onePixelPng, mimeType: "image/png" }],+ isError: false,+ });++ const rendered = stripAnsi(component.render(120).join("\n"));+ expect(rendered).toContain("[Image: [image/png]]");+ expect(rendered).not.toContain("1x1");+ } finally {+ resetCapabilitiesCache();+ }+ });++ test("pending history components can be restored to live inline image rendering", () => {+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });+ try {+ const component = new ToolExecutionComponent(+ "custom_tool",+ "tool-image-pending-history",+ {},+ { showImages: true, allowInlineImages: false },+ undefined,+ createFakeTui(),+ process.cwd(),+ );+ component.updateResult({ content: [{ type: "image", data: "AAAA", mimeType: "image/png" }], isError: false });+ expect(component.render(120).join("\n")).not.toContain("\x1b_G");++ component.setAllowInlineImages(true);++ expect(component.render(120).join("\n")).toContain("\x1b_G");+ } finally {+ resetCapabilitiesCache();+ }+ });++ test("converts kitty images even if image display is toggled off when the result arrives", async () => {+ setCapabilities({ images: "kitty", trueColor: true, hyperlinks: true });+ try {+ const tinyJpeg =+ "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAACAAIDAREAAhEBAxEB/8QAFAABAAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/xAAVAQEBAAAAAAAAAAAAAAAAAAAGCf/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AD3VTB3/2Q==";+ const component = new ToolExecutionComponent(+ "custom_tool",+ "tool-image-hidden-conversion",+ {},+ { showImages: false },+ undefined,+ createFakeTui(),+ process.cwd(),+ );+ component.updateResult({+ content: [{ type: "image", data: tinyJpeg, mimeType: "image/jpeg" }],+ isError: false,+ });++ await waitForCondition(() => (component as any).convertedImages.size === 1);+ component.setShowImages(true);++ expect(component.render(120).join("\n")).toContain("\x1b_G");+ } finally {+ resetCapabilitiesCache();+ }+ });+test("uses built-in rendering for built-in overrides without custom renderers", () => {const overrideDefinition: ToolDefinition = {...createBaseToolDefinition("edit"),