Skip to content

AI-native knowledge base & Evernote/Obsidian alternative with instant sync & native MCP. Free on Cloudflare or self-hosted via Docker.

fix(desktop): render local agent images in the sidebar

tianma-if2633d08acd14
diff --git a/apps/desktop/src/main/acp-host.mjs b/apps/desktop/src/main/acp-host.mjs
index 633e565a..92490cd3 100644
--- a/apps/desktop/src/main/acp-host.mjs
+++ b/apps/desktop/src/main/acp-host.mjs
@@ -1,5 +1,5 @@
import { spawn as nodeSpawn } from "node:child_process";
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
import { accessSync, constants as fsConstants, existsSync, readFileSync, realpathSync, statSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { homedir, tmpdir } from "node:os";
@@ -397,18 +397,59 @@ const safeToolToken = (value) => {
return trimmed;
};
+const MAX_GENERATED_IMAGES = 8;
+
+const compactBase64 = (value) => {
+ if (typeof value !== "string") return "";
+ const trimmed = value.trim();
+ if (trimmed.startsWith("data:") && !trimmed.includes(",")) return "";
+ const payload = trimmed.startsWith("data:") ? trimmed.slice(trimmed.indexOf(",") + 1) : trimmed;
+ const compact = payload.replace(/\s+/g, "");
+ if (!compact || compact.length > MAX_ATTACHMENT_CHARS || compact.length % 4 === 1) return "";
+ if (!/^[A-Za-z0-9+/]+={0,2}$/.test(compact)) return "";
+ return compact;
+};
+
+const imageFingerprint = (base64) => createHash("sha256").update(base64).digest("hex").slice(0, 16);
+
+const imageEventFromBlock = (requestId, block, id) => {
+ if (!block || typeof block !== "object" || block.type !== "image") return null;
+ const mediaType = mediaTypeBase(block.mimeType);
+ if (!IMAGE_TYPES.has(mediaType)) return null;
+ const base64 = compactBase64(block.data);
+ if (!base64) return null;
+ return { requestId, type: "image", id, mediaType, base64 };
+};
+
+const toolImageEvents = (requestId, update) => {
+ if (!Array.isArray(update.content)) return [];
+ const toolId = safeToolToken(update.toolCallId) || "tool";
+ const events = [];
+ for (const item of update.content) {
+ if (events.length >= MAX_GENERATED_IMAGES) break;
+ const block = item?.type === "content" ? item.content : item;
+ const image = imageEventFromBlock(requestId, block, `${toolId}:${events.length}`);
+ if (image) events.push(image);
+ }
+ return events;
+};
+
export function eventsFromSessionUpdate(requestId, params) {
const update = params?.update;
if (!update || typeof update !== "object") return [];
- if (update.sessionUpdate === "agent_message_chunk" || update.sessionUpdate === "agent_thought_chunk") {
+ if (update.sessionUpdate === "agent_thought_chunk") {
if (update.content?.type !== "text" || typeof update.content.text !== "string" || update.content.text.length === 0) {
return [];
}
- return [{
- requestId,
- type: update.sessionUpdate === "agent_message_chunk" ? "text-delta" : "reasoning",
- text: update.content.text,
- }];
+ return [{ requestId, type: "reasoning", text: update.content.text }];
+ }
+ if (update.sessionUpdate === "agent_message_chunk") {
+ if (update.content?.type === "text" && typeof update.content.text === "string" && update.content.text.length > 0) {
+ return [{ requestId, type: "text-delta", text: update.content.text }];
+ }
+ const image = imageEventFromBlock(requestId, update.content, "");
+ if (!image) return [];
+ return [{ ...image, id: `message:${imageFingerprint(image.base64)}` }];
}
if (update.sessionUpdate !== "tool_call" && update.sessionUpdate !== "tool_call_update") return [];
const name = safeToolToken(update.name) || safeToolToken(update.toolCallId) || "tool";
@@ -416,7 +457,7 @@ export function eventsFromSessionUpdate(requestId, params) {
const event = { requestId, type: "tool", name, status };
const title = safeToolToken(update.title);
if (title) event.title = title;
- return [event];
+ return [event, ...toolImageEvents(requestId, update)];
}
export function acpInitializeParams(version = clientVersion()) {
diff --git a/apps/desktop/src/main/acp-host.test.mjs b/apps/desktop/src/main/acp-host.test.mjs
index fa055216..d8056cb6 100644
--- a/apps/desktop/src/main/acp-host.test.mjs
+++ b/apps/desktop/src/main/acp-host.test.mjs
@@ -477,6 +477,54 @@ describe("ACP spawn and failure mapping", () => {
update: { sessionUpdate: "tool_call", toolCallId: "call_2", title: `${home}/secret`, status: "completed" },
})).toEqual([{ requestId: "r1", type: "tool", name: "call_2", status: "completed" }]);
});
+
+ test("forwards generated images without paths, diffs, or non-image payloads", () => {
+ const png = Buffer.from("png-bytes").toString("base64");
+ const wrapped = `data:image/png;base64,${png.slice(0, 4)}\n${png.slice(4)}`;
+ const message = eventsFromSessionUpdate("r1", {
+ update: {
+ sessionUpdate: "agent_message_chunk",
+ content: { type: "image", mimeType: "image/png", data: wrapped, uri: `${home}/cat.png` },
+ },
+ });
+ const again = eventsFromSessionUpdate("r1", {
+ update: {
+ sessionUpdate: "agent_message_chunk",
+ content: { type: "image", mimeType: "image/png", data: png, uri: `file://${home}/cat.png` },
+ },
+ });
+ expect(message).toEqual([{ requestId: "r1", type: "image", id: again[0].id, mediaType: "image/png", base64: png }]);
+ expect(message[0].id.startsWith("message:")).toBe(true);
+ expect(JSON.stringify(message)).not.toContain(home);
+ expect(JSON.stringify(message)).not.toContain("file:");
+
+ const tool = eventsFromSessionUpdate("r1", {
+ update: {
+ sessionUpdate: "tool_call_update",
+ toolCallId: "call_img",
+ title: "Image generation",
+ status: "completed",
+ content: [
+ { type: "diff", path: `${home}/note.md`, oldText: "a", newText: "b" },
+ { type: "content", content: { type: "text", text: `saved ${home}/cat.png` } },
+ { type: "content", content: { type: "image", mimeType: "image/jpeg", data: png, uri: `${home}/cat.png` } },
+ { type: "content", content: { type: "image", mimeType: "image/svg+xml", data: png } },
+ { type: "image", mimeType: "image/webp", data: `${home}/cat.webp` },
+ ],
+ },
+ });
+ expect(tool).toEqual([
+ { requestId: "r1", type: "tool", name: "call_img", status: "completed", title: "Image generation" },
+ { requestId: "r1", type: "image", id: "call_img:0", mediaType: "image/jpeg", base64: png },
+ ]);
+ expect(JSON.stringify(tool)).not.toContain(home);
+ expect(eventsFromSessionUpdate("r1", {
+ update: { sessionUpdate: "agent_thought_chunk", content: { type: "image", mimeType: "image/png", data: png } },
+ })).toEqual([]);
+ expect(eventsFromSessionUpdate("r1", {
+ update: { sessionUpdate: "agent_message_chunk", content: { type: "image", mimeType: "image/png", data: "@@@@" } },
+ })).toEqual([]);
+ });
});
// bun test started at the workspace root drops child stdin and stdout pipes
diff --git a/apps/web/src/components/ai-elements/image.tsx b/apps/web/src/components/ai-elements/image.tsx
new file mode 100644
index 00000000..545ccd00
--- /dev/null
+++ b/apps/web/src/components/ai-elements/image.tsx
@@ -0,0 +1,26 @@
+"use client";
+
+import { cn } from "@/lib/utils";
+import type { Experimental_GeneratedImage } from "ai";
+
+export type ImageProps = Experimental_GeneratedImage & {
+ className?: string;
+ alt?: string;
+};
+
+export const Image = ({
+ base64,
+ uint8Array: _uint8Array,
+ mediaType,
+ ...props
+}: ImageProps) => (
+ <img
+ {...props}
+ alt={props.alt}
+ className={cn(
+ "h-auto max-w-full overflow-hidden rounded-md",
+ props.className
+ )}
+ src={`data:${mediaType};base64,${base64}`}
+ />
+);
diff --git a/apps/web/src/components/ai-sidebar/AiSidebar.tsx b/apps/web/src/components/ai-sidebar/AiSidebar.tsx
index c85260b3..0e2a0406 100644
--- a/apps/web/src/components/ai-sidebar/AiSidebar.tsx
+++ b/apps/web/src/components/ai-sidebar/AiSidebar.tsx
@@ -22,6 +22,7 @@ import {
Attachments,
} from "@/components/ai-elements/attachments";
import { Conversation, ConversationContent, ConversationEmptyState, ConversationScrollButton } from "@/components/ai-elements/conversation";
+import { Image } from "@/components/ai-elements/image";
import { Message, MessageContent } from "@/components/ai-elements/message";
import {
PromptInput,
@@ -107,12 +108,18 @@ type PendingAttachment = PreparedAiAttachment & {
type LocalToolRow = { id: string; name: string; status: string; title?: string };
+type LocalImage = { id: string; mediaType: string; base64: string };
+
+const EMPTY_IMAGE_BYTES = new Uint8Array();
+const MAX_LOCAL_IMAGES = 8;
+
type LocalTurn = {
id: string;
message: string;
response: string;
reasoning: string;
tools: LocalToolRow[];
+ images: LocalImage[];
attachments: Array<{ id: string; filename: string; mediaType: string; byteLength: number }>;
status: "running" | "completed" | "failed" | "cancelled";
};
@@ -848,6 +855,22 @@ function AiSidebarSession({
}));
return;
}
+ if (event.type === "image") {
+ setLocalTurns((previous) => previous.map((turn) => {
+ if (turn.id !== turnId || turn.status === "cancelled") return turn;
+ if (turn.images.some((image) => image.mediaType === event.mediaType && image.base64 === event.base64)) return turn;
+ const index = turn.images.findIndex((image) => image.id === event.id);
+ const next = { id: event.id, mediaType: event.mediaType, base64: event.base64 };
+ if (index >= 0) {
+ const images = turn.images.slice();
+ images[index] = next;
+ return { ...turn, images };
+ }
+ if (turn.images.length >= MAX_LOCAL_IMAGES) return turn;
+ return { ...turn, images: [...turn.images, next] };
+ }));
+ return;
+ }
if (event.type === "error" || event.type === "done") {
setLocalTurns((previous) => previous.map((turn) => turn.id === turnId && turn.status === "running"
? { ...turn, status: event.type === "error" ? "failed" : "completed" }
@@ -1024,6 +1047,7 @@ function AiSidebarSession({
response: "",
reasoning: "",
tools: [],
+ images: [],
attachments: localAttachments,
status: "running",
}]);
@@ -1488,6 +1512,20 @@ function AiSidebarSession({
))}
</ul>
) : null}
+ {turn.images.length ? (
+ <div className="flex flex-col gap-2">
+ {turn.images.map((image) => (
+ <Image
+ key={image.id}
+ alt={t("aiAssistant.sidebar.generatedImage")}
+ base64={image.base64}
+ className="max-h-96"
+ mediaType={image.mediaType}
+ uint8Array={EMPTY_IMAGE_BYTES}
+ />
+ ))}
+ </div>
+ ) : null}
{turn.response ? <AiSidebarMessage isAnimating={turn.status === "running"}>{turn.response}</AiSidebarMessage> : null}
{renderSelectionReply(turn.id, turn.response, turn.status)}
{turn.status === "failed" || turn.status === "cancelled" ? (
diff --git a/apps/web/src/lib/desktop-acp.ts b/apps/web/src/lib/desktop-acp.ts
index 473905ca..641ecd7b 100644
--- a/apps/web/src/lib/desktop-acp.ts
+++ b/apps/web/src/lib/desktop-acp.ts
@@ -41,6 +41,7 @@ export type DesktopAcpEvent =
| { requestId: string; type: "text-delta"; text: string }
| { requestId: string; type: "reasoning"; text: string }
| { requestId: string; type: "tool"; name: string; status: string; title?: string }
+ | { requestId: string; type: "image"; id: string; mediaType: string; base64: string }
| { requestId: string; type: "done" }
| { requestId: string; type: "error"; message: string };
diff --git a/apps/web/src/vite-env.d.ts b/apps/web/src/vite-env.d.ts
index 1374debe..3ee28d05 100644
--- a/apps/web/src/vite-env.d.ts
+++ b/apps/web/src/vite-env.d.ts
@@ -139,6 +139,7 @@ interface EdgeEverDesktopBridge {
| { requestId: string; type: "text-delta"; text: string }
| { requestId: string; type: "reasoning"; text: string }
| { requestId: string; type: "tool"; name: string; status: string; title?: string }
+ | { requestId: string; type: "image"; id: string; mediaType: string; base64: string }
| { requestId: string; type: "done" }
| { requestId: string; type: "error"; message: string }
) => void): () => void;
diff --git a/packages/shared/src/i18n/en-US.ts b/packages/shared/src/i18n/en-US.ts
index 8f98b5d9..462c9981 100644
--- a/packages/shared/src/i18n/en-US.ts
+++ b/packages/shared/src/i18n/en-US.ts
@@ -1367,6 +1367,7 @@ export const enUS = {
proposalConfirmOther: "Confirm",
proposalDismiss: "Dismiss",
toolProgress: "{{name}} · {{status}}",
+ generatedImage: "Generated image",
},
agentSource: {
title: "Agent mode",
diff --git a/packages/shared/src/i18n/ja.ts b/packages/shared/src/i18n/ja.ts
index 82003717..6dfdd28b 100644
--- a/packages/shared/src/i18n/ja.ts
+++ b/packages/shared/src/i18n/ja.ts
@@ -1367,6 +1367,7 @@ export const ja = {
proposalConfirmOther: "確認",
proposalDismiss: "無視",
toolProgress: "{{name}} · {{status}}",
+ generatedImage: "生成された画像",
},
agentSource: {
title: "エージェントの動作モード",
diff --git a/packages/shared/src/i18n/zh-CN.ts b/packages/shared/src/i18n/zh-CN.ts
index 8a7a3801..fda6eb9b 100644
--- a/packages/shared/src/i18n/zh-CN.ts
+++ b/packages/shared/src/i18n/zh-CN.ts
@@ -1367,6 +1367,7 @@ export const zhCN = {
proposalConfirmOther: "确认",
proposalDismiss: "忽略",
toolProgress: "{{name}} · {{status}}",
+ generatedImage: "生成的图片",
},
agentSource: {
title: "Agent 运行模式",