Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

feat: retain native tool images and videos in work details

Chrisf77c1af9bb84
diff --git a/apps/desktop/src/main/__tests__/messagePersistBroadcaster.test.ts b/apps/desktop/src/main/__tests__/messagePersistBroadcaster.test.ts
index 71338ddf1d..1211effad4 100644
--- a/apps/desktop/src/main/__tests__/messagePersistBroadcaster.test.ts
+++ b/apps/desktop/src/main/__tests__/messagePersistBroadcaster.test.ts
@@ -1,3 +1,8 @@
+import { materializeToolProcessMedia } from '../cindy-media/toolProcessMedia.js';
+vi.mock('../cindy-media/toolProcessMedia.js', () => ({ materializeToolProcessMedia: vi.fn(async () => [
+ { kind: 'image', url: 'cindy-media://blobs/' + 'a'.repeat(64) + '.png' },
+]) }));
+vi.mock('../localDb/client/current.js', () => ({ getDbClient: vi.fn(() => ({ drizzle: {} })) }));
vi.mock('../botTaskReplyResults.js', () => ({ readTaskResultsForReply: vi.fn(async () => []) }));
import { readTaskResultsForReply } from '../botTaskReplyResults.js';
/**
@@ -76,6 +81,7 @@ import {
__resetMediaToolResultPoolForTesting,
} from '../mcp-integrations/mediaToolResultFallback.js';
import {
+ persistToolProcessMedia,
onToolUseEvent,
onAgentTaskUpdateEvent,
persistCodexPlanOnDone,
@@ -4085,3 +4091,17 @@ it('persists bound results with the final seal, and a failed result lookup canno
await expect(markAssistantTurnCompleted(SESSION, 'other', undefined, ['bot-delegation-completion:job'])).resolves.toBe(true);
expect(patchMessageAgentMetaWithResult).toHaveBeenCalledWith(SESSION, 'other', { turnCompleted: true });
});
+
+// Unit regression: a late import updates the original result, never the next turn.
+it('persists process media on its accepted result and broadcasts the durable row', async () => {
+ onToolUseEvent(SESSION, { toolUseId: 'native-image', toolName: 'Read', input: {} }, null);
+ const result = onToolResultFullEvent(SESSION, { toolUseId: 'native-image', fullText: 'read' }, null)!;
+ persistToolProcessMedia(SESSION, result.persistId, result.content, [{ kind: 'image', data: 'bytes' }]);
+ await drainPersistQueue();
+ const updated = vi.mocked(updateMessageContent).mock.calls.find(([, id]) => id === result.persistId);
+ expect(JSON.parse(updated![2] as string)).toEqual({ text: 'read', cindy_process_media: [
+ { kind: 'image', url: 'cindy-media://blobs/' + 'a'.repeat(64) + '.png' },
+ ] });
+ expect(broadcastMessageRow).toHaveBeenCalledWith(SESSION, expect.anything(), ownerScopeState.scope);
+ expect(materializeToolProcessMedia).toHaveBeenCalled();
+});
diff --git a/apps/desktop/src/main/cindy-media/__tests__/toolProcessMedia.test.ts b/apps/desktop/src/main/cindy-media/__tests__/toolProcessMedia.test.ts
new file mode 100644
index 0000000000..c5313d0335
--- /dev/null
+++ b/apps/desktop/src/main/cindy-media/__tests__/toolProcessMedia.test.ts
@@ -0,0 +1,81 @@
+// Unit: native bytes become managed refs, never raw base64 in message history.
+// Existing generated-media tests do not cover native tool blocks or owner invalidation.
+import { describe, expect, it, vi } from 'vitest';
+import { materializeToolProcessMedia } from '../toolProcessMedia';
+import { ingestMedia } from '../ingest';
+import type { LedgerDb } from '../ledger';
+import fs from 'node:fs/promises';
+import { materializeSshRemoteMedia } from '../../file-browser/ssh-media';
+vi.mock('node:fs/promises', () => ({ default: { readFile: vi.fn() } }));
+vi.mock('../../file-browser/ssh-media', () => ({ materializeSshRemoteMedia: vi.fn() }));
+vi.mock('../ingest', () => ({
+ ingestMedia: vi.fn(async () => ({ url: 'cindy-media://blobs/' + 'a'.repeat(64) + '.png' })),
+}));
+const db = {} as LedgerDb;
+const png = Buffer.from(
+ 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aL1kAAAAASUVORK5CYII=',
+ 'base64',
+);
+describe('native tool media materialization', () => {
+ it('ingests verified bytes and collapses repeated refs', async () => {
+ const image = { kind: 'image' as const, data: png.toString('base64'), mimeType: 'image/png' };
+ expect(await materializeToolProcessMedia([image, image], db, () => {})).toEqual([
+ { kind: 'image', url: 'cindy-media://blobs/' + 'a'.repeat(64) + '.png' },
+ ]);
+ expect(ingestMedia).toHaveBeenCalledWith(
+ expect.objectContaining({ buffer: png, mimeType: 'image/png', refs: [] }),
+ db,
+ );
+ });
+ it('rejects false media types without importing them', async () => {
+ vi.mocked(ingestMedia).mockClear();
+ expect(
+ await materializeToolProcessMedia(
+ [{ kind: 'video', data: png.toString('base64'), mimeType: 'video/mp4' }],
+ db,
+ () => {},
+ ),
+ ).toEqual([]);
+ expect(ingestMedia).not.toHaveBeenCalled();
+ });
+ it('keeps playable video references in the tool step', async () => {
+ const video = { kind: 'video' as const, url: 'https://example.com/clip.mp4' };
+ expect(await materializeToolProcessMedia([video], db, () => {})).toEqual([video]);
+ });
+ it('stops before writing when the owner changed', async () => {
+ vi.mocked(ingestMedia).mockClear();
+ await expect(
+ materializeToolProcessMedia([{ kind: 'image', data: png.toString('base64') }], db, () => {
+ throw new Error('owner changed');
+ }),
+ ).rejects.toThrow('owner changed');
+ expect(ingestMedia).not.toHaveBeenCalled();
+ });
+ it('reads remote paths only through the SSH cache and keeps other media after a missing file', async () => {
+ vi.mocked(materializeSshRemoteMedia).mockResolvedValueOnce({
+ ok: true,
+ cachePath: '/cache/image',
+ sourceVersion: 'v',
+ size: png.length,
+ mime: 'image/png',
+ relPath: 'image.png',
+ });
+ vi.mocked(fs.readFile)
+ .mockResolvedValueOnce(png as never)
+ .mockRejectedValueOnce(new Error('missing'));
+ const origin = { workdir: '/remote', remoteHostId: 'ssh-host' };
+ const refs = await materializeToolProcessMedia(
+ [{ kind: 'image', path: '/remote/image.png' }],
+ db,
+ () => {},
+ origin,
+ );
+ expect(fs.readFile).toHaveBeenCalledWith('/cache/image');
+ expect(fs.readFile).not.toHaveBeenCalledWith('/remote/image.png');
+ expect(refs).toHaveLength(1);
+ const video = { kind: 'video' as const, url: 'https://example.com/video.mp4' };
+ expect(
+ await materializeToolProcessMedia([{ kind: 'image', path: '/missing' }, video], db, () => {}),
+ ).toEqual([video]);
+ });
+});
diff --git a/apps/desktop/src/main/cindy-media/toolProcessMedia.ts b/apps/desktop/src/main/cindy-media/toolProcessMedia.ts
new file mode 100644
index 0000000000..c2891d1391
--- /dev/null
+++ b/apps/desktop/src/main/cindy-media/toolProcessMedia.ts
@@ -0,0 +1,65 @@
+import fs from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import type { NativeToolMedia, ProcessMediaRef } from '@cindy/maker-shared/tool-process-media';
+import { isProcessMediaUrl } from '@cindy/maker-shared/tool-process-media';
+import { ingestMedia } from './ingest';
+import { sniffMediaMime } from './sniffMediaMime';
+import type { LedgerDb } from './ledger';
+import { createLogger } from '../logger';
+
+const log = createLogger('tool-process-media');
+export interface ToolMediaOrigin {
+ workdir: string;
+ remoteHostId: string | null;
+}
+
+/** Native bytes never enter chat history. Store one managed copy for all clients. */
+export async function materializeToolProcessMedia(
+ items: NativeToolMedia[],
+ db: LedgerDb,
+ assertStillValid: () => void,
+ origin?: ToolMediaOrigin,
+): Promise<ProcessMediaRef[]> {
+ const result: ProcessMediaRef[] = [];
+ for (const item of items) {
+ assertStillValid();
+ try {
+ if (item.url && isProcessMediaUrl(item.url)) {
+ result.push({ kind: item.kind, url: item.url });
+ continue;
+ }
+ const dataUrl = item.url?.match(
+ /^data:((?:image|video)\/[a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+)$/,
+ );
+ const encoded = item.data ?? dataUrl?.[2];
+ const sourcePath =
+ item.path ?? (item.url?.startsWith('file:') ? fileURLToPath(item.url) : undefined);
+ let buffer: Buffer;
+ if (encoded) buffer = Buffer.from(encoded, 'base64');
+ else if (sourcePath) {
+ let localPath = sourcePath;
+ if (origin?.remoteHostId) {
+ const { materializeSshRemoteMedia } = await import('../file-browser/ssh-media');
+ const remote = await materializeSshRemoteMedia(
+ { workdir: origin.workdir, remoteHostId: origin.remoteHostId },
+ `xdt-file://?path=${encodeURIComponent(sourcePath)}`,
+ );
+ assertStillValid();
+ if (!remote.ok) continue;
+ localPath = remote.cachePath;
+ }
+ buffer = await fs.readFile(localPath);
+ } else continue;
+ assertStillValid();
+ const mimeType = sniffMediaMime(buffer, item.mimeType ?? dataUrl?.[1]);
+ if (!mimeType?.startsWith(item.kind + '/')) continue;
+ const media = await ingestMedia({ buffer, mimeType, refs: [], assertStillValid }, db);
+ result.push({ kind: item.kind, url: media.url });
+ } catch {
+ // One unavailable attachment must not hide the rest; owner changes still abort.
+ assertStillValid();
+ log.warn('Native tool attachment could not be stored', { kind: item.kind });
+ }
+ }
+ return [...new Map(result.map((item) => [item.kind + ':' + item.url, item])).values()];
+}
diff --git a/apps/desktop/src/main/maker-ipc/register.ts b/apps/desktop/src/main/maker-ipc/register.ts
index 2c91088646..99fc9469db 100644
--- a/apps/desktop/src/main/maker-ipc/register.ts
+++ b/apps/desktop/src/main/maker-ipc/register.ts
@@ -23470,6 +23470,10 @@ function redactEventForRenderer(event: AgentEvent): AgentEvent {
const data = event.data as Record<string, unknown>;
const safeData = { ...data };
let changed = false;
+ if ('processMedia' in safeData) {
+ delete safeData.processMedia;
+ changed = true;
+ }
if (
typeof safeData.toolName === 'string' &&
Object.prototype.hasOwnProperty.call(safeData, 'input')
diff --git a/apps/desktop/src/main/maker-ipc/sessionEventStream.ts b/apps/desktop/src/main/maker-ipc/sessionEventStream.ts
index 824536c4ac..11c94f1faa 100644
--- a/apps/desktop/src/main/maker-ipc/sessionEventStream.ts
+++ b/apps/desktop/src/main/maker-ipc/sessionEventStream.ts
@@ -1,3 +1,4 @@
+import type { NativeToolMedia } from '@cindy/maker-shared/tool-process-media';
import type { Session } from '@cindy/maker-core';
import { persistSubagentTaskUpdate } from '../localDb/subagentRuns.js';
import { broadcastSubagentRunsChanged } from '../localDb/ipc/subagentRuns.js';
@@ -15,6 +16,7 @@ import {
onThinkingEvent,
onToolResultEvent,
onToolResultFullEvent,
+ persistToolProcessMedia,
onToolUseEvent,
} from '../messagePersistBroadcaster.js';
import { type OrcaTeamService } from './orcaTeamService.js';
@@ -91,6 +93,12 @@ export function persistSessionStreamEvent(
);
persistId = r?.persistId;
resolvedContent = r?.content;
+ const processMedia = (event.data as { processMedia?: NativeToolMedia[] }).processMedia;
+ if (r && Array.isArray(processMedia) && processMedia.length) {
+ persistToolProcessMedia(session.id, r.persistId, r.content, processMedia, {
+ workdir: session.workDir, remoteHostId: session.remoteHostId,
+ });
+ }
} else if (event.type === 'thinking') {
// thinking final/redacted 落库收口 main;clientId=blockId(renderer 同源),无需
// persistId 回传。start/delta 不落库。
diff --git a/apps/desktop/src/main/messagePersistBroadcaster.ts b/apps/desktop/src/main/messagePersistBroadcaster.ts
index 2be06612c2..3cff46f127 100644
--- a/apps/desktop/src/main/messagePersistBroadcaster.ts
+++ b/apps/desktop/src/main/messagePersistBroadcaster.ts
@@ -1,3 +1,5 @@
+import { materializeToolProcessMedia } from './cindy-media/toolProcessMedia.js';
+import { withToolProcessMedia, type NativeToolMedia } from '@cindy/maker-shared/tool-process-media';
import { readTaskResultsForReply } from './botTaskReplyResults.js';
/**
* messagePersistBroadcaster — 把 agent 消息的持久化从 renderer 收口到 main 单点。
@@ -1745,6 +1747,40 @@ export function onToolResultEvent(
return { persistId, content };
}
+/** Attach native media to the already accepted result row, never to the current turn.
+ * The existing owner-scoped FIFO prevents clear/delete or the next turn from stealing it.
+ */
+export function persistToolProcessMedia(
+ sessionId: string,
+ persistId: string,
+ text: string,
+ items: NativeToolMedia[],
+ origin?: import('./cindy-media/toolProcessMedia').ToolMediaOrigin,
+): void {
+ if (!items.length) return;
+ enqueueWrite(`tool_media:${sessionId}:${persistId}`, async (ownerScope) => {
+ const assertStillValid = () => {
+ if (!isOwnerScopeCurrent(ownerScope)) throw ownerScopeSupersededError();
+ };
+ const db = getDbClient().drizzle;
+ const media = await materializeToolProcessMedia(items, db, assertStillValid, origin);
+ assertStillValid();
+ if (!media.length) return;
+ // Keep media refs outside the text truncation budget so reopening retains images.
+ const content = withToolProcessMedia(text, media);
+ const updated = await updateDbMessageContent(sessionId, persistId, content);
+ if (updated) {
+ toolResultContentByClientId.get(sessionId)?.set(persistId, content);
+ broadcastMessageRow(sessionId, updated, ownerScope);
+ }
+ });
+}
+
+function markFailedToolResultText(text: string, isError: boolean): string {
+ if (!isError || text.includes('<tool_use_error>')) return text;
+ return `<tool_use_error>${text}</tool_use_error>`;
+}
+
/**
* 处理 tool_result_full 事件(toolUseId + 全文)。返回 { persistId, content } 让
* renderer 把对应 tool_result 气泡内容更新成全文;返回 null 表示无需显示变更
@@ -1753,11 +1789,6 @@ export function onToolResultEvent(
* 对齐老 renderer:有映射 → 覆盖更新;无映射但 tool_use 已到 → eager-create;
* tool_use 也没到 → buffer。
*/
-function markFailedToolResultText(text: string, isError: boolean): string {
- if (!isError || text.includes('<tool_use_error>')) return text;
- return `<tool_use_error>${text}</tool_use_error>`;
-}
-
export function onToolResultFullEvent(
sessionId: string,
data: { toolUseId?: unknown; fullText?: unknown; isError?: unknown },
diff --git a/apps/desktop/src/renderer/components/chat/AgentActionRow.tsx b/apps/desktop/src/renderer/components/chat/AgentActionRow.tsx
index bed70363e9..c714a61971 100644
--- a/apps/desktop/src/renderer/components/chat/AgentActionRow.tsx
+++ b/apps/desktop/src/renderer/components/chat/AgentActionRow.tsx
@@ -1,3 +1,6 @@
+import { readToolProcessMedia } from '@cindy/maker-shared/tool-process-media';
+import { ChatVideoView } from './ChatVideoView';
+import { ChatImageView } from './ChatImageView';
import { FileTypeIcon } from '@/components/ui/file-type-icon';
/**
* AgentActionRow
@@ -788,6 +791,7 @@ export function AgentActionRow({
onActivateOverride,
}: AgentActionRowProps) {
const { t } = useTranslation();
+ const processMedia = useMemo(() => readToolProcessMedia(toolResult), [toolResult]);
const compactedToolResult = parseToolResultCompactionMarker(toolResult);
const displayedToolResult = compactedToolResult
? t('chat.toolResultCompacted', {
@@ -1172,6 +1176,13 @@ export function AgentActionRow({
{rawCommand}
</div>
)}
+ {processMedia.length > 0 && (
+ <div className="flex min-w-0 flex-col gap-3 py-2 pl-8" data-tool-process-media="true">
+ {processMedia.map((media) => media.kind === 'image'
+ ? <ChatImageView key={media.url} src={media.url} variant="tool-output" sessionId={fileCtx.sessionId} filename={media.url.split('/').pop() ?? media.kind} />
+ : <ChatVideoView key={media.url} src={media.url} variant="tool-output" sessionId={fileCtx.sessionId} filename={media.url.split('/').pop() ?? media.kind} />)}
+ </div>
+ )}
{chipFilePath ? fileChipMenu.menu : null}
{/* v10 就地展开内容:命令类工具(Bash/Grep/Glob/WebFetch/WebSearch/...)
diff --git a/apps/mobile/src/session/MessageRenderer.tsx b/apps/mobile/src/session/MessageRenderer.tsx
index 8457d2a602..1b78f3e2a9 100644
--- a/apps/mobile/src/session/MessageRenderer.tsx
+++ b/apps/mobile/src/session/MessageRenderer.tsx
@@ -1,3 +1,4 @@
+import { readToolProcessMedia } from '@cindy/maker-shared/tool-process-media';
import { PluginCardActions } from '@/plugins/PluginCardActions';
import { McpAppCard } from './McpAppCard';
import { RichContentContext } from './richContentContext';
@@ -4240,7 +4241,7 @@ function ToolGroupCard({
* 媒体条 / 结果预览,再点收起。展开态走共享进程内记忆(blockId 前缀 `toolrow-`,
* 与组级 `tools-` key 空间天然隔离)。无详情可展的行不显示 chevron、不可点击。
*/
-type ToolDetailActions = Pick<MessageActions, 'onLoadToolInput' | 'onOpenPayload'>;
+type ToolDetailActions = Pick<MessageActions, 'onLoadToolInput' | 'onOpenPayload' | 'onResolveRemoteMedia'>;
function ToolActionRow({
actions,
@@ -4287,6 +4288,15 @@ function ToolActionRow({
<Text style={[styles.toolName, styles.toolNameFlex]} numberOfLines={1}>{row.label}</Text>
{chevronNode}
</Pressable>
+ {readToolProcessMedia(tool.secondaryBody).map((entry) => (
+ <MediaPreview key={entry.url} label={mediaLabel({ ...entry, previewable: false })}
+ layout={contentLayout} media={{ ...entry, previewable: false }}
+ onOpen={actions.onOpenPayload ? () => actions.onOpenPayload!(buildMediaPayload(
+ { ...entry, previewable: false }, mediaLabel({ ...entry, previewable: false }),
+ )) : undefined}
+ onResolveRemoteMedia={actions.onResolveRemoteMedia}
+ variant={entry.kind === 'image' ? 'attachment' : 'card'} />
+ ))}
{showDetails ? (
<View style={styles.toolRowDetails}>
{row.detail || tool.body ? (
diff --git a/packages/maker-core/src/agents/claude-code/__tests__/translator-tool-output.test.ts b/packages/maker-core/src/agents/claude-code/__tests__/translator-tool-output.test.ts
index d6b3644989..49a118aee8 100644
--- a/packages/maker-core/src/agents/claude-code/__tests__/translator-tool-output.test.ts
+++ b/packages/maker-core/src/agents/claude-code/__tests__/translator-tool-output.test.ts
@@ -358,3 +358,17 @@ describe('Claude Code translator tool output normalization', () => {
});
});
});
+
+// Regression: image-only tool results used to disappear from the GUI; unit tier.
+it('retains native image-only results and their tool identity', async () => {
+ const queue = createAsyncQueue<AgentEvent>();
+ translateSdkMessage({ type: 'user', message: { content: [{ type: 'tool_result',
+ tool_use_id: 'read-image', content: [{ type: 'image', source: {
+ type: 'base64', media_type: 'image/png', data: 'aW1hZ2U=',
+ } }],
+ }] } }, queue, createCtx());
+ expect((await drain(queue)).find(e => e.type === 'tool_result_full')?.data).toMatchObject({
+ toolUseId: 'read-image', fullText: '',
+ processMedia: [{ kind: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' }],
+ });
+});
diff --git a/packages/maker-core/src/agents/claude-code/translator.ts b/packages/maker-core/src/agents/claude-code/translator.ts
index 8f5a6e52d3..b71c8a465d 100644
--- a/packages/maker-core/src/agents/claude-code/translator.ts
+++ b/packages/maker-core/src/agents/claude-code/translator.ts
@@ -1,3 +1,4 @@
+import { collectNativeToolMedia, type NativeToolMedia } from '@cindy/maker-shared/tool-process-media';
/**
* Claude Code SDKMessage → maker-core 语义 AgentEvent 翻译器。
*
@@ -481,6 +482,7 @@ export function extractAssistantMeta(rawMsg: unknown): Record<string, unknown> {
const TERMINAL_OUTPUT_TOOL_NAMES = new Set(['Bash', 'PowerShell']);
interface ToolResultPair {
+ processMedia: NativeToolMedia[];
toolUseId: string;
fullText: string;
isError: boolean;
@@ -493,7 +495,7 @@ function normalizeToolResultFullText(
const toolName = rt.toolUseIdToName.get(pair.toolUseId);
if (!toolName || !TERMINAL_OUTPUT_TOOL_NAMES.has(toolName)) return pair;
return {
- toolUseId: pair.toolUseId,
+ ...pair,
fullText: stripTerminalControlSequences(pair.fullText),
isError: pair.isError,
};
@@ -510,7 +512,7 @@ function extractToolResultFullText(
const pair = readToolResultFullText(block);
if (pair) {
const normalized = normalizeToolResultFullText(pair, rt);
- if (normalized.fullText.length > 0) out.push(normalized);
+ if (normalized.fullText.length > 0 || normalized.processMedia.length > 0) out.push(normalized);
}
}
return out;
@@ -535,7 +537,7 @@ function readToolResultFullText(blockRaw: unknown): ToolResultPair | null {
}
fullText = parts.join('\n');
}
- return { toolUseId: b.tool_use_id, fullText, isError: b.is_error === true };
+ return { toolUseId: b.tool_use_id, fullText, isError: b.is_error === true, processMedia: collectNativeToolMedia(inner) };
}
/**
@@ -811,7 +813,7 @@ export function translateSdkMessage(
for (const pair of fullPairs) {
queue.push({
type: 'tool_result_full',
- data: { toolUseId: pair.toolUseId, fullText: pair.fullText },
+ data: { toolUseId: pair.toolUseId, fullText: pair.fullText, ...(pair.processMedia.length ? { processMedia: pair.processMedia } : {}) },
source: 'claude-code',
...(parentToolUseId ? { agentMeta: { parentUuid: parentToolUseId } } : {}),
});
diff --git a/packages/maker-core/src/agents/codex/translator.test.ts b/packages/maker-core/src/agents/codex/translator.test.ts
index 3b96eadcac..9eb2e4f98e 100644
--- a/packages/maker-core/src/agents/codex/translator.test.ts
+++ b/packages/maker-core/src/agents/codex/translator.test.ts
@@ -2505,3 +2505,23 @@ it('does not count visible summaries as output and pauses the usage clock while
vi.useRealTimers();
}
});
+
+// Regression: imageView was discarded by the host; now a paired process step. Unit.
+it('projects imageView once and retains mixed MCP media', async () => {
+ const q = createAsyncQueue<AgentEvent>();
+ const ctx = makeCtx(newCodexRuntimeState());
+ const viewed = { threadId: 't', turnId: 'turn', item: { type: 'imageView', id: 'view', path: '/tmp/preview.png' } };
+ translateItemNotification('started', viewed, q, ctx);
+ translateItemNotification('completed', viewed, q, ctx);
+ translateItemNotification('completed', { threadId: 't', turnId: 'turn', item: {
+ type: 'mcpToolCall', id: 'mcp', server: 'preview', tool: 'capture', status: 'completed',
+ result: { content: [{ type: 'text', text: 'captured' }, { type: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' }] },
+ } }, q, ctx);
+ q.end(); const events: AgentEvent[] = []; for await (const e of q) events.push(e);
+ expect(events.filter(e => e.type === 'tool_use')).toHaveLength(1);
+ expect(events.filter(e => e.type === 'tool_result_full').map(e => e.data)).toEqual([
+ { toolUseId: 'view', fullText: '/tmp/preview.png', processMedia: [{ kind: 'image', path: '/tmp/preview.png' }] },
+ { toolUseId: 'mcp', fullText: 'captured', isError: false,
+ processMedia: [{ kind: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' }] },
+ ]);
+});
diff --git a/packages/maker-core/src/agents/codex/translator.ts b/packages/maker-core/src/agents/codex/translator.ts
index 726a47dc63..305138d79a 100644
--- a/packages/maker-core/src/agents/codex/translator.ts
+++ b/packages/maker-core/src/agents/codex/translator.ts
@@ -1,3 +1,4 @@
+import { nativeToolMediaFields, type NativeToolMedia } from '@cindy/maker-shared/tool-process-media';
/**
* Codex app-server v2 Notification → maker-core 语义 AgentEvent 翻译器。
*
@@ -1500,7 +1501,7 @@ function handleMcpToolCall(
}
queue.push({
type: 'tool_result_full',
- data: { toolUseId: item.id, fullText, isError },
+ data: { toolUseId: item.id, fullText, isError, ...nativeToolMediaFields(item.result?.content) },
source: 'codex',
});
queue.push({
@@ -1821,7 +1822,7 @@ function handleDynamicToolCall(
const fullText = dynamicContentItemsToText(item.contentItems);
queue.push({
type: 'tool_result_full',
- data: { toolUseId: item.id, fullText, isError },
+ data: { toolUseId: item.id, fullText, isError, ...nativeToolMediaFields(item.contentItems) },
source: 'codex',
});
queue.push({
@@ -2147,21 +2148,22 @@ function toCodexTaskUpdate(
};
}
-// ── imageView → image event (kind='view') ────────────────────────────────────
-// 模型读了一张本地图。codex 一般只发 completed (path 必有), started/updated 防御性同样发 — 同一
-// blockId, renderer 自己判重。
+// Native image reads are tool steps, with their preview inside the work group.
+// Only completed items have usable media; emit the step once.
function handleImageView(
phase: ItemPhase,
item: ImageViewItem,
queue: AsyncQueue<AgentEvent>,
): void {
- if (phase === 'updated') return;
- queue.push({
- type: 'image',
- data: { kind: 'view', blockId: item.id, path: item.path },
- source: 'codex',
- });
+ if (phase !== 'completed') return;
+ queue.push({ type: 'tool_use', source: 'codex',
+ data: { toolUseId: item.id, toolName: 'view_image', input: { path: item.path } } });
+ queue.push({ type: 'tool_result_full', source: 'codex',
+ data: { toolUseId: item.id, fullText: item.path,
+ processMedia: [{ kind: 'image', path: item.path } satisfies NativeToolMedia] } });
+ queue.push({ type: 'tool_result', source: 'codex',
+ data: { toolUseIds: [item.id], summary: 'done' } });
}
// ── imageGeneration → image event (kind='generation') ────────────────────────
diff --git a/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts b/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts
index 979735aebf..3564987cdb 100644
--- a/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts
+++ b/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts
@@ -2359,3 +2359,21 @@ describe('pi translator live token speed', () => {
}
});
});
+
+// Regression: [image] retained text but lost the actual media. Unit, native RPC entry.
+it('retains tool image bytes and a video resource alongside text', () => {
+ const ctx = createPiTranslateContext(noopLogger);
+ const { queue, events } = makeQueue();
+ translatePiEvent(ev({ type: 'tool_execution_end', toolCallId: 'media', result: {
+ content: [{ type: 'text', text: 'preview' },
+ { type: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' },
+ { type: 'resource_link', mimeType: 'video/mp4', uri: 'https://example.com/clip.mp4' }],
+ } }), queue, ctx);
+ expect(events.find(e => e.type === 'tool_result_full')?.data).toMatchObject({
+ toolUseId: 'media', fullText: 'preview\n[image]', processMedia: [
+ { kind: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' },
+ { kind: 'video', url: 'https://example.com/clip.mp4' },
+ ],
+ });
+ disposePiTranslateContext(ctx);
+});
diff --git a/packages/maker-core/src/agents/pi/translator.ts b/packages/maker-core/src/agents/pi/translator.ts
index b2af2bfaee..76fa2fb281 100644
--- a/packages/maker-core/src/agents/pi/translator.ts
+++ b/packages/maker-core/src/agents/pi/translator.ts
@@ -1,3 +1,4 @@
+import { nativeToolMediaFields } from '@cindy/maker-shared/tool-process-media';
/**
* pi RPC 事件 → AgentEvent 翻译层。
*
@@ -918,7 +919,7 @@ export function translatePiEvent(
: rawText;
queue.push({
type: 'tool_result_full',
- data: { toolUseId, fullText, isError },
+ data: { toolUseId, fullText, isError, ...nativeToolMediaFields((event.result as { content?: unknown } | null)?.content) },
source: 'pi',
});
queue.push({
diff --git a/packages/maker-shared/package.json b/packages/maker-shared/package.json
index 6264836188..3995906bb5 100644
--- a/packages/maker-shared/package.json
+++ b/packages/maker-shared/package.json
@@ -51,6 +51,7 @@
"./message-window": "./src/messageWindow.ts",
"./orca-team": "./src/orcaTeam.ts",
"./mobile-home": "./src/mobileHome.ts",
+ "./tool-process-media": "./src/toolProcessMedia.ts",
"./payload-summary": "./src/payloadSummary.ts",
"./project-order-sync": "./src/projectOrderSync.ts",
"./path-text": "./src/pathText.ts",
diff --git a/packages/maker-shared/src/__tests__/toolProcessMedia.test.ts b/packages/maker-shared/src/__tests__/toolProcessMedia.test.ts
new file mode 100644
index 0000000000..c427a54195
--- /dev/null
+++ b/packages/maker-shared/src/__tests__/toolProcessMedia.test.ts
@@ -0,0 +1,63 @@
+// User-requested process previews: native extraction + persisted refs used by both UIs.
+// Unit coverage missing from delivered-media tests; image-only and mixed video are minimal inputs.
+import { describe, it, expect } from "vitest";
+import {
+ collectNativeToolMedia,
+ readToolProcessMedia,
+ withToolProcessMedia,
+} from "../toolProcessMedia";
+describe("tool process media", () => {
+ it("keeps MCP embedded video and Codex dynamic images", () => {
+ expect(
+ collectNativeToolMedia([
+ {
+ type: "resource",
+ resource: { mimeType: "video/mp4", blob: "dmlkZW8=" },
+ },
+ { type: "inputImage", imageUrl: "data:image/png;base64,aW1hZ2U=" },
+ { type: "text", text: "view /tmp/not-an-image.png" },
+ ]),
+ ).toEqual([
+ { kind: "video", data: "dmlkZW8=", mimeType: "video/mp4" },
+ { kind: "image", url: "data:image/png;base64,aW1hZ2U=" },
+ ]);
+ });
+ it("round trips media beside existing delivered results", () => {
+ const refs = [
+ {
+ kind: "image" as const,
+ url: "cindy-media://blobs/" + "a".repeat(64) + ".png",
+ },
+ ];
+ const text = withToolProcessMedia(
+ '{"xdt_video_url":"https://example.com/final.mp4"}',
+ refs,
+ );
+ expect(readToolProcessMedia(text)).toEqual(refs);
+ expect(JSON.parse(text).xdt_video_url).toBe(
+ "https://example.com/final.mp4",
+ );
+ });
+ it("rejects arbitrary native paths and active schemes in rendered metadata", () => {
+ expect(
+ readToolProcessMedia(
+ JSON.stringify({
+ cindy_process_media: [
+ { kind: "image", url: "file:///etc/passwd" },
+ { kind: "video", url: "javascript:alert(1)" },
+ ],
+ }),
+ ),
+ ).toEqual([]);
+ expect(readToolProcessMedia("broken cindy_process_media")).toEqual([]);
+ });
+ it("keeps multilingual output within the remote byte budget without breaking the media JSON", () => {
+ const refs = [
+ { kind: "video" as const, url: "https://example.com/clip.mp4" },
+ ];
+ const text = withToolProcessMedia("截图说明😀".repeat(3000), refs);
+ expect(new TextEncoder().encode(text).length).toBeLessThanOrEqual(8192);
+ expect(readToolProcessMedia(text)).toEqual(refs);
+ expect(JSON.parse(text).text).toMatch(/^截图说明😀/);
+ });
+});
diff --git a/packages/maker-shared/src/toolProcessMedia.ts b/packages/maker-shared/src/toolProcessMedia.ts
new file mode 100644
index 0000000000..d9fb12066c
--- /dev/null
+++ b/packages/maker-shared/src/toolProcessMedia.ts
@@ -0,0 +1,112 @@
+/** Native tool media stays with its tool step; explicit delivered media is unchanged. */
+export interface NativeToolMedia {
+ kind: "image" | "video";
+ data?: string;
+ mimeType?: string;
+ url?: string;
+ path?: string;
+}
+export interface ProcessMediaRef {
+ kind: "image" | "video";
+ url: string;
+}
+
+const record = (v: unknown): Record<string, unknown> | null =>
+ v && typeof v === "object" && !Array.isArray(v)
+ ? (v as Record<string, unknown>)
+ : null;
+
+/** MCP, Anthropic and Pi content blocks. Never infer media from prose or commands. */
+export function collectNativeToolMedia(content: unknown): NativeToolMedia[] {
+ if (!Array.isArray(content)) return [];
+ const result: NativeToolMedia[] = [];
+ for (const value of content) {
+ const block = record(value);
+ if (!block) continue;
+ const source = record(block.source);
+ const resource = record(block.resource);
+ const mime = source?.media_type ?? block.mimeType ?? resource?.mimeType;
+ const kind =
+ block.type === "video" ||
+ (typeof mime === "string" && mime.startsWith("video/"))
+ ? "video"
+ : block.type === "image" ||
+ block.type === "inputImage" ||
+ (typeof mime === "string" && mime.startsWith("image/"))
+ ? "image"
+ : null;
+ if (!kind) continue;
+ const data = source?.data ?? block.data ?? resource?.blob;
+ const url = source?.url ?? block.imageUrl ?? resource?.uri ?? block.uri;
+ if (typeof data === "string" && typeof mime === "string") {
+ result.push({ kind, data, mimeType: mime });
+ } else if (typeof url === "string") result.push({ kind, url });
+ }
+ return result;
+}
+
+export function isProcessMediaUrl(url: string): boolean {
+ return (
+ /^cindy-media:\/\/blobs\/[a-f0-9]{64}\.(?:png|jpg|jpeg|gif|webp|mp4|webm|mov)$/.test(
+ url,
+ ) ||
+ /^xdt-(?:image|video):\/\//.test(url) ||
+ /^https?:\/\//.test(url)
+ );
+}
+
+export function readToolProcessMedia(
+ text: string | undefined,
+): ProcessMediaRef[] {
+ if (!text?.includes("cindy_process_media")) return [];
+ try {
+ const parsed = record(JSON.parse(text));
+ if (!Array.isArray(parsed?.cindy_process_media)) return [];
+ return parsed.cindy_process_media.flatMap((value): ProcessMediaRef[] => {
+ const item = record(value);
+ return item &&
+ (item.kind === "image" || item.kind === "video") &&
+ typeof item.url === "string" &&
+ isProcessMediaUrl(item.url)
+ ? [{ kind: item.kind, url: item.url }]
+ : [];
+ });
+ } catch {
+ return [];
+ }
+}
+
+export function withToolProcessMedia(
+ text: string,
+ media: ProcessMediaRef[],
+): string {
+ let payload: Record<string, unknown> = { text };
+ try {
+ payload = record(JSON.parse(text)) ?? payload;
+ } catch {
+ /* ordinary tool text */
+ }
+ const encode = (value: Record<string, unknown>) =>
+ JSON.stringify({ ...value, cindy_process_media: media });
+ const encoded = encode(payload);
+ // Remote history has an 8KB byte budget. Truncate prose, never cut JSON or a URL.
+ const bytes = new TextEncoder();
+ if (bytes.encode(encoded).length <= 8192) return encoded;
+ let low = 0;
+ let high = text.length;
+ while (low < high) {
+ const mid = Math.ceil((low + high) / 2);
+ if (bytes.encode(encode({ text: text.slice(0, mid) })).length <= 8192)
+ low = mid;
+ else high = mid - 1;
+ }
+ if (low > 0 && /[\uD800-\uDBFF]/.test(text[low - 1])) low--;
+ return encode({ text: text.slice(0, low) });
+}
+
+export function nativeToolMediaFields(content: unknown): {
+ processMedia?: NativeToolMedia[];
+} {
+ const processMedia = collectNativeToolMedia(content);
+ return processMedia.length ? { processMedia } : {};
+}