Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
feat: retain native tool images and videos in work details
Chrisf77c1af9bb84
diff --git a/apps/desktop/src/main/__tests__/messagePersistBroadcaster.test.ts b/apps/desktop/src/main/__tests__/messagePersistBroadcaster.test.tsindex 71338ddf1d..1211effad4 100644--- a/apps/desktop/src/main/__tests__/messagePersistBroadcaster.test.ts+++ b/apps/desktop/src/main/__tests__/messagePersistBroadcaster.test.ts@@ -1,3 +1,8 @@+import { materializeToolProcessMedia } from '../cindy-media/toolProcessMedia.js';+vi.mock('../cindy-media/toolProcessMedia.js', () => ({ materializeToolProcessMedia: vi.fn(async () => [+ { kind: 'image', url: 'cindy-media://blobs/' + 'a'.repeat(64) + '.png' },+]) }));+vi.mock('../localDb/client/current.js', () => ({ getDbClient: vi.fn(() => ({ drizzle: {} })) }));vi.mock('../botTaskReplyResults.js', () => ({ readTaskResultsForReply: vi.fn(async () => []) }));import { readTaskResultsForReply } from '../botTaskReplyResults.js';/**@@ -76,6 +81,7 @@ import {__resetMediaToolResultPoolForTesting,} from '../mcp-integrations/mediaToolResultFallback.js';import {+ persistToolProcessMedia,onToolUseEvent,onAgentTaskUpdateEvent,persistCodexPlanOnDone,@@ -4085,3 +4091,17 @@ it('persists bound results with the final seal, and a failed result lookup cannoawait expect(markAssistantTurnCompleted(SESSION, 'other', undefined, ['bot-delegation-completion:job'])).resolves.toBe(true);expect(patchMessageAgentMetaWithResult).toHaveBeenCalledWith(SESSION, 'other', { turnCompleted: true });});++// Unit regression: a late import updates the original result, never the next turn.+it('persists process media on its accepted result and broadcasts the durable row', async () => {+ onToolUseEvent(SESSION, { toolUseId: 'native-image', toolName: 'Read', input: {} }, null);+ const result = onToolResultFullEvent(SESSION, { toolUseId: 'native-image', fullText: 'read' }, null)!;+ persistToolProcessMedia(SESSION, result.persistId, result.content, [{ kind: 'image', data: 'bytes' }]);+ await drainPersistQueue();+ const updated = vi.mocked(updateMessageContent).mock.calls.find(([, id]) => id === result.persistId);+ expect(JSON.parse(updated![2] as string)).toEqual({ text: 'read', cindy_process_media: [+ { kind: 'image', url: 'cindy-media://blobs/' + 'a'.repeat(64) + '.png' },+ ] });+ expect(broadcastMessageRow).toHaveBeenCalledWith(SESSION, expect.anything(), ownerScopeState.scope);+ expect(materializeToolProcessMedia).toHaveBeenCalled();+});diff --git a/apps/desktop/src/main/cindy-media/__tests__/toolProcessMedia.test.ts b/apps/desktop/src/main/cindy-media/__tests__/toolProcessMedia.test.tsnew file mode 100644index 0000000000..c5313d0335--- /dev/null+++ b/apps/desktop/src/main/cindy-media/__tests__/toolProcessMedia.test.ts@@ -0,0 +1,81 @@+// Unit: native bytes become managed refs, never raw base64 in message history.+// Existing generated-media tests do not cover native tool blocks or owner invalidation.+import { describe, expect, it, vi } from 'vitest';+import { materializeToolProcessMedia } from '../toolProcessMedia';+import { ingestMedia } from '../ingest';+import type { LedgerDb } from '../ledger';+import fs from 'node:fs/promises';+import { materializeSshRemoteMedia } from '../../file-browser/ssh-media';+vi.mock('node:fs/promises', () => ({ default: { readFile: vi.fn() } }));+vi.mock('../../file-browser/ssh-media', () => ({ materializeSshRemoteMedia: vi.fn() }));+vi.mock('../ingest', () => ({+ ingestMedia: vi.fn(async () => ({ url: 'cindy-media://blobs/' + 'a'.repeat(64) + '.png' })),+}));+const db = {} as LedgerDb;+const png = Buffer.from(+ 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aL1kAAAAASUVORK5CYII=',+ 'base64',+);+describe('native tool media materialization', () => {+ it('ingests verified bytes and collapses repeated refs', async () => {+ const image = { kind: 'image' as const, data: png.toString('base64'), mimeType: 'image/png' };+ expect(await materializeToolProcessMedia([image, image], db, () => {})).toEqual([+ { kind: 'image', url: 'cindy-media://blobs/' + 'a'.repeat(64) + '.png' },+ ]);+ expect(ingestMedia).toHaveBeenCalledWith(+ expect.objectContaining({ buffer: png, mimeType: 'image/png', refs: [] }),+ db,+ );+ });+ it('rejects false media types without importing them', async () => {+ vi.mocked(ingestMedia).mockClear();+ expect(+ await materializeToolProcessMedia(+ [{ kind: 'video', data: png.toString('base64'), mimeType: 'video/mp4' }],+ db,+ () => {},+ ),+ ).toEqual([]);+ expect(ingestMedia).not.toHaveBeenCalled();+ });+ it('keeps playable video references in the tool step', async () => {+ const video = { kind: 'video' as const, url: 'https://example.com/clip.mp4' };+ expect(await materializeToolProcessMedia([video], db, () => {})).toEqual([video]);+ });+ it('stops before writing when the owner changed', async () => {+ vi.mocked(ingestMedia).mockClear();+ await expect(+ materializeToolProcessMedia([{ kind: 'image', data: png.toString('base64') }], db, () => {+ throw new Error('owner changed');+ }),+ ).rejects.toThrow('owner changed');+ expect(ingestMedia).not.toHaveBeenCalled();+ });+ it('reads remote paths only through the SSH cache and keeps other media after a missing file', async () => {+ vi.mocked(materializeSshRemoteMedia).mockResolvedValueOnce({+ ok: true,+ cachePath: '/cache/image',+ sourceVersion: 'v',+ size: png.length,+ mime: 'image/png',+ relPath: 'image.png',+ });+ vi.mocked(fs.readFile)+ .mockResolvedValueOnce(png as never)+ .mockRejectedValueOnce(new Error('missing'));+ const origin = { workdir: '/remote', remoteHostId: 'ssh-host' };+ const refs = await materializeToolProcessMedia(+ [{ kind: 'image', path: '/remote/image.png' }],+ db,+ () => {},+ origin,+ );+ expect(fs.readFile).toHaveBeenCalledWith('/cache/image');+ expect(fs.readFile).not.toHaveBeenCalledWith('/remote/image.png');+ expect(refs).toHaveLength(1);+ const video = { kind: 'video' as const, url: 'https://example.com/video.mp4' };+ expect(+ await materializeToolProcessMedia([{ kind: 'image', path: '/missing' }, video], db, () => {}),+ ).toEqual([video]);+ });+});diff --git a/apps/desktop/src/main/cindy-media/toolProcessMedia.ts b/apps/desktop/src/main/cindy-media/toolProcessMedia.tsnew file mode 100644index 0000000000..c2891d1391--- /dev/null+++ b/apps/desktop/src/main/cindy-media/toolProcessMedia.ts@@ -0,0 +1,65 @@+import fs from 'node:fs/promises';+import { fileURLToPath } from 'node:url';+import type { NativeToolMedia, ProcessMediaRef } from '@cindy/maker-shared/tool-process-media';+import { isProcessMediaUrl } from '@cindy/maker-shared/tool-process-media';+import { ingestMedia } from './ingest';+import { sniffMediaMime } from './sniffMediaMime';+import type { LedgerDb } from './ledger';+import { createLogger } from '../logger';++const log = createLogger('tool-process-media');+export interface ToolMediaOrigin {+ workdir: string;+ remoteHostId: string | null;+}++/** Native bytes never enter chat history. Store one managed copy for all clients. */+export async function materializeToolProcessMedia(+ items: NativeToolMedia[],+ db: LedgerDb,+ assertStillValid: () => void,+ origin?: ToolMediaOrigin,+): Promise<ProcessMediaRef[]> {+ const result: ProcessMediaRef[] = [];+ for (const item of items) {+ assertStillValid();+ try {+ if (item.url && isProcessMediaUrl(item.url)) {+ result.push({ kind: item.kind, url: item.url });+ continue;+ }+ const dataUrl = item.url?.match(+ /^data:((?:image|video)\/[a-zA-Z0-9.+-]+);base64,([A-Za-z0-9+/=\s]+)$/,+ );+ const encoded = item.data ?? dataUrl?.[2];+ const sourcePath =+ item.path ?? (item.url?.startsWith('file:') ? fileURLToPath(item.url) : undefined);+ let buffer: Buffer;+ if (encoded) buffer = Buffer.from(encoded, 'base64');+ else if (sourcePath) {+ let localPath = sourcePath;+ if (origin?.remoteHostId) {+ const { materializeSshRemoteMedia } = await import('../file-browser/ssh-media');+ const remote = await materializeSshRemoteMedia(+ { workdir: origin.workdir, remoteHostId: origin.remoteHostId },+ `xdt-file://?path=${encodeURIComponent(sourcePath)}`,+ );+ assertStillValid();+ if (!remote.ok) continue;+ localPath = remote.cachePath;+ }+ buffer = await fs.readFile(localPath);+ } else continue;+ assertStillValid();+ const mimeType = sniffMediaMime(buffer, item.mimeType ?? dataUrl?.[1]);+ if (!mimeType?.startsWith(item.kind + '/')) continue;+ const media = await ingestMedia({ buffer, mimeType, refs: [], assertStillValid }, db);+ result.push({ kind: item.kind, url: media.url });+ } catch {+ // One unavailable attachment must not hide the rest; owner changes still abort.+ assertStillValid();+ log.warn('Native tool attachment could not be stored', { kind: item.kind });+ }+ }+ return [...new Map(result.map((item) => [item.kind + ':' + item.url, item])).values()];+}diff --git a/apps/desktop/src/main/maker-ipc/register.ts b/apps/desktop/src/main/maker-ipc/register.tsindex 2c91088646..99fc9469db 100644--- a/apps/desktop/src/main/maker-ipc/register.ts+++ b/apps/desktop/src/main/maker-ipc/register.ts@@ -23470,6 +23470,10 @@ function redactEventForRenderer(event: AgentEvent): AgentEvent {const data = event.data as Record<string, unknown>;const safeData = { ...data };let changed = false;+ if ('processMedia' in safeData) {+ delete safeData.processMedia;+ changed = true;+ }if (typeof safeData.toolName === 'string' &&Object.prototype.hasOwnProperty.call(safeData, 'input')diff --git a/apps/desktop/src/main/maker-ipc/sessionEventStream.ts b/apps/desktop/src/main/maker-ipc/sessionEventStream.tsindex 824536c4ac..11c94f1faa 100644--- a/apps/desktop/src/main/maker-ipc/sessionEventStream.ts+++ b/apps/desktop/src/main/maker-ipc/sessionEventStream.ts@@ -1,3 +1,4 @@+import type { NativeToolMedia } from '@cindy/maker-shared/tool-process-media';import type { Session } from '@cindy/maker-core';import { persistSubagentTaskUpdate } from '../localDb/subagentRuns.js';import { broadcastSubagentRunsChanged } from '../localDb/ipc/subagentRuns.js';@@ -15,6 +16,7 @@ import {onThinkingEvent,onToolResultEvent,onToolResultFullEvent,+ persistToolProcessMedia,onToolUseEvent,} from '../messagePersistBroadcaster.js';import { type OrcaTeamService } from './orcaTeamService.js';@@ -91,6 +93,12 @@ export function persistSessionStreamEvent();persistId = r?.persistId;resolvedContent = r?.content;+ const processMedia = (event.data as { processMedia?: NativeToolMedia[] }).processMedia;+ if (r && Array.isArray(processMedia) && processMedia.length) {+ persistToolProcessMedia(session.id, r.persistId, r.content, processMedia, {+ workdir: session.workDir, remoteHostId: session.remoteHostId,+ });+ }} else if (event.type === 'thinking') {// thinking final/redacted 落库收口 main;clientId=blockId(renderer 同源),无需// persistId 回传。start/delta 不落库。diff --git a/apps/desktop/src/main/messagePersistBroadcaster.ts b/apps/desktop/src/main/messagePersistBroadcaster.tsindex 2be06612c2..3cff46f127 100644--- a/apps/desktop/src/main/messagePersistBroadcaster.ts+++ b/apps/desktop/src/main/messagePersistBroadcaster.ts@@ -1,3 +1,5 @@+import { materializeToolProcessMedia } from './cindy-media/toolProcessMedia.js';+import { withToolProcessMedia, type NativeToolMedia } from '@cindy/maker-shared/tool-process-media';import { readTaskResultsForReply } from './botTaskReplyResults.js';/*** messagePersistBroadcaster — 把 agent 消息的持久化从 renderer 收口到 main 单点。@@ -1745,6 +1747,40 @@ export function onToolResultEvent(return { persistId, content };}+/** Attach native media to the already accepted result row, never to the current turn.+ * The existing owner-scoped FIFO prevents clear/delete or the next turn from stealing it.+ */+export function persistToolProcessMedia(+ sessionId: string,+ persistId: string,+ text: string,+ items: NativeToolMedia[],+ origin?: import('./cindy-media/toolProcessMedia').ToolMediaOrigin,+): void {+ if (!items.length) return;+ enqueueWrite(`tool_media:${sessionId}:${persistId}`, async (ownerScope) => {+ const assertStillValid = () => {+ if (!isOwnerScopeCurrent(ownerScope)) throw ownerScopeSupersededError();+ };+ const db = getDbClient().drizzle;+ const media = await materializeToolProcessMedia(items, db, assertStillValid, origin);+ assertStillValid();+ if (!media.length) return;+ // Keep media refs outside the text truncation budget so reopening retains images.+ const content = withToolProcessMedia(text, media);+ const updated = await updateDbMessageContent(sessionId, persistId, content);+ if (updated) {+ toolResultContentByClientId.get(sessionId)?.set(persistId, content);+ broadcastMessageRow(sessionId, updated, ownerScope);+ }+ });+}++function markFailedToolResultText(text: string, isError: boolean): string {+ if (!isError || text.includes('<tool_use_error>')) return text;+ return `<tool_use_error>${text}</tool_use_error>`;+}+/*** 处理 tool_result_full 事件(toolUseId + 全文)。返回 { persistId, content } 让* renderer 把对应 tool_result 气泡内容更新成全文;返回 null 表示无需显示变更@@ -1753,11 +1789,6 @@ export function onToolResultEvent(* 对齐老 renderer:有映射 → 覆盖更新;无映射但 tool_use 已到 → eager-create;* tool_use 也没到 → buffer。*/-function markFailedToolResultText(text: string, isError: boolean): string {- if (!isError || text.includes('<tool_use_error>')) return text;- return `<tool_use_error>${text}</tool_use_error>`;-}-export function onToolResultFullEvent(sessionId: string,data: { toolUseId?: unknown; fullText?: unknown; isError?: unknown },diff --git a/apps/desktop/src/renderer/components/chat/AgentActionRow.tsx b/apps/desktop/src/renderer/components/chat/AgentActionRow.tsxindex bed70363e9..c714a61971 100644--- a/apps/desktop/src/renderer/components/chat/AgentActionRow.tsx+++ b/apps/desktop/src/renderer/components/chat/AgentActionRow.tsx@@ -1,3 +1,6 @@+import { readToolProcessMedia } from '@cindy/maker-shared/tool-process-media';+import { ChatVideoView } from './ChatVideoView';+import { ChatImageView } from './ChatImageView';import { FileTypeIcon } from '@/components/ui/file-type-icon';/*** AgentActionRow@@ -788,6 +791,7 @@ export function AgentActionRow({onActivateOverride,}: AgentActionRowProps) {const { t } = useTranslation();+ const processMedia = useMemo(() => readToolProcessMedia(toolResult), [toolResult]);const compactedToolResult = parseToolResultCompactionMarker(toolResult);const displayedToolResult = compactedToolResult? t('chat.toolResultCompacted', {@@ -1172,6 +1176,13 @@ export function AgentActionRow({{rawCommand}</div>)}+ {processMedia.length > 0 && (+ <div className="flex min-w-0 flex-col gap-3 py-2 pl-8" data-tool-process-media="true">+ {processMedia.map((media) => media.kind === 'image'+ ? <ChatImageView key={media.url} src={media.url} variant="tool-output" sessionId={fileCtx.sessionId} filename={media.url.split('/').pop() ?? media.kind} />+ : <ChatVideoView key={media.url} src={media.url} variant="tool-output" sessionId={fileCtx.sessionId} filename={media.url.split('/').pop() ?? media.kind} />)}+ </div>+ )}{chipFilePath ? fileChipMenu.menu : null}{/* v10 就地展开内容:命令类工具(Bash/Grep/Glob/WebFetch/WebSearch/...)diff --git a/apps/mobile/src/session/MessageRenderer.tsx b/apps/mobile/src/session/MessageRenderer.tsxindex 8457d2a602..1b78f3e2a9 100644--- a/apps/mobile/src/session/MessageRenderer.tsx+++ b/apps/mobile/src/session/MessageRenderer.tsx@@ -1,3 +1,4 @@+import { readToolProcessMedia } from '@cindy/maker-shared/tool-process-media';import { PluginCardActions } from '@/plugins/PluginCardActions';import { McpAppCard } from './McpAppCard';import { RichContentContext } from './richContentContext';@@ -4240,7 +4241,7 @@ function ToolGroupCard({* 媒体条 / 结果预览,再点收起。展开态走共享进程内记忆(blockId 前缀 `toolrow-`,* 与组级 `tools-` key 空间天然隔离)。无详情可展的行不显示 chevron、不可点击。*/-type ToolDetailActions = Pick<MessageActions, 'onLoadToolInput' | 'onOpenPayload'>;+type ToolDetailActions = Pick<MessageActions, 'onLoadToolInput' | 'onOpenPayload' | 'onResolveRemoteMedia'>;function ToolActionRow({actions,@@ -4287,6 +4288,15 @@ function ToolActionRow({<Text style={[styles.toolName, styles.toolNameFlex]} numberOfLines={1}>{row.label}</Text>{chevronNode}</Pressable>+ {readToolProcessMedia(tool.secondaryBody).map((entry) => (+ <MediaPreview key={entry.url} label={mediaLabel({ ...entry, previewable: false })}+ layout={contentLayout} media={{ ...entry, previewable: false }}+ onOpen={actions.onOpenPayload ? () => actions.onOpenPayload!(buildMediaPayload(+ { ...entry, previewable: false }, mediaLabel({ ...entry, previewable: false }),+ )) : undefined}+ onResolveRemoteMedia={actions.onResolveRemoteMedia}+ variant={entry.kind === 'image' ? 'attachment' : 'card'} />+ ))}{showDetails ? (<View style={styles.toolRowDetails}>{row.detail || tool.body ? (diff --git a/packages/maker-core/src/agents/claude-code/__tests__/translator-tool-output.test.ts b/packages/maker-core/src/agents/claude-code/__tests__/translator-tool-output.test.tsindex d6b3644989..49a118aee8 100644--- a/packages/maker-core/src/agents/claude-code/__tests__/translator-tool-output.test.ts+++ b/packages/maker-core/src/agents/claude-code/__tests__/translator-tool-output.test.ts@@ -358,3 +358,17 @@ describe('Claude Code translator tool output normalization', () => {});});});++// Regression: image-only tool results used to disappear from the GUI; unit tier.+it('retains native image-only results and their tool identity', async () => {+ const queue = createAsyncQueue<AgentEvent>();+ translateSdkMessage({ type: 'user', message: { content: [{ type: 'tool_result',+ tool_use_id: 'read-image', content: [{ type: 'image', source: {+ type: 'base64', media_type: 'image/png', data: 'aW1hZ2U=',+ } }],+ }] } }, queue, createCtx());+ expect((await drain(queue)).find(e => e.type === 'tool_result_full')?.data).toMatchObject({+ toolUseId: 'read-image', fullText: '',+ processMedia: [{ kind: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' }],+ });+});diff --git a/packages/maker-core/src/agents/claude-code/translator.ts b/packages/maker-core/src/agents/claude-code/translator.tsindex 8f5a6e52d3..b71c8a465d 100644--- a/packages/maker-core/src/agents/claude-code/translator.ts+++ b/packages/maker-core/src/agents/claude-code/translator.ts@@ -1,3 +1,4 @@+import { collectNativeToolMedia, type NativeToolMedia } from '@cindy/maker-shared/tool-process-media';/*** Claude Code SDKMessage → maker-core 语义 AgentEvent 翻译器。*@@ -481,6 +482,7 @@ export function extractAssistantMeta(rawMsg: unknown): Record<string, unknown> {const TERMINAL_OUTPUT_TOOL_NAMES = new Set(['Bash', 'PowerShell']);interface ToolResultPair {+ processMedia: NativeToolMedia[];toolUseId: string;fullText: string;isError: boolean;@@ -493,7 +495,7 @@ function normalizeToolResultFullText(const toolName = rt.toolUseIdToName.get(pair.toolUseId);if (!toolName || !TERMINAL_OUTPUT_TOOL_NAMES.has(toolName)) return pair;return {- toolUseId: pair.toolUseId,+ ...pair,fullText: stripTerminalControlSequences(pair.fullText),isError: pair.isError,};@@ -510,7 +512,7 @@ function extractToolResultFullText(const pair = readToolResultFullText(block);if (pair) {const normalized = normalizeToolResultFullText(pair, rt);- if (normalized.fullText.length > 0) out.push(normalized);+ if (normalized.fullText.length > 0 || normalized.processMedia.length > 0) out.push(normalized);}}return out;@@ -535,7 +537,7 @@ function readToolResultFullText(blockRaw: unknown): ToolResultPair | null {}fullText = parts.join('\n');}- return { toolUseId: b.tool_use_id, fullText, isError: b.is_error === true };+ return { toolUseId: b.tool_use_id, fullText, isError: b.is_error === true, processMedia: collectNativeToolMedia(inner) };}/**@@ -811,7 +813,7 @@ export function translateSdkMessage(for (const pair of fullPairs) {queue.push({type: 'tool_result_full',- data: { toolUseId: pair.toolUseId, fullText: pair.fullText },+ data: { toolUseId: pair.toolUseId, fullText: pair.fullText, ...(pair.processMedia.length ? { processMedia: pair.processMedia } : {}) },source: 'claude-code',...(parentToolUseId ? { agentMeta: { parentUuid: parentToolUseId } } : {}),});diff --git a/packages/maker-core/src/agents/codex/translator.test.ts b/packages/maker-core/src/agents/codex/translator.test.tsindex 3b96eadcac..9eb2e4f98e 100644--- a/packages/maker-core/src/agents/codex/translator.test.ts+++ b/packages/maker-core/src/agents/codex/translator.test.ts@@ -2505,3 +2505,23 @@ it('does not count visible summaries as output and pauses the usage clock whilevi.useRealTimers();}});++// Regression: imageView was discarded by the host; now a paired process step. Unit.+it('projects imageView once and retains mixed MCP media', async () => {+ const q = createAsyncQueue<AgentEvent>();+ const ctx = makeCtx(newCodexRuntimeState());+ const viewed = { threadId: 't', turnId: 'turn', item: { type: 'imageView', id: 'view', path: '/tmp/preview.png' } };+ translateItemNotification('started', viewed, q, ctx);+ translateItemNotification('completed', viewed, q, ctx);+ translateItemNotification('completed', { threadId: 't', turnId: 'turn', item: {+ type: 'mcpToolCall', id: 'mcp', server: 'preview', tool: 'capture', status: 'completed',+ result: { content: [{ type: 'text', text: 'captured' }, { type: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' }] },+ } }, q, ctx);+ q.end(); const events: AgentEvent[] = []; for await (const e of q) events.push(e);+ expect(events.filter(e => e.type === 'tool_use')).toHaveLength(1);+ expect(events.filter(e => e.type === 'tool_result_full').map(e => e.data)).toEqual([+ { toolUseId: 'view', fullText: '/tmp/preview.png', processMedia: [{ kind: 'image', path: '/tmp/preview.png' }] },+ { toolUseId: 'mcp', fullText: 'captured', isError: false,+ processMedia: [{ kind: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' }] },+ ]);+});diff --git a/packages/maker-core/src/agents/codex/translator.ts b/packages/maker-core/src/agents/codex/translator.tsindex 726a47dc63..305138d79a 100644--- a/packages/maker-core/src/agents/codex/translator.ts+++ b/packages/maker-core/src/agents/codex/translator.ts@@ -1,3 +1,4 @@+import { nativeToolMediaFields, type NativeToolMedia } from '@cindy/maker-shared/tool-process-media';/*** Codex app-server v2 Notification → maker-core 语义 AgentEvent 翻译器。*@@ -1500,7 +1501,7 @@ function handleMcpToolCall(}queue.push({type: 'tool_result_full',- data: { toolUseId: item.id, fullText, isError },+ data: { toolUseId: item.id, fullText, isError, ...nativeToolMediaFields(item.result?.content) },source: 'codex',});queue.push({@@ -1821,7 +1822,7 @@ function handleDynamicToolCall(const fullText = dynamicContentItemsToText(item.contentItems);queue.push({type: 'tool_result_full',- data: { toolUseId: item.id, fullText, isError },+ data: { toolUseId: item.id, fullText, isError, ...nativeToolMediaFields(item.contentItems) },source: 'codex',});queue.push({@@ -2147,21 +2148,22 @@ function toCodexTaskUpdate(};}-// ── imageView → image event (kind='view') ────────────────────────────────────-// 模型读了一张本地图。codex 一般只发 completed (path 必有), started/updated 防御性同样发 — 同一-// blockId, renderer 自己判重。+// Native image reads are tool steps, with their preview inside the work group.+// Only completed items have usable media; emit the step once.function handleImageView(phase: ItemPhase,item: ImageViewItem,queue: AsyncQueue<AgentEvent>,): void {- if (phase === 'updated') return;- queue.push({- type: 'image',- data: { kind: 'view', blockId: item.id, path: item.path },- source: 'codex',- });+ if (phase !== 'completed') return;+ queue.push({ type: 'tool_use', source: 'codex',+ data: { toolUseId: item.id, toolName: 'view_image', input: { path: item.path } } });+ queue.push({ type: 'tool_result_full', source: 'codex',+ data: { toolUseId: item.id, fullText: item.path,+ processMedia: [{ kind: 'image', path: item.path } satisfies NativeToolMedia] } });+ queue.push({ type: 'tool_result', source: 'codex',+ data: { toolUseIds: [item.id], summary: 'done' } });}// ── imageGeneration → image event (kind='generation') ────────────────────────diff --git a/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts b/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.tsindex 979735aebf..3564987cdb 100644--- a/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts+++ b/packages/maker-core/src/agents/pi/__tests__/pi-translator.test.ts@@ -2359,3 +2359,21 @@ describe('pi translator live token speed', () => {}});});++// Regression: [image] retained text but lost the actual media. Unit, native RPC entry.+it('retains tool image bytes and a video resource alongside text', () => {+ const ctx = createPiTranslateContext(noopLogger);+ const { queue, events } = makeQueue();+ translatePiEvent(ev({ type: 'tool_execution_end', toolCallId: 'media', result: {+ content: [{ type: 'text', text: 'preview' },+ { type: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' },+ { type: 'resource_link', mimeType: 'video/mp4', uri: 'https://example.com/clip.mp4' }],+ } }), queue, ctx);+ expect(events.find(e => e.type === 'tool_result_full')?.data).toMatchObject({+ toolUseId: 'media', fullText: 'preview\n[image]', processMedia: [+ { kind: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' },+ { kind: 'video', url: 'https://example.com/clip.mp4' },+ ],+ });+ disposePiTranslateContext(ctx);+});diff --git a/packages/maker-core/src/agents/pi/translator.ts b/packages/maker-core/src/agents/pi/translator.tsindex b2af2bfaee..76fa2fb281 100644--- a/packages/maker-core/src/agents/pi/translator.ts+++ b/packages/maker-core/src/agents/pi/translator.ts@@ -1,3 +1,4 @@+import { nativeToolMediaFields } from '@cindy/maker-shared/tool-process-media';/*** pi RPC 事件 → AgentEvent 翻译层。*@@ -918,7 +919,7 @@ export function translatePiEvent(: rawText;queue.push({type: 'tool_result_full',- data: { toolUseId, fullText, isError },+ data: { toolUseId, fullText, isError, ...nativeToolMediaFields((event.result as { content?: unknown } | null)?.content) },source: 'pi',});queue.push({diff --git a/packages/maker-shared/package.json b/packages/maker-shared/package.jsonindex 6264836188..3995906bb5 100644--- a/packages/maker-shared/package.json+++ b/packages/maker-shared/package.json@@ -51,6 +51,7 @@"./message-window": "./src/messageWindow.ts","./orca-team": "./src/orcaTeam.ts","./mobile-home": "./src/mobileHome.ts",+ "./tool-process-media": "./src/toolProcessMedia.ts","./payload-summary": "./src/payloadSummary.ts","./project-order-sync": "./src/projectOrderSync.ts","./path-text": "./src/pathText.ts",diff --git a/packages/maker-shared/src/__tests__/toolProcessMedia.test.ts b/packages/maker-shared/src/__tests__/toolProcessMedia.test.tsnew file mode 100644index 0000000000..c427a54195--- /dev/null+++ b/packages/maker-shared/src/__tests__/toolProcessMedia.test.ts@@ -0,0 +1,63 @@+// User-requested process previews: native extraction + persisted refs used by both UIs.+// Unit coverage missing from delivered-media tests; image-only and mixed video are minimal inputs.+import { describe, it, expect } from "vitest";+import {+ collectNativeToolMedia,+ readToolProcessMedia,+ withToolProcessMedia,+} from "../toolProcessMedia";+describe("tool process media", () => {+ it("keeps MCP embedded video and Codex dynamic images", () => {+ expect(+ collectNativeToolMedia([+ {+ type: "resource",+ resource: { mimeType: "video/mp4", blob: "dmlkZW8=" },+ },+ { type: "inputImage", imageUrl: "data:image/png;base64,aW1hZ2U=" },+ { type: "text", text: "view /tmp/not-an-image.png" },+ ]),+ ).toEqual([+ { kind: "video", data: "dmlkZW8=", mimeType: "video/mp4" },+ { kind: "image", url: "data:image/png;base64,aW1hZ2U=" },+ ]);+ });+ it("round trips media beside existing delivered results", () => {+ const refs = [+ {+ kind: "image" as const,+ url: "cindy-media://blobs/" + "a".repeat(64) + ".png",+ },+ ];+ const text = withToolProcessMedia(+ '{"xdt_video_url":"https://example.com/final.mp4"}',+ refs,+ );+ expect(readToolProcessMedia(text)).toEqual(refs);+ expect(JSON.parse(text).xdt_video_url).toBe(+ "https://example.com/final.mp4",+ );+ });+ it("rejects arbitrary native paths and active schemes in rendered metadata", () => {+ expect(+ readToolProcessMedia(+ JSON.stringify({+ cindy_process_media: [+ { kind: "image", url: "file:///etc/passwd" },+ { kind: "video", url: "javascript:alert(1)" },+ ],+ }),+ ),+ ).toEqual([]);+ expect(readToolProcessMedia("broken cindy_process_media")).toEqual([]);+ });+ it("keeps multilingual output within the remote byte budget without breaking the media JSON", () => {+ const refs = [+ { kind: "video" as const, url: "https://example.com/clip.mp4" },+ ];+ const text = withToolProcessMedia("截图说明😀".repeat(3000), refs);+ expect(new TextEncoder().encode(text).length).toBeLessThanOrEqual(8192);+ expect(readToolProcessMedia(text)).toEqual(refs);+ expect(JSON.parse(text).text).toMatch(/^截图说明😀/);+ });+});diff --git a/packages/maker-shared/src/toolProcessMedia.ts b/packages/maker-shared/src/toolProcessMedia.tsnew file mode 100644index 0000000000..d9fb12066c--- /dev/null+++ b/packages/maker-shared/src/toolProcessMedia.ts@@ -0,0 +1,112 @@+/** Native tool media stays with its tool step; explicit delivered media is unchanged. */+export interface NativeToolMedia {+ kind: "image" | "video";+ data?: string;+ mimeType?: string;+ url?: string;+ path?: string;+}+export interface ProcessMediaRef {+ kind: "image" | "video";+ url: string;+}++const record = (v: unknown): Record<string, unknown> | null =>+ v && typeof v === "object" && !Array.isArray(v)+ ? (v as Record<string, unknown>)+ : null;++/** MCP, Anthropic and Pi content blocks. Never infer media from prose or commands. */+export function collectNativeToolMedia(content: unknown): NativeToolMedia[] {+ if (!Array.isArray(content)) return [];+ const result: NativeToolMedia[] = [];+ for (const value of content) {+ const block = record(value);+ if (!block) continue;+ const source = record(block.source);+ const resource = record(block.resource);+ const mime = source?.media_type ?? block.mimeType ?? resource?.mimeType;+ const kind =+ block.type === "video" ||+ (typeof mime === "string" && mime.startsWith("video/"))+ ? "video"+ : block.type === "image" ||+ block.type === "inputImage" ||+ (typeof mime === "string" && mime.startsWith("image/"))+ ? "image"+ : null;+ if (!kind) continue;+ const data = source?.data ?? block.data ?? resource?.blob;+ const url = source?.url ?? block.imageUrl ?? resource?.uri ?? block.uri;+ if (typeof data === "string" && typeof mime === "string") {+ result.push({ kind, data, mimeType: mime });+ } else if (typeof url === "string") result.push({ kind, url });+ }+ return result;+}++export function isProcessMediaUrl(url: string): boolean {+ return (+ /^cindy-media:\/\/blobs\/[a-f0-9]{64}\.(?:png|jpg|jpeg|gif|webp|mp4|webm|mov)$/.test(+ url,+ ) ||+ /^xdt-(?:image|video):\/\//.test(url) ||+ /^https?:\/\//.test(url)+ );+}++export function readToolProcessMedia(+ text: string | undefined,+): ProcessMediaRef[] {+ if (!text?.includes("cindy_process_media")) return [];+ try {+ const parsed = record(JSON.parse(text));+ if (!Array.isArray(parsed?.cindy_process_media)) return [];+ return parsed.cindy_process_media.flatMap((value): ProcessMediaRef[] => {+ const item = record(value);+ return item &&+ (item.kind === "image" || item.kind === "video") &&+ typeof item.url === "string" &&+ isProcessMediaUrl(item.url)+ ? [{ kind: item.kind, url: item.url }]+ : [];+ });+ } catch {+ return [];+ }+}++export function withToolProcessMedia(+ text: string,+ media: ProcessMediaRef[],+): string {+ let payload: Record<string, unknown> = { text };+ try {+ payload = record(JSON.parse(text)) ?? payload;+ } catch {+ /* ordinary tool text */+ }+ const encode = (value: Record<string, unknown>) =>+ JSON.stringify({ ...value, cindy_process_media: media });+ const encoded = encode(payload);+ // Remote history has an 8KB byte budget. Truncate prose, never cut JSON or a URL.+ const bytes = new TextEncoder();+ if (bytes.encode(encoded).length <= 8192) return encoded;+ let low = 0;+ let high = text.length;+ while (low < high) {+ const mid = Math.ceil((low + high) / 2);+ if (bytes.encode(encode({ text: text.slice(0, mid) })).length <= 8192)+ low = mid;+ else high = mid - 1;+ }+ if (low > 0 && /[\uD800-\uDBFF]/.test(text[low - 1])) low--;+ return encode({ text: text.slice(0, low) });+}++export function nativeToolMediaFields(content: unknown): {+ processMedia?: NativeToolMedia[];+} {+ const processMedia = collectNativeToolMedia(content);+ return processMedia.length ? { processMedia } : {};+}