Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(companion-cloud): D10 send the images tools made during a turn with the reply
nana8a2dc32cb615
diff --git a/apps/desktop/src/main/companion-cloud/__tests__/replyArtifacts.test.ts b/apps/desktop/src/main/companion-cloud/__tests__/replyArtifacts.test.tsindex 21d87243c5..8bf7f52759 100644--- a/apps/desktop/src/main/companion-cloud/__tests__/replyArtifacts.test.ts+++ b/apps/desktop/src/main/companion-cloud/__tests__/replyArtifacts.test.ts@@ -2,7 +2,7 @@ import fs from 'node:fs/promises';import os from 'node:os';import path from 'node:path';import { afterEach, beforeEach, describe, expect, it } from 'vitest';-import { collectReplyReferences, replyArtifactAttachments, replyTextWithFiles, REPLY_ARTIFACTS_MAX } from '../replyArtifacts.js';+import { collectReplyReferences, replyArtifactAttachments, replyTextWithFiles, toolResultImageUrls, REPLY_ARTIFACTS_MAX } from '../replyArtifacts.js';const HASH = 'a'.repeat(64);@@ -75,6 +75,36 @@ describe('replyArtifactAttachments', () => {});});+describe('images made by tools during the turn', () => {+ const url = (hash: string) => `cindy-media://blobs/${hash.repeat(64)}.png`;+ it('reads the stored images a tool result carries, whether it is stored as an object or as a JSON string', () => {+ const result = { ok: true, xdt_image_url: url('a'), xdt_image_urls: [url('b'), url('a'), 'xdt-image://old/x.png', 'https://example.com/x.png'] };+ expect(toolResultImageUrls(JSON.stringify(result))).toEqual([url('a'), url('b')]);+ expect(toolResultImageUrls(JSON.stringify(JSON.stringify(result)))).toEqual([url('a'), url('b')]);+ expect(toolResultImageUrls(JSON.stringify({ ...result, _xdt_render_image: false }))).toEqual([]);+ expect(toolResultImageUrls('not json')).toEqual([]);+ expect(toolResultImageUrls(null)).toEqual([]);+ });++ it('sends them with the reply although its text never names them, once and within the limit', async () => {+ const known = new Map([[url('a'), 5], [url('b'), 7]]);+ const asked: unknown[] = [];+ const files = await replyArtifactAttachments({ id: 'm1', createdAt: 100 }, `看看 ${url('b')}`, 's1', {+ roots: async () => [], maxBytes: 1000,+ blob: async address => (known.has(address) ? { mimeType: 'image/png', size: known.get(address)! } : null),+ turnImages: async (...args) => { asked.push(args); return [url('a'), url('b'), url('c')]; },+ });+ expect(asked).toEqual([['s1', { id: 'm1', createdAt: 100 }]]);+ expect(files.map(file => [file.url, file.category])).toEqual([[url('b'), 'image'], [url('a'), 'image']]);+ // A failure to look costs the reply nothing else.+ const plain = await replyArtifactAttachments({ id: 'm2', createdAt: 100 }, `看看 ${url('b')}`, 's1', {+ roots: async () => [], maxBytes: 1000, blob: async () => ({ mimeType: 'image/png', size: 1 }),+ turnImages: async () => { throw new Error('boom'); },+ });+ expect(plain.map(file => file.url)).toEqual([url('b')]);+ });+});+describe('replyTextWithFiles', () => {it('shows media the text already names as before, and links everything else below the reply', () => {const named = `cindy-media://blobs/${HASH}.png`;diff --git a/apps/desktop/src/main/companion-cloud/host.ts b/apps/desktop/src/main/companion-cloud/host.tsindex 4aa03936a4..909afbedf6 100644--- a/apps/desktop/src/main/companion-cloud/host.ts+++ b/apps/desktop/src/main/companion-cloud/host.ts@@ -3,7 +3,7 @@ import { createHash, randomUUID } from 'node:crypto';import fs from 'node:fs/promises';import path from 'node:path';import { app } from 'electron';-import { and, asc, desc, eq, gt, gte, inArray, isNull, or, sql } from 'drizzle-orm';+import { and, asc, desc, eq, gt, gte, inArray, isNull, lt, lte, or, sql } from 'drizzle-orm';import { portableBotAvatarOrFallback, BOT_AVATAR_MAX_BYTES } from '../../shared/botAvatarValue.js';import type { DeviceLinkDeviceView } from '../../shared/deviceLinkIpc.js';import { activeOwnerScopeKey, ownerScopedUserDataPath } from '../appSessionState.js';@@ -36,7 +36,7 @@ import type { CompanionMaterializeHost } from './materialize.js';import { companionHomes } from './homes.js';import { companionCapabilitiesPatch, portableCompanionCapabilities, type CompanionArrivalNotice } from './capabilities.js';import { MIRROR_ATTACHMENT_MAX_BYTES, type CompanionMirrorCursor, type CompanionMirrorRow } from './mirror.js';-import { replyArtifactAttachments, replyTextWithFiles } from './replyArtifacts.js';+import { replyArtifactAttachments, replyTextWithFiles, toolResultImageUrls } from './replyArtifacts.js';import type { CompanionCloudHost } from './runtime.js';import { createCompanionCloudState } from './state.js';import type { CompanionLog } from './types.js';@@ -299,6 +299,22 @@ async function readTaskFileRoots(sessionId: string): Promise<string[]> {return session ? taskImageRoots(session) : [];}+/**+ * Stored images the tools produced since the row before this reply (the previous user or assistant+ * row): what the chat shows as images of the turn. Each lands on the first reply after it.+ */+async function readTurnImageUrls(sessionId: string, row: { id: string; createdAt: number }): Promise<string[]> {+ const { drizzle } = getDbClient();+ const [previous] = await drizzle.select({ createdAt: messages.createdAt }).from(messages)+ .where(and(visibleRows(sessionId), lt(messages.createdAt, row.createdAt)))+ .orderBy(desc(messages.createdAt)).limit(1);+ const results = await drizzle.select({ content: messages.content }).from(messages)+ .where(and(eq(messages.sessionId, sessionId), isNull(messages.rewindAt), eq(messages.role, 'tool_result'),+ gt(messages.createdAt, previous?.createdAt ?? 0), lte(messages.createdAt, row.createdAt)))+ .orderBy(asc(messages.createdAt)).limit(50);+ return [...new Set(results.flatMap(result => toolResultImageUrls(result.content)))];+}+/** A media blob this task may read and this computer still has (the reply only names its address). */async function readReplyBlob(url: string, sessionId: string): Promise<{ mimeType: string; size: number } | null> {const blob = parseBlobUrl(url);@@ -401,6 +417,7 @@ export function createCompanionCloudHost(input: {replyAttachments: (_botId, sessionId, row, text) => replyArtifactAttachments(row, text, sessionId, {roots: readTaskFileRoots,blob: readReplyBlob,+ turnImages: readTurnImageUrls,maxBytes: MIRROR_ATTACHMENT_MAX_BYTES,...(input.log ? { log: input.log } : {}),}),diff --git a/apps/desktop/src/main/companion-cloud/replyArtifacts.ts b/apps/desktop/src/main/companion-cloud/replyArtifacts.tsindex 620ee01999..506cb45163 100644--- a/apps/desktop/src/main/companion-cloud/replyArtifacts.ts+++ b/apps/desktop/src/main/companion-cloud/replyArtifacts.ts@@ -62,6 +62,24 @@ export function collectReplyReferences(text: string): ReplyReferences {return { blobs, paths };}+/**+ * The stored images a tool handed back (`xdt_image_url` / `xdt_image_urls` in its result), which the+ * chat shows as images of the turn although the reply text never names them. A result that asked not+ * to be shown (`_xdt_render_image: false`) and any address that is not a stored blob give nothing.+ */+export function toolResultImageUrls(rawContent: string | null): string[] {+ let value: unknown = rawContent;+ // A result is stored as JSON, often a JSON string holding the tool's own JSON.+ for (let depth = 0; depth < 2 && typeof value === 'string'; depth += 1) {+ try { value = JSON.parse(value); } catch { return []; }+ }+ if (!value || typeof value !== 'object' || Array.isArray(value)) return [];+ const record = value as Record<string, unknown>;+ if (record._xdt_render_image === false) return [];+ const candidates = [record.xdt_image_url, ...(Array.isArray(record.xdt_image_urls) ? record.xdt_image_urls : [])];+ return [...new Set(candidates.filter((url): url is string => typeof url === 'string' && /^cindy-media:\/\/blobs\/[0-9a-f]{64}\.[a-z0-9]+$/.test(url)))];+}+function safeDecode(value: string): string {try { return decodeURIComponent(value); } catch { return value; }}@@ -83,15 +101,24 @@ export interface ReplyArtifactDeps {roots: (sessionId: string) => Promise<string[]>;/** A stored media blob: its type and size, or null when this computer does not have it. */blob: (url: string, sessionId: string) => Promise<{ mimeType: string; size: number } | null>;+ /** Stored images the tools of the turn this reply ends produced (see `toolResultImageUrls`). */+ turnImages?: (sessionId: string, row: { id: string; createdAt: number }) => Promise<string[]>;maxBytes: number;log?: { warn: (message: string, meta?: Record<string, unknown>) => void };}/** Files the reply names that can go to the room with it (see the file comment). */export async function replyArtifactAttachments(- row: { id: string }, text: string, sessionId: string, deps: ReplyArtifactDeps,+ row: { id: string; createdAt?: number }, text: string, sessionId: string, deps: ReplyArtifactDeps,): Promise<CompanionMirrorAttachment[]> {const refs = collectReplyReferences(text);+ // Images the turn's tools made are part of what the chat shows with this reply, named or not.+ const turn = deps.turnImages && row.createdAt !== undefined+ ? await deps.turnImages(sessionId, { id: row.id, createdAt: row.createdAt }).catch((error: unknown) => {+ deps.log?.warn('Turn images could not be listed', { code: (error as NodeJS.ErrnoException)?.code ?? 'UNKNOWN' });+ return [] as string[];+ }) : [];+ for (const url of turn) if (!refs.blobs.includes(url)) refs.blobs.push(url);if (!refs.blobs.length && !refs.paths.length) return [];const out: CompanionMirrorAttachment[] = [];const full = () => out.length >= REPLY_ARTIFACTS_MAX;diff --git a/docs/product-rules/cloud-companions.md b/docs/product-rules/cloud-companions.mdindex 0cd747161a..c170d2be70 100644--- a/docs/product-rules/cloud-companions.md+++ b/docs/product-rules/cloud-companions.md@@ -136,6 +136,7 @@只认 docx / xlsx / pptx / pdf / csv / zip 等交付格式),代码块里的示例不算。文件必须真实存在于该任务自己的工作目录内(符号链接按真实路径复核,隐藏文件与目录如 `.env`、`.git` 不上传,目录外与 UNC 路径不读),单个不超过 25 MB,每条回复最多 8 个;经与主人附件同一条房间附件通道上传,永久 / 暂时失败的区分与主人附件一致(暂时失败这一轮停下重试,永久失败只发正文)。+ 工具在这一轮生成并存入媒体仓的图片(工具结果里的 `xdt_image_url` / `xdt_image_urls`,聊天里作为这一轮的图片独立显示,回复正文可以不提)同样作为附件随回复上云:取上一条用户 / 伙伴消息之后到这条回复为止的工具结果,落在其后第一条被导入的回复上,计入每条回复 8 个的上限,只带图片本身,不带工具调用明细(§1 第 2 条);结果要求不显示(`_xdt_render_image: false`)或不是媒体仓地址的不带。换电脑导入时,房间里的附件先下载并入库(图片入媒体仓,同一字节得到同一个 `cindy-media://` 地址)、挂上任务引用,再写消息,所以回复里原有的 `cindy-media://` 图片在新电脑直接显示;正文没提到的图片和非图片文件作为链接追加在回复下面(文件按房间附件落盘方式保存在本机,链接指向它),不再只写 `[附件: …]` 占位。