Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(companion-cloud): D10 send the images tools made during a turn with the reply

nana8a2dc32cb615
diff --git a/apps/desktop/src/main/companion-cloud/__tests__/replyArtifacts.test.ts b/apps/desktop/src/main/companion-cloud/__tests__/replyArtifacts.test.ts
index 21d87243c5..8bf7f52759 100644
--- a/apps/desktop/src/main/companion-cloud/__tests__/replyArtifacts.test.ts
+++ b/apps/desktop/src/main/companion-cloud/__tests__/replyArtifacts.test.ts
@@ -2,7 +2,7 @@ import fs from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-import { collectReplyReferences, replyArtifactAttachments, replyTextWithFiles, REPLY_ARTIFACTS_MAX } from '../replyArtifacts.js';
+import { collectReplyReferences, replyArtifactAttachments, replyTextWithFiles, toolResultImageUrls, REPLY_ARTIFACTS_MAX } from '../replyArtifacts.js';
const HASH = 'a'.repeat(64);
@@ -75,6 +75,36 @@ describe('replyArtifactAttachments', () => {
});
});
+describe('images made by tools during the turn', () => {
+ const url = (hash: string) => `cindy-media://blobs/${hash.repeat(64)}.png`;
+ it('reads the stored images a tool result carries, whether it is stored as an object or as a JSON string', () => {
+ const result = { ok: true, xdt_image_url: url('a'), xdt_image_urls: [url('b'), url('a'), 'xdt-image://old/x.png', 'https://example.com/x.png'] };
+ expect(toolResultImageUrls(JSON.stringify(result))).toEqual([url('a'), url('b')]);
+ expect(toolResultImageUrls(JSON.stringify(JSON.stringify(result)))).toEqual([url('a'), url('b')]);
+ expect(toolResultImageUrls(JSON.stringify({ ...result, _xdt_render_image: false }))).toEqual([]);
+ expect(toolResultImageUrls('not json')).toEqual([]);
+ expect(toolResultImageUrls(null)).toEqual([]);
+ });
+
+ it('sends them with the reply although its text never names them, once and within the limit', async () => {
+ const known = new Map([[url('a'), 5], [url('b'), 7]]);
+ const asked: unknown[] = [];
+ const files = await replyArtifactAttachments({ id: 'm1', createdAt: 100 }, `看看 ${url('b')}`, 's1', {
+ roots: async () => [], maxBytes: 1000,
+ blob: async address => (known.has(address) ? { mimeType: 'image/png', size: known.get(address)! } : null),
+ turnImages: async (...args) => { asked.push(args); return [url('a'), url('b'), url('c')]; },
+ });
+ expect(asked).toEqual([['s1', { id: 'm1', createdAt: 100 }]]);
+ expect(files.map(file => [file.url, file.category])).toEqual([[url('b'), 'image'], [url('a'), 'image']]);
+ // A failure to look costs the reply nothing else.
+ const plain = await replyArtifactAttachments({ id: 'm2', createdAt: 100 }, `看看 ${url('b')}`, 's1', {
+ roots: async () => [], maxBytes: 1000, blob: async () => ({ mimeType: 'image/png', size: 1 }),
+ turnImages: async () => { throw new Error('boom'); },
+ });
+ expect(plain.map(file => file.url)).toEqual([url('b')]);
+ });
+});
+
describe('replyTextWithFiles', () => {
it('shows media the text already names as before, and links everything else below the reply', () => {
const named = `cindy-media://blobs/${HASH}.png`;
diff --git a/apps/desktop/src/main/companion-cloud/host.ts b/apps/desktop/src/main/companion-cloud/host.ts
index 4aa03936a4..909afbedf6 100644
--- a/apps/desktop/src/main/companion-cloud/host.ts
+++ b/apps/desktop/src/main/companion-cloud/host.ts
@@ -3,7 +3,7 @@ import { createHash, randomUUID } from 'node:crypto';
import fs from 'node:fs/promises';
import path from 'node:path';
import { app } from 'electron';
-import { and, asc, desc, eq, gt, gte, inArray, isNull, or, sql } from 'drizzle-orm';
+import { and, asc, desc, eq, gt, gte, inArray, isNull, lt, lte, or, sql } from 'drizzle-orm';
import { portableBotAvatarOrFallback, BOT_AVATAR_MAX_BYTES } from '../../shared/botAvatarValue.js';
import type { DeviceLinkDeviceView } from '../../shared/deviceLinkIpc.js';
import { activeOwnerScopeKey, ownerScopedUserDataPath } from '../appSessionState.js';
@@ -36,7 +36,7 @@ import type { CompanionMaterializeHost } from './materialize.js';
import { companionHomes } from './homes.js';
import { companionCapabilitiesPatch, portableCompanionCapabilities, type CompanionArrivalNotice } from './capabilities.js';
import { MIRROR_ATTACHMENT_MAX_BYTES, type CompanionMirrorCursor, type CompanionMirrorRow } from './mirror.js';
-import { replyArtifactAttachments, replyTextWithFiles } from './replyArtifacts.js';
+import { replyArtifactAttachments, replyTextWithFiles, toolResultImageUrls } from './replyArtifacts.js';
import type { CompanionCloudHost } from './runtime.js';
import { createCompanionCloudState } from './state.js';
import type { CompanionLog } from './types.js';
@@ -299,6 +299,22 @@ async function readTaskFileRoots(sessionId: string): Promise<string[]> {
return session ? taskImageRoots(session) : [];
}
+/**
+ * Stored images the tools produced since the row before this reply (the previous user or assistant
+ * row): what the chat shows as images of the turn. Each lands on the first reply after it.
+ */
+async function readTurnImageUrls(sessionId: string, row: { id: string; createdAt: number }): Promise<string[]> {
+ const { drizzle } = getDbClient();
+ const [previous] = await drizzle.select({ createdAt: messages.createdAt }).from(messages)
+ .where(and(visibleRows(sessionId), lt(messages.createdAt, row.createdAt)))
+ .orderBy(desc(messages.createdAt)).limit(1);
+ const results = await drizzle.select({ content: messages.content }).from(messages)
+ .where(and(eq(messages.sessionId, sessionId), isNull(messages.rewindAt), eq(messages.role, 'tool_result'),
+ gt(messages.createdAt, previous?.createdAt ?? 0), lte(messages.createdAt, row.createdAt)))
+ .orderBy(asc(messages.createdAt)).limit(50);
+ return [...new Set(results.flatMap(result => toolResultImageUrls(result.content)))];
+}
+
/** A media blob this task may read and this computer still has (the reply only names its address). */
async function readReplyBlob(url: string, sessionId: string): Promise<{ mimeType: string; size: number } | null> {
const blob = parseBlobUrl(url);
@@ -401,6 +417,7 @@ export function createCompanionCloudHost(input: {
replyAttachments: (_botId, sessionId, row, text) => replyArtifactAttachments(row, text, sessionId, {
roots: readTaskFileRoots,
blob: readReplyBlob,
+ turnImages: readTurnImageUrls,
maxBytes: MIRROR_ATTACHMENT_MAX_BYTES,
...(input.log ? { log: input.log } : {}),
}),
diff --git a/apps/desktop/src/main/companion-cloud/replyArtifacts.ts b/apps/desktop/src/main/companion-cloud/replyArtifacts.ts
index 620ee01999..506cb45163 100644
--- a/apps/desktop/src/main/companion-cloud/replyArtifacts.ts
+++ b/apps/desktop/src/main/companion-cloud/replyArtifacts.ts
@@ -62,6 +62,24 @@ export function collectReplyReferences(text: string): ReplyReferences {
return { blobs, paths };
}
+/**
+ * The stored images a tool handed back (`xdt_image_url` / `xdt_image_urls` in its result), which the
+ * chat shows as images of the turn although the reply text never names them. A result that asked not
+ * to be shown (`_xdt_render_image: false`) and any address that is not a stored blob give nothing.
+ */
+export function toolResultImageUrls(rawContent: string | null): string[] {
+ let value: unknown = rawContent;
+ // A result is stored as JSON, often a JSON string holding the tool's own JSON.
+ for (let depth = 0; depth < 2 && typeof value === 'string'; depth += 1) {
+ try { value = JSON.parse(value); } catch { return []; }
+ }
+ if (!value || typeof value !== 'object' || Array.isArray(value)) return [];
+ const record = value as Record<string, unknown>;
+ if (record._xdt_render_image === false) return [];
+ const candidates = [record.xdt_image_url, ...(Array.isArray(record.xdt_image_urls) ? record.xdt_image_urls : [])];
+ return [...new Set(candidates.filter((url): url is string => typeof url === 'string' && /^cindy-media:\/\/blobs\/[0-9a-f]{64}\.[a-z0-9]+$/.test(url)))];
+}
+
function safeDecode(value: string): string {
try { return decodeURIComponent(value); } catch { return value; }
}
@@ -83,15 +101,24 @@ export interface ReplyArtifactDeps {
roots: (sessionId: string) => Promise<string[]>;
/** A stored media blob: its type and size, or null when this computer does not have it. */
blob: (url: string, sessionId: string) => Promise<{ mimeType: string; size: number } | null>;
+ /** Stored images the tools of the turn this reply ends produced (see `toolResultImageUrls`). */
+ turnImages?: (sessionId: string, row: { id: string; createdAt: number }) => Promise<string[]>;
maxBytes: number;
log?: { warn: (message: string, meta?: Record<string, unknown>) => void };
}
/** Files the reply names that can go to the room with it (see the file comment). */
export async function replyArtifactAttachments(
- row: { id: string }, text: string, sessionId: string, deps: ReplyArtifactDeps,
+ row: { id: string; createdAt?: number }, text: string, sessionId: string, deps: ReplyArtifactDeps,
): Promise<CompanionMirrorAttachment[]> {
const refs = collectReplyReferences(text);
+ // Images the turn's tools made are part of what the chat shows with this reply, named or not.
+ const turn = deps.turnImages && row.createdAt !== undefined
+ ? await deps.turnImages(sessionId, { id: row.id, createdAt: row.createdAt }).catch((error: unknown) => {
+ deps.log?.warn('Turn images could not be listed', { code: (error as NodeJS.ErrnoException)?.code ?? 'UNKNOWN' });
+ return [] as string[];
+ }) : [];
+ for (const url of turn) if (!refs.blobs.includes(url)) refs.blobs.push(url);
if (!refs.blobs.length && !refs.paths.length) return [];
const out: CompanionMirrorAttachment[] = [];
const full = () => out.length >= REPLY_ARTIFACTS_MAX;
diff --git a/docs/product-rules/cloud-companions.md b/docs/product-rules/cloud-companions.md
index 0cd747161a..c170d2be70 100644
--- a/docs/product-rules/cloud-companions.md
+++ b/docs/product-rules/cloud-companions.md
@@ -136,6 +136,7 @@
只认 docx / xlsx / pptx / pdf / csv / zip 等交付格式),代码块里的示例不算。文件必须真实存在于该任务自己的工作目录内
(符号链接按真实路径复核,隐藏文件与目录如 `.env`、`.git` 不上传,目录外与 UNC 路径不读),单个不超过 25 MB,每条回复最多 8 个;
经与主人附件同一条房间附件通道上传,永久 / 暂时失败的区分与主人附件一致(暂时失败这一轮停下重试,永久失败只发正文)。
+ 工具在这一轮生成并存入媒体仓的图片(工具结果里的 `xdt_image_url` / `xdt_image_urls`,聊天里作为这一轮的图片独立显示,回复正文可以不提)同样作为附件随回复上云:取上一条用户 / 伙伴消息之后到这条回复为止的工具结果,落在其后第一条被导入的回复上,计入每条回复 8 个的上限,只带图片本身,不带工具调用明细(§1 第 2 条);结果要求不显示(`_xdt_render_image: false`)或不是媒体仓地址的不带。
换电脑导入时,房间里的附件先下载并入库(图片入媒体仓,同一字节得到同一个 `cindy-media://` 地址)、挂上任务引用,再写消息,
所以回复里原有的 `cindy-media://` 图片在新电脑直接显示;正文没提到的图片和非图片文件作为链接追加在回复下面
(文件按房间附件落盘方式保存在本机,链接指向它),不再只写 `[附件: …]` 占位。