Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(chat): preserve shared image references until byte classification

ylliuuu66c230b20d166e
diff --git a/apps/desktop/src/main/cindy-media/__tests__/taskImageDelivery.test.ts b/apps/desktop/src/main/cindy-media/__tests__/taskImageDelivery.test.ts
index 709c08f8d7..d1572456b8 100644
--- a/apps/desktop/src/main/cindy-media/__tests__/taskImageDelivery.test.ts
+++ b/apps/desktop/src/main/cindy-media/__tests__/taskImageDelivery.test.ts
@@ -116,17 +116,35 @@ function insert(text: string) {
describe('task image delivery', () => {
// #6773: run both production preparation passes; a shared PDF definition
// must not be interpreted as an image after the file has been extracted.
- it.each(['nested', 'sibling'])('delivers shared file/image references through both preparation passes (%s)', async mode => {
- const pdf = path.join(work, 'report.pdf');
+ it.each(['nested', 'sibling'].flatMap(mode => ['pdf', 'png'].map(ext => [mode, ext])))('delivers shared non-image references through both preparation passes (%s, .%s)', async (mode, ext) => {
+ const pdf = path.join(work, `report.${ext}`);
const bytes = Buffer.from('%PDF-1.4\nreport');
await fs.writeFile(pdf, bytes);
const label = mode === 'nested' ? '[![preview][report]][report]' : '[report][report] ![preview][report]';
const files = await prepareChatReplyFiles(`${label}\n\n[report]: <${pdf}>`, 's', () => {});
expect(files.attachments).toHaveLength(1);
expect(await files.attachments[0]!.readBytes!()).toEqual(bytes);
- const images = await prepareChatReplyImages(files.text, 's', () => {});
+ const images = await prepareChatReplyImages(files.text, 's', () => {}, files.attachments);
expect(images.attachments).toEqual([]);
expect(images.text).not.toContain(pdf);
+ expect(images.text).not.toContain('![');
+ expect(images.text).not.toContain('[report]');
+ });
+
+ it.each(['nested', 'sibling'])('keeps a real shared-reference image through both reply passes (%s)', async mode => {
+ // Content, not the extension, decides whether this shared target is an image.
+ const source = path.join(work, 'asset.bin');
+ await fs.writeFile(source, PNG);
+ const body = mode === 'nested' ? '[![preview][asset]][asset]' : '[download][asset] ![preview][asset]';
+ const files = await prepareChatReplyFiles(`${body}\n\n[asset]: <${source}>`, 's', () => {});
+ const images = await prepareChatReplyImages(files.text, 's', () => {}, files.attachments);
+ const attachments = mergeChatReplyAttachments(images.attachments, files.attachments);
+ expect(attachments).toHaveLength(1);
+ expect(attachments[0]).toMatchObject({ category: 'image', mimeType: 'image/png', sourceOffset: 0 });
+ expect(images.text).toContain('preview');
+ expect(images.text).not.toContain('![');
+ expect(images.text).not.toContain('[asset]');
+ expect(images.text).not.toContain(source);
});
it.each(['image-first', 'file-first'])('uploads eleven real images once through both reply passes, in source order (%s)', async order => {
@@ -140,7 +158,7 @@ describe('task image delivery', () => {
const body = `${order === 'image-first' ? image(0) : file(0)} [report](<${pdf}>) ${order === 'image-first' ? file(0) : image(0)} `
+ targets.slice(1).map((_, index) => `${image(index + 1)} ${file(index + 1)}`).join(' ');
const files = await prepareChatReplyFiles(body, 's', () => {});
- const images = await prepareChatReplyImages(files.text, 's', () => {});
+ const images = await prepareChatReplyImages(files.text, 's', () => {}, files.attachments);
const attachments = mergeChatReplyAttachments(images.attachments, files.attachments);
expect(attachments).toHaveLength(12);
expect(attachments.map(a => a.category)).toEqual(['image', 'file', ...Array(10).fill('image')]);
@@ -154,7 +172,7 @@ describe('task image delivery', () => {
await fs.writeFile(first, PNG);
await fs.writeFile(second, PNG);
const files = await prepareChatReplyFiles(`[download](<${second}>) ![a](<${first}>) ![b](<${second}>)`, 's', () => {});
- const images = await prepareChatReplyImages(files.text, 's', () => {});
+ const images = await prepareChatReplyImages(files.text, 's', () => {}, files.attachments);
const attachments = mergeChatReplyAttachments(images.attachments, files.attachments);
expect(attachments).toHaveLength(1);
expect(attachments[0]).toMatchObject({ category: 'image', sourceOffset: 0 });
@@ -166,7 +184,7 @@ describe('task image delivery', () => {
await fs.writeFile(source, PNG);
await fs.link(source, path.join(state.root, 'outside.png'));
const files = await prepareChatReplyFiles(`![preview](<${source}>) [download](<${source}>)`, 's', () => {});
- const images = await prepareChatReplyImages(files.text, 's', () => {});
+ const images = await prepareChatReplyImages(files.text, 's', () => {}, files.attachments);
const attachments = mergeChatReplyAttachments(images.attachments, files.attachments);
expect(attachments.map(a => a.category)).toEqual(['image', 'file']);
await expect(attachments[1]!.readBytes!()).rejects.toThrow('ATTACHMENT_UNAVAILABLE');
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/chatServer.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/chatServer.test.ts
index d6c020db92..a7a388a9e0 100644
--- a/apps/desktop/src/main/maker-ipc/__tests__/chatServer.test.ts
+++ b/apps/desktop/src/main/maker-ipc/__tests__/chatServer.test.ts
@@ -654,6 +654,8 @@ describe('Chat Server result delivery and refresh', () => {
const uploads = fixture.upload.mock.calls.length;
await service.settleLaneTurn({ ...terminal, resultText: '[Report](./report.pdf)' });
expect(fixture.upload.mock.calls.length).toBe(uploads);
+ // Shared-reference classification can read bytes; it must wait for current authority.
+ expect(replyImages).not.toHaveBeenCalled();
expect(deliveries()).toHaveLength(0);
if (change === 'temporary') revoked = false;
await vi.advanceTimersByTimeAsync(16000);
diff --git a/apps/desktop/src/main/maker-ipc/__tests__/chatServerReplyFiles.test.ts b/apps/desktop/src/main/maker-ipc/__tests__/chatServerReplyFiles.test.ts
index b44d425bf2..9d948c5e57 100644
--- a/apps/desktop/src/main/maker-ipc/__tests__/chatServerReplyFiles.test.ts
+++ b/apps/desktop/src/main/maker-ipc/__tests__/chatServerReplyFiles.test.ts
@@ -64,7 +64,7 @@ describe('group reply file delivery', () => {
expect((await result.attachments[0]!.readBytes!()).toString()).toContain('%PDF');
expect(result.text).toContain(image);
expect(result.text).not.toContain(`[${image}]`);
- if (mode === 'shared-reference') expect(result.text).not.toContain(`[image]: <${pdf}>`);
+ if (mode === 'shared-reference') expect(result.text).toContain(`[image]: <${pdf}>`);
});
it.each(['file', 'xdt-file', 'xdt-query'].flatMap(scheme => ['100% done.pdf', 'encoded%20.pdf'].map(name => [scheme, name])))('preserves literal or once-encoded percent paths: %s %s', async (scheme, name) => {
diff --git a/apps/desktop/src/main/maker-ipc/chatServer.ts b/apps/desktop/src/main/maker-ipc/chatServer.ts
index ad372dfe4b..06a3a98df7 100644
--- a/apps/desktop/src/main/maker-ipc/chatServer.ts
+++ b/apps/desktop/src/main/maker-ipc/chatServer.ts
@@ -1505,6 +1505,19 @@ function createChatServer(local: BotGroupChatService, deps: BotGroupChatServiceD
let contentText = text;
let files: ChatContentBlock[] = [];
const ordinaryReply = !run.plan || run.plan.kind === 'talk';
+ const validateUpload = async () => {
+ assertCurrent();
+ if (pending.fileReply?.attachments.length) {
+ try { await validateExecutionAuthority(run); }
+ catch (error) {
+ // Definitive revocation ends delivery; transport failures retain
+ // the same result for retry without running the Agent again.
+ if (error instanceof GroupToolAuthorizationError) throw new ChatResponseError('PERMISSION_DENIED', 403);
+ throw error;
+ }
+ }
+ assertCurrent();
+ };
try {
if (terminal.outcome !== 'error') {
if (ordinaryReply) {
@@ -1512,24 +1525,15 @@ function createChatServer(local: BotGroupChatService, deps: BotGroupChatServiceD
assertCurrent();
if (run.execution.access_mode === 'chat') throw new Error('ATTACHMENT_UNAVAILABLE');
}, !terminal.resultText.trim() ? terminal.resultMessageClientId : undefined);
- pending.reply ??= { sourceText: text, ...await prepareChatReplyImages(pending.fileReply.text, run.sessionId, assertCurrent) };
+ // Shared reference classification reads file bytes before image import.
+ // Retain the later upload/seal checks and authorize this read as well.
+ if (!pending.reply && pending.fileReply.attachments.length) await validateUpload();
+ pending.reply ??= { sourceText: text, ...await prepareChatReplyImages(pending.fileReply.text, run.sessionId, assertCurrent,
+ pending.fileReply.attachments) };
contentText = pending.reply.text;
produced.push(...mergeChatReplyAttachments(pending.reply.attachments, pending.fileReply.attachments));
if (produced.length > BOT_GROUP_ATTACHMENTS_MAX) throw new Error('ATTACHMENT_UNAVAILABLE');
}
- const validateUpload = async () => {
- assertCurrent();
- if (pending.fileReply?.attachments.length) {
- try { await validateExecutionAuthority(run); }
- catch (error) {
- // Definitive revocation ends delivery; transport failures retain
- // the same result for retry without running the Agent again.
- if (error instanceof GroupToolAuthorizationError) throw new ChatResponseError('PERMISSION_DENIED', 403);
- throw error;
- }
- }
- assertCurrent();
- };
if (ordinaryReply) await validateUpload();
files = await media.upload(run.execution.conversation_id, `step:${run.execution.id}:${run.execution.epoch}`,
produced, run.execution.bot_id, ordinaryReply ? validateUpload : undefined);
diff --git a/apps/desktop/src/main/maker-ipc/chatServerMedia.ts b/apps/desktop/src/main/maker-ipc/chatServerMedia.ts
index 72da8c701c..0d50848053 100644
--- a/apps/desktop/src/main/maker-ipc/chatServerMedia.ts
+++ b/apps/desktop/src/main/maker-ipc/chatServerMedia.ts
@@ -19,6 +19,8 @@ export type ChatUploadAttachment = BotGroupAttachment & {
sourceOffset?: number;
/** Canonical local image/file sources, used only to reconcile the two reply passes. */
sourcePaths?: string[];
+ /** Original Markdown target, for shared reference definitions; never serialized. */
+ sourceUrl?: string;
readBytes?: () => Promise<Buffer>;
assertCurrent?: () => void;
};
diff --git a/apps/desktop/src/main/maker-ipc/chatServerReplyFiles.ts b/apps/desktop/src/main/maker-ipc/chatServerReplyFiles.ts
index 45a2eadcbd..2a8c7024b6 100644
--- a/apps/desktop/src/main/maker-ipc/chatServerReplyFiles.ts
+++ b/apps/desktop/src/main/maker-ipc/chatServerReplyFiles.ts
@@ -62,6 +62,8 @@ export function replyFileLinks(text: string) {
});
const links: Array<{ url: string; start: number; end: number; label: string }> = [];
const usedDefinitions = new Set<string>();
+ const imageDefinitions = new Set<string>();
+ visit(tree, 'imageReference', node => { imageDefinitions.add(node.identifier); });
visit(tree, node => {
if (node.type !== 'link' && node.type !== 'linkReference') return;
const target = node.type === 'link' ? node : definitions.get(node.identifier);
@@ -76,8 +78,10 @@ export function replyFileLinks(text: string) {
});
const edits = links.map(link => ({ start: link.start, end: link.end, text: link.label }));
for (const identifier of usedDefinitions) {
+ // The image pass decides from authorized bytes whether a shared target is
+ // an image or a file-only preview. Keep its definition until then.
+ if (imageDefinitions.has(identifier)) continue;
const node = definitions.get(identifier)!;
- // A shared image reference must not send this already-delivered file through image import.
if (node.position?.start.offset !== undefined && node.position.end.offset !== undefined)
edits.push({ start: node.position.start.offset, end: node.position.end.offset, text: '' });
}
@@ -246,6 +250,7 @@ export async function prepareChatReplyFiles(text: string, sessionId: string, ass
assertCurrent();
let digest: string | undefined;
attachments.push({ id: canonicalSource, sourceOffset: link.outputStart, name: path.basename(source), category: 'file',
+ sourceUrl: link.url,
sourcePaths: realSource && isPathWithin(root, realSource) ? [realSource] : [],
mimeType: path.extname(source).toLowerCase() === '.pdf' ? 'application/pdf' : 'application/octet-stream',
size: 0, path: null, url: null, assertCurrent,
diff --git a/apps/desktop/src/main/maker-ipc/chatServerReplyImages.ts b/apps/desktop/src/main/maker-ipc/chatServerReplyImages.ts
index 3308db782f..415e5c5901 100644
--- a/apps/desktop/src/main/maker-ipc/chatServerReplyImages.ts
+++ b/apps/desktop/src/main/maker-ipc/chatServerReplyImages.ts
@@ -4,12 +4,16 @@ import { BOT_GROUP_ATTACHMENTS_MAX } from '../../shared/botGroupChat.js';
import { parseBlobUrl, resolveSafe } from '../cindy-media/blobStore.js';
import { sessionCanRead } from '../cindy-media/ledger.js';
import { materializeTaskImageTextResult } from '../cindy-media/taskImageDelivery.js';
+import { sniffMediaMime } from '../cindy-media/sniffMediaMime.js';
import { hasLocalTaskImages, localTaskImagePath, taskImageReferences, rewriteTaskImageReferences } from '../cindy-media/taskImageMarkdown.js';
import type { ChatUploadAttachment } from './chatServerMedia.js';
import { getDbClient } from '../localDb/client/current.js';
+const imageLabel = (image: { alt: string }) => image.alt.replace(/[\\`*_{}[\]<>#!|]/g, '\\$&');
+
/** Prepare only rendered, task-owned generated images. Never reads legacy caches. */
-export async function prepareChatReplyImages(text: string, sessionId: string, assertCurrent: () => void) {
+export async function prepareChatReplyImages(text: string, sessionId: string, assertCurrent: () => void,
+ authorizedFiles: ChatUploadAttachment[] = []) {
assertCurrent();
const originalImages = taskImageReferences(text);
const localSources = new Map<string, string>();
@@ -20,6 +24,25 @@ export async function prepareChatReplyImages(text: string, sessionId: string, as
assertCurrent();
if (real) localSources.set(ref.url, real);
}
+ const fileOnly = new Map<string, string>();
+ const inspected = new Set<string>();
+ for (const ref of originalImages) {
+ if (!ref.identifier || inspected.has(ref.url)) continue;
+ const real = localSources.get(ref.url);
+ const file = authorizedFiles.find(file => file.sourceUrl === ref.url
+ || (real !== undefined && file.sourcePaths?.includes(real)));
+ if (!file) continue;
+ inspected.add(ref.url);
+ // A shared PDF definition must not enter image import, while real images
+ // (including extensionless ones) retain the existing materialization path.
+ // The caller verifies live file authority before these bounded descriptor reads.
+ if (!localTaskImagePath(ref.url)) { fileOnly.set(ref.url, ref.url); continue; }
+ if (!file.readBytes) throw new Error('ATTACHMENT_UNAVAILABLE');
+ const bytes = await file.readBytes();
+ assertCurrent();
+ if (!sniffMediaMime(bytes)?.startsWith('image/')) fileOnly.set(ref.url, ref.url);
+ }
+ text = rewriteTaskImageReferences(text, fileOnly, imageLabel);
let replacements = new Map<string, string>();
if (hasLocalTaskImages(text)) {
const imported = await materializeTaskImageTextResult(sessionId, text);
@@ -57,6 +80,5 @@ export async function prepareChatReplyImages(text: string, sessionId: string, as
mimeType: image.mimeType, size: stat.size, url: ref.url, path: null });
delivered.set(ref.url, ref.url);
}
- return { attachments, text: rewriteTaskImageReferences(text, delivered, image =>
- image.alt.replace(/[\\`*_{}[\]<>#!|]/g, '\\$&')) };
+ return { attachments, text: rewriteTaskImageReferences(text, delivered, imageLabel) };
}