Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(desktop): 图片标注修复静默降级、预览不一致等问题并补齐编辑体验

Dash0f1a4a29114f
diff --git a/apps/desktop/src/main/__tests__/agentInputQueue.test.ts b/apps/desktop/src/main/__tests__/agentInputQueue.test.ts
index 736b8ac7bf..9d110d585d 100644
--- a/apps/desktop/src/main/__tests__/agentInputQueue.test.ts
+++ b/apps/desktop/src/main/__tests__/agentInputQueue.test.ts
@@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest';
import type { AgentInputQueuedMessage } from '../../shared/agentInputQueue.js';
import {
ANNOTATED_IMAGE_NOTE,
+ ANNOTATED_IMAGE_REGIONS_PREFIX,
buildMakerUserMessage,
getAgentFacingText,
reconcileSessionRefsForText,
@@ -155,6 +156,118 @@ describe('agentInputQueue', () => {
});
});
+ it('keeps the plain note byte-identical and pins its exact wording', () => {
+ expect(ANNOTATED_IMAGE_NOTE).toBe(
+ 'Note: the red freehand marks on the attached image(s) are annotations drawn by the user ' +
+ 'to highlight the region(s) they are referring to; they are not part of the original image.',
+ );
+ });
+
+ it('appends marked regions indexed by image order when annotated images carry them', () => {
+ const image = (id: string, extra: Record<string, unknown> = {}) => ({
+ id,
+ name: `${id}.png`,
+ path: `/repo/${id}.png`,
+ ext: '.png',
+ size: 64,
+ category: 'image' as const,
+ mimeType: 'image/png',
+ url: `xdt-image://session/${id}.png`,
+ ...extra,
+ });
+ const message = buildMakerUserMessage(
+ queuedMessage([
+ image('plain'),
+ {
+ id: 'gif',
+ name: 'clip.gif',
+ path: '/repo/clip.gif',
+ ext: '.gif',
+ size: 1,
+ category: 'image',
+ mimeType: 'image/gif',
+ url: 'xdt-image://session/clip.gif',
+ },
+ image('marked', {
+ annotated: true,
+ annotationRegions: [
+ { x0: 0.31, y0: 0.12, x1: 0.46, y1: 0.2 },
+ { x0: 0.7, y0: 0.55, x1: 0.9, y1: 0.61 },
+ ],
+ }),
+ // 标注图但区域缺失:只贡献固定说明,不出现在区域描述里。
+ image('legacy', { annotated: true }),
+ ]),
+ );
+ expect(message).toEqual({
+ type: 'user',
+ content: [
+ { type: 'text', text: 'inspect attachment' },
+ { type: 'image', path: 'xdt-image://session/plain.png', mimeType: 'image/png' },
+ { type: 'file', path: 'xdt-image://session/clip.gif', mimeType: 'image/gif' },
+ { type: 'image', path: 'xdt-image://session/marked.png', mimeType: 'image/png' },
+ { type: 'image', path: 'xdt-image://session/legacy.png', mimeType: 'image/png' },
+ {
+ type: 'text',
+ text:
+ `${ANNOTATED_IMAGE_NOTE}\n${ANNOTATED_IMAGE_REGIONS_PREFIX}` +
+ 'image 2: x 0.31–0.46, y 0.12–0.20; x 0.70–0.90, y 0.55–0.61.',
+ },
+ ],
+ });
+ });
+
+ it('falls back to the plain note when region data is missing or malformed', () => {
+ const annotatedImage = (annotationRegions: unknown) => ({
+ id: 'image-1',
+ name: 'shot.png',
+ path: '/repo/shot.png',
+ ext: '.png',
+ size: 128,
+ category: 'image' as const,
+ mimeType: 'image/png',
+ url: 'xdt-image://session/shot.png',
+ annotated: true,
+ annotationRegions: annotationRegions as never,
+ });
+ for (const regions of [undefined, [], 'bad', [{ x0: 'a', y0: 0, x1: 1, y1: 1 }], [{ x0: 0.9, y0: 0, x1: 0.1, y1: 1 }]]) {
+ const message = buildMakerUserMessage(queuedMessage([annotatedImage(regions)]));
+ expect(message).toEqual({
+ type: 'user',
+ content: [
+ { type: 'text', text: 'inspect attachment' },
+ { type: 'image', path: 'xdt-image://session/shot.png', mimeType: 'image/png' },
+ { type: 'text', text: ANNOTATED_IMAGE_NOTE },
+ ],
+ });
+ }
+ });
+
+ it('ignores region data on images that are not annotated', () => {
+ const message = buildMakerUserMessage(
+ queuedMessage([
+ {
+ id: 'image-1',
+ name: 'shot.png',
+ path: '/repo/shot.png',
+ ext: '.png',
+ size: 128,
+ category: 'image',
+ mimeType: 'image/png',
+ url: 'xdt-image://session/shot.png',
+ annotationRegions: [{ x0: 0.1, y0: 0.1, x1: 0.2, y1: 0.2 }],
+ },
+ ]),
+ );
+ expect(message).toEqual({
+ type: 'user',
+ content: [
+ { type: 'text', text: 'inspect attachment' },
+ { type: 'image', path: 'xdt-image://session/shot.png', mimeType: 'image/png' },
+ ],
+ });
+ });
+
it('does not inject the annotation note for plain images or annotated GIF-as-file blocks', () => {
expect(
buildMakerUserMessage(
diff --git a/apps/desktop/src/renderer/__tests__/messageAttachmentPayload.test.ts b/apps/desktop/src/renderer/__tests__/messageAttachmentPayload.test.ts
index b39d0e5fdc..2ce94ac3b1 100644
--- a/apps/desktop/src/renderer/__tests__/messageAttachmentPayload.test.ts
+++ b/apps/desktop/src/renderer/__tests__/messageAttachmentPayload.test.ts
@@ -72,6 +72,39 @@ describe('messageAttachmentPayload', () => {
]);
});
+ it('puts annotation regions on the wire only for burned (annotated) images', () => {
+ const regions = [{ x0: 0.1, y0: 0.2, x1: 0.3, y1: 0.4 }];
+ const burned = attachment({
+ id: 'burned',
+ name: 'shot-annotated.png',
+ ext: '.png',
+ category: 'image',
+ mimeType: 'image/png',
+ url: 'cindy-media://blobs/burned.png',
+ annotated: true,
+ annotationSourceUrl: 'cindy-media://blobs/source.png',
+ annotationStrokes: [{ points: [{ x: 0.1, y: 0.2 }] }],
+ annotationRegions: regions,
+ });
+ const draft = attachment({
+ id: 'draft',
+ name: 'draft.png',
+ ext: '.png',
+ category: 'image',
+ mimeType: 'image/png',
+ url: 'cindy-media://blobs/draft.png',
+ annotationRegions: regions,
+ });
+
+ const payload = buildUserMessageAttachmentPayload([burned, draft]);
+
+ expect(payload.serializedFiles?.[0]).toMatchObject({ annotated: true, annotationRegions: regions });
+ // 编辑期元数据(原图 / 笔迹)不上 wire。
+ expect(payload.serializedFiles?.[0]).not.toHaveProperty('annotationSourceUrl');
+ expect(payload.serializedFiles?.[0]).not.toHaveProperty('annotationStrokes');
+ expect(payload.serializedFiles?.[1]).not.toHaveProperty('annotationRegions');
+ });
+
it('uses base64 only for image fallback and keeps it out of persisted refs', () => {
const image = attachment({
name: 'fallback.png',
diff --git a/apps/desktop/src/renderer/__tests__/pendingQueueDefer.test.ts b/apps/desktop/src/renderer/__tests__/pendingQueueDefer.test.ts
index cbc49e66ad..3750766106 100644
--- a/apps/desktop/src/renderer/__tests__/pendingQueueDefer.test.ts
+++ b/apps/desktop/src/renderer/__tests__/pendingQueueDefer.test.ts
@@ -81,6 +81,8 @@ vi.mock('@/lib/composerDraftStore', () => ({
}));
import { makerChatStore, type QueuedMessage } from '@/lib/makerChatStore';
+import { AnnotationBurnInError } from '@/lib/annotationBurnIn';
+import { toast } from '@/lib/toast';
const MODEL = 'claude-opus-4-7';
const EFFORT = 'medium';
@@ -1165,3 +1167,216 @@ describe('renderer input queue facade', () => {
expect(ok).toBe(false);
});
});
+
+describe('annotation burn-in in the renderer send / queue-edit paths', () => {
+ /** 最近一次物化调用的选项(mock 签名只声明了 files,这里按真实签名取第三参)。 */
+ function materializeOptions(): Record<string, unknown> | undefined {
+ const call = annotationBurnInMocks.materialize.mock.calls.at(-1) as unknown as
+ | [unknown, unknown, Record<string, unknown> | undefined]
+ | undefined;
+ return call?.[2];
+ }
+ const strokes = [{ points: [{ x: 0.2, y: 0.8 }, { x: 0.4, y: 0.6 }] }];
+ const editContent = {
+ text: 'edited text',
+ mentions: [],
+ hasQuotes: false,
+ agentReferences: [],
+ pastedTextRanges: [],
+ slashCommandRanges: [],
+ };
+
+ function annotatedQueueItem(burnedUrl: string) {
+ const sourceUrl = 'cindy-media://blobs/source.jpg';
+ const item = queued(`q-annotated-${Math.random().toString(36).slice(2, 8)}`, 'old text');
+ item.files = [{
+ id: 'annotated-image',
+ name: 'shot-annotated.png',
+ originalName: 'shot-annotated.png',
+ path: 'C:\\images\\shot.jpg',
+ ext: '.png',
+ size: 123,
+ category: 'image',
+ mimeType: 'image/png',
+ url: burnedUrl,
+ annotated: true,
+ annotationRegions: [{ x0: 0.2, y0: 0.6, x1: 0.4, y1: 0.8 }],
+ }];
+ const retryFile: AttachedFile = {
+ ...item.files[0],
+ annotationSourceUrl: sourceUrl,
+ annotationStrokes: strokes,
+ cacheUrlShared: true,
+ };
+ item.chatMessage.retryFiles = [retryFile];
+ // 队列编辑草稿里的可编辑态(queueMessageToComposerEditDraft 的产物形态)。
+ const editableFile: AttachedFile = {
+ ...item.files[0],
+ path: sourceUrl,
+ url: sourceUrl,
+ ext: '.jpg',
+ mimeType: 'image/jpeg',
+ annotated: undefined,
+ annotationRegions: undefined,
+ annotationStrokes: strokes.map((stroke) => ({
+ points: stroke.points.map((point) => ({ ...point })),
+ })),
+ cacheUrlShared: true,
+ stagedPathShared: true,
+ };
+ return { item, retryFile, editableFile };
+ }
+
+ it('reuses the queued burned bitmap when only the text of a queued message changed', async () => {
+ const sid = `annotated-reuse-${Math.random().toString(36).slice(2, 8)}`;
+ const burnedUrl = 'cindy-media://blobs/burned.png';
+ const { item, retryFile, editableFile } = annotatedQueueItem(burnedUrl);
+ makerChatStore.initGlobalListeners();
+ projectionHandler?.(projection(sid, { pendingQueue: [item] }));
+
+ const saved = await makerChatStore.updateQueueItemContent(sid, item.clientId, {
+ content: editContent,
+ files: [editableFile],
+ });
+
+ expect(saved).toBe(true);
+ const [materializeInput] = annotationBurnInMocks.materialize.mock.calls.at(-1) ?? [];
+ // 交给物化的是队列里的烧录附件本身(retryFile,含所有权标记与元数据),不再重烧。
+ expect(materializeInput?.[0]).toBe(retryFile);
+ const replacement = input.updateContent.mock.calls.at(-1)?.[2] as unknown as QueuedMessage;
+ expect(replacement.files?.[0]).toMatchObject({
+ url: burnedUrl,
+ annotated: true,
+ annotationRegions: [{ x0: 0.2, y0: 0.6, x1: 0.4, y1: 0.8 }],
+ });
+ expect(replacement.chatMessage.retryFiles?.[0]).toMatchObject({
+ url: burnedUrl,
+ annotationSourceUrl: 'cindy-media://blobs/source.jpg',
+ cacheUrlShared: true,
+ });
+ for (const call of cleanupCachedImages.mock.calls as unknown as Array<[string[]]>) {
+ expect(call[0]).not.toContain(burnedUrl);
+ expect(call[0]).not.toContain('cindy-media://blobs/source.jpg');
+ }
+ });
+
+ it('re-burns when the strokes changed or the burned file is a legacy xdt-image', async () => {
+ const changedSid = `annotated-changed-${Math.random().toString(36).slice(2, 8)}`;
+ const changed = annotatedQueueItem('cindy-media://blobs/burned.png');
+ makerChatStore.initGlobalListeners();
+ projectionHandler?.(projection(changedSid, { pendingQueue: [changed.item] }));
+ const changedFile = {
+ ...changed.editableFile,
+ annotationStrokes: [{ points: [{ x: 0.5, y: 0.5 }] }],
+ };
+ await makerChatStore.updateQueueItemContent(changedSid, changed.item.clientId, {
+ content: editContent,
+ files: [changedFile],
+ });
+ expect(annotationBurnInMocks.materialize.mock.calls.at(-1)?.[0]?.[0]).toBe(changedFile);
+
+ const legacySid = `annotated-legacy-${Math.random().toString(36).slice(2, 8)}`;
+ const legacy = annotatedQueueItem('xdt-image://session/burned.png');
+ projectionHandler?.(projection(legacySid, { pendingQueue: [legacy.item] }));
+ await makerChatStore.updateQueueItemContent(legacySid, legacy.item.clientId, {
+ content: editContent,
+ files: [legacy.editableFile],
+ });
+ expect(annotationBurnInMocks.materialize.mock.calls.at(-1)?.[0]?.[0]).toBe(
+ legacy.editableFile,
+ );
+ });
+
+ it('aborts a queue edit on burn-in failure without touching the queued item', async () => {
+ const sid = `annotated-abort-${Math.random().toString(36).slice(2, 8)}`;
+ const { item, editableFile } = annotatedQueueItem('cindy-media://blobs/burned.png');
+ makerChatStore.initGlobalListeners();
+ projectionHandler?.(projection(sid, { pendingQueue: [item] }));
+ annotationBurnInMocks.materialize.mockRejectedValueOnce(
+ new AnnotationBurnInError('shot.png', new Error('decode failed')),
+ );
+ const changedFile = { ...editableFile, annotationStrokes: [{ points: [{ x: 0.5, y: 0.5 }] }] };
+
+ await expect(
+ makerChatStore.updateQueueItemContent(sid, item.clientId, {
+ content: editContent,
+ files: [changedFile],
+ }),
+ ).rejects.toBeInstanceOf(AnnotationBurnInError);
+ expect(materializeOptions()).toMatchObject({
+ burnFailure: 'abort',
+ });
+ expect(input.updateContent).not.toHaveBeenCalled();
+ expect(input.updateText).not.toHaveBeenCalled();
+ expect(makerChatStore.getSnapshot(sid).pendingQueue[0]).toBe(item);
+ });
+
+ it('aborts an interactive local send on burn-in failure: returns false, no enqueue, error toast', async () => {
+ const sid = `annotated-send-abort-${Math.random().toString(36).slice(2, 8)}`;
+ const errorToast = vi.spyOn(toast, 'error');
+ annotationBurnInMocks.materialize.mockRejectedValueOnce(
+ new AnnotationBurnInError('shot.png', new Error('decode failed')),
+ );
+ const file: AttachedFile = {
+ id: 'tray-image',
+ name: 'shot.png',
+ path: 'C:\\images\\shot.png',
+ ext: '.png',
+ size: 1,
+ category: 'image',
+ mimeType: 'image/png',
+ url: 'cindy-media://blobs/tray.png',
+ annotationStrokes: strokes,
+ };
+
+ const ok = await makerChatStore.sendMessage(sid, 'look here', MODEL, EFFORT, PERM, WD, [file], undefined, {
+ annotationBurnFailure: 'abort',
+ });
+ await flushPromises();
+
+ expect(ok).toBe(false);
+ expect(materializeOptions()).toMatchObject({
+ burnFailure: 'abort',
+ });
+ expect(input.enqueue).not.toHaveBeenCalled();
+ expect(makerChatStore.getSnapshot(sid).messages).toHaveLength(0);
+ expect(errorToast).toHaveBeenCalledTimes(1);
+ });
+
+ it('keeps the fallback (and warns) for sends that did not opt into aborting', async () => {
+ const sid = `annotated-send-fallback-${Math.random().toString(36).slice(2, 8)}`;
+ const warningToast = vi.spyOn(toast, 'warning');
+ (
+ annotationBurnInMocks.materialize as unknown as ReturnType<typeof vi.fn>
+ ).mockImplementationOnce(
+ async (
+ files: readonly AttachedFile[] | undefined,
+ _sessionId: string,
+ opts?: { onFallback?: (count: number) => void },
+ ) => {
+ opts?.onFallback?.(1);
+ return files ? files.map((f) => ({ ...f, annotationStrokes: undefined })) : undefined;
+ },
+ );
+ const file: AttachedFile = {
+ id: 'tray-image',
+ name: 'shot.png',
+ path: 'C:\\images\\shot.png',
+ ext: '.png',
+ size: 1,
+ category: 'image',
+ mimeType: 'image/png',
+ url: 'cindy-media://blobs/tray.png',
+ annotationStrokes: strokes,
+ };
+
+ await makerChatStore.sendMessage(sid, 'look here', MODEL, EFFORT, PERM, WD, [file]);
+ await flushPromises();
+
+ expect(materializeOptions()).toMatchObject({
+ burnFailure: 'fallback',
+ });
+ expect(input.enqueue).toHaveBeenCalledTimes(1);
+ expect(warningToast).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/apps/desktop/src/renderer/__tests__/rewindDraftAttachments.test.ts b/apps/desktop/src/renderer/__tests__/rewindDraftAttachments.test.ts
index 963380dd39..5e2efac636 100644
--- a/apps/desktop/src/renderer/__tests__/rewindDraftAttachments.test.ts
+++ b/apps/desktop/src/renderer/__tests__/rewindDraftAttachments.test.ts
@@ -10,7 +10,11 @@ import { resolve } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { cleanupRemovedCachedImage } from '@/hooks/useAttachments';
-import { buildRewindDraftAttachments } from '@/lib/rewindDraftAttachments';
+import {
+ buildRewindDraftAttachments,
+ dropMissingAnnotationSources,
+ hasRestorableAnnotationSources,
+} from '@/lib/rewindDraftAttachments';
describe('buildRewindDraftAttachments', () => {
it('restores cached image refs and persisted file refs into composer attachments', () => {
@@ -77,6 +81,38 @@ describe('buildRewindDraftAttachments', () => {
});
});
+ it('restores an annotated history image as editable source + strokes (shared source)', () => {
+ const strokes = [{ points: [{ x: 0.1, y: 0.2 }, { x: 0.3, y: 0.4 }] }];
+ const attachments = buildRewindDraftAttachments({
+ images: [
+ {
+ url: 'cindy-media://blobs/burned.png',
+ mimeType: 'image/png',
+ originalName: 'shot-annotated.png',
+ annotationSourceUrl: 'cindy-media://blobs/source.jpg',
+ annotationStrokes: strokes,
+ },
+ ],
+ });
+
+ expect(attachments).toHaveLength(1);
+ expect(attachments[0]).toMatchObject({
+ name: 'shot-annotated.png',
+ path: 'cindy-media://blobs/source.jpg',
+ url: 'cindy-media://blobs/source.jpg',
+ ext: '.jpg',
+ mimeType: 'image/jpeg',
+ category: 'image',
+ cacheUrlShared: true,
+ annotationStrokes: strokes,
+ });
+ expect(attachments[0].annotated).toBeUndefined();
+ expect(attachments[0].annotationSourceUrl).toBeUndefined();
+ // 深拷贝:草稿里的笔迹不与历史消息共享可变对象。
+ expect(attachments[0].annotationStrokes?.[0]).not.toBe(strokes[0]);
+ expect(attachments[0].annotationStrokes?.[0].points[0]).not.toBe(strokes[0].points[0]);
+ });
+
it('restores unknown historical file refs as generic composer attachments', () => {
const attachments = buildRewindDraftAttachments({
files: [
@@ -121,7 +157,13 @@ describe('rewind draft attachment wiring', () => {
);
it('rewind prefill writes text plus attachments to the composer draft', () => {
- expect(userMessageSrc).toMatch(/buildRewindDraftAttachments\(\{\s*images,\s*files\s*\}\)/);
+ expect(userMessageSrc).toMatch(
+ /buildRewindDraftAttachments\(\{\s*images:\s*draftImages,\s*files\s*\}\)/,
+ );
+ // 带可再编辑标注的历史图先探测原图,丢失的退回烧录图后再写草稿。
+ expect(userMessageSrc).toMatch(
+ /hasRestorableAnnotationSources\(images\)[\s\S]{0,120}dropMissingAnnotationSources\(images\)\.then\(saveRewindDraft\)/,
+ );
expect(userMessageSrc).toMatch(
/saveComposerDraft\(sessionId,\s*\{\s*text:\s*draftText,\s*attachments:\s*draftAttachments/s,
);
@@ -199,3 +241,52 @@ describe('cleanupRemovedCachedImage', () => {
).not.toThrow();
});
});
+
+describe('dropMissingAnnotationSources', () => {
+ const annotated = {
+ url: 'cindy-media://blobs/burned.png',
+ mimeType: 'image/png',
+ originalName: 'shot-annotated.png',
+ annotationSourceUrl: 'cindy-media://blobs/gone.png',
+ annotationStrokes: [{ points: [{ x: 0.5, y: 0.5 }] }],
+ };
+ const plain = { url: 'cindy-media://blobs/plain.png', mimeType: 'image/png', originalName: 'p.png' };
+
+ it('detects which image lists need a source probe', () => {
+ expect(hasRestorableAnnotationSources([plain])).toBe(false);
+ expect(hasRestorableAnnotationSources([{ base64: 'x', mimeType: 'image/png' }])).toBe(false);
+ expect(hasRestorableAnnotationSources([plain, annotated])).toBe(true);
+ });
+
+ it('falls back to the burned image (no strokes) when the unburned source is gone', async () => {
+ const probe = vi.fn(async () => false);
+ const images = await dropMissingAnnotationSources([plain, annotated], probe);
+
+ expect(probe).toHaveBeenCalledTimes(1);
+ expect(probe).toHaveBeenCalledWith('cindy-media://blobs/gone.png');
+ expect(images[0]).toBe(plain);
+ expect(images[1]).toEqual({
+ url: 'cindy-media://blobs/burned.png',
+ mimeType: 'image/png',
+ originalName: 'shot-annotated.png',
+ });
+ const [attachment] = buildRewindDraftAttachments({ images: [images[1]] });
+ expect(attachment).toMatchObject({
+ url: 'cindy-media://blobs/burned.png',
+ path: 'cindy-media://blobs/burned.png',
+ });
+ expect(attachment.annotationStrokes).toBeUndefined();
+ expect(attachment.cacheUrlShared).toBeUndefined();
+ });
+
+ it('keeps the editable restore when the source still exists or the probe cannot decide', async () => {
+ await expect(dropMissingAnnotationSources([annotated], async () => true)).resolves.toEqual([
+ annotated,
+ ]);
+ await expect(
+ dropMissingAnnotationSources([annotated], async () => {
+ throw new Error('probe crashed');
+ }),
+ ).resolves.toEqual([annotated]);
+ });
+});
diff --git a/apps/desktop/src/renderer/components/chat/AnnotationStrokesSvg.tsx b/apps/desktop/src/renderer/components/chat/AnnotationStrokesSvg.tsx
new file mode 100644
index 0000000000..ff44af06c8
--- /dev/null
+++ b/apps/desktop/src/renderer/components/chat/AnnotationStrokesSvg.tsx
@@ -0,0 +1,102 @@
+/**
+ * AnnotationStrokesSvg — 矢量标注笔迹的 SVG 叠加层(lightbox / 托盘缩略图 /
+ * hover 预览共用)。
+ *
+ * 分层顺序必须与 canvas 烧录(`drawAnnotationStrokes`)一致:**先全部白描边、
+ * 再全部红线**。旧实现按笔迹逐条画「描边 + 红线」,后画笔迹的白描边会盖住
+ * 先画笔迹的红线,预览与发出去的烧录图在笔迹交叉处不一致。
+ *
+ * 坐标系:viewBox = 图片自然尺寸,归一化点 × 自然尺寸 = path 坐标,线宽也按
+ * 自然尺寸算,与烧录像素级一致。`preserveAspectRatio` 默认 `none`(容器与图片
+ * 盒同尺寸);托盘缩略图是 object-cover,传 `xMidYMid slice` 即与其裁切一致。
+ *
+ * 性能:已提交笔迹的 path 串按 strokes 身份 memo,组件本身 memo——绘制中父组件
+ * 的高频重渲染不会重算全部笔迹。进行中的笔迹不走 React:调用方传 `draftRefs`,
+ * 本组件在两个图层末尾各挂一条空 path,由调用方直接写 `d`(rAF 合帧),与已提交
+ * 笔迹处于同一图层,绘制中的交叉观感与松手后一致。
+ */
+
+import { memo, useMemo, type CSSProperties, type Ref } from 'react';
+import {
+ ANNOTATION_OUTLINE_COLOR,
+ ANNOTATION_STROKE_COLOR,
+ annotationOutlineWidth,
+ annotationStrokeWidth,
+ strokeToSvgPath,
+ type AnnotationStroke,
+} from './lightboxAnnotations';
+
+export interface AnnotationDraftPathRefs {
+ outline: Ref<SVGPathElement>;
+ stroke: Ref<SVGPathElement>;
+}
+
+interface AnnotationStrokesSvgProps {
+ strokes: readonly AnnotationStroke[];
+ /** 图片自然尺寸(viewBox 与线宽基准)。 */
+ naturalWidth: number;
+ naturalHeight: number;
+ preserveAspectRatio?: string;
+ className?: string;
+ style?: CSSProperties;
+ /** 进行中笔迹的两条 path(描边层 / 红线层),`d` 由调用方直接写入。 */
+ draftRefs?: AnnotationDraftPathRefs;
+}
+
+export const AnnotationStrokesSvg = memo(function AnnotationStrokesSvg({
+ strokes,
+ naturalWidth,
+ naturalHeight,
+ preserveAspectRatio = 'none',
+ className,
+ style,
+ draftRefs,
+}: AnnotationStrokesSvgProps) {
+ const paths = useMemo(
+ () =>
+ strokes
+ .map((stroke) => strokeToSvgPath(stroke, naturalWidth, naturalHeight))
+ .filter((d) => d.length > 0),
+ [strokes, naturalWidth, naturalHeight],
+ );
+ const strokeWidth = annotationStrokeWidth(naturalWidth, naturalHeight);
+ const outlineWidth = annotationOutlineWidth(strokeWidth);
+ return (
+ <svg
+ viewBox={`0 0 ${naturalWidth} ${naturalHeight}`}
+ preserveAspectRatio={preserveAspectRatio}
+ className={className}
+ style={style}
+ aria-hidden
+ >
+ <g
+ data-annotation-layer="outline"
+ fill="none"
+ stroke={ANNOTATION_OUTLINE_COLOR}
+ strokeWidth={outlineWidth}
+ strokeLinecap="round"
+ strokeLinejoin="round"
+ >
+ {paths.map((d, i) => (
+ // biome-ignore lint/suspicious/noArrayIndexKey: 笔迹列表只增/尾删,index 稳定。
+ <path key={i} d={d} />
+ ))}
+ {draftRefs ? <path ref={draftRefs.outline} data-annotation-draft="" /> : null}
+ </g>
+ <g
+ data-annotation-layer="stroke"
+ fill="none"
+ stroke={ANNOTATION_STROKE_COLOR}
+ strokeWidth={strokeWidth}
+ strokeLinecap="round"
+ strokeLinejoin="round"
+ >
+ {paths.map((d, i) => (
+ // biome-ignore lint/suspicious/noArrayIndexKey: 同上。
+ <path key={i} d={d} />
+ ))}
+ {draftRefs ? <path ref={draftRefs.stroke} data-annotation-draft="" /> : null}
+ </g>
+ </svg>
+ );
+});
diff --git a/apps/desktop/src/renderer/components/chat/ChatImageView.tsx b/apps/desktop/src/renderer/components/chat/ChatImageView.tsx
index 04d2d3bec9..d52cae7cb3 100644
--- a/apps/desktop/src/renderer/components/chat/ChatImageView.tsx
+++ b/apps/desktop/src/renderer/components/chat/ChatImageView.tsx
@@ -313,9 +313,11 @@ export function ChatImageView({
// 续画,出口是"发送到对话"生成新消息)。displaySrc !== src 说明是
// 远程会话改写,原图在被控端拿不到,退回普通烧录图预览。
// 单张展示:gallery 按烧录图 src 定位,换成原图后无法匹配。
+ // 原图若已被清理,lightbox 退回烧录图(annotationFallbackSrc)供查看 / 叠加新标。
<ImageLightbox
src={annotationSourceUrl}
initialStrokes={annotationStrokes}
+ annotationFallbackSrc={displaySrc}
onClose={closePreview}
/>
) : (
diff --git a/apps/desktop/src/renderer/components/chat/ImageHoverPreview.tsx b/apps/desktop/src/renderer/components/chat/ImageHoverPreview.tsx
index d13146e743..7890c20898 100644
--- a/apps/desktop/src/renderer/components/chat/ImageHoverPreview.tsx
+++ b/apps/desktop/src/renderer/components/chat/ImageHoverPreview.tsx
@@ -1,6 +1,8 @@
import { useEffect, useLayoutEffect, useState, type RefObject } from 'react';
import { createPortal } from 'react-dom';
import { containImageSize, type ImageDimensions } from './imageDisplaySize';
+import { AnnotationStrokesSvg } from './AnnotationStrokesSvg';
+import type { AnnotationStroke } from './lightboxAnnotations';
const PREVIEW_MAX_WIDTH = 224;
const PREVIEW_MAX_HEIGHT = 168;
@@ -20,6 +22,8 @@ interface ImageHoverPreviewProps {
anchorRef: RefObject<HTMLElement | null>;
src: string;
alt: string;
+ /** 托盘里带标注的图片:叠加矢量笔迹(与烧录同一分层渲染),所见即将发送的。 */
+ annotationStrokes?: readonly AnnotationStroke[];
}
/**
@@ -28,7 +32,13 @@ interface ImageHoverPreviewProps {
* portal 到 body,避免被消息流 / composer 的 overflow 裁掉;默认位于锚点上方
* 12px,空间不足时翻到下方,并始终收在视口内。最大 224×168。
*/
-export function ImageHoverPreview({ open, anchorRef, src, alt }: ImageHoverPreviewProps) {
+export function ImageHoverPreview({
+ open,
+ anchorRef,
+ src,
+ alt,
+ annotationStrokes,
+}: ImageHoverPreviewProps) {
const [position, setPosition] = useState<PreviewPosition | null>(null);
const [naturalSize, setNaturalSize] = useState<ImageDimensions | null>(null);
@@ -119,6 +129,23 @@ export function ImageHoverPreview({ open, anchorRef, src, alt }: ImageHoverPrevi
});
}}
/>
+ {annotationStrokes?.length && naturalSize && displaySize ? (
+ // 与图片盒同尺寸(img 是 contain 结果,宽高比与自然尺寸一致),不用 inset:0——
+ // 行内 img 的基线留白会让容器比图片略高。
+ <AnnotationStrokesSvg
+ strokes={annotationStrokes}
+ naturalWidth={naturalSize.width}
+ naturalHeight={naturalSize.height}
+ style={{
+ position: 'absolute',
+ top: 0,
+ left: 0,
+ width: displaySize.width,
+ height: displaySize.height,
+ pointerEvents: 'none',
+ }}
+ />
+ ) : null}
</div>,
document.body,
);
diff --git a/apps/desktop/src/renderer/components/chat/ImageLightbox.tsx b/apps/desktop/src/renderer/components/chat/ImageLightbox.tsx
index 4354f4f7ba..ba45912abe 100644
--- a/apps/desktop/src/renderer/components/chat/ImageLightbox.tsx
+++ b/apps/desktop/src/renderer/components/chat/ImageLightbox.tsx
@@ -25,6 +25,10 @@
* composerDraftStore(与手机版语义一致——加入输入框附件托盘,不直接发消息)。
* 当前会话 id 来自 ChatSessionFileContext(portal 不断 context 链);聊天流外
* 复用本组件(文件浏览器 / 输入框预览)拿到默认值 undefined,不显示该动作。
+ * - 标注模式(笔按钮 / autoAnnotate / 托盘编辑):Pointer Events 落笔(鼠标、触屏、
+ * 笔统一,pointer capture 保证移出图片仍能收尾),绘制中不经 React 重渲染;
+ * ⌘/Ctrl+Z 撤销、⇧⌘Z / Ctrl+Shift+Z(非 mac 另有 Ctrl+Y)重做;放大后按住 Space
+ * 或中键拖拽平移;Esc / X 放弃——笔迹偏离基线时先经统一确认框确认。
* - 右键:菜单关闭后的同一次背景点击不能顺带关闭 lightbox,见 `lastMenuCloseAt`。
* - 动画:200ms 透明度淡入淡出。
* - 焦点:FocusScope 打开时把焦点移入 overlay 并圈禁 Tab;卸载时还给打开前的焦点元素。
@@ -36,7 +40,7 @@
* `data-gallery-active` 定位,细节见 `collectGallery` / `resolveStartIndexInFull`。
*/
-import { useCallback, useContext, useEffect, useRef, useState } from 'react';
+import { useCallback, useContext, useEffect, useLayoutEffect, useRef, useState } from 'react';
import { createPortal } from 'react-dom';
import { FocusScope } from '@radix-ui/react-focus-scope';
import {
@@ -50,11 +54,14 @@ import {
Globe,
MessageSquarePlus,
Pen,
+ Redo2,
Undo2,
X,
} from 'lucide-react';
import { useTranslation } from 'react-i18next';
import { toast } from '@/lib/toast';
+import { createLogger } from '@/lib/logger';
+import { isEditableKeyboardTarget } from '@/lib/editableKeyboardTarget';
import { cn } from '@/lib/utils';
import { extractIpcError } from '@/utils/ipcError';
import { getMimeType, extractExt, type AttachedFile } from '@/lib/fileTypes';
@@ -69,14 +76,14 @@ import { toWorkdirRel } from '../../../shared/workdirPath';
import { openFileInSidebarFileBrowser } from '@/features/right-sidebar/lib/openInSidebarFileBrowser';
import { useSidebarTargetSessionId } from '@/features/cc-agent/embeddedSessionNavigation';
import {
- ANNOTATION_OUTLINE_COLOR,
- ANNOTATION_STROKE_COLOR,
- annotationStrokeWidth,
+ extendDraftSvgPath,
normalizePoint,
shouldAppendPoint,
- strokeToSvgPath,
type AnnotationStroke,
+ type DraftSvgPathCache,
} from './lightboxAnnotations';
+import { AnnotationStrokesSvg, type AnnotationDraftPathRefs } from './AnnotationStrokesSvg';
+import { useOptionalConfirmDialog } from '@/components/ui/confirm-dialog-provider';
import { getDraft, saveDraft } from '@/lib/composerDraftStore';
import {
DropdownMenu,
@@ -107,6 +114,48 @@ const DRAG_CLICK_SUPPRESS_PX = 4;
/** 视口初始态:适配窗口、无平移。 */
const FIT_VIEWPORT: LightboxViewport = { scale: 1, tx: 0, ty: 0 };
+const log = createLogger('ImageLightbox');
+
+/** 标注层与图片盒严格同尺寸;事件由 transform 容器接管。 */
+const ANNOTATION_OVERLAY_STYLE: React.CSSProperties = {
+ position: 'absolute',
+ inset: 0,
+ width: '100%',
+ height: '100%',
+ pointerEvents: 'none',
+};
+
+const EMPTY_STROKES: readonly AnnotationStroke[] = [];
+
+/**
+ * 画廊项可编辑的持久化笔迹:原图(annotationSourceUrl)与笔迹成对存在、且原图
+ * 未被判定丢失时才返回笔迹;否则返回空(显示烧录图本身,在其上叠新标)。
+ */
+function editableItemStrokes(
+ item: GalleryImage | undefined,
+ failedSources: ReadonlySet<string>,
+): readonly AnnotationStroke[] {
+ if (!item?.annotationSourceUrl || !item.annotationStrokes?.length) return EMPTY_STROKES;
+ return failedSources.has(item.annotationSourceUrl) ? EMPTY_STROKES : item.annotationStrokes;
+}
+
+/**
+ * 标注模式下 Space 按住 = 平移修饰键;但焦点在可编辑控件或按钮类控件上时让位
+ * (Space 在按钮上是激活键,键盘用户仍需用它操作工具栏 / 确认框)。
+ */
+function isSpacePanExemptTarget(target: EventTarget | null): boolean {
+ if (isEditableKeyboardTarget(target)) return true;
+ return (
+ target instanceof Element &&
+ target.closest('button, a[href], [role="button"], [role="menuitem"], [role="checkbox"]') !==
+ null
+ );
+}
+
+function isMacPlatform(): boolean {
+ return typeof window !== 'undefined' && window.electronAPI?.platform === 'darwin';
+}
+
interface ImageLightboxProps {
src: string;
onClose: () => void;
@@ -152,6 +201,12 @@ interface ImageLightboxProps {
* 持久化的笔迹)。与 annotationEdit 互斥使用;出口仍是"发送到对话"。
*/
initialStrokes?: readonly AnnotationStroke[];
+ /**
+ * 与 initialStrokes 配套:该标注图的**烧录图** url。未烧录原图(src)加载失败
+ * (原图缓存已被清理)时退回显示烧录图、清空预置笔迹——用户仍能查看并在其上
+ * 叠加新标注,而不是提示"图片已丢失"后直接关闭(imageRef 注释的降级语义)。
+ */
+ annotationFallbackSrc?: string;
/**
* 打开即进入标注模式(等价于用户点了笔按钮)。「Mermaid/表格/公式 → 标注」
* 这类"光栅化后直奔涂画"的入口用它省一次点击;默认 false 不影响既有场景。
@@ -336,6 +391,7 @@ export function ImageLightbox({
sessionId: sessionIdProp,
annotationEdit,
initialStrokes,
+ annotationFallbackSrc,
autoAnnotate = false,
}: ImageLightboxProps) {
const { t } = useTranslation();
@@ -356,7 +412,23 @@ export function ImageLightbox({
// 本地文件**链接**复用同一个 lightbox 打开,但它们不是画廊图片)退回单张,
// 不然会套用会话画廊、起始落到不相干的第 0 张图。
const [gallery] = useState<{ items: readonly GalleryImage[]; start: number }>(() => {
- if (!enableGallery) return { items: [{ src }], start: 0 };
+ if (!enableGallery) {
+ // 历史标注图再编辑:单张项带上烧录图 + 原图元数据,原图丢失时可退回烧录图
+ // (与画廊翻到标注图走同一套 currentSrc 推导)。
+ if (annotationFallbackSrc && !annotationEdit && initialStrokes?.length) {
+ return {
+ items: [
+ {
+ src: annotationFallbackSrc,
+ annotationSourceUrl: src,
+ annotationStrokes: initialStrokes,
+ },
+ ],
+ start: 0,
+ };
+ }
+ return { items: [{ src }], start: 0 };
+ }
const keyedIndex =
galleryId && sessionImages
? sessionImages.findIndex((item) => item.galleryId === galleryId)
@@ -382,8 +454,16 @@ export function ImageLightbox({
// 再编辑分支同语义),而非把烧录图当普通图降级(review P2)。笔迹预置在
// index effect 里(挂载首帧由 initialStrokes 承担)。
const currentItem = galleryItems[index] ?? { src };
+ // 已确认加载失败(被清理)的未烧录原图:该项退回烧录图 + 空笔迹。
+ const [failedAnnotationSources, setFailedAnnotationSources] = useState<ReadonlySet<string>>(
+ () => new Set(),
+ );
+ const failedAnnotationSourcesRef = useRef(failedAnnotationSources);
+ failedAnnotationSourcesRef.current = failedAnnotationSources;
const currentSrc =
- currentItem.annotationSourceUrl && currentItem.annotationStrokes?.length
+ currentItem.annotationSourceUrl &&
+ currentItem.annotationStrokes?.length &&
+ !failedAnnotationSources.has(currentItem.annotationSourceUrl)
? currentItem.annotationSourceUrl
: currentItem.src;
const isClosingRef = useRef(false);
@@ -422,8 +502,23 @@ export function ImageLightbox({
// 已保存的笔迹起步(托盘编辑或历史图再编辑)——"撤销之前的编辑"就是从
// 这里往回 pop。
const [strokes, setStrokes] = useState<AnnotationStroke[]>(() => [...baselineStrokesRef.current]);
- const [draftStroke, setDraftStroke] = useState<AnnotationStroke | null>(null);
- const [isDrawing, setIsDrawing] = useState(false);
+ // 重做栈:撤销弹出的笔迹按序压栈;画新笔迹、放弃标注、翻页时清空。只在事件
+ // handler 中读写(不驱动渲染),用 ref 即可。
+ const redoStackRef = useRef<AnnotationStroke[]>([]);
+ // 标注模式下按住 Space = 平移修饰键(state 驱动光标,ref 供事件 handler 同步读)。
+ const [spaceHeld, setSpaceHeld] = useState(false);
+ const spaceHeldRef = useRef(false);
+ // 放弃确认框在途:期间忽略 lightbox 自己的全部快捷键,避免同一次 Esc 既关确认框
+ // 又再次触发放弃(Radix 在 capture 阶段处理 Esc,本组件的 document 监听随后
+ // 仍会收到同一事件)。
+ const confirmPendingRef = useRef(false);
+ const mountedRef = useRef(true);
+ useEffect(() => {
+ mountedRef.current = true;
+ return () => {
+ mountedRef.current = false;
+ };
+ }, []);
// 图片自然尺寸:SVG viewBox 与烧录 canvas 的坐标基准。onLoad 时设置。
const [naturalSize, setNaturalSize] = useState<{ src: string; w: number; h: number } | null>(
null,
@@ -439,6 +534,13 @@ export function ImageLightbox({
isAnnotatingRef.current = isAnnotating;
strokesRef.current = strokes;
+ // 退出标注模式时复位 Space 平移修饰(keyup 可能落在模式切换之后)。
+ useEffect(() => {
+ if (isAnnotating || !spaceHeldRef.current) return;
+ spaceHeldRef.current = false;
+ setSpaceHeld(false);
+ }, [isAnnotating]);
+
useEffect(() => {
const updateViewportSize = () => {
setViewportSize({ width: window.innerWidth, height: window.innerHeight });
@@ -458,10 +560,26 @@ export function ImageLightbox({
)
: null;
- const undoLastStroke = useCallback(() => {
- setStrokes((s) => s.slice(0, -1));
+ /** 同步更新 ref 与 state:连续快捷键(按键连发)在重渲染前也读到最新笔迹。 */
+ const replaceStrokes = useCallback((next: AnnotationStroke[]) => {
+ strokesRef.current = next;
+ setStrokes(next);
}, []);
+ const undoLastStroke = useCallback(() => {
+ const current = strokesRef.current;
+ if (current.length === 0) return;
+ redoStackRef.current = [...redoStackRef.current, current[current.length - 1]];
+ replaceStrokes(current.slice(0, -1));
+ }, [replaceStrokes]);
+
+ const redoStroke = useCallback(() => {
+ const redo = redoStackRef.current;
+ if (redo.length === 0) return;
+ redoStackRef.current = redo.slice(0, -1);
+ replaceStrokes([...strokesRef.current, redo[redo.length - 1]]);
+ }, [replaceStrokes]);
+
/**
* 放弃标注:恢复到打开时的笔迹(编辑模式=上次保存的;发送模式=空)并退出
* 标注模式。X 按钮与标注中的 Esc 都走这里。
@@ -470,16 +588,64 @@ export function ImageLightbox({
* 为标注而开的临时光栅化图,放弃标注后留在图片界面没有意义——直接关闭
* 整个 lightbox 回到原界面。
*/
- const discardAnnotation = useCallback(() => {
+ const performDiscardAnnotation = () => {
if (autoAnnotate) {
handleClose();
return;
}
- setStrokes([...baselineStrokesRef.current]);
- setDraftStroke(null);
- setIsDrawing(false);
+ resetDraftStroke();
+ redoStackRef.current = [];
+ replaceStrokes([...baselineStrokesRef.current]);
setIsAnnotating(false);
- }, [autoAnnotate, handleClose]);
+ };
+
+ /**
+ * 当前笔迹是否偏离基线(或有进行中的笔迹)。笔迹对象不可变、基线复制的是
+ * 同一批对象引用,所以按长度 + 逐项身份比较即可:撤销后再重做回来的笔迹
+ * 仍视为未改动。
+ */
+ const hasUnsavedStrokeChanges = (): boolean => {
+ if (draftStrokeRef.current) return true;
+ const current = strokesRef.current;
+ const baseline = baselineStrokesRef.current;
+ return current.length !== baseline.length || current.some((s, i) => s !== baseline[i]);
+ };
+
+ const confirmDialog = useOptionalConfirmDialog();
+ /**
+ * 放弃标注入口(X 按钮 / 标注中的 Esc)。笔迹与基线相同时行为与以往完全一致
+ * (立即恢复并退出);有改动时先经应用统一确认框确认,避免误触丢掉笔迹。
+ * 裸渲染环境(无 ConfirmDialogProvider)退回旧行为。
+ */
+ const discardAnnotation = async (): Promise<void> => {
+ if (confirmPendingRef.current) return;
+ if (!confirmDialog || !hasUnsavedStrokeChanges()) {
+ performDiscardAnnotation();
+ return;
+ }
+ confirmPendingRef.current = true;
+ let confirmed = false;
+ try {
+ confirmed = await confirmDialog.confirm({
+ title: t('chat.media.annotateDiscardConfirmTitle'),
+ description: t('chat.media.annotateDiscardConfirmDescription'),
+ confirmText: t('chat.media.annotateDiscardConfirmAction'),
+ cancelText: t('chat.media.annotateDiscardConfirmCancel'),
+ confirmVariant: 'destructive',
+ });
+ } finally {
+ // 推迟到下一个宏任务再解除:关闭确认框的那次 Esc 仍在派发中(微任务会在
+ // 两个监听之间清空),立即解除会让本组件的 keydown 再放弃一次。
+ setTimeout(() => {
+ confirmPendingRef.current = false;
+ }, 0);
+ }
+ if (confirmed && mountedRef.current) performDiscardAnnotation();
+ };
+ // once-bound keydown handler 通过 ref 调用最新的放弃逻辑(依赖 confirmDialog 等
+ // 每次渲染的值)。
+ const discardAnnotationRef = useRef(discardAnnotation);
+ discardAnnotationRef.current = discardAnnotation;
const applyViewport = useCallback((next: LightboxViewport) => {
viewportRef.current = next;
@@ -542,10 +708,13 @@ export function ImageLightbox({
setIsAnnotating(false);
// 翻到持久化标注图时预置其笔迹(配合 currentSrc 换到原图,进入可编辑
// 视图);普通图照旧清空。基线同步更新,放弃标注恢复到该图的基线。
- baselineStrokesRef.current = galleryItems[index]?.annotationStrokes ?? [];
- setStrokes([...baselineStrokesRef.current]);
- setDraftStroke(null);
- setIsDrawing(false);
+ baselineStrokesRef.current = editableItemStrokes(
+ galleryItems[index],
+ failedAnnotationSourcesRef.current,
+ );
+ resetDraftStroke();
+ redoStackRef.current = [];
+ replaceStrokes([...baselineStrokesRef.current]);
setNaturalSize(null);
// biome-ignore lint/correctness/useExhaustiveDependencies: galleryItems 挂载时固定。
}, [index, resetViewport]);
@@ -584,16 +753,21 @@ export function ImageLightbox({
return () => overlay.removeEventListener('wheel', onWheel);
}, [applyViewport, markWheeling]);
+ /** 放大状态下开始拖拽平移(适配态没有可平移的范围,不响应)。 */
+ function startPan(clientX: number, clientY: number) {
+ if (viewportRef.current.scale <= 1) return;
+ dragMovedRef.current = false;
+ setIsDragging(true);
+ const current = viewportRef.current;
+ dragStartRef.current = { x: clientX, y: clientY, tx: current.tx, ty: current.ty };
+ }
+
/** 放大状态下按住图片开始拖拽平移。 */
function handleImageMouseDown(e: React.MouseEvent) {
if (e.button !== 0) return;
// preventDefault 同时抑制 <img> 的原生拖拽 ghost 与文字选区。
e.preventDefault();
- if (viewportRef.current.scale <= 1) return;
- dragMovedRef.current = false;
- setIsDragging(true);
- const current = viewportRef.current;
- dragStartRef.current = { x: e.clientX, y: e.clientY, tx: current.tx, ty: current.ty };
+ startPan(e.clientX, e.clientY);
}
// 拖拽期间 move/up 绑到 window,光标移出图片仍能继续拖;仅 isDragging 时挂载。
@@ -652,54 +826,150 @@ export function ImageLightbox({
// ---- 标注模式手势与烧录(状态声明在视口区,几何/烧录纯函数在 lightboxAnnotations) ----
- // 进行中笔迹的事实源:ref(事件 handler 同步读写),state 仅驱动 SVG 渲染。
+ // 进行中笔迹的事实源:ref(事件 handler 同步读写)。绘制中不经 React state——
+ // 每个 move 事件只往 ref 里追加一点,再由 rAF 合帧直接改写两条草稿 path 的
+ // `d`(描边层 / 红线层,见 AnnotationStrokesSvg),避免每次移动都复制整条点
+ // 数组并重渲染整个 lightbox。
// ⚠️ 提交笔迹绝不能写在 setState updater 内部——updater 必须是纯函数,
// StrictMode(dev)会双调 updater,曾导致每笔被 push 两次:两条笔迹完全
// 重叠肉眼不可见,表现为"撤销要点两次才生效"。
const draftStrokeRef = useRef<AnnotationStroke | null>(null);
+ /** 正在绘制的指针(pointer capture 的所有者);null = 未在绘制。 */
+ const drawingPointerIdRef = useRef<number | null>(null);
+ const draftPathCacheRef = useRef<DraftSvgPathCache | null>(null);
+ const draftRafRef = useRef<number | null>(null);
+ const draftOutlinePathRef = useRef<SVGPathElement>(null);
+ const draftStrokePathRef = useRef<SVGPathElement>(null);
+ const draftPathRefs = useRef<AnnotationDraftPathRefs>({
+ outline: draftOutlinePathRef,
+ stroke: draftStrokePathRef,
+ }).current;
+ // rAF 回调读取的自然尺寸(与 SVG viewBox 同源)。
+ const naturalSizeRef = useRef<{ w: number; h: number } | null>(null);
+ naturalSizeRef.current = currentNaturalSize;
- /** 标注模式画笔:mousedown 起笔(替代平移拖拽)。 */
- function handleAnnotateMouseDown(e: React.MouseEvent) {
- if (e.button !== 0) return;
- e.preventDefault();
+ function writeDraftPath(d: string) {
+ draftOutlinePathRef.current?.setAttribute('d', d);
+ draftStrokePathRef.current?.setAttribute('d', d);
+ }
+
+ function paintDraftStroke() {
+ draftRafRef.current = null;
+ const draft = draftStrokeRef.current;
+ const size = naturalSizeRef.current;
+ if (!draft || !size) return;
+ draftPathCacheRef.current = extendDraftSvgPath(
+ draft.points,
+ size.w,
+ size.h,
+ draftPathCacheRef.current,
+ );
+ writeDraftPath(draftPathCacheRef.current.d);
+ }
+
+ function scheduleDraftPaint() {
+ if (draftRafRef.current !== null) return;
+ draftRafRef.current = requestAnimationFrame(paintDraftStroke);
+ }
+
+ /** 丢弃进行中的笔迹(放弃标注 / 翻页 / 卸载)。 */
+ function resetDraftStroke() {
+ if (draftRafRef.current !== null) {
+ cancelAnimationFrame(draftRafRef.current);
+ draftRafRef.current = null;
+ }
+ draftStrokeRef.current = null;
+ drawingPointerIdRef.current = null;
+ draftPathCacheRef.current = null;
+ writeDraftPath('');
+ }
+
+ useEffect(
+ () => () => {
+ if (draftRafRef.current !== null) cancelAnimationFrame(draftRafRef.current);
+ },
+ [],
+ );
+
+ // 提交后的笔迹由 React 渲染进已提交图层;在同一次提交的 layout 阶段(绘制前)
+ // 再清空草稿 path,松手瞬间不会出现"笔迹消失一帧"的闪烁。
+ // biome-ignore lint/correctness/useExhaustiveDependencies: 只在笔迹列表变化时收尾草稿。
+ useLayoutEffect(() => {
+ if (drawingPointerIdRef.current === null && draftStrokeRef.current === null) {
+ draftPathCacheRef.current = null;
+ writeDraftPath('');
+ }
+ }, [strokes]);
+
+ /** 当前指针位置 → 归一化图片坐标(rect 已含缩放/平移 transform)。 */
+ function pointFromClient(clientX: number, clientY: number) {
const rect = imgRef.current?.getBoundingClientRect();
- if (!rect) return;
- const point = normalizePoint(e.clientX, e.clientY, rect);
+ if (!rect) return null;
+ return normalizePoint(clientX, clientY, rect);
+ }
+
+ /**
+ * 标注模式的鼠标按下:只负责平移手势(中键拖拽 / 按住 Space 左键拖拽)与
+ * 抑制原生拖拽 ghost、文字选区;画笔由 pointer 事件负责(鼠标 / 触屏 / 笔统一)。
+ */
+ function handleAnnotateMouseDown(e: React.MouseEvent) {
+ if (e.button === 1 || (e.button === 0 && spaceHeldRef.current)) {
+ // 中键 preventDefault 同时阻止 Windows / Linux 的中键自动滚动。
+ e.preventDefault();
+ startPan(e.clientX, e.clientY);
+ return;
+ }
+ if (e.button === 0) e.preventDefault();
+ }
+
+ /** 画笔起笔:仅主键(鼠标左键 / 触屏 / 笔尖接触),平移修饰键按住时不画。 */
+ function handleAnnotatePointerDown(e: React.PointerEvent<HTMLDivElement>) {
+ if (e.button !== 0 || spaceHeldRef.current) return;
+ // 多点触控:已有指针在画时忽略其它手指。
+ if (drawingPointerIdRef.current !== null) return;
+ const point = pointFromClient(e.clientX, e.clientY);
if (!point) return;
draftStrokeRef.current = { points: [point] };
- setDraftStroke(draftStrokeRef.current);
- setIsDrawing(true);
+ draftPathCacheRef.current = null;
+ drawingPointerIdRef.current = e.pointerId;
+ // pointer capture:移出图片 / 窗口仍能继续画与收尾(等价旧实现的 window 监听)。
+ try {
+ e.currentTarget.setPointerCapture?.(e.pointerId);
+ } catch {
+ // 指针已失效(极端竞态)时 capture 失败不影响本笔绘制。
+ }
+ scheduleDraftPaint();
}
- // 画笔进行中:move/up 绑 window(移出图片仍能收尾),仅 isDrawing 时挂载。
- useEffect(() => {
- if (!isDrawing) return;
- const onMove = (e: globalThis.MouseEvent) => {
- const draft = draftStrokeRef.current;
- if (!draft) return;
- const rect = imgRef.current?.getBoundingClientRect();
- if (!rect) return;
- const point = normalizePoint(e.clientX, e.clientY, rect);
- if (!point || !shouldAppendPoint(draft, point)) return;
- draftStrokeRef.current = { points: [...draft.points, point] };
- setDraftStroke(draftStrokeRef.current);
- };
- const onUp = () => {
- const draft = draftStrokeRef.current;
- draftStrokeRef.current = null;
- setIsDrawing(false);
- setDraftStroke(null);
- if (draft && draft.points.length > 0) {
- setStrokes((s) => [...s, draft]);
- }
- };
- window.addEventListener('mousemove', onMove);
- window.addEventListener('mouseup', onUp);
- return () => {
- window.removeEventListener('mousemove', onMove);
- window.removeEventListener('mouseup', onUp);
- };
- }, [isDrawing]);
+ function handleAnnotatePointerMove(e: React.PointerEvent<HTMLDivElement>) {
+ if (e.pointerId !== drawingPointerIdRef.current) return;
+ const draft = draftStrokeRef.current;
+ if (!draft) return;
+ const point = pointFromClient(e.clientX, e.clientY);
+ if (!point || !shouldAppendPoint(draft, point)) return;
+ // 草稿笔迹提交前只归本 ref 所有,原地追加(O(1));提交后不再修改。
+ draft.points.push(point);
+ scheduleDraftPaint();
+ }
+
+ /** 收笔(pointerup / pointercancel / 失去 capture 均走这里,幂等)。 */
+ function finishAnnotateStroke(e: React.PointerEvent<HTMLDivElement>) {
+ if (e.pointerId !== drawingPointerIdRef.current) return;
+ const draft = draftStrokeRef.current;
+ drawingPointerIdRef.current = null;
+ draftStrokeRef.current = null;
+ if (draftRafRef.current !== null) {
+ cancelAnimationFrame(draftRafRef.current);
+ draftRafRef.current = null;
+ }
+ if (draft && draft.points.length > 0) {
+ redoStackRef.current = [];
+ replaceStrokes([...strokesRef.current, draft]);
+ } else {
+ draftPathCacheRef.current = null;
+ writeDraftPath('');
+ }
+ }
/** 即时烧录当前所见(复制/另存为用),实现在共享模块 annotationBurnIn。 */
async function materializeAnnotatedImage(
@@ -725,25 +995,47 @@ export function ImageLightbox({
// biome-ignore lint/correctness/useExhaustiveDependencies: galleryItems 在挂载时固定;这里只订阅一次全局键盘事件,避免反复绑定 DOM listener。
useEffect(() => {
const handler = (e: KeyboardEvent) => {
+ // 放弃确认框打开期间,按键归确认框(含关闭它的那次 Esc)。
+ if (confirmPendingRef.current) return;
if (e.key === 'Escape') {
if (performance.now() - lastMenuCloseAt.current < 150) return;
// 标注模式中 Esc = 放弃标注(清笔迹退出),再按一次才关 lightbox。
if (isAnnotatingRef.current) {
- discardAnnotation();
+ void discardAnnotationRef.current();
return;
}
handleClose();
return;
}
- // 标注模式撤销上一笔(在修饰键放行守卫之前拦截)。
+ // 标注模式按住 Space = 平移修饰键(焦点在输入 / 按钮类控件上时让位)。
+ if (e.code === 'Space' || e.key === ' ') {
+ if (isAnnotatingRef.current && !isSpacePanExemptTarget(e.target)) {
+ e.preventDefault();
+ if (!spaceHeldRef.current) {
+ spaceHeldRef.current = true;
+ setSpaceHeld(true);
+ }
+ }
+ return;
+ }
+ // 标注模式撤销 / 重做(在修饰键放行守卫之前拦截):⌘/Ctrl+Z 撤销,
+ // ⇧⌘Z / Ctrl+Shift+Z 重做,非 mac 另支持 Ctrl+Y。
+ if (isAnnotatingRef.current && (e.metaKey || e.ctrlKey) && (e.key === 'z' || e.key === 'Z')) {
+ e.preventDefault();
+ if (e.shiftKey) redoStroke();
+ else undoLastStroke();
+ return;
+ }
if (
isAnnotatingRef.current &&
- (e.metaKey || e.ctrlKey) &&
+ !isMacPlatform() &&
+ e.ctrlKey &&
+ !e.metaKey &&
!e.shiftKey &&
- (e.key === 'z' || e.key === 'Z')
+ (e.key === 'y' || e.key === 'Y')
) {
e.preventDefault();
- undoLastStroke();
+ redoStroke();
return;
}
// 步进缩放。zoomBy / resetViewport 只读写 ref + 稳定 setter,首帧闭包即终身有效。
@@ -778,8 +1070,28 @@ export function ImageLightbox({
setIndex((i) => (i - 1 + galleryItems.length) % galleryItems.length);
}
};
+ // Space 松开:结束平移修饰(只处理本组件接管过的那次按键)。
+ const keyUpHandler = (e: KeyboardEvent) => {
+ if ((e.code === 'Space' || e.key === ' ') && spaceHeldRef.current) {
+ e.preventDefault();
+ spaceHeldRef.current = false;
+ setSpaceHeld(false);
+ }
+ };
+ // 窗口失焦时收不到 keyup,修饰状态必须复位,否则回来后无法落笔。
+ const blurHandler = () => {
+ if (!spaceHeldRef.current) return;
+ spaceHeldRef.current = false;
+ setSpaceHeld(false);
+ };
document.addEventListener('keydown', handler);
- return () => document.removeEventListener('keydown', handler);
+ document.addEventListener('keyup', keyUpHandler);
+ window.addEventListener('blur', blurHandler);
+ return () => {
+ document.removeEventListener('keydown', handler);
+ document.removeEventListener('keyup', keyUpHandler);
+ window.removeEventListener('blur', blurHandler);
+ };
// handleClose / gallery 都在组件生命周期内稳定(onClose 是调用方传的稳定
// setState dispatch;gallery 由 useState 初始化后不再变)。只在挂载/卸载
// 订阅一次是有意为之——每次渲染重订阅会带来无谓的 DOM listener 抖动。
@@ -1130,14 +1442,27 @@ export function ImageLightbox({
// 拖拽/连续滚轮期间关过渡,跟手;步进缩放(键盘/双击)时 80ms 平滑。
transition: isDragging || isWheeling ? 'none' : 'transform 80ms ease-out',
cursor: isAnnotating
- ? 'crosshair'
+ ? isDragging
+ ? 'grabbing'
+ : spaceHeld
+ ? scale > 1
+ ? 'grab'
+ : 'default'
+ : 'crosshair'
: scale > 1
? isDragging
? 'grabbing'
: 'grab'
: 'default',
+ // 标注模式下触屏 / 笔的拖动全部交给画笔,不触发浏览器滚动 / 缩放手势。
+ touchAction: isAnnotating ? 'none' : undefined,
}}
onMouseDown={isAnnotating ? handleAnnotateMouseDown : handleImageMouseDown}
+ onPointerDown={isAnnotating ? handleAnnotatePointerDown : undefined}
+ onPointerMove={isAnnotating ? handleAnnotatePointerMove : undefined}
+ onPointerUp={isAnnotating ? finishAnnotateStroke : undefined}
+ onPointerCancel={isAnnotating ? finishAnnotateStroke : undefined}
+ onLostPointerCapture={isAnnotating ? finishAnnotateStroke : undefined}
onDoubleClick={isAnnotating ? undefined : handleImageDoubleClick}
>
<img
@@ -1174,53 +1499,33 @@ export function ImageLightbox({
// image-local-cache F4:lightbox 只放大已知可用图片。底层文件消失时
// (例如缓存被清理),关闭 lightbox 并提示;背后的缩略图卡片会显示自己的
// ImageMissingPlaceholder,用户仍能看到上下文。
+ // 例外:历史标注图的未烧录原图丢失而烧录图仍在时,退回烧录图、清空预置
+ // 笔迹(在烧录图上叠新标),不关闭。
onError={() => {
+ const sourceUrl = currentItem.annotationSourceUrl;
+ if (sourceUrl && currentSrc === sourceUrl && currentItem.src !== sourceUrl) {
+ log.info('annotation source missing, falling back to burned image');
+ setFailedAnnotationSources((prev) => new Set(prev).add(sourceUrl));
+ baselineStrokesRef.current = EMPTY_STROKES;
+ resetDraftStroke();
+ redoStackRef.current = [];
+ replaceStrokes([]);
+ return;
+ }
toast.warning(t('chat.media.imageMissing'));
handleClose();
}}
/>
{/* 标注层:viewBox = 图片自然尺寸,归一化笔迹 × 自然尺寸 = path 坐标,
与烧录坐标一致(所见即所得)。pointerEvents 关闭,事件由容器接管。 */}
- {currentNaturalSize && (strokes.length > 0 || draftStroke) ? (
- <svg
- viewBox={`0 0 ${currentNaturalSize.w} ${currentNaturalSize.h}`}
- preserveAspectRatio="none"
- style={{
- position: 'absolute',
- inset: 0,
- width: '100%',
- height: '100%',
- pointerEvents: 'none',
- }}
- aria-hidden
- >
- {[...strokes, ...(draftStroke ? [draftStroke] : [])].map((stroke, i) => {
- const d = strokeToSvgPath(stroke, currentNaturalSize.w, currentNaturalSize.h);
- if (!d) return null;
- const w = annotationStrokeWidth(currentNaturalSize.w, currentNaturalSize.h);
- return (
- // biome-ignore lint/suspicious/noArrayIndexKey: 笔迹列表只增/尾删,index 稳定。
- <g key={i}>
- <path
- d={d}
- fill="none"
- stroke={ANNOTATION_OUTLINE_COLOR}
- strokeWidth={Math.round(w * 1.8)}
- strokeLinecap="round"
- strokeLinejoin="round"
- />
- <path
- d={d}
- fill="none"
- stroke={ANNOTATION_STROKE_COLOR}
- strokeWidth={w}
- strokeLinecap="round"
- strokeLinejoin="round"
- />
- </g>
- );
- })}
- </svg>
+ {currentNaturalSize && (strokes.length > 0 || isAnnotating) ? (
+ <AnnotationStrokesSvg
+ strokes={strokes}
+ naturalWidth={currentNaturalSize.w}
+ naturalHeight={currentNaturalSize.h}
+ style={ANNOTATION_OVERLAY_STYLE}
+ draftRefs={draftPathRefs}
+ />
) : null}
</div>
{hasAnyAction ? (
@@ -1324,24 +1629,41 @@ export function ImageLightbox({
onClick={(e) => e.stopPropagation()}
>
<LightboxToolbarButton
- onClick={discardAnnotation}
+ onClick={() => void discardAnnotation()}
label={t('chat.media.annotateDiscard')}
+ keepPointerFocus
>
<X className="h-4 w-4" />
</LightboxToolbarButton>
- <LightboxToolbarButton onClick={undoLastStroke} label={t('chat.media.annotateUndo')}>
+ <LightboxToolbarButton
+ onClick={undoLastStroke}
+ label={t('chat.media.annotateUndo')}
+ keepPointerFocus
+ >
<Undo2 className="h-4 w-4" />
</LightboxToolbarButton>
+ <LightboxToolbarButton
+ onClick={redoStroke}
+ label={t('chat.media.annotateRedo')}
+ keepPointerFocus
+ >
+ <Redo2 className="h-4 w-4" />
+ </LightboxToolbarButton>
<div className="mx-1 h-5 w-px bg-[var(--lightbox-toolbar-border)]" />
{annotationEdit ? (
<LightboxToolbarButton
onClick={handleAnnotationSave}
label={t('chat.media.annotateSave')}
+ keepPointerFocus
>
<Check className="h-4 w-4" />
</LightboxToolbarButton>
) : (
- <LightboxToolbarButton onClick={handleSendToChat} label={t('chat.media.sendToChat')}>
+ <LightboxToolbarButton
+ onClick={handleSendToChat}
+ label={t('chat.media.sendToChat')}
+ keepPointerFocus
+ >
<MessageSquarePlus className="h-4 w-4" />
</LightboxToolbarButton>
)}
@@ -1451,10 +1773,16 @@ export function ImageLightbox({
function LightboxToolbarButton({
onClick,
label,
+ keepPointerFocus = false,
children,
}: {
onClick: () => void;
label: string;
+ /**
+ * 鼠标点击不把焦点停在按钮上(焦点留在画布)。标注工具栏用:否则点过「撤销」
+ * 后按住 Space 平移会变成再次激活该按钮。键盘 Tab 聚焦与 Space / Enter 激活不受影响。
+ */
+ keepPointerFocus?: boolean;
children: React.ReactNode;
}) {
return (
@@ -1462,6 +1790,7 @@ function LightboxToolbarButton({
<button
type="button"
onClick={onClick}
+ onMouseDown={keepPointerFocus ? (e) => e.preventDefault() : undefined}
aria-label={label}
className={cn(
'inline-flex h-7 w-7 items-center justify-center',
diff --git a/apps/desktop/src/renderer/components/chat/UserMessage.tsx b/apps/desktop/src/renderer/components/chat/UserMessage.tsx
index 0474b0f927..13e694bf3c 100644
--- a/apps/desktop/src/renderer/components/chat/UserMessage.tsx
+++ b/apps/desktop/src/renderer/components/chat/UserMessage.tsx
@@ -56,7 +56,11 @@ import type {
import type { PastedTextRange, SlashCommandRange } from '@/lib/imageRef';
import type { AgentInputReference } from '../../../shared/agentInputQueue';
import type { PersistedSessionReferenceMetadata } from '../../../shared/sessionReferenceMetadata';
-import { buildRewindDraftAttachments } from '@/lib/rewindDraftAttachments';
+import {
+ buildRewindDraftAttachments,
+ dropMissingAnnotationSources,
+ hasRestorableAnnotationSources,
+} from '@/lib/rewindDraftAttachments';
import {
useAgentCapabilities,
type AgentKind as MakerAgentKind,
@@ -1251,12 +1255,21 @@ export function UserMessage({
// reload it disappears from the list; the composer keeps the draft so
// the user can edit and re-send.
const draftText = quoteDraftDocument ?? textToTiptapDoc(bubbleBody);
- const draftAttachments = buildRewindDraftAttachments({ images, files });
- if (draftText || draftAttachments.length > 0) {
- saveComposerDraft(sessionId, {
- text: draftText,
- attachments: draftAttachments,
- });
+ const saveRewindDraft = (draftImages: typeof images) => {
+ const draftAttachments = buildRewindDraftAttachments({ images: draftImages, files });
+ if (draftText || draftAttachments.length > 0) {
+ saveComposerDraft(sessionId, {
+ text: draftText,
+ attachments: draftAttachments,
+ });
+ }
+ };
+ // 带可再编辑标注的历史图:先确认未烧录原图仍在,丢失的退回烧录图(否则
+ // 草稿里是一张打不开的原图)。其余消息保持同步写草稿,时序与以往一致。
+ if (images && hasRestorableAnnotationSources(images)) {
+ void dropMissingAnnotationSources(images).then(saveRewindDraft);
+ } else {
+ saveRewindDraft(images);
}
// Patch sidebar: tokens reset, sdkSessionId may have changed, bump
// updatedAt so this session sorts back to top.
diff --git a/apps/desktop/src/renderer/components/chat/__tests__/ImageHoverPreview.test.tsx b/apps/desktop/src/renderer/components/chat/__tests__/ImageHoverPreview.test.tsx
index 17f7198b19..2726159406 100644
--- a/apps/desktop/src/renderer/components/chat/__tests__/ImageHoverPreview.test.tsx
+++ b/apps/desktop/src/renderer/components/chat/__tests__/ImageHoverPreview.test.tsx
@@ -136,4 +136,48 @@ describe('ImageHoverPreview', () => {
expect(parseFloat(image.style.width)).toBeCloseTo(176);
expect(parseFloat(image.style.height)).toBeCloseTo(117.33);
});
+
+ it('overlays tray annotation strokes at the displayed image size (outline layer first)', () => {
+ const anchor = document.createElement('span');
+ anchor.getBoundingClientRect = () =>
+ ({
+ top: 400,
+ left: 100,
+ width: 56,
+ height: 56,
+ right: 156,
+ bottom: 456,
+ x: 100,
+ y: 400,
+ toJSON: () => ({}),
+ }) as DOMRect;
+ const anchorRef = { current: anchor } as RefObject<HTMLElement | null>;
+ render(
+ <ImageHoverPreview
+ open
+ anchorRef={anchorRef}
+ src="cindy-media://blobs/tray.png"
+ alt="tray.png"
+ annotationStrokes={[{ points: [{ x: 0.1, y: 0.1 }, { x: 0.9, y: 0.9 }] }]}
+ />,
+ );
+ const image = screen.getByRole('img', { name: 'tray.png' });
+ // 尺寸未知前不画叠加层(避免错位的一帧)。
+ expect(document.body.querySelector('[data-annotation-layer]')).toBeNull();
+ Object.defineProperties(image, {
+ naturalWidth: { configurable: true, value: 400 },
+ naturalHeight: { configurable: true, value: 200 },
+ });
+ fireEvent.load(image);
+
+ const svg = document.body.querySelector('svg');
+ expect(svg?.getAttribute('viewBox')).toBe('0 0 400 200');
+ expect(parseFloat(svg?.style.width ?? '')).toBeCloseTo(parseFloat(image.style.width));
+ expect(parseFloat(svg?.style.height ?? '')).toBeCloseTo(parseFloat(image.style.height));
+ const layers = Array.from(svg?.querySelectorAll('[data-annotation-layer]') ?? []).map((el) =>
+ el.getAttribute('data-annotation-layer'),
+ );
+ expect(layers).toEqual(['outline', 'stroke']);
+ });
});
+
diff --git a/apps/desktop/src/renderer/components/chat/__tests__/ImageLightboxAnnotate.test.tsx b/apps/desktop/src/renderer/components/chat/__tests__/ImageLightboxAnnotate.test.tsx
new file mode 100644
index 0000000000..3a83695de2
--- /dev/null
+++ b/apps/desktop/src/renderer/components/chat/__tests__/ImageLightboxAnnotate.test.tsx
@@ -0,0 +1,322 @@
+// @vitest-environment jsdom
+/**
+ * ImageLightbox 标注模式的 DOM 行为。
+ *
+ * - 画笔走 Pointer Events(鼠标 / 触屏 / 笔统一),归一化坐标交给保存回调;
+ * - 撤销 / 重做(按钮与快捷键),画新笔迹清空重做栈;
+ * - SVG 预览与 canvas 烧录同一分层:先全部白描边、后全部红线;
+ * - Esc / X:笔迹未偏离基线时行为同旧版(立即退出,不弹确认);有改动时经统一
+ * 确认框,关闭确认框的那次 Esc 不会再次触发放弃;
+ * - Space / 中键拖拽平移时不落笔;
+ * - 历史标注图原图丢失时退回烧录图,而不是提示丢失并关闭。
+ */
+
+import { act, cleanup, fireEvent, render, screen } from '@testing-library/react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { i18n } from '@/i18n';
+import { ImageLightbox } from '@/components/chat/ImageLightbox';
+import { ConfirmDialogProvider } from '@/components/ui/confirm-dialog-provider';
+
+const SRC = 'cindy-media://blobs/source.png';
+
+type Stroke = { points: Array<{ x: number; y: number }> };
+
+function label(key: string): string {
+ return i18n.t(key);
+}
+
+/** 让 <img> 以 200×100 的自然尺寸"加载完成",显示矩形同为 (0,0,200,100)。 */
+function loadImage(img: HTMLImageElement) {
+ Object.defineProperty(img, 'naturalWidth', { configurable: true, value: 200 });
+ Object.defineProperty(img, 'naturalHeight', { configurable: true, value: 100 });
+ vi.spyOn(img, 'getBoundingClientRect').mockReturnValue({
+ left: 0,
+ top: 0,
+ width: 200,
+ height: 100,
+ right: 200,
+ bottom: 100,
+ x: 0,
+ y: 0,
+ toJSON: () => ({}),
+ } as DOMRect);
+ fireEvent.load(img);
+}
+
+function lightboxImage(): HTMLImageElement {
+ const img = document.body.querySelector<HTMLImageElement>('img');
+ if (!img) throw new Error('lightbox image not rendered');
+ return img;
+}
+
+function surface(): HTMLElement {
+ const parent = lightboxImage().parentElement;
+ if (!parent) throw new Error('transform container missing');
+ return parent;
+}
+
+function drawStroke(
+ from: { x: number; y: number },
+ to: { x: number; y: number },
+ pointerId = 1,
+ button = 0,
+) {
+ const el = surface();
+ fireEvent.pointerDown(el, { pointerId, button, clientX: from.x, clientY: from.y });
+ fireEvent.pointerMove(el, { pointerId, clientX: to.x, clientY: to.y });
+ fireEvent.pointerUp(el, { pointerId, button, clientX: to.x, clientY: to.y });
+}
+
+function committedPaths(layer: 'outline' | 'stroke'): Element[] {
+ return Array.from(
+ document.body.querySelectorAll(
+ `[data-annotation-layer="${layer}"] path:not([data-annotation-draft])`,
+ ),
+ );
+}
+
+function openAnnotate() {
+ fireEvent.click(screen.getByRole('button', { name: label('chat.media.annotate') }));
+}
+
+function pressKey(init: KeyboardEventInit & { key: string }) {
+ fireEvent.keyDown(document.activeElement ?? document.body, init);
+}
+
+beforeEach(() => {
+ vi.useRealTimers();
+});
+
+afterEach(() => {
+ cleanup();
+ vi.restoreAllMocks();
+ vi.useRealTimers();
+});
+
+describe('ImageLightbox annotate mode', () => {
+ it('draws with pointer events and saves normalized strokes', async () => {
+ const onSave = vi.fn();
+ render(
+ <ImageLightbox src={SRC} onClose={vi.fn()} annotationEdit={{ onSave }} />,
+ );
+ loadImage(lightboxImage());
+ openAnnotate();
+
+ drawStroke({ x: 20, y: 20 }, { x: 100, y: 50 });
+
+ expect(committedPaths('stroke')).toHaveLength(1);
+ expect(surface().style.touchAction).toBe('none');
+
+ await act(async () => {
+ fireEvent.click(screen.getByRole('button', { name: label('chat.media.annotateSave') }));
+ });
+ expect(onSave).toHaveBeenCalledWith({
+ strokes: [
+ {
+ points: [
+ { x: 0.1, y: 0.2 },
+ { x: 0.5, y: 0.5 },
+ ],
+ },
+ ],
+ });
+ });
+
+ it('renders all white outlines before all red strokes (burn-in parity)', () => {
+ render(<ImageLightbox src={SRC} onClose={vi.fn()} annotationEdit={{ onSave: vi.fn() }} />);
+ loadImage(lightboxImage());
+ openAnnotate();
+ drawStroke({ x: 10, y: 10 }, { x: 190, y: 90 });
+ drawStroke({ x: 190, y: 10 }, { x: 10, y: 90 });
+
+ const layers = Array.from(document.body.querySelectorAll('[data-annotation-layer]')).map(
+ (el) => el.getAttribute('data-annotation-layer'),
+ );
+ expect(layers).toEqual(['outline', 'stroke']);
+ expect(committedPaths('outline')).toHaveLength(2);
+ expect(committedPaths('stroke')).toHaveLength(2);
+ const outlineWidth = Number(
+ document.body.querySelector('[data-annotation-layer="outline"]')?.getAttribute('stroke-width'),
+ );
+ const strokeWidth = Number(
+ document.body.querySelector('[data-annotation-layer="stroke"]')?.getAttribute('stroke-width'),
+ );
+ expect(outlineWidth).toBe(Math.round(strokeWidth * 1.8));
+ });
+
+ it('undoes and redoes via toolbar and shortcuts; a new stroke clears redo', () => {
+ render(<ImageLightbox src={SRC} onClose={vi.fn()} annotationEdit={{ onSave: vi.fn() }} />);
+ loadImage(lightboxImage());
+ openAnnotate();
+ drawStroke({ x: 10, y: 10 }, { x: 50, y: 50 });
+ drawStroke({ x: 60, y: 10 }, { x: 90, y: 50 });
+ expect(committedPaths('stroke')).toHaveLength(2);
+
+ fireEvent.click(screen.getByRole('button', { name: label('chat.media.annotateUndo') }));
+ expect(committedPaths('stroke')).toHaveLength(1);
+ pressKey({ key: 'z', ctrlKey: true });
+ expect(committedPaths('stroke')).toHaveLength(0);
+
+ fireEvent.click(screen.getByRole('button', { name: label('chat.media.annotateRedo') }));
+ expect(committedPaths('stroke')).toHaveLength(1);
+ pressKey({ key: 'Z', ctrlKey: true, shiftKey: true });
+ expect(committedPaths('stroke')).toHaveLength(2);
+
+ pressKey({ key: 'z', metaKey: true });
+ expect(committedPaths('stroke')).toHaveLength(1);
+ drawStroke({ x: 100, y: 10 }, { x: 150, y: 50 });
+ expect(committedPaths('stroke')).toHaveLength(2);
+ pressKey({ key: 'y', ctrlKey: true });
+ expect(committedPaths('stroke')).toHaveLength(2);
+ });
+
+ it('Esc without changes exits annotate mode immediately (no confirm)', () => {
+ const baseline: Stroke[] = [{ points: [{ x: 0.5, y: 0.5 }] }];
+ render(
+ <ConfirmDialogProvider>
+ <ImageLightbox
+ src={SRC}
+ onClose={vi.fn()}
+ annotationEdit={{ initialStrokes: baseline, onSave: vi.fn() }}
+ />
+ </ConfirmDialogProvider>,
+ );
+ loadImage(lightboxImage());
+ openAnnotate();
+ // 撤销再重做回到基线:仍视为未改动。
+ pressKey({ key: 'z', ctrlKey: true });
+ pressKey({ key: 'z', ctrlKey: true, shiftKey: true });
+
+ pressKey({ key: 'Escape' });
+
+ expect(screen.queryByRole('alertdialog')).toBeNull();
+ expect(screen.getByRole('button', { name: label('chat.media.annotate') })).toBeTruthy();
+ expect(committedPaths('stroke')).toHaveLength(1);
+ });
+
+ it('Esc with changes asks first; the Esc that closes the dialog does not discard', async () => {
+ render(
+ <ConfirmDialogProvider>
+ <ImageLightbox src={SRC} onClose={vi.fn()} annotationEdit={{ onSave: vi.fn() }} />
+ </ConfirmDialogProvider>,
+ );
+ loadImage(lightboxImage());
+ openAnnotate();
+ drawStroke({ x: 10, y: 10 }, { x: 50, y: 50 });
+
+ await act(async () => {
+ pressKey({ key: 'Escape' });
+ });
+ const dialog = await screen.findByRole('alertdialog');
+ expect(dialog).toBeTruthy();
+ expect(committedPaths('stroke')).toHaveLength(1);
+
+ // 关闭确认框的 Esc:取消放弃,笔迹与标注模式都保留。
+ await act(async () => {
+ fireEvent.keyDown(dialog, { key: 'Escape' });
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ });
+ expect(committedPaths('stroke')).toHaveLength(1);
+ expect(
+ screen.getByRole('button', { name: label('chat.media.annotateUndo') }),
+ ).toBeTruthy();
+
+ // 等确认框退场后再次放弃并确认:恢复到基线(空)并退出标注模式。
+ await act(async () => {
+ await new Promise((resolve) => setTimeout(resolve, 250));
+ });
+ await act(async () => {
+ fireEvent.click(screen.getByRole('button', { name: label('chat.media.annotateDiscard') }));
+ });
+ const confirmButton = await screen.findByRole('button', {
+ name: label('chat.media.annotateDiscardConfirmAction'),
+ });
+ await act(async () => {
+ fireEvent.click(confirmButton);
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ });
+ expect(committedPaths('stroke')).toHaveLength(0);
+ expect(screen.getByRole('button', { name: label('chat.media.annotate') })).toBeTruthy();
+ });
+
+ it('autoAnnotate: Esc without changes closes the lightbox as before', () => {
+ vi.useFakeTimers();
+ const onClose = vi.fn();
+ render(
+ <ConfirmDialogProvider>
+ <ImageLightbox src={SRC} onClose={onClose} sessionId="s1" autoAnnotate />
+ </ConfirmDialogProvider>,
+ );
+ loadImage(lightboxImage());
+ // 越过 Esc 的"菜单刚关闭"吞键窗口(假时钟下 performance.now 从 0 起步)。
+ act(() => {
+ vi.advanceTimersByTime(200);
+ });
+ pressKey({ key: 'Escape' });
+ act(() => {
+ vi.advanceTimersByTime(250);
+ });
+ expect(onClose).toHaveBeenCalledTimes(1);
+ expect(screen.queryByRole('alertdialog')).toBeNull();
+ });
+
+ it('does not draw while panning with Space held or the middle button', () => {
+ render(<ImageLightbox src={SRC} onClose={vi.fn()} annotationEdit={{ onSave: vi.fn() }} />);
+ loadImage(lightboxImage());
+ openAnnotate();
+ expect(surface().style.cursor).toBe('crosshair');
+
+ drawStroke({ x: 10, y: 10 }, { x: 50, y: 50 }, 1, 1);
+ expect(committedPaths('stroke')).toHaveLength(0);
+
+ // 放大后按住 Space:光标变为抓手,拖动不落笔。
+ pressKey({ key: '+' });
+ fireEvent.keyDown(document.activeElement ?? document.body, { key: ' ', code: 'Space' });
+ expect(surface().style.cursor).toBe('grab');
+ drawStroke({ x: 10, y: 10 }, { x: 50, y: 50 });
+ expect(committedPaths('stroke')).toHaveLength(0);
+ fireEvent.keyUp(document.activeElement ?? document.body, { key: ' ', code: 'Space' });
+ expect(surface().style.cursor).toBe('crosshair');
+
+ drawStroke({ x: 10, y: 10 }, { x: 50, y: 50 });
+ expect(committedPaths('stroke')).toHaveLength(1);
+ });
+
+ it('falls back to the burned image when the unburned source is missing', () => {
+ const onClose = vi.fn();
+ render(
+ <ImageLightbox
+ src={SRC}
+ onClose={onClose}
+ initialStrokes={[{ points: [{ x: 0.2, y: 0.2 }, { x: 0.4, y: 0.4 }] }]}
+ annotationFallbackSrc="cindy-media://blobs/burned.png"
+ />,
+ );
+ expect(lightboxImage().getAttribute('src')).toBe(SRC);
+
+ fireEvent.error(lightboxImage());
+
+ expect(lightboxImage().getAttribute('src')).toBe('cindy-media://blobs/burned.png');
+ loadImage(lightboxImage());
+ expect(committedPaths('stroke')).toHaveLength(0);
+ expect(onClose).not.toHaveBeenCalled();
+ });
+
+ it('still reports a missing image and closes when there is no burned fallback', () => {
+ vi.useFakeTimers();
+ const onClose = vi.fn();
+ render(
+ <ImageLightbox
+ src={SRC}
+ onClose={onClose}
+ initialStrokes={[{ points: [{ x: 0.2, y: 0.2 }] }]}
+ />,
+ );
+ fireEvent.error(lightboxImage());
+ act(() => {
+ vi.advanceTimersByTime(250);
+ });
+ expect(onClose).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/apps/desktop/src/renderer/components/chat/__tests__/lightboxAnnotations.test.ts b/apps/desktop/src/renderer/components/chat/__tests__/lightboxAnnotations.test.ts
index 89faa485e6..2e048aff2e 100644
--- a/apps/desktop/src/renderer/components/chat/__tests__/lightboxAnnotations.test.ts
+++ b/apps/desktop/src/renderer/components/chat/__tests__/lightboxAnnotations.test.ts
@@ -4,6 +4,7 @@ import {
ANNOTATION_STROKE_COLOR,
annotationStrokeWidth,
drawStrokesOnCanvas,
+ extendDraftSvgPath,
normalizePoint,
shouldAppendPoint,
strokeToSvgPath,
@@ -25,6 +26,10 @@ describe('normalizePoint', () => {
it('returns null for a degenerate rect', () => {
expect(normalizePoint(1, 1, { left: 0, top: 0, width: 0, height: 100 })).toBeNull();
});
+
+ it('quantizes captured coordinates to 4 decimals (shared core)', () => {
+ expect(normalizePoint(100 + 400 / 3, 50 + 200 / 7, rect)).toEqual({ x: 0.3333, y: 0.1429 });
+ });
});
describe('shouldAppendPoint', () => {
@@ -43,6 +48,11 @@ describe('annotationStrokeWidth', () => {
expect(annotationStrokeWidth(4000, 2000)).toBe(10);
expect(annotationStrokeWidth(20000, 20000)).toBe(24); // capped
});
+
+ it('thickens very long screenshots so the mark survives model downscaling', () => {
+ // 1000×8000:旧公式只有 5px,缩到 1568 长边后不足 1px;共享核心保证 ≥3px。
+ expect(annotationStrokeWidth(1000, 8000)).toBeGreaterThan(5);
+ });
});
describe('strokeToSvgPath', () => {
@@ -103,3 +113,25 @@ describe('drawStrokesOnCanvas', () => {
expect(calls.filter((c) => c === 'beginPath')).toHaveLength(0);
});
});
+
+describe('extendDraftSvgPath', () => {
+ it('builds the in-progress path incrementally, identical to strokeToSvgPath', () => {
+ const points: Array<{ x: number; y: number }> = [];
+ let cache = null as ReturnType<typeof extendDraftSvgPath> | null;
+ const samples = [
+ { x: 0.1, y: 0.1 },
+ { x: 0.2, y: 0.15 },
+ { x: 0.33333, y: 0.4 },
+ { x: 0.9, y: 0.95 },
+ { x: 1, y: 0 },
+ ];
+ for (const sample of samples) {
+ points.push(sample);
+ cache = extendDraftSvgPath(points, 640, 480, cache);
+ expect(cache.d).toBe(strokeToSvgPath({ points: [...points] }, 640, 480));
+ }
+ // 尺寸变化时全量重算。
+ cache = extendDraftSvgPath(points, 320, 240, cache);
+ expect(cache.d).toBe(strokeToSvgPath({ points: [...points] }, 320, 240));
+ });
+});
diff --git a/apps/desktop/src/renderer/components/chat/lightboxAnnotations.ts b/apps/desktop/src/renderer/components/chat/lightboxAnnotations.ts
index 73b68ab7df..883a2f04bb 100644
--- a/apps/desktop/src/renderer/components/chat/lightboxAnnotations.ts
+++ b/apps/desktop/src/renderer/components/chat/lightboxAnnotations.ts
@@ -1,144 +1,78 @@
/**
- * lightboxAnnotations — 图片 lightbox 标注模式的纯函数层。
+ * lightboxAnnotations — 图片 lightbox 标注模式的纯函数层(desktop 侧薄封装)。
*
- * 职责:笔迹坐标归一化、SVG path 构造、相对线宽计算、canvas 烧录重放。
- * 全部无副作用,React 组件(ImageLightbox)只做事件采集与状态管理。
+ * 笔迹坐标归一化、SVG path 构造、线宽计算与 canvas 两遍烧录的**唯一实现**在
+ * 跨端共享模块 `@cindy/maker-shared/image-annotation`(desktop + mobile 共用,
+ * 保证两端所见即所得且不漂移)。本文件只保留 desktop 既有导出名,让调用点
+ * (ImageLightbox / annotationBurnIn / 测试)不必随共享模块改名而改动。
*
* 坐标系约定:笔迹点一律存**归一化坐标**(0..1,相对图片自然尺寸)。
* - 采集:屏幕坐标 ÷ 图片元素的 getBoundingClientRect(rect 已含 CSS transform,
- * 因此缩放/平移状态下画的笔迹无需手动逆变换)。
+ * 因此缩放/平移状态下画的笔迹无需手动逆变换);共享实现会钳制到边缘并量化
+ * 到 4 位小数(1e-4 在万像素边长上也只有 1px)。
* - 显示:SVG overlay 的 viewBox 即图片自然尺寸,归一化点 × 自然尺寸 = path 坐标,
- * 与烧录坐标完全一致(所见即所得,线宽也一致)。
+ * 与烧录坐标完全一致(所见即所得,线宽也一致)。显示必须与烧录同一分层顺序:
+ * 先全部白描边、再全部红线(见 AnnotationStrokesSvg)。
* - 烧录:canvas 以自然尺寸绘制原图后按同一映射重放笔迹。
*/
+import { annotationStrokeToSvgPath as strokeToSvgPathImpl } from '@cindy/maker-shared/image-annotation';
import type { ImageAnnotationStroke } from '@/lib/fileTypes';
+export {
+ ANNOTATION_OUTLINE_COLOR,
+ ANNOTATION_STROKE_COLOR,
+ MIN_POINT_DISTANCE_RATIO,
+ annotationOutlineWidth,
+ annotationStrokeWidth,
+ drawAnnotationStrokes as drawStrokesOnCanvas,
+ normalizeAnnotationPoint as normalizePoint,
+ shouldAppendAnnotationPoint as shouldAppendPoint,
+ annotationStrokeToSvgPath as strokeToSvgPath,
+ type AnnotationCanvasContext as StrokeCanvasContext,
+} from '@cindy/maker-shared/image-annotation';
+
/** 一条手绘笔迹:归一化坐标点序列(0..1,相对图片自然尺寸)。 */
export type AnnotationStroke = ImageAnnotationStroke;
-/** 标注笔迹主色(红,与语义豁免色同族;白描边保证深色/红色背景上仍醒目)。 */
-export const ANNOTATION_STROKE_COLOR = '#FF3B30';
-export const ANNOTATION_OUTLINE_COLOR = 'rgba(255,255,255,0.9)';
-
-/** 白描边相对红线的宽度倍率。 */
-const OUTLINE_WIDTH_RATIO = 1.8;
-/** 采集时的最小点距(归一化):小于该距离的 move 点丢弃,抑制点数爆炸。 */
-export const MIN_POINT_DISTANCE_RATIO = 0.002;
-
-function clamp01(v: number): number {
- return Math.min(1, Math.max(0, v));
-}
-
-/**
- * 屏幕坐标 → 归一化图片坐标。`rect` 传图片元素的 getBoundingClientRect()
- * (已含 transform 后的实际显示矩形)。越界点钳制到边缘(画到图外时贴边)。
- */
-export function normalizePoint(
- clientX: number,
- clientY: number,
- rect: { left: number; top: number; width: number; height: number },
-): { x: number; y: number } | null {
- if (rect.width <= 0 || rect.height <= 0) return null;
- return {
- x: clamp01((clientX - rect.left) / rect.width),
- y: clamp01((clientY - rect.top) / rect.height),
- };
-}
-
-/**
- * 是否应把新点追加进笔迹:与上一点距离(归一化)超过阈值才收,
- * 抑制高频 mousemove 造成的点数爆炸。首点恒收。
- */
-export function shouldAppendPoint(
- stroke: AnnotationStroke,
- point: { x: number; y: number },
- minDistance = MIN_POINT_DISTANCE_RATIO,
-): boolean {
- const last = stroke.points[stroke.points.length - 1];
- if (!last) return true;
- const dx = point.x - last.x;
- const dy = point.y - last.y;
- return Math.hypot(dx, dy) >= minDistance;
-}
-
-/**
- * 烧录/显示用线宽(像素,相对图片自然尺寸):大图上足够醒目、小图上不糊脸。
- * 0.5% 短边,4px 下限、24px 上限。
- */
-export function annotationStrokeWidth(naturalWidth: number, naturalHeight: number): number {
- const base = Math.min(naturalWidth, naturalHeight) * 0.005;
- return Math.min(24, Math.max(4, Math.round(base)));
+/** 进行中笔迹 path 的增量构建缓存(见 {@link extendDraftSvgPath})。 */
+export interface DraftSvgPathCache {
+ /** 已格式化进 `d` 的点数。 */
+ count: number;
+ width: number;
+ height: number;
+ d: string;
}
/**
- * 归一化笔迹 → SVG path `d`(映射到 width×height 像素空间)。
- * 单点笔迹(点按)画一个极短线段,配合 round linecap 呈现为圆点。
+ * 进行中笔迹的 path `d` 增量构建:绘制时每帧只格式化新增的点,避免长笔迹
+ * 每帧全量重算(O(n²))。输出与 `strokeToSvgPath` 逐字相同——前两点及尺寸
+ * 变化时回退全量计算(单点笔迹的"圆点"形态与多点形态不同)。
*/
-export function strokeToSvgPath(
- stroke: AnnotationStroke,
+export function extendDraftSvgPath(
+ points: readonly AnnotationStroke['points'][number][],
width: number,
height: number,
-): string {
- const pts = stroke.points;
- if (pts.length === 0) return '';
- const fmt = (p: { x: number; y: number }) =>
- `${(p.x * width).toFixed(1)} ${(p.y * height).toFixed(1)}`;
- if (pts.length === 1) {
- const x = pts[0].x * width;
- const y = pts[0].y * height;
- return `M ${x.toFixed(1)} ${y.toFixed(1)} L ${(x + 0.1).toFixed(1)} ${y.toFixed(1)}`;
+ cache: DraftSvgPathCache | null,
+): DraftSvgPathCache {
+ const reusable =
+ cache !== null &&
+ cache.width === width &&
+ cache.height === height &&
+ cache.count >= 2 &&
+ cache.count <= points.length;
+ if (!reusable) {
+ return {
+ count: points.length,
+ width,
+ height,
+ d: strokeToSvgPathImpl({ points: [...points] }, width, height),
+ };
}
- return `M ${fmt(pts[0])} ${pts.slice(1).map((p) => `L ${fmt(p)}`).join(' ')}`;
-}
-
-/** canvas 2D context 中烧录所需的最小接口(便于单测注入 fake)。 */
-export interface StrokeCanvasContext {
- lineCap: string;
- lineJoin: string;
- strokeStyle: string | CanvasGradient | CanvasPattern;
- lineWidth: number;
- beginPath(): void;
- moveTo(x: number, y: number): void;
- lineTo(x: number, y: number): void;
- stroke(): void;
-}
-
-/**
- * 把笔迹重放到 canvas context(width×height 为图片自然尺寸)。
- * 两遍绘制:先全部白描边、再全部红线——保证笔迹交叉处不会出现
- * 白边盖住相邻红线的断裂感。
- */
-export function drawStrokesOnCanvas(
- ctx: StrokeCanvasContext,
- strokes: readonly AnnotationStroke[],
- width: number,
- height: number,
-): void {
- const strokeWidth = annotationStrokeWidth(width, height);
- ctx.lineCap = 'round';
- ctx.lineJoin = 'round';
-
- const drawPass = (color: string, lineWidth: number) => {
- ctx.strokeStyle = color;
- ctx.lineWidth = lineWidth;
- for (const stroke of strokes) {
- const pts = stroke.points;
- if (pts.length === 0) continue;
- ctx.beginPath();
- ctx.moveTo(pts[0].x * width, pts[0].y * height);
- if (pts.length === 1) {
- // 单点:极短线段 + round cap = 圆点,与 SVG 显示一致。
- ctx.lineTo(pts[0].x * width + 0.1, pts[0].y * height);
- } else {
- for (let i = 1; i < pts.length; i++) {
- ctx.lineTo(pts[i].x * width, pts[i].y * height);
- }
- }
- ctx.stroke();
- }
- };
-
- drawPass(ANNOTATION_OUTLINE_COLOR, Math.round(strokeWidth * OUTLINE_WIDTH_RATIO));
- drawPass(ANNOTATION_STROKE_COLOR, strokeWidth);
+ let d = cache.d;
+ for (let i = cache.count; i < points.length; i++) {
+ const p = points[i];
+ d += ` L ${(p.x * width).toFixed(1)} ${(p.y * height).toFixed(1)}`;
+ }
+ return { count: points.length, width, height, d };
}
diff --git a/apps/desktop/src/renderer/components/new-chat/ChatInput.tsx b/apps/desktop/src/renderer/components/new-chat/ChatInput.tsx
index 524b62f50a..a1face06d3 100644
--- a/apps/desktop/src/renderer/components/new-chat/ChatInput.tsx
+++ b/apps/desktop/src/renderer/components/new-chat/ChatInput.tsx
@@ -20,6 +20,8 @@ import { useStableTranslation as useTranslation } from '@/hooks/useStableTransla
import type { AgentInputReference } from '@cindy/maker-shared/agent-input-projection';
import { requiresFullAccessConfirmation } from '@cindy/maker-shared/permission-mode';
import { ImageLightbox } from '@/components/chat/ImageLightbox';
+import { AnnotationStrokesSvg } from '@/components/chat/AnnotationStrokesSvg';
+import { isAnnotationBurnInError } from '@/lib/annotationBurnIn';
import { ImageHoverPreview } from '@/components/chat/ImageHoverPreview';
import { formatBytes, TextLightbox } from '@/components/chat/TextLightbox';
import { AttachmentTypeThumb } from './AttachmentTypeThumb';
@@ -489,6 +491,11 @@ interface ChatInputProps {
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
/** 发送因补选目录暂缓时,由父组件在后续真正受理后完成原 composer 的清理。 */
onDeferredAccepted?: () => void;
+ /**
+ * 标注烧录失败时中止发送而非降级发原图。仅在已有任务的发送上传:此时
+ * 返回 false 会由本组件把点击时的正文、附件与笔迹原样恢复,用户可直接重试。
+ */
+ annotationBurnFailure?: 'abort';
},
) => boolean | void | Promise<boolean | void>;
/** Session ID for binding workingDir. When absent, folder picker is hidden. */
@@ -5906,6 +5913,7 @@ export function ChatInput({
...(usedGhost ? { onAccepted: markRecentPluginUsage } : {}),
...(onRemoteOptimisticFailure ? { onRemoteOptimisticFailure } : {}),
onDeferredAccepted,
+ ...(sourceSessionId ? { annotationBurnFailure: 'abort' as const } : {}),
},
);
} catch (error) {
@@ -6000,7 +6008,11 @@ export function ChatInput({
if (!saved) toast.error(t('ipcError.INTERNAL'));
} catch (error) {
log.warn('queue edit rejected:', error instanceof Error ? error.message : String(error));
- toast.error(t(mapIpcErrorToI18nKey(error, { fallback: 'ipcError.INTERNAL' })));
+ toast.error(
+ isAnnotationBurnInError(error)
+ ? t('chat.media.annotateBurnFailedNotSaved')
+ : t(mapIpcErrorToI18nKey(error, { fallback: 'ipcError.INTERNAL' })),
+ );
} finally {
setSendDispatchInFlight(false);
}
@@ -9819,6 +9831,12 @@ function ThumbnailItem({
// Lightbox state is local to each item so multiple thumbnails don't fight.
const [lightboxSrc, setLightboxSrc] = useState<string | null>(null);
const [textLightboxOpen, setTextLightboxOpen] = useState(false);
+ // 缩略图自然尺寸:标注叠加层的 viewBox 基准(onLoad 取得,换图时重置)。
+ const [thumbNaturalSize, setThumbNaturalSize] = useState<{
+ src: string;
+ width: number;
+ height: number;
+ } | null>(null);
const isDownloadOnly =
isDangerousAttachmentName(file.name) || isDangerousAttachmentName(file.path);
@@ -9865,6 +9883,7 @@ function ThumbnailItem({
// (上限 220px)。判定条件必须与下面渲染分支一致——缓存写失败、既无 url 也无
// base64 的图片同样落到文件卡分支。
const isImageThumb = file.category === 'image' && Boolean(file.url || file.base64);
+ const thumbSrc = file.url ?? `data:${file.mimeType};base64,${file.base64}`;
// 副行是「类型 · 大小」;无扩展名(Makefile 之类)或 size 缺失时按存在的部分给。
// file.size 是拖入那一刻的快照:文件在托盘期间被改写后,发出去的是新内容,卡片
// 却还报旧字节数。缩略图复核时 main 会把当前 stat 大小一并带回,这里优先用它。
@@ -9906,11 +9925,37 @@ function ThumbnailItem({
{file.category === 'image' && (file.url || file.base64) ? (
<span className="relative block h-full w-full">
<img
- src={file.url ?? `data:${file.mimeType};base64,${file.base64}`}
+ src={thumbSrc}
alt={file.name}
className="h-full w-full rounded-lg object-cover"
draggable={false}
+ onLoad={(event) => {
+ setThumbNaturalSize({
+ src: thumbSrc,
+ width: event.currentTarget.naturalWidth,
+ height: event.currentTarget.naturalHeight,
+ });
+ }}
/>
+ {file.annotationStrokes?.length &&
+ thumbNaturalSize?.src === thumbSrc &&
+ thumbNaturalSize.width > 0 &&
+ thumbNaturalSize.height > 0 ? (
+ // 缩略图是 object-cover:SVG 以 xMidYMid slice 做同样的居中裁切,笔迹与
+ // 图片严格对齐;外层圆角裁剪与图片圆角一致。
+ <span
+ className="pointer-events-none absolute inset-0 overflow-hidden rounded-lg"
+ aria-hidden
+ >
+ <AnnotationStrokesSvg
+ strokes={file.annotationStrokes}
+ naturalWidth={thumbNaturalSize.width}
+ naturalHeight={thumbNaturalSize.height}
+ preserveAspectRatio="xMidYMid slice"
+ className="block h-full w-full"
+ />
+ </span>
+ ) : null}
{file.annotationStrokes && file.annotationStrokes.length > 0 ? (
<span
className="absolute right-0.5 top-0.5 flex h-4 w-4 items-center justify-center rounded-full"
@@ -9974,8 +10019,9 @@ function ThumbnailItem({
<ImageHoverPreview
open={isHovered}
anchorRef={thumbRef}
- src={file.url ?? `data:${file.mimeType};base64,${file.base64}`}
+ src={thumbSrc}
alt={file.name}
+ annotationStrokes={file.annotationStrokes}
/>
) : null}
{isHovered &&
diff --git a/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx b/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx
index 4bd9034263..1ff6e14ea5 100644
--- a/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx
+++ b/apps/desktop/src/renderer/features/cc-agent/CCAgentSessionView.tsx
@@ -3641,6 +3641,7 @@ export function CCAgentSessionView({
slashCommandRanges?: SlashCommandRange[];
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
onDeferredAccepted?: () => void;
+ annotationBurnFailure?: 'abort';
},
) => {
if (readOnly) return false;
@@ -3865,6 +3866,10 @@ export function CCAgentSessionView({
? { onRemoteOptimisticFailure: opts.onRemoteOptimisticFailure }
: {}),
...(opts?.onDeferredAccepted ? { onDeferredAccepted: opts.onDeferredAccepted } : {}),
+ // 输入框在返回 false 时会原样恢复草稿:标注烧录失败可安全中止(见 SendMessageOpts)。
+ ...(opts?.annotationBurnFailure
+ ? { annotationBurnFailure: opts.annotationBurnFailure }
+ : {}),
};
if (deliveryMode === 'steer') {
const followStartGeneration = readSendFollowCancelGeneration(sessionId);
diff --git a/apps/desktop/src/renderer/hooks/useCCAgentChat.ts b/apps/desktop/src/renderer/hooks/useCCAgentChat.ts
index b3dc00a787..89a47f0d55 100644
--- a/apps/desktop/src/renderer/hooks/useCCAgentChat.ts
+++ b/apps/desktop/src/renderer/hooks/useCCAgentChat.ts
@@ -134,6 +134,7 @@ interface UseCCAgentChatReturn {
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
+ annotationBurnFailure?: 'abort';
},
) => Promise<boolean>;
compactSession: (
@@ -159,6 +160,7 @@ interface UseCCAgentChatReturn {
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
+ annotationBurnFailure?: 'abort';
},
) => Promise<boolean>;
steerQueuedMessage: (clientId: string) => Promise<boolean>;
@@ -422,6 +424,7 @@ export function useCCAgentChat(
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
+ annotationBurnFailure?: 'abort';
},
): Promise<boolean> => {
if (!sessionId) return Promise.resolve(false);
@@ -478,6 +481,7 @@ export function useCCAgentChat(
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
+ annotationBurnFailure?: 'abort';
},
) => {
if (!sessionId) return Promise.resolve(false);
diff --git a/apps/desktop/src/renderer/i18n/locales/en/common.json b/apps/desktop/src/renderer/i18n/locales/en/common.json
index ced085aa0b..e784f0f5f6 100644
--- a/apps/desktop/src/renderer/i18n/locales/en/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/en/common.json
@@ -9240,6 +9240,14 @@
"annotateDiscard": "Discard annotations",
"annotateSave": "Save annotations",
"annotateSaveFailed": "Failed to save annotations",
+ "annotateRedo": "Redo",
+ "annotateDiscardConfirmTitle": "Discard annotation changes?",
+ "annotateDiscardConfirmDescription": "Strokes you drew or undid since opening this image won't be kept.",
+ "annotateDiscardConfirmAction": "Discard",
+ "annotateDiscardConfirmCancel": "Keep annotating",
+ "annotateBurnFailedNotSent": "Couldn't apply annotations to the image, so the message wasn't sent. Try again, or remove the annotations and send.",
+ "annotateBurnFailedNotSaved": "Couldn't apply annotations to the image, so your changes weren't saved. Try again, or remove the annotations and save.",
+ "annotateBurnFailedSentOriginal": "Couldn't apply annotations to the image. The original image was sent without them.",
"prevImage": "Previous image",
"nextImage": "Next image",
"openModel": "Open model",
diff --git a/apps/desktop/src/renderer/i18n/locales/ja/common.json b/apps/desktop/src/renderer/i18n/locales/ja/common.json
index beaebf0f79..3c4e98256c 100644
--- a/apps/desktop/src/renderer/i18n/locales/ja/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/ja/common.json
@@ -9234,6 +9234,14 @@
"annotateDiscard": "注釈を破棄",
"annotateSave": "注釈を保存",
"annotateSaveFailed": "注釈の保存に失敗しました",
+ "annotateRedo": "やり直す",
+ "annotateDiscardConfirmTitle": "注釈の変更を破棄しますか?",
+ "annotateDiscardConfirmDescription": "この画像を開いてから描いた線や取り消した線は保持されません。",
+ "annotateDiscardConfirmAction": "破棄",
+ "annotateDiscardConfirmCancel": "注釈を続ける",
+ "annotateBurnFailedNotSent": "画像への注釈の合成に失敗したため、メッセージは送信されませんでした。再試行するか、注釈を削除してから送信してください。",
+ "annotateBurnFailedNotSaved": "画像への注釈の合成に失敗したため、変更は保存されませんでした。再試行するか、注釈を削除してから保存してください。",
+ "annotateBurnFailedSentOriginal": "画像への注釈の合成に失敗したため、注釈なしの元画像を送信しました。",
"prevImage": "前の画像",
"nextImage": "次の画像",
"openModel": "モデルを開く",
diff --git a/apps/desktop/src/renderer/i18n/locales/ko/common.json b/apps/desktop/src/renderer/i18n/locales/ko/common.json
index 9e54cb6e05..9f3b8322e4 100644
--- a/apps/desktop/src/renderer/i18n/locales/ko/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/ko/common.json
@@ -9234,6 +9234,14 @@
"annotateDiscard": "주석 취소",
"annotateSave": "주석 저장",
"annotateSaveFailed": "주석 저장에 실패했습니다",
+ "annotateRedo": "다시 실행",
+ "annotateDiscardConfirmTitle": "주석 변경 사항을 버릴까요?",
+ "annotateDiscardConfirmDescription": "이 이미지를 연 후 그리거나 실행 취소한 획은 유지되지 않습니다.",
+ "annotateDiscardConfirmAction": "버리기",
+ "annotateDiscardConfirmCancel": "계속 주석 달기",
+ "annotateBurnFailedNotSent": "이미지에 주석을 합성하지 못해 메시지를 보내지 않았습니다. 다시 시도하거나 주석을 제거한 후 보내세요.",
+ "annotateBurnFailedNotSaved": "이미지에 주석을 합성하지 못해 변경 사항을 저장하지 않았습니다. 다시 시도하거나 주석을 제거한 후 저장하세요.",
+ "annotateBurnFailedSentOriginal": "이미지에 주석을 합성하지 못해 주석 없는 원본 이미지를 보냈습니다.",
"prevImage": "이전 이미지",
"nextImage": "다음 이미지",
"openModel": "모델 열기",
diff --git a/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json b/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json
index 0b803bdc50..2b9073c00c 100644
--- a/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/zh-CN/common.json
@@ -9234,6 +9234,14 @@
"annotateDiscard": "放弃标注",
"annotateSave": "保存标注",
"annotateSaveFailed": "保存标注失败",
+ "annotateRedo": "重做",
+ "annotateDiscardConfirmTitle": "放弃本次标注?",
+ "annotateDiscardConfirmDescription": "打开后新画或撤销的笔迹不会保留。",
+ "annotateDiscardConfirmAction": "放弃",
+ "annotateDiscardConfirmCancel": "继续标注",
+ "annotateBurnFailedNotSent": "标注图片合成失败,消息未发送。可重试,或移除标注后发送。",
+ "annotateBurnFailedNotSaved": "标注图片合成失败,修改未保存。可重试,或移除标注后保存。",
+ "annotateBurnFailedSentOriginal": "标注图片合成失败,已发送不含标注的原图。",
"prevImage": "上一张",
"nextImage": "下一张",
"openModel": "打开模型",
diff --git a/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json b/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json
index 4d9b21535e..0e90bf6884 100644
--- a/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json
+++ b/apps/desktop/src/renderer/i18n/locales/zh-TW/common.json
@@ -9233,6 +9233,14 @@
"annotateDiscard": "放棄標註",
"annotateSave": "儲存標註",
"annotateSaveFailed": "儲存標註失敗",
+ "annotateRedo": "重做",
+ "annotateDiscardConfirmTitle": "放棄本次標註?",
+ "annotateDiscardConfirmDescription": "開啟後新畫或撤銷的筆跡不會保留。",
+ "annotateDiscardConfirmAction": "放棄",
+ "annotateDiscardConfirmCancel": "繼續標註",
+ "annotateBurnFailedNotSent": "標註圖片合成失敗,訊息未傳送。可重試,或移除標註後傳送。",
+ "annotateBurnFailedNotSaved": "標註圖片合成失敗,修改未儲存。可重試,或移除標註後儲存。",
+ "annotateBurnFailedSentOriginal": "標註圖片合成失敗,已傳送不含標註的原圖。",
"prevImage": "上一張",
"nextImage": "下一張",
"openModel": "開啟模型",
diff --git a/apps/desktop/src/renderer/lib/__tests__/annotationBurnInMaterialize.test.ts b/apps/desktop/src/renderer/lib/__tests__/annotationBurnInMaterialize.test.ts
new file mode 100644
index 0000000000..b8f26a53fe
--- /dev/null
+++ b/apps/desktop/src/renderer/lib/__tests__/annotationBurnInMaterialize.test.ts
@@ -0,0 +1,245 @@
+// @vitest-environment jsdom
+/**
+ * materializeAnnotatedAttachmentsForSend — 发送前标注物化的直接单测。
+ *
+ * 覆盖:烧录成功的产物形态(原图引用、笔迹、标注区域、文件名)、幂等(已烧录
+ * 附件原样返回)、remote 剥离元数据、烧录失败的两种处理(降级 + 通知 / 中止抛错)、
+ * 共享引用附件的私有化。canvas 与图片解码在 jsdom 中不可用,这里只替换这两个
+ * 浏览器原语,materialize 本身的逻辑全部真实执行。
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import {
+ AnnotationBurnInError,
+ annotatedFileName,
+ isAnnotationBurnInError,
+ materializeAnnotatedAttachmentsForSend,
+} from '@/lib/annotationBurnIn';
+import type { AttachedFile } from '@/lib/fileTypes';
+
+const strokes = [{ points: [{ x: 0.1, y: 0.2 }, { x: 0.3, y: 0.4 }] }];
+
+function imageFile(overrides: Partial<AttachedFile> = {}): AttachedFile {
+ return {
+ id: 'img-1',
+ name: 'shot.png',
+ originalName: 'shot.png',
+ path: '/Users/sam/shot.png',
+ ext: '.png',
+ size: 10,
+ category: 'image',
+ mimeType: 'image/png',
+ url: 'cindy-media://blobs/source.png',
+ ...overrides,
+ };
+}
+
+const api = {
+ readCachedImageAsBase64: vi.fn(),
+ cacheImageFromBuffer: vi.fn(),
+ cacheMediaForSession: vi.fn(),
+};
+
+beforeEach(() => {
+ api.readCachedImageAsBase64.mockResolvedValue({ base64: 'AAAA', mimeType: 'image/png' });
+ api.cacheImageFromBuffer.mockResolvedValue({
+ url: 'cindy-media://blobs/burned.png',
+ filename: 'burned.png',
+ });
+ api.cacheMediaForSession.mockResolvedValue({
+ url: 'cindy-media://blobs/private.png',
+ name: 'private.png',
+ ext: '.png',
+ mimeType: 'image/png',
+ size: 99,
+ });
+ (window as unknown as { electronAPI: typeof api }).electronAPI = api;
+ // jsdom 未实现 HTMLImageElement.decode:按浏览器语义补一个立即 resolve 的版本。
+ Object.defineProperty(HTMLImageElement.prototype, 'decode', {
+ configurable: true,
+ value: () => Promise.resolve(),
+ });
+ vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockImplementation(
+ () =>
+ ({
+ drawImage: vi.fn(),
+ beginPath: vi.fn(),
+ moveTo: vi.fn(),
+ lineTo: vi.fn(),
+ stroke: vi.fn(),
+ }) as unknown as CanvasRenderingContext2D,
+ );
+ vi.spyOn(HTMLCanvasElement.prototype, 'toBlob').mockImplementation(function toBlob(
+ callback: BlobCallback,
+ type?: string,
+ ) {
+ callback(new Blob(['burned-bytes'], { type: type ?? 'image/png' }));
+ });
+});
+
+afterEach(() => {
+ vi.restoreAllMocks();
+ vi.clearAllMocks();
+ delete (HTMLImageElement.prototype as { decode?: unknown }).decode;
+});
+
+describe('materializeAnnotatedAttachmentsForSend', () => {
+ it('returns a plain copy without any IPC when nothing needs burning', async () => {
+ const files = [imageFile()];
+ const result = await materializeAnnotatedAttachmentsForSend(files, 's1');
+ expect(result).toEqual(files);
+ expect(result).not.toBe(files);
+ expect(api.readCachedImageAsBase64).not.toHaveBeenCalled();
+ await expect(materializeAnnotatedAttachmentsForSend(undefined, 's1')).resolves.toBeUndefined();
+ });
+
+ it('burns strokes into a new cached bitmap and keeps the editable source + regions', async () => {
+ const [result] =
+ (await materializeAnnotatedAttachmentsForSend(
+ [imageFile({ annotationStrokes: strokes })],
+ 's1',
+ )) ?? [];
+
+ expect(api.readCachedImageAsBase64).toHaveBeenCalledWith({
+ url: 'cindy-media://blobs/source.png',
+ });
+ expect(api.cacheImageFromBuffer).toHaveBeenCalledTimes(1);
+ expect(result).toMatchObject({
+ url: 'cindy-media://blobs/burned.png',
+ name: 'shot-annotated.png',
+ originalName: 'shot-annotated.png',
+ ext: '.png',
+ mimeType: 'image/png',
+ annotated: true,
+ annotationSourceUrl: 'cindy-media://blobs/source.png',
+ annotationStrokes: strokes,
+ annotationRegions: [{ x0: 0.1, y0: 0.2, x1: 0.3, y1: 0.4 }],
+ });
+ // 原图磁盘路径不变(队列 / 历史仍以 url 为准)。
+ expect(result.path).toBe('/Users/sam/shot.png');
+ });
+
+ it('is idempotent for already-burned attachments (auth-retry resend)', async () => {
+ const burned = imageFile({
+ url: 'cindy-media://blobs/burned.png',
+ annotated: true,
+ annotationSourceUrl: 'cindy-media://blobs/source.png',
+ annotationStrokes: strokes,
+ });
+ const [result] = (await materializeAnnotatedAttachmentsForSend([burned], 's1')) ?? [];
+ expect(result).toBe(burned);
+ expect(api.readCachedImageAsBase64).not.toHaveBeenCalled();
+ expect(api.cacheImageFromBuffer).not.toHaveBeenCalled();
+ });
+
+ it('strips controller-local annotation meta for remote sessions but keeps regions', async () => {
+ const [result] =
+ (await materializeAnnotatedAttachmentsForSend(
+ [imageFile({ annotationStrokes: strokes })],
+ 's1',
+ { stripAnnotationMeta: true },
+ )) ?? [];
+ expect(result.annotated).toBe(true);
+ expect(result.annotationSourceUrl).toBeUndefined();
+ expect(result.annotationStrokes).toBeUndefined();
+ expect(result.annotationRegions).toEqual([{ x0: 0.1, y0: 0.2, x1: 0.3, y1: 0.4 }]);
+ });
+
+ it('burns base64 draft attachments in place', async () => {
+ const [result] =
+ (await materializeAnnotatedAttachmentsForSend(
+ [imageFile({ url: undefined, base64: 'QUJD', annotationStrokes: strokes })],
+ 's1',
+ )) ?? [];
+ expect(api.cacheImageFromBuffer).not.toHaveBeenCalled();
+ expect(result.annotated).toBe(true);
+ expect(result.base64).toBeTruthy();
+ expect(result.base64).not.toBe('QUJD');
+ expect(result.annotationRegions).toHaveLength(1);
+ });
+
+ it('falls back to the original image and notifies once when burning fails (default)', async () => {
+ api.readCachedImageAsBase64.mockRejectedValue(new Error('gone'));
+ const onFallback = vi.fn();
+ const files = [
+ imageFile({ id: 'a', annotationStrokes: strokes }),
+ imageFile({ id: 'b', annotationStrokes: strokes }),
+ ];
+ const result = (await materializeAnnotatedAttachmentsForSend(files, 's1', { onFallback })) ?? [];
+
+ expect(onFallback).toHaveBeenCalledTimes(1);
+ expect(onFallback).toHaveBeenCalledWith(2);
+ for (const file of result) {
+ expect(file.url).toBe('cindy-media://blobs/source.png');
+ expect(file.annotated).toBeUndefined();
+ expect(file.annotationStrokes).toBeUndefined();
+ expect(file.annotationSourceUrl).toBeUndefined();
+ expect(file.annotationRegions).toBeUndefined();
+ }
+ });
+
+ it('aborts with AnnotationBurnInError in abort mode and never notifies a fallback', async () => {
+ api.readCachedImageAsBase64.mockRejectedValue(new Error('gone'));
+ const onFallback = vi.fn();
+ const promise = materializeAnnotatedAttachmentsForSend(
+ [imageFile({ annotationStrokes: strokes })],
+ 's1',
+ { burnFailure: 'abort', onFallback },
+ );
+ await expect(promise).rejects.toBeInstanceOf(AnnotationBurnInError);
+ await promise.catch((error) => expect(isAnnotationBurnInError(error)).toBe(true));
+ expect(onFallback).not.toHaveBeenCalled();
+ expect(api.cacheImageFromBuffer).not.toHaveBeenCalled();
+ });
+
+ it('privatizes a shared history image whose strokes were all removed', async () => {
+ const [result] =
+ (await materializeAnnotatedAttachmentsForSend(
+ [imageFile({ url: 'cindy-media://blobs/history.png', cacheUrlShared: true })],
+ 's1',
+ )) ?? [];
+ expect(api.cacheMediaForSession).toHaveBeenCalledWith({
+ url: 'cindy-media://blobs/history.png',
+ sessionId: 's1',
+ });
+ expect(result).toMatchObject({ url: 'cindy-media://blobs/private.png', size: 99 });
+ expect(result.cacheUrlShared).toBeUndefined();
+ });
+
+ it('privatizes the unannotated fallback of a shared history image', async () => {
+ api.readCachedImageAsBase64.mockRejectedValue(new Error('decode failed'));
+ const onFallback = vi.fn();
+ const [result] =
+ (await materializeAnnotatedAttachmentsForSend(
+ [
+ imageFile({
+ url: 'cindy-media://blobs/history.png',
+ cacheUrlShared: true,
+ annotationStrokes: strokes,
+ }),
+ ],
+ 's1',
+ { onFallback },
+ )) ?? [];
+ expect(onFallback).toHaveBeenCalledWith(1);
+ expect(result).toMatchObject({ url: 'cindy-media://blobs/private.png' });
+ expect(result.cacheUrlShared).toBeUndefined();
+ expect(result.annotated).toBeUndefined();
+ });
+});
+
+describe('annotatedFileName', () => {
+ it('keeps the user base name and never stacks the suffix', () => {
+ expect(annotatedFileName('截图.png', '.png', 1)).toBe('截图-annotated.png');
+ expect(annotatedFileName('photo.jpeg', '.jpg', 1)).toBe('photo-annotated.jpg');
+ expect(annotatedFileName('photo-annotated.png', '.png', 1)).toBe('photo-annotated.png');
+ expect(annotatedFileName('C:\\\\shots\\\\a.webp', '.png', 1)).toBe('a-annotated.png');
+ });
+
+ it('falls back to the implementation name for synthetic or missing names', () => {
+ expect(annotatedFileName('annotated-1753.png', '.png', 42)).toBe('annotated-42.png');
+ expect(annotatedFileName(undefined, '.png', 42)).toBe('annotated-42.png');
+ expect(annotatedFileName('.png', '.jpg', 42)).toBe('annotated-42.jpg');
+ });
+});
diff --git a/apps/desktop/src/renderer/lib/__tests__/annotationRestore.test.ts b/apps/desktop/src/renderer/lib/__tests__/annotationRestore.test.ts
new file mode 100644
index 0000000000..777686713a
--- /dev/null
+++ b/apps/desktop/src/renderer/lib/__tests__/annotationRestore.test.ts
@@ -0,0 +1,108 @@
+/**
+ * annotationRestore —「烧录附件 → 可编辑态」唯一实现的行为锁定。
+ *
+ * 队列编辑草稿、队列编辑"是否未改"比较、回退草稿三处共用本模块;这里锁住配对
+ * 条件(防止把无关原图换进来)与还原形态(原图 url / 扩展名 / MIME、去 annotated、
+ * 笔迹深拷贝)。
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import {
+ annotationStrokesEqual,
+ cloneAnnotationStrokes,
+ queuedAnnotationEditMeta,
+ toEditableAnnotatedAttachment,
+} from '@/lib/annotationRestore';
+import type { AttachedFile } from '@/lib/fileTypes';
+
+const strokes = [{ points: [{ x: 0.1, y: 0.2 }, { x: 0.3, y: 0.4 }] }];
+
+const burned: AttachedFile = {
+ id: 'img-1',
+ name: 'shot-annotated.png',
+ originalName: 'shot-annotated.png',
+ path: '/Users/sam/shot.jpg',
+ ext: '.png',
+ size: 42,
+ category: 'image',
+ mimeType: 'image/png',
+ url: 'cindy-media://blobs/burned.png',
+ annotated: true,
+ annotationRegions: [{ x0: 0.1, y0: 0.2, x1: 0.3, y1: 0.4 }],
+};
+
+const retry: AttachedFile = {
+ ...burned,
+ annotationSourceUrl: 'cindy-media://blobs/source.jpg',
+ annotationStrokes: strokes,
+};
+
+describe('queuedAnnotationEditMeta', () => {
+ it('pairs a burned queue file with the retry file that describes it', () => {
+ expect(queuedAnnotationEditMeta(burned, retry)).toEqual({
+ sourceUrl: 'cindy-media://blobs/source.jpg',
+ strokes,
+ });
+ });
+
+ it('refuses mismatched or incomplete pairs', () => {
+ expect(queuedAnnotationEditMeta(burned, undefined)).toBeNull();
+ expect(queuedAnnotationEditMeta({ ...burned, annotated: undefined }, retry)).toBeNull();
+ expect(queuedAnnotationEditMeta(burned, { ...retry, annotated: undefined })).toBeNull();
+ expect(queuedAnnotationEditMeta(burned, { ...retry, url: 'cindy-media://blobs/x.png' })).toBeNull();
+ expect(queuedAnnotationEditMeta(burned, { ...retry, path: '/other.png' })).toBeNull();
+ expect(
+ queuedAnnotationEditMeta(burned, { ...retry, annotationSourceUrl: undefined }),
+ ).toBeNull();
+ expect(queuedAnnotationEditMeta(burned, { ...retry, annotationStrokes: [] })).toBeNull();
+ });
+});
+
+describe('toEditableAnnotatedAttachment', () => {
+ it('points the attachment back at the unburned source with deep-copied strokes', () => {
+ const editable = toEditableAnnotatedAttachment(burned, {
+ sourceUrl: 'cindy-media://blobs/source.jpg',
+ strokes,
+ });
+
+ expect(editable).toMatchObject({
+ id: 'img-1',
+ name: 'shot-annotated.png',
+ path: 'cindy-media://blobs/source.jpg',
+ url: 'cindy-media://blobs/source.jpg',
+ ext: '.jpg',
+ mimeType: 'image/jpeg',
+ annotationStrokes: strokes,
+ });
+ expect(editable).not.toHaveProperty('annotated');
+ expect(editable).not.toHaveProperty('annotationRegions');
+ expect(editable).not.toHaveProperty('annotationSourceUrl');
+ expect(editable.annotationStrokes?.[0]).not.toBe(strokes[0]);
+ // 输入不被修改。
+ expect(burned.annotated).toBe(true);
+ });
+
+ it('keeps the attachment extension when the source url has none', () => {
+ const editable = toEditableAnnotatedAttachment(burned, {
+ sourceUrl: 'cindy-media://blobs/noext',
+ strokes,
+ });
+ expect(editable.ext).toBe('.png');
+ expect(editable.mimeType).toBe('image/png');
+ });
+});
+
+describe('stroke helpers', () => {
+ it('clones only coordinates and compares point by point', () => {
+ const cloned = cloneAnnotationStrokes(strokes);
+ expect(cloned).toEqual(strokes);
+ expect(annotationStrokesEqual(cloned, strokes)).toBe(true);
+ expect(annotationStrokesEqual(undefined, [])).toBe(true);
+ expect(annotationStrokesEqual(strokes, [])).toBe(false);
+ expect(
+ annotationStrokesEqual(strokes, [{ points: [{ x: 0.1, y: 0.2 }, { x: 0.3, y: 0.41 }] }]),
+ ).toBe(false);
+ expect(annotationStrokesEqual(strokes, [{ points: [{ x: 0.1, y: 0.2 }] }])).toBe(false);
+ });
+});
diff --git a/apps/desktop/src/renderer/lib/annotationBurnIn.ts b/apps/desktop/src/renderer/lib/annotationBurnIn.ts
index b2d5c31d85..f494c1346d 100644
--- a/apps/desktop/src/renderer/lib/annotationBurnIn.ts
+++ b/apps/desktop/src/renderer/lib/annotationBurnIn.ts
@@ -9,6 +9,7 @@
* 打进 canvas 后 toBlob 被跨源 taint 拦截)。
*/
+import { summarizeAnnotationRegions } from '@cindy/maker-shared/image-annotation';
import {
drawStrokesOnCanvas,
type AnnotationStroke,
@@ -18,6 +19,68 @@ import { createLogger } from '@/lib/logger';
const log = createLogger('AnnotationBurnIn');
+/**
+ * 发送前烧录失败(`burnFailure: 'abort'` 时抛出)。调用方据此中止本次发送 /
+ * 保存并保留草稿里的矢量笔迹,让用户重试,而不是悄悄发出一张没有圈的图。
+ */
+export class AnnotationBurnInError extends Error {
+ constructor(
+ readonly fileName: string,
+ readonly cause?: unknown,
+ ) {
+ super(`annotation burn-in failed: ${fileName}`);
+ this.name = 'AnnotationBurnInError';
+ }
+}
+
+export function isAnnotationBurnInError(error: unknown): error is AnnotationBurnInError {
+ return (
+ error instanceof AnnotationBurnInError ||
+ (error instanceof Error && error.name === 'AnnotationBurnInError')
+ );
+}
+
+/** {@link materializeAnnotatedAttachmentsForSend} 的选项。 */
+export interface MaterializeAnnotatedOptions {
+ /**
+ * remote(device-link/SSH)会话发送时剥离 annotationSourceUrl/annotationStrokes——
+ * 它们指向控制端本地缓存,被控端无法解析。annotated 与 annotationRegions 保留。
+ */
+ stripAnnotationMeta?: boolean;
+ /**
+ * 烧录失败时的处理:
+ * - `'fallback'`(默认,历史行为):降级发原图并剥离全部标注字段,随后调用一次
+ * `onFallback` 让调用方给出用户可见提示。用于无法安全中止的路径(例如
+ * device-link 乐观发送:气泡已上屏、输入框已清空)。
+ * - `'abort'`:整体抛 {@link AnnotationBurnInError},调用方中止发送并保留草稿。
+ */
+ burnFailure?: 'fallback' | 'abort';
+ /** fallback 模式下本次有图片降级时回调一次(参数为降级张数)。 */
+ onFallback?: (failedCount: number) => void;
+}
+
+/**
+ * 烧录产物文件名:保留用户原图的基础名(`截图.png` → `截图-annotated.png`),
+ * 自动标题 / 缺图占位等处能认出是哪张图。重复烧录(队列编辑、回退重发)先剥掉
+ * 已有的 `-annotated` 后缀,不会越叠越长;本来就是实现名(`annotated-<ts>`)或
+ * 取不到基础名时沿用 `annotated-<ts>`。文件名只作展示用途——烧录字节按内容
+ * 寻址入库,文件名不参与存储路径、去重与缓存解析。
+ */
+export function annotatedFileName(
+ originalName: string | undefined,
+ ext: string,
+ nowMs: number,
+): string {
+ const name = (originalName ?? '').split(/[\\/]/).pop() ?? '';
+ const dot = name.lastIndexOf('.');
+ // 点号开头(只有扩展名)视为无基础名。
+ const stem = (dot > 0 ? name.slice(0, dot) : dot === 0 ? '' : name)
+ .replace(/-annotated$/i, '')
+ .trim();
+ if (!stem || /^annotated-\d+$/i.test(stem)) return `annotated-${nowMs}${ext}`;
+ return `${stem}-annotated${ext}`;
+}
+
/**
* 剥掉 MIME 参数(`image/svg+xml;charset=utf-8` → `image/svg+xml`)。
* 字节层所有出口统一输出**无参数** MIME:下游会拿它重构 `data:<mime>;base64,`
@@ -169,8 +232,12 @@ export function needsAnnotationMaterialize(files?: readonly AttachedFile[]): boo
* - 缓存附件(有 url):烧录图写入会话缓存成为发送 url;物化前的 url(原图)
* 记入 annotationSourceUrl、笔迹保留——两者随消息持久化,历史图可再编辑。
* - base64 草稿附件:烧录结果直接替换 base64 随消息走。
- * - 烧录失败:降级发原图并剥离全部标注字段(不能带着 annotated 标发一张
- * 没有圈的图误导模型),只记日志不阻塞发送。
+ * - 两种形态都附带 `annotationRegions`(笔迹归纳出的标注区域,归一化坐标),
+ * 供 buildMakerUserMessage 给模型说明每张图圈在哪;remote 剥离元数据时保留。
+ * - 烧录失败:按 `opts.burnFailure` 处理——`'abort'` 抛 AnnotationBurnInError
+ * (调用方中止并保留草稿);默认 `'fallback'` 降级发原图并剥离全部标注字段
+ * (不能带着 annotated 标发一张没有圈的图误导模型),并经 `onFallback` 通知
+ * 调用方提示用户。
* 幂等:`f.annotated` 的附件已是烧录产物,原样返回——远程 auth-retry 会把
* 已物化的 retryFiles 重新送进本函数,若再烧一遍会把笔迹叠画到烧录图上、
* annotationSourceUrl 也会被错误重指到烧录副本(review P2)。
@@ -183,12 +250,14 @@ export function needsAnnotationMaterialize(files?: readonly AttachedFile[]): boo
export async function materializeAnnotatedAttachmentsForSend(
files: readonly AttachedFile[] | undefined,
sessionId: string,
- opts?: { stripAnnotationMeta?: boolean },
+ opts?: MaterializeAnnotatedOptions,
): Promise<AttachedFile[] | undefined> {
if (!needsAnnotationMaterialize(files)) {
return files ? [...files] : undefined;
}
- return Promise.all(
+ const abortOnFailure = opts?.burnFailure === 'abort';
+ let fallbackCount = 0;
+ const materialized = await Promise.all(
files!.map(async (f): Promise<AttachedFile> => {
const strokes = f.annotationStrokes;
if (f.category !== 'image' || f.annotated) return f;
@@ -212,7 +281,9 @@ export async function materializeAnnotatedAttachmentsForSend(
if (!source) return f;
const { blob, mimeType } = await burnInAnnotations(source, strokes);
const ext = mimeType === 'image/jpeg' ? '.jpg' : '.png';
- const name = `annotated-${Date.now()}${ext}`;
+ const name = annotatedFileName(f.originalName ?? f.name, ext, Date.now());
+ const regions = summarizeAnnotationRegions(strokes);
+ const annotationRegions = regions.length > 0 ? { annotationRegions: regions } : {};
if (f.url) {
const cached = await window.electronAPI.cacheImageFromBuffer({
sessionId,
@@ -229,6 +300,7 @@ export async function materializeAnnotatedAttachmentsForSend(
mimeType,
size: blob.size,
annotated: true,
+ ...annotationRegions,
...(opts?.stripAnnotationMeta
? { annotationSourceUrl: undefined, annotationStrokes: undefined }
: { annotationSourceUrl: f.url }),
@@ -243,17 +315,27 @@ export async function materializeAnnotatedAttachmentsForSend(
mimeType,
size: blob.size,
annotated: true,
+ ...annotationRegions,
};
} catch (err) {
+ if (abortOnFailure) {
+ log.warn('burn-in failed, aborting send', {
+ name: f.name,
+ error: err instanceof Error ? err.message : String(err),
+ });
+ throw new AnnotationBurnInError(f.name, err);
+ }
log.warn('burn-in failed, sending original image without annotations', {
name: f.name,
error: err instanceof Error ? err.message : String(err),
});
+ fallbackCount += 1;
const stripped: AttachedFile = {
...f,
annotated: undefined,
annotationSourceUrl: undefined,
annotationStrokes: undefined,
+ annotationRegions: undefined,
};
// 共享引用附件烧录失败:降级发的是**历史消息的原图文件**,与上方撤光
// 笔迹分支同一悬空引用问题——同样先私有化,失败再原样降级(review P2)。
@@ -264,6 +346,16 @@ export async function materializeAnnotatedAttachmentsForSend(
}
}),
);
+ if (fallbackCount > 0) {
+ try {
+ opts?.onFallback?.(fallbackCount);
+ } catch (notifyError) {
+ log.warn('burn-in fallback notify failed', {
+ error: notifyError instanceof Error ? notifyError.message : String(notifyError),
+ });
+ }
+ }
+ return materialized;
}
/**
diff --git a/apps/desktop/src/renderer/lib/annotationRestore.ts b/apps/desktop/src/renderer/lib/annotationRestore.ts
new file mode 100644
index 0000000000..b046f66e0f
--- /dev/null
+++ b/apps/desktop/src/renderer/lib/annotationRestore.ts
@@ -0,0 +1,110 @@
+/**
+ * annotationRestore — 「烧录标注附件 → 可编辑态(原图 + 矢量笔迹)」的唯一实现。
+ *
+ * 三处入口共用本模块,避免还原规则各写一份后漂移:
+ * - 队列消息进入输入框编辑(queueComposerEdit.queueMessageToComposerEditDraft);
+ * - 队列编辑保存时判断附件是否"实际未改"(makerChatStore.queueEditFilesRemainUnchanged);
+ * - 回退 / 编辑重发把历史消息附件恢复进草稿(rewindDraftAttachments)。
+ *
+ * 可编辑态约定:`url` / `path` 指向未烧录原图,扩展名与 MIME 随原图,`annotated`
+ * 去掉(发送时会按笔迹重新烧录),笔迹深拷贝(草稿与历史 / 队列数据互不共享可变
+ * 对象)。所有权标记(cacheUrlShared / stagedPathShared)与来源相关,由调用方加。
+ */
+
+import {
+ extractExt,
+ getMimeType,
+ type AttachedFile,
+ type ImageAnnotationStroke,
+} from '@/lib/fileTypes';
+
+/** 可还原的标注元数据:未烧录原图 url + 持久化笔迹。 */
+export interface AnnotationEditMeta {
+ sourceUrl: string;
+ strokes: readonly ImageAnnotationStroke[];
+}
+
+/** 深拷贝笔迹,只保留坐标字段。 */
+export function cloneAnnotationStrokes(
+ strokes: readonly ImageAnnotationStroke[],
+): ImageAnnotationStroke[] {
+ return strokes.map((stroke) => ({
+ points: stroke.points.map((point) => ({ x: point.x, y: point.y })),
+ }));
+}
+
+/** 两组笔迹坐标是否逐点相同(用于判断用户是否改过标注)。 */
+export function annotationStrokesEqual(
+ left: readonly ImageAnnotationStroke[] | undefined,
+ right: readonly ImageAnnotationStroke[] | undefined,
+): boolean {
+ const a = left ?? [];
+ const b = right ?? [];
+ if (a.length !== b.length) return false;
+ for (let i = 0; i < a.length; i++) {
+ const pa = a[i].points;
+ const pb = b[i].points;
+ if (pa.length !== pb.length) return false;
+ for (let j = 0; j < pa.length; j++) {
+ if (pa[j].x !== pb[j].x || pa[j].y !== pb[j].y) return false;
+ }
+ }
+ return true;
+}
+
+/**
+ * 队列附件(wire 形态,不含编辑元数据)与同 id 的 retryFile(完整 AttachedFile,
+ * 保留烧录前的原图与笔迹)配对,得出可还原的标注元数据。
+ *
+ * 必须两者都标 annotated、path 与 url 完全一致(retryFile 描述的正是这张烧录图),
+ * 且原图与笔迹齐备;任一不满足返回 null——附件按普通图片处理,绝不把一张
+ * 无关的原图换进来。
+ */
+export function queuedAnnotationEditMeta(
+ file: { annotated?: boolean; path: string; url?: string },
+ retryFile: AttachedFile | undefined,
+): AnnotationEditMeta | null {
+ if (
+ file.annotated !== true ||
+ retryFile?.annotated !== true ||
+ retryFile.path !== file.path ||
+ retryFile.url !== file.url ||
+ !retryFile.annotationSourceUrl ||
+ !retryFile.annotationStrokes?.length
+ ) {
+ return null;
+ }
+ return { sourceUrl: retryFile.annotationSourceUrl, strokes: retryFile.annotationStrokes };
+}
+
+/**
+ * 烧录附件 → 可编辑附件。原图扩展名推不出时沿用附件原扩展名。烧录期派生
+ * 字段(annotated / annotationSourceUrl / annotationRegions)一并去掉:编辑后
+ * 以笔迹为唯一事实源,发送时重新物化。
+ */
+export function toEditableAnnotatedAttachment<
+ T extends {
+ path: string;
+ ext: string;
+ mimeType: string;
+ url?: string;
+ annotated?: boolean;
+ annotationSourceUrl?: string;
+ annotationStrokes?: ImageAnnotationStroke[];
+ annotationRegions?: unknown;
+ },
+>(file: T, meta: AnnotationEditMeta): T {
+ const sourceExt = extractExt(meta.sourceUrl) || file.ext;
+ const editable = { ...file };
+ delete editable.annotated;
+ delete editable.annotationSourceUrl;
+ delete editable.annotationRegions;
+ return {
+ ...editable,
+ path: meta.sourceUrl,
+ url: meta.sourceUrl,
+ ext: sourceExt,
+ mimeType: getMimeType(sourceExt, 'image'),
+ annotationStrokes: cloneAnnotationStrokes(meta.strokes),
+ };
+}
diff --git a/apps/desktop/src/renderer/lib/editLastUserMessage.ts b/apps/desktop/src/renderer/lib/editLastUserMessage.ts
index 6f20589ed4..1ca0ff8649 100644
--- a/apps/desktop/src/renderer/lib/editLastUserMessage.ts
+++ b/apps/desktop/src/renderer/lib/editLastUserMessage.ts
@@ -43,6 +43,8 @@ import { expandGhostCommand } from '@/cindy-brain/ghostCommand';
import { filterGhostsForWorkdir } from '@/cindy-brain/ghostWorkdirFilter';
import {
buildRewindDraftAttachments,
+ dropMissingAnnotationSources,
+ hasRestorableAnnotationSources,
type RewindDraftImage,
} from '@/lib/rewindDraftAttachments';
import type { AttachedFile } from '@/lib/fileTypes';
@@ -205,8 +207,13 @@ export async function commitEditAndResend(
opts: CommitEditAndResendOptions,
deps: CommitEditAndResendDeps = defaultDeps,
): Promise<boolean> {
+ // 带可再编辑标注的历史图:原图已被清理的退回烧录图(否则重发时无法重新烧录)。
+ const images =
+ opts.images && hasRestorableAnnotationSources(opts.images)
+ ? await dropMissingAnnotationSources(opts.images)
+ : opts.images;
const attachments = buildRewindDraftAttachments({
- images: opts.images,
+ images,
files: opts.files,
});
// 空文本 + 无附件的重发会被 sendMessage 静默 no-op,那样就变成"只回退没重发"
diff --git a/apps/desktop/src/renderer/lib/fileTypes.ts b/apps/desktop/src/renderer/lib/fileTypes.ts
index 2ad0f55a60..f3bf872daa 100644
--- a/apps/desktop/src/renderer/lib/fileTypes.ts
+++ b/apps/desktop/src/renderer/lib/fileTypes.ts
@@ -11,6 +11,7 @@
* `@/lib/fileTypes` 的现存 import 不用改。
*/
+import type { AnnotationRegion } from '@cindy/maker-shared/image-annotation';
import {
SUPPORTED_TEXT_EXTS,
COMPOUND_EXTS,
@@ -100,14 +101,22 @@ export interface AttachedFile {
*/
annotated?: boolean;
/**
- * 非破坏性标注:未烧录**原图**的 xdt-image:// 缓存 url。`annotated` 为真时
- * 存在;托盘预览用它(而非烧录位图)+ `annotationStrokes` 矢量叠加显示,
- * 使标注可继续编辑 / 撤销。仅编辑期数据,不进 SerializedAttachedFile。
+ * 非破坏性标注:发送物化(烧录)后才存在——`url` 已是烧录位图,这里记录未烧录
+ * **原图**的缓存 url(xdt-image:// / cindy-media://),随消息持久化进 ImageRef,
+ * 供历史图"再编辑"还原成"原图 + 矢量笔迹"。托盘(编辑期)附件不设本字段:
+ * 其 `url` 本身就是原图,缩略图 / 预览直接以 `url` + `annotationStrokes` 矢量叠加。
+ * 不进 SerializedAttachedFile(wire 只带 `annotated` 与 `annotationRegions`)。
*/
annotationSourceUrl?: string;
/** 非破坏性标注:归一化笔迹(0..1 相对原图自然尺寸)。托盘期唯一事实源;
* 发送时据此烧录位图,烧录前的原图 url 记入 annotationSourceUrl。 */
annotationStrokes?: ImageAnnotationStroke[];
+ /**
+ * 烧录时由笔迹归纳出的标注区域(归一化外接框,见 summarizeAnnotationRegions)。
+ * 只在 `annotated` 附件上存在,随 wire 透传给 buildMakerUserMessage 生成
+ * "每张图圈在哪"的隐藏说明;remote 会话剥离笔迹时仍保留。
+ */
+ annotationRegions?: AnnotationRegion[];
/**
* url 指向的缓存文件是**共享引用**(如历史消息的原图),不归本附件所有:
* 删除附件时不清理该文件。缺省(false)为附件私有,删除时照常清理。
@@ -170,6 +179,8 @@ export interface SerializedAttachedFile {
truncated?: boolean;
/** 图片带用户手绘标注,见 {@link AttachedFile.annotated}。 */
annotated?: boolean;
+ /** 标注区域(可选、向后兼容),见 {@link AttachedFile.annotationRegions}。 */
+ annotationRegions?: AnnotationRegion[];
}
export interface MentionedResource {
diff --git a/apps/desktop/src/renderer/lib/makerChatStore.ts b/apps/desktop/src/renderer/lib/makerChatStore.ts
index 31bcec8918..6788372b33 100644
--- a/apps/desktop/src/renderer/lib/makerChatStore.ts
+++ b/apps/desktop/src/renderer/lib/makerChatStore.ts
@@ -81,8 +81,6 @@ import type { ToolLoopErrorDetails } from '@cindy/maker-core';
import type { AgentMeta, MessageRole, Message, MessageAutomationOrigin } from '@/lib/ccAgent.types';
import { toMessageAutomationOrigin } from '@/lib/messageAutomationOrigin';
import {
- extractExt,
- getMimeType,
type AttachedFile,
type MentionedResource,
type SerializedAttachedFile,
@@ -210,9 +208,15 @@ import {
import { parseReconnectAttemptMessage } from '@/utils/networkError';
import {
+ isAnnotationBurnInError,
materializeAnnotatedAttachmentsForSend,
needsAnnotationMaterialize,
} from '@/lib/annotationBurnIn';
+import {
+ annotationStrokesEqual,
+ queuedAnnotationEditMeta,
+ toEditableAnnotatedAttachment,
+} from '@/lib/annotationRestore';
const log = createLogger('CcAgentChatStore');
// perf-baseline(与 MessageStream / sidebar 的 perf/session-switch 探针同通道):
@@ -13686,8 +13690,12 @@ function queuedContentProjectionMatches(
function cleanupUnacceptedQueueEditMaterialization(
originalFiles: readonly AttachedFile[],
preparedFiles: readonly AttachedFile[],
+ queuedFiles: readonly AttachedFile[] = [],
): void {
- const originalUrls = new Set(originalFiles.map((file) => file.url).filter(Boolean));
+ // 仍被原队列消息引用的文件(含免重烧复用的烧录图)绝不是本次编辑新生成的。
+ const originalUrls = new Set(
+ [...originalFiles, ...queuedFiles].map((file) => file.url).filter(Boolean),
+ );
const generatedUrls = preparedFiles
.map((file) => file.url)
.filter((url): url is string => Boolean(url) && !originalUrls.has(url));
@@ -13777,6 +13785,28 @@ function cleanupAcceptedQueueEditMaterializationSources(
});
}
+/**
+ * 队列中可"免重烧复用"的烧录附件(按附件 id):队列消息的 retryFile 描述的正是
+ * 这张烧录图(annotationRestore 配对规则),且烧录图是 `cindy-media://` 内容寻址
+ * 文件——渲染进程的缓存清理 IPC 从不物理删除 cindy-media(见 image-cache:cleanup-files),
+ * 复用后无论编辑被接受还是拒绝,都不会有清理路径删掉仍被队列 / 新消息引用的字节。
+ * 旧版 xdt-image:// 烧录图按文件删除,保守起见不复用、照旧重烧。
+ * 返回的是 retryFile 本身:保留原图与笔迹(下次编辑仍可还原)和标注区域。
+ */
+function reusableQueuedAnnotatedFiles(queued: QueuedMessage): Map<string, AttachedFile> {
+ const retryFilesById = new Map(
+ (queued.chatMessage.retryFiles ?? []).map((file) => [file.id, file]),
+ );
+ const reusable = new Map<string, AttachedFile>();
+ for (const file of queued.files ?? []) {
+ const retryFile = retryFilesById.get(file.id);
+ if (!retryFile || !queuedAnnotationEditMeta(file, retryFile)) continue;
+ if (!file.url?.startsWith('cindy-media://')) continue;
+ reusable.set(file.id, retryFile);
+ }
+ return reusable;
+}
+
function queueEditFilesMatch(
left: readonly AttachedFile[] | undefined,
right: readonly AttachedFile[] | undefined,
@@ -13810,29 +13840,10 @@ function queueEditFilesRemainUnchanged(
const retryFilesById = new Map(
(queued.chatMessage.retryFiles ?? []).map((file) => [file.id, file]),
);
+ // 与输入框编辑草稿同一套还原规则(annotationRestore),比较口径才一致。
const editableQueuedFiles = (queued.files ?? []).map((file) => {
- const retryFile = retryFilesById.get(file.id);
- const annotationSourceUrl =
- file.annotated === true &&
- retryFile?.annotated === true &&
- retryFile.path === file.path &&
- retryFile.url === file.url &&
- retryFile.annotationSourceUrl &&
- retryFile.annotationStrokes?.length
- ? retryFile.annotationSourceUrl
- : null;
- if (!annotationSourceUrl || !retryFile?.annotationStrokes) return file;
- const sourceExt = extractExt(annotationSourceUrl) || file.ext;
- const editableFile = { ...file };
- delete editableFile.annotated;
- return {
- ...editableFile,
- path: annotationSourceUrl,
- url: annotationSourceUrl,
- ext: sourceExt,
- mimeType: getMimeType(sourceExt, 'image'),
- annotationStrokes: retryFile.annotationStrokes,
- };
+ const meta = queuedAnnotationEditMeta(file, retryFilesById.get(file.id));
+ return meta ? toEditableAnnotatedAttachment(file, meta) : file;
});
return queueEditFilesMatch(editableQueuedFiles, editedFiles);
}
@@ -13867,15 +13878,30 @@ async function updateQueueItemContent(
const { content, files } = update;
if (!content.text.trim() && files.length === 0) return false;
const queuedFilesById = new Map((queued.files ?? []).map((file) => [file.id, file]));
+ const reusableBurnedFiles = reusableQueuedAnnotatedFiles(queued);
const filesForMaterialization = files.map((file) => {
+ // 只改了文字、标注原样的图:直接沿用队列里已烧录的附件(retryFile 原样,含其
+ // 所有权标记与原图 / 笔迹元数据),不重新解码 / 编码。
+ const reused = reusableBurnedFiles.get(file.id);
+ if (
+ reused &&
+ file.url === reused.annotationSourceUrl &&
+ file.path === reused.annotationSourceUrl &&
+ annotationStrokesEqual(file.annotationStrokes, reused.annotationStrokes)
+ ) {
+ return reused;
+ }
const queuedFile = queuedFilesById.get(file.id);
if (!queuedFile || queuedFile.url !== file.url || queuedFile.path !== file.path) return file;
return { ...file, cacheUrlShared: undefined, stagedPathShared: undefined };
});
const remoteMediaSession = isRemoteMediaSession(sessionId);
+ // 队列编辑是交互式保存:烧录失败即中止(抛出,输入框里的编辑与笔迹原样保留,
+ // 由 ChatInput 提示),不再悄悄把原图存进队列。此时尚未改动任何队列状态。
const preparedFiles =
(await materializeAnnotatedAttachmentsForSend(filesForMaterialization, sessionId, {
stripAnnotationMeta: remoteMediaSession,
+ burnFailure: 'abort',
})) ?? [];
const textUnchanged = content.text === queued.text;
@@ -13951,11 +13977,11 @@ async function updateQueueItemContent(
));
usedTextFallback = true;
} catch (fallbackError) {
- cleanupUnacceptedQueueEditMaterialization(files, preparedFiles);
+ cleanupUnacceptedQueueEditMaterialization(files, preparedFiles, queued.files ?? []);
throw fallbackError;
}
} else {
- cleanupUnacceptedQueueEditMaterialization(files, preparedFiles);
+ cleanupUnacceptedQueueEditMaterialization(files, preparedFiles, queued.files ?? []);
throw error;
}
}
@@ -13966,7 +13992,7 @@ async function updateQueueItemContent(
const updated = queuedContentProjectionMatches(accepted, acceptedReplacement);
if (updated && accepted) {
if (usedTextFallback) {
- cleanupUnacceptedQueueEditMaterialization(files, preparedFiles);
+ cleanupUnacceptedQueueEditMaterialization(files, preparedFiles, queued.files ?? []);
} else {
cleanupAcceptedQueueEditReplacements(
queued.files ?? [],
@@ -13983,7 +14009,7 @@ async function updateQueueItemContent(
);
}
} else {
- cleanupUnacceptedQueueEditMaterialization(files, preparedFiles);
+ cleanupUnacceptedQueueEditMaterialization(files, preparedFiles, queued.files ?? []);
}
return updated;
}
@@ -14179,6 +14205,13 @@ type SendMessageOpts = {
beforeEnqueue?: () => Promise<boolean>;
/** 远程乐观发送在稍后确认永久失败时恢复 composer。 */
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
+ /**
+ * 标注烧录失败时中止本次发送(返回 false、提示用户、不产生任何消息),而不是
+ * 降级发原图。只有「返回 false 必定原样保留草稿」的调用方才能传(输入框对
+ * 已有任务的发送);缺省保持降级 + 提示。device-link 乐观发送气泡已上屏,
+ * 一律降级。
+ */
+ annotationBurnFailure?: 'abort';
};
/** remote(SSH / device-link)会话:标注编辑数据指向控制端本地缓存,发送时剥离。 */
@@ -14186,6 +14219,37 @@ function isRemoteMediaSession(sessionId: string): boolean {
return Boolean(getOrCreateState(sessionId).remoteHostId ?? getStickySessionDeviceId(sessionId));
}
+/** 烧录降级提示:无法安全中止的发送路径已发出不含标注的原图,必须让用户知道。 */
+function notifyAnnotationBurnFallback(): void {
+ toast.warning(i18n.t('chat.media.annotateBurnFailedSentOriginal'));
+}
+
+/**
+ * 本机 / SSH 发送的标注物化:`annotationBurnFailure: 'abort'` 时烧录失败即中止
+ * (提示 + 返回 false,调用方原样恢复草稿与笔迹供重试);否则降级发原图并提示。
+ * 物化发生在 dispatch 之前,中止时尚未产生气泡、队列项或落库。
+ */
+function runLocalAnnotatedSend(
+ sessionId: string,
+ files: AttachedFile[] | undefined,
+ burnFailure: 'abort' | undefined,
+ dispatch: (prepared: AttachedFile[] | undefined) => Promise<boolean>,
+): Promise<boolean> {
+ return runRemoteOptimisticMaterialization(
+ null,
+ materializeAnnotatedAttachmentsForSend(files, sessionId, {
+ stripAnnotationMeta: isRemoteMediaSession(sessionId),
+ burnFailure: burnFailure === 'abort' ? 'abort' : 'fallback',
+ onFallback: notifyAnnotationBurnFallback,
+ }),
+ dispatch,
+ ).catch((error: unknown) => {
+ if (!isAnnotationBurnInError(error)) throw error;
+ toast.error(i18n.t('chat.media.annotateBurnFailedNotSent'));
+ return false;
+ });
+}
+
/**
* 共享发送边界:同步完成「无切换在途」检查与发送 token 登记,并保证同步抛错和
* Promise settle 两条路径都会释放。`task` 仍在当前调用栈内启动,保留 sendMessage
@@ -14290,8 +14354,10 @@ function sendMessage(
void accepted.then(
(optimisticallyAccepted) => {
if (!optimisticallyAccepted) return;
+ // 气泡已上屏、输入框已清空:烧录失败不中止,降级发原图并提示。
void materializeAnnotatedAttachmentsForSend(files, sessionId, {
stripAnnotationMeta,
+ onFallback: notifyAnnotationBurnFallback,
})
.then((prepared) => {
completeRemoteOptimisticMaterialization(sessionId, identity.clientId, () =>
@@ -14320,11 +14386,10 @@ function sendMessage(
);
return accepted;
}
- return runRemoteOptimisticMaterialization(
- null,
- materializeAnnotatedAttachmentsForSend(files, sessionId, {
- stripAnnotationMeta: isRemoteMediaSession(sessionId),
- }),
+ return runLocalAnnotatedSend(
+ sessionId,
+ files,
+ opts?.annotationBurnFailure,
(prepared) =>
sendMessageCore(
sessionId,
@@ -14632,6 +14697,8 @@ function steerMessage(
slashCommandRanges?: SlashCommandRange[];
beforeEnqueue?: () => Promise<boolean>;
onRemoteOptimisticFailure?: (clientId: string, error?: unknown) => void;
+ /** 见 SendMessageOpts.annotationBurnFailure。 */
+ annotationBurnFailure?: 'abort';
},
): Promise<boolean> {
if (!sessionId || (!text.trim() && (!files || files.length === 0)) || !workingDir) {
@@ -14705,8 +14772,10 @@ function steerMessage(
void accepted.then(
(optimisticallyAccepted) => {
if (!optimisticallyAccepted) return;
+ // 气泡已上屏、输入框已清空:烧录失败不中止,降级发原图并提示。
void materializeAnnotatedAttachmentsForSend(files, sessionId, {
stripAnnotationMeta,
+ onFallback: notifyAnnotationBurnFallback,
})
.then((prepared) => {
completeRemoteOptimisticMaterialization(sessionId, identity.clientId, () =>
@@ -14735,11 +14804,10 @@ function steerMessage(
);
return accepted;
}
- return runRemoteOptimisticMaterialization(
- null,
- materializeAnnotatedAttachmentsForSend(files, sessionId, {
- stripAnnotationMeta: isRemoteMediaSession(sessionId),
- }),
+ return runLocalAnnotatedSend(
+ sessionId,
+ files,
+ opts?.annotationBurnFailure,
(prepared) =>
steerMessageCore(
sessionId,
diff --git a/apps/desktop/src/renderer/lib/messageAttachmentPayload.ts b/apps/desktop/src/renderer/lib/messageAttachmentPayload.ts
index cf83dd898f..c198fe4da2 100644
--- a/apps/desktop/src/renderer/lib/messageAttachmentPayload.ts
+++ b/apps/desktop/src/renderer/lib/messageAttachmentPayload.ts
@@ -86,6 +86,10 @@ export function serializeAttachedFiles(
originalName: file.originalName ?? file.name,
...(file.url ? { url: file.url } : {}),
...(file.annotated ? { annotated: true } : {}),
+ // 标注区域只随烧录产物(annotated)上 wire;缺省时被控端退回纯说明。
+ ...(file.annotated && file.annotationRegions?.length
+ ? { annotationRegions: file.annotationRegions }
+ : {}),
...(legacy.base64 ? { base64: legacy.base64 } : {}),
...(legacy.textContent !== undefined ? { textContent: legacy.textContent } : {}),
...(legacy.truncated !== undefined ? { truncated: legacy.truncated } : {}),
diff --git a/apps/desktop/src/renderer/lib/queueComposerEdit.ts b/apps/desktop/src/renderer/lib/queueComposerEdit.ts
index 0c9d615897..6279ebbde1 100644
--- a/apps/desktop/src/renderer/lib/queueComposerEdit.ts
+++ b/apps/desktop/src/renderer/lib/queueComposerEdit.ts
@@ -5,7 +5,8 @@ import type { ComposerDraft } from '@/lib/composerDraftStore';
import { formatQuoteForSend, parseChatQuoteSegments, type ChatQuoteSegment } from '@/lib/chatQuotes';
import { COMPOSER_QUOTE_NODE_TYPE } from '@/lib/composerQuoteDocument';
import { normalizeComposerDocumentJSON } from '@/lib/composerListDocument';
-import { extractExt, getMimeType, type AttachedFile } from '@/lib/fileTypes';
+import type { AttachedFile } from '@/lib/fileTypes';
+import { queuedAnnotationEditMeta, toEditableAnnotatedAttachment } from '@/lib/annotationRestore';
import type { QueuedMessage, QueueItemContentUpdate } from '@/lib/makerChatStore';
import { rebaseInlineRangesAfterSlashCommandRewrite } from '@/lib/slashCommands';
import { formatMentionRef } from '@/lib/mentionRefFormat';
@@ -343,34 +344,10 @@ export function queueMessageToComposerEditDraft(
(entry.chatMessage.retryFiles ?? []).map((file) => [file.id, file]),
);
const attachments: AttachedFile[] = (entry.files ?? []).map(({ pathOrigin: _, ...file }) => {
- const retryFile = retryFilesById.get(file.id);
- const annotationSourceUrl =
- file.annotated === true &&
- retryFile?.annotated === true &&
- retryFile.path === file.path &&
- retryFile.url === file.url &&
- retryFile.annotationSourceUrl &&
- retryFile.annotationStrokes?.length
- ? retryFile.annotationSourceUrl
- : null;
- if (!annotationSourceUrl || !retryFile?.annotationStrokes) {
- return { ...file, cacheUrlShared: true, stagedPathShared: true };
- }
- const sourceExt = extractExt(annotationSourceUrl) || file.ext;
- const editableFile = { ...file };
- delete editableFile.annotated;
- return {
- ...editableFile,
- path: annotationSourceUrl,
- url: annotationSourceUrl,
- ext: sourceExt,
- mimeType: getMimeType(sourceExt, 'image'),
- annotationStrokes: retryFile.annotationStrokes.map((stroke) => ({
- points: stroke.points.map((point) => ({ ...point })),
- })),
- cacheUrlShared: true,
- stagedPathShared: true,
- };
+ // 队列里的文件归队列 / 历史所有:草稿侧移除附件不得清理它们(共享引用)。
+ const meta = queuedAnnotationEditMeta(file, retryFilesById.get(file.id));
+ const restored = meta ? toEditableAnnotatedAttachment(file, meta) : file;
+ return { ...restored, cacheUrlShared: true, stagedPathShared: true };
});
const document = structuredQueueDocument(entry, text);
diff --git a/apps/desktop/src/renderer/lib/rewindDraftAttachments.ts b/apps/desktop/src/renderer/lib/rewindDraftAttachments.ts
index 6d2fdc0f3c..68632e8b93 100644
--- a/apps/desktop/src/renderer/lib/rewindDraftAttachments.ts
+++ b/apps/desktop/src/renderer/lib/rewindDraftAttachments.ts
@@ -6,6 +6,7 @@ import {
type AttachedFile,
} from '@/lib/fileTypes';
import type { FileRef, ImageRef } from '@/lib/imageRef';
+import { toEditableAnnotatedAttachment } from '@/lib/annotationRestore';
export type RewindDraftImage =
| ImageRef
@@ -90,17 +91,13 @@ export function buildRewindDraftAttachments(input: {
// 不可撤销、重发也不再注入标注说明)。原图可能是更早历史消息的共享
// 缓存文件(ImageRef 无法区分来源),一律标记 cacheUrlShared——删除
// 草稿不清理它;若原图实为本消息私有,rewind 后的孤儿由 sweep 兜底。
- const sourceExt = extractExt(image.annotationSourceUrl) || common.ext;
+ // 原图已被清理时由调用方先经 dropMissingAnnotationSources 退回烧录图。
attachments.push({
- ...common,
- path: image.annotationSourceUrl,
- ext: sourceExt,
- mimeType: getMimeType(sourceExt, 'image'),
- url: image.annotationSourceUrl,
+ ...toEditableAnnotatedAttachment(common, {
+ sourceUrl: image.annotationSourceUrl,
+ strokes: image.annotationStrokes,
+ }),
cacheUrlShared: true,
- annotationStrokes: image.annotationStrokes.map((stroke) => ({
- points: stroke.points.map((pt) => ({ x: pt.x, y: pt.y })),
- })),
});
continue;
}
@@ -116,3 +113,76 @@ export function buildRewindDraftAttachments(input: {
return attachments;
}
+
+/** 这批历史图里是否有可还原为"原图 + 笔迹"的标注图(需要探测原图是否还在)。 */
+export function hasRestorableAnnotationSources(
+ images: readonly RewindDraftImage[] | undefined,
+): boolean {
+ return Boolean(
+ images?.some(
+ (image) =>
+ 'url' in image && image.annotationSourceUrl && image.annotationStrokes?.length,
+ ),
+ );
+}
+
+/** 探测单张原图时的超时(ms):超时按"仍存在"处理,保持可编辑还原(旧行为)。 */
+const SOURCE_PROBE_TIMEOUT_MS = 3000;
+
+/**
+ * 用 `<img>` 同款加载链路探测图片地址是否可读(自定义协议在 renderer 内由 main
+ * 流式提供,无需额外 IPC)。加载成功或无法判定(无 DOM / 超时)均返回 true,
+ * 只有明确的加载失败返回 false——宁可保留可编辑态,也不误把可用原图降级。
+ */
+export function probeImageSource(
+ url: string,
+ timeoutMs = SOURCE_PROBE_TIMEOUT_MS,
+): Promise<boolean> {
+ if (typeof Image === 'undefined') return Promise.resolve(true);
+ return new Promise((resolve) => {
+ const image = new Image();
+ let settled = false;
+ const finish = (ok: boolean) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ image.onload = null;
+ image.onerror = null;
+ resolve(ok);
+ };
+ const timer = setTimeout(() => finish(true), timeoutMs);
+ image.onload = () => finish(true);
+ image.onerror = () => finish(false);
+ image.src = url;
+ });
+}
+
+/**
+ * 历史标注图的未烧录原图已被清理时,去掉该图的标注元数据:恢复出的草稿附件
+ * 退回烧录图本身(可在其上叠加新标注),而不是指向一张打不开的原图——后者在
+ * 托盘里显示为坏图,发送时也无法重新烧录。烧录图不存在的情况不在此处理
+ * (与无标注元数据的历史图同一行为)。从不 reject。
+ */
+export async function dropMissingAnnotationSources<T extends RewindDraftImage>(
+ images: readonly T[],
+ probe: (url: string) => Promise<boolean> = probeImageSource,
+): Promise<T[]> {
+ return Promise.all(
+ images.map(async (image) => {
+ if (!('url' in image) || !image.annotationSourceUrl || !image.annotationStrokes?.length) {
+ return image;
+ }
+ let exists = true;
+ try {
+ exists = await probe(image.annotationSourceUrl);
+ } catch {
+ exists = true;
+ }
+ if (exists) return image;
+ const burnedOnly: ImageRef = { ...image };
+ delete burnedOnly.annotationSourceUrl;
+ delete burnedOnly.annotationStrokes;
+ return burnedOnly as T;
+ }),
+ );
+}
diff --git a/apps/desktop/src/shared/agentInputQueue.ts b/apps/desktop/src/shared/agentInputQueue.ts
index 3ac1e259b7..d8d3b81722 100644
--- a/apps/desktop/src/shared/agentInputQueue.ts
+++ b/apps/desktop/src/shared/agentInputQueue.ts
@@ -12,6 +12,11 @@ import { stripChatQuoteMarkerLines } from '@cindy/maker-shared/chat-quotes';
import type { SharedTaskAuthor } from '@cindy/maker-shared';
import { UI_ACTION_TRIGGER_PREFIX } from '@cindy/maker-shared/synthetic-trigger';
import { MENTION_TOKEN_SPLIT, parseMentionToken } from '@cindy/maker-shared/mention-ref';
+import {
+ formatAnnotationRegion,
+ sanitizeAnnotationRegions,
+ type AnnotationRegion,
+} from '@cindy/maker-shared/image-annotation';
import {
describeAgentInputReference,
projectAgentFacingText,
@@ -70,6 +75,13 @@ export interface AgentInputSerializedFile {
* 在附件 block 后注入一句固定说明,告诉模型红色笔迹是用户标注、非原图内容。
*/
annotated?: boolean;
+ /**
+ * 可选(向后兼容):标注区域,归一化坐标外接框(0..1,原点左上)。新控制端烧录时
+ * 由笔迹归纳;buildMakerUserMessage 经 sanitizeAnnotationRegions 校验后在标注
+ * 说明末尾补一句"每张图圈在哪"。旧主机忽略该字段;缺省 / 非法时新主机退回
+ * 原固定说明(字节不变)。
+ */
+ annotationRegions?: AnnotationRegion[];
}
export interface AgentInputMention {
@@ -762,6 +774,22 @@ export const ANNOTATED_IMAGE_NOTE =
'Note: the red freehand marks on the attached image(s) are annotations drawn by the user ' +
'to highlight the region(s) they are referring to; they are not part of the original image.';
+/**
+ * 标注区域说明的前缀(接在 ANNOTATED_IMAGE_NOTE 之后另起一行)。只有至少一张
+ * 标注图带合法 `annotationRegions` 时才出现;图片按本条消息内 image block 的
+ * 顺序从 1 编号,帮助模型在多张图 / 多处标注时对上用户所指。
+ */
+export const ANNOTATED_IMAGE_REGIONS_PREFIX =
+ 'Marked regions (normalized image coordinates, origin top-left): ';
+
+/** 一张标注图的区域描述,如 `image 2: x 0.31–0.46, y 0.12–0.20; x 0.70–0.90, y 0.55–0.61`。 */
+function describeAnnotatedImageRegions(
+ imageNumber: number,
+ regions: readonly AnnotationRegion[],
+): string {
+ return `image ${imageNumber}: ${regions.map(formatAnnotationRegion).join('; ')}`;
+}
+
/** Stable serialization shared by the resolver's final budget check and agent injection. */
export function serializeSessionReferencePayload(
sessionReferenceContexts: readonly AgentInputSessionReferenceContext[],
@@ -1037,6 +1065,9 @@ export function buildMakerUserMessage(
blocks.push({ type: 'mention', name: m.name, path: m.path, kind: m.type });
}
let hasAnnotatedImage = false;
+ // 本条消息内 image block 的序号(从 1 起),与模型看到的图片顺序一致。
+ let imageBlockCount = 0;
+ const regionDescriptions: string[] = [];
for (const f of queued.files ?? []) {
const type = getAgentInputAttachmentBlockType(f.category, f.ext);
const pathOrigin = type === 'image' && f.pathOrigin === 'desktop-host'
@@ -1051,12 +1082,25 @@ export function buildMakerUserMessage(
} else {
continue;
}
- if (type === 'image' && f.annotated) hasAnnotatedImage = true;
+ if (type !== 'image') continue;
+ imageBlockCount += 1;
+ if (!f.annotated) continue;
+ hasAnnotatedImage = true;
+ // wire 数据不可信(device-link / 持久化队列):只用校验后的区域。
+ const regions = sanitizeAnnotationRegions(f.annotationRegions);
+ if (regions) regionDescriptions.push(describeAnnotatedImageRegions(imageBlockCount, regions));
}
// 标注说明放在全部附件 block 之后、每条消息至多一条:codex 侧 inputs 保序,
// 文本紧随图片;claude 侧所有 text 会合并进文本前缀,红色笔迹自身即区分符。
+ // 没有任何区域数据(旧控制端 / mobile)时说明与旧版逐字节相同。
if (hasAnnotatedImage) {
- blocks.push({ type: 'text', text: ANNOTATED_IMAGE_NOTE });
+ blocks.push({
+ type: 'text',
+ text:
+ regionDescriptions.length > 0
+ ? `${ANNOTATED_IMAGE_NOTE}\n${ANNOTATED_IMAGE_REGIONS_PREFIX}${regionDescriptions.join('; ')}.`
+ : ANNOTATED_IMAGE_NOTE,
+ });
}
if (sessionReferenceContexts.length > 0) {
const payload = serializeSessionReferencePayload(sessionReferenceContexts);
diff --git a/docs/dev-rules/protocol-compatibility.md b/docs/dev-rules/protocol-compatibility.md
index bf89e8ff2e..8708443d0e 100644
--- a/docs/dev-rules/protocol-compatibility.md
+++ b/docs/dev-rules/protocol-compatibility.md
@@ -153,6 +153,18 @@ link-accept 双向声明,不改 relay)。Desktop 控制端在本机没有订
受信 renderer 开放。不改 relay、帧限制或服务器权限,服务端无需改动。实现见
`apps/desktop/src/main/usage/usageDeviceRows.ts` 与 `peerUsageSync.ts`。
+## 图片标注区域说明
+
+`maker:input:enqueue` / `maker:input:steer` / `maker:input:update-content` 的队列附件
+(`AgentInputSerializedFile`)追加可选 `annotationRegions: { x0, y0, x1, y1 }[]`:标注图
+(`annotated: true`)烧录时由笔迹归纳的外接框,归一化坐标(0..1,原点左上,两位小数,
+每张图至多 6 处)。只经既有 device-link 隧道与 IPC 透传,不新增 channel、relay 类型或
+持久化 schema。消费端 `buildMakerUserMessage` 一律经 `sanitizeAnnotationRegions` 校验,
+有合法区域时在原标注说明后另起一行按本条消息内图片顺序描述区域;仍然每条消息至多一条说明。
+旧主机忽略该字段,只注入原固定说明;新主机收到旧控制端或 mobile(不带该字段)的消息时,
+说明与旧版逐字节相同。remote 会话剥离 `annotationSourceUrl` / `annotationStrokes` 时保留
+区域字段。服务端无需改动。
+
## 事实来源
| 内容 | 权威来源 |