Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

feat(mobile): 标注图把标注区域发给模型,规则与桌面一致

Dash4c543963a852
diff --git a/apps/mobile/src/__tests__/durableOutboxFiles.test.ts b/apps/mobile/src/__tests__/durableOutboxFiles.test.ts
index 05f98e0ea9..6b2def75cf 100644
--- a/apps/mobile/src/__tests__/durableOutboxFiles.test.ts
+++ b/apps/mobile/src/__tests__/durableOutboxFiles.test.ts
@@ -162,6 +162,23 @@ describe("outbox-owned attachment bytes", () => {
});
expect(upload.size).toBe(123);
expect(durableOutboxDirectory(record)).not.toContain("/../");
+ expect(upload.annotated).toBeUndefined();
+ expect(upload.annotationRegions).toBeUndefined();
+ });
+ it("records annotation regions for annotated uploads, but not for re-saved burns whose old marks are unknown", async () => {
+ const strokes = [{ points: [{ x: 0.1, y: 0.2 }, { x: 0.3, y: 0.4 }] }];
+ const annotated = await retainOutboxFile(record, 0, {
+ ...source,
+ annotation: { strokes, sourceUri: "file:///cache/src.jpg", sourceMimeType: "image/jpeg" },
+ });
+ expect(annotated.annotated).toBe(true);
+ expect(annotated.annotationRegions).toEqual([{ x0: 0.1, y0: 0.2, x1: 0.3, y1: 0.4 }]);
+ const reburned = await retainOutboxFile(record, 1, {
+ ...source,
+ annotation: { strokes, sourceUri: "file:///cache/src.jpg", sourceMimeType: "image/jpeg", baseAnnotated: true },
+ });
+ expect(reburned.annotated).toBe(true);
+ expect(reburned.annotationRegions).toBeUndefined();
});
it("rejects a partial copy rather than accepting a message with missing bytes", async () => {
fs.getInfoAsync.mockResolvedValue({
diff --git a/apps/mobile/src/__tests__/useComposerImageAnnotations.test.tsx b/apps/mobile/src/__tests__/useComposerImageAnnotations.test.tsx
index 267d4e103e..ea2c9fb48a 100644
--- a/apps/mobile/src/__tests__/useComposerImageAnnotations.test.tsx
+++ b/apps/mobile/src/__tests__/useComposerImageAnnotations.test.tsx
@@ -171,6 +171,8 @@ describe('generated file lifecycle', () => {
const candidate = enqueued()[0];
const decorated = api().decorateUploadedAttachment(attachment('att-1'), candidate);
expect(decorated.annotated).toBe(true);
+ // 与桌面同一归纳算法:给模型的标注区域随附件发给被控端。
+ expect(decorated.annotationRegions).toEqual([{ x0: 0.1, y0: 0.1, x1: 0.4, y1: 0.4 }]);
candidate.onAbandoned?.();
expect(runtime.deleted).toEqual([]);
act(() => api().forgetAttachment('att-1'));
@@ -265,7 +267,10 @@ describe('re-edit replacement', () => {
const candidate = hook.enqueued()[0];
expect(runtime.burnIn).not.toHaveBeenCalled();
expect(candidate.annotation).toMatchObject({ strokes: [], baseAnnotated: true });
- expect(hook.api().decorateUploadedAttachment(attachment('R2'), candidate).annotated).toBe(true);
+ const resaved = hook.api().decorateUploadedAttachment(attachment('R2'), candidate);
+ expect(resaved.annotated).toBe(true);
+ // 旧红线位置不可知:不带区域,退回固定说明。
+ expect(resaved.annotationRegions).toBeUndefined();
// 之后再次保存(已有内存真相但底图仍是烧录图):标仍保留。
await traySubmit(hook.api(), 'R2', hook.api().trayImageSourceUri('R2', 'preview'), []);
const again = hook.enqueued()[1];
diff --git a/apps/mobile/src/session/MobileOutboxBridge.tsx b/apps/mobile/src/session/MobileOutboxBridge.tsx
index f9bf9c14ee..c690c31928 100644
--- a/apps/mobile/src/session/MobileOutboxBridge.tsx
+++ b/apps/mobile/src/session/MobileOutboxBridge.tsx
@@ -268,7 +268,13 @@ export function MobileOutboxBridge() {
throw new Error("OUTBOX_OWNER_CHANGED");
}
return upload.annotated
- ? { ...attachment, annotated: true }
+ ? {
+ ...attachment,
+ annotated: true,
+ ...(upload.annotationRegions?.length
+ ? { annotationRegions: upload.annotationRegions }
+ : {}),
+ }
: attachment;
},
enqueue: (r) =>
diff --git a/apps/mobile/src/session/durableOutbox.ts b/apps/mobile/src/session/durableOutbox.ts
index c42cab939f..2ec3ab50ac 100644
--- a/apps/mobile/src/session/durableOutbox.ts
+++ b/apps/mobile/src/session/durableOutbox.ts
@@ -1,3 +1,4 @@
+import type { AnnotationRegion } from "@cindy/maker-shared/image-annotation";
import type { NewSessionDraft } from "./newSession";
import type { ComposerDocument } from './composerDocument';
import type { MobileOutboxItem } from "./sessionOutbox";
@@ -11,6 +12,8 @@ export interface DurableUpload {
kind: "image" | "file";
size: number;
annotated?: boolean;
+ /** 标注区域(可选,见 RemoteSerializedAttachment.annotationRegions)。 */
+ annotationRegions?: AnnotationRegion[];
}
export interface DurableOutboxRecord {
diff --git a/apps/mobile/src/session/durableOutboxFiles.ts b/apps/mobile/src/session/durableOutboxFiles.ts
index 86b1c91806..e413494c88 100644
--- a/apps/mobile/src/session/durableOutboxFiles.ts
+++ b/apps/mobile/src/session/durableOutboxFiles.ts
@@ -5,6 +5,7 @@ import { DURABLE_OUTBOX_PREFIX, type DurableOutboxRecord, type DurableUpload } f
import type { MobileLocalAttachmentUploadCandidate } from "./mobileLocalAttachmentUpload";
import { isAttachmentOssRef } from './attachmentOssRef';
import { isPeerAttachmentRef } from '@cindy/device-link';
+import { summarizeAnnotationRegions } from '@cindy/maker-shared/image-annotation';
/** Desktop path references already have their source on the controlled device. */
export function outboxAttachmentNeedsLocalBytes(attachment: { path: string }): boolean {
@@ -29,6 +30,16 @@ export function durableOutboxUploadUri(
throw new Error("OUTBOX_FILE_INVALID");
return durableOutboxDirectory(record) + upload.fileName;
}
+/** 标注区域随发件箱落盘(与乐观上传路径同一归纳算法),重投时照样带给被控端。 */
+function annotationRegionsFor(
+ source: MobileLocalAttachmentUploadCandidate,
+): Pick<DurableUpload, "annotationRegions"> {
+ // 底图已是烧录图(旧红线位置不可知)时不带区域,见 annotationRegionsForUpload。
+ if (!source.annotation || source.annotation.baseAnnotated) return {};
+ const regions = summarizeAnnotationRegions(source.annotation.strokes);
+ return regions.length > 0 ? { annotationRegions: regions } : {};
+}
+
export async function retainOutboxFile(
record: DurableOutboxRecord,
slot: number,
@@ -45,6 +56,7 @@ export async function retainOutboxFile(
kind: source.kind,
size: source.size,
...(source.annotation ? { annotated: true } : {}),
+ ...annotationRegionsFor(source),
};
await FileSystem.makeDirectoryAsync(durableOutboxDirectory(record), {
intermediates: true,
diff --git a/apps/mobile/src/session/types.ts b/apps/mobile/src/session/types.ts
index 485b7fcccb..14e90cb0cb 100644
--- a/apps/mobile/src/session/types.ts
+++ b/apps/mobile/src/session/types.ts
@@ -3,6 +3,7 @@ import type { AgentInputReference } from '@cindy/maker-shared/agent-input-projec
import type { RemoteMoney } from '@/session/remoteMoney';
import type { MobileToolLoopErrorDetails } from '@/session/agentErrorI18n';
import type { MobileToolInputProjection } from '@/session/messageToolPayloadProjection';
+import type { AnnotationRegion } from '@cindy/maker-shared/image-annotation';
export type RemoteSessionStatus = 'active' | 'archived' | 'deleted';
export type RemoteMessageRole =
@@ -180,6 +181,13 @@ export interface RemoteSerializedAttachment {
* 与桌面 AgentInputSerializedFile.annotated 同一契约。
*/
annotated?: boolean;
+ /**
+ * 可选(向后兼容):标注区域,由笔迹归纳的归一化外接框(0..1,原点左上),与桌面
+ * AgentInputSerializedFile.annotationRegions 同一契约、同一归纳算法
+ * (summarizeAnnotationRegions)。新被控端校验后在标注说明里补一句每张图圈在哪;
+ * 旧被控端忽略。
+ */
+ annotationRegions?: AnnotationRegion[];
}
export interface QueuedRemoteMessage {
diff --git a/apps/mobile/src/session/useComposerImageAnnotations.ts b/apps/mobile/src/session/useComposerImageAnnotations.ts
index f04c5a6786..8cfb6cf86e 100644
--- a/apps/mobile/src/session/useComposerImageAnnotations.ts
+++ b/apps/mobile/src/session/useComposerImageAnnotations.ts
@@ -25,6 +25,7 @@ import { useCallback, useMemo, useRef } from 'react';
import { useTranslation } from 'react-i18next';
import { Alert, Platform } from 'react-native';
import * as FileSystem from 'expo-file-system/legacy';
+import { summarizeAnnotationRegions } from '@cindy/maker-shared/image-annotation';
import { useAnnotationBurnIn } from '@/session/AnnotationBurnInWebView';
import {
annotationBurnedFileName,
@@ -153,6 +154,19 @@ function positiveHintDimension(value: number | undefined): number | undefined {
return typeof value === 'number' && Number.isFinite(value) && value > 0 ? Math.round(value) : undefined;
}
+/**
+ * 给模型的标注区域(与桌面同一归纳算法)。底图本身已是烧录图(再编辑真相丢失)时,
+ * 旧红线的位置不可知,只描述新笔迹会让说明与图上红线不符——此时不带区域,
+ * 退回固定说明。
+ */
+export function annotationRegionsForUpload(
+ annotation: { strokes: readonly AnnotationStroke[]; baseAnnotated?: boolean } | undefined,
+): Pick<RemoteSerializedAttachment, 'annotationRegions'> {
+ if (!annotation || annotation.baseAnnotated) return {};
+ const regions = summarizeAnnotationRegions(annotation.strokes);
+ return regions.length > 0 ? { annotationRegions: regions } : {};
+}
+
function extForMime(mimeType: string): string {
const lower = mimeType.toLowerCase();
if (lower === 'image/png') return 'png';
@@ -553,7 +567,7 @@ export function useComposerImageAnnotations(
sourceMimeType: candidate.annotation.sourceMimeType,
...(candidate.annotation.baseAnnotated ? { baseAnnotated: true } : {}),
});
- return { ...attachment, annotated: true };
+ return { ...attachment, annotated: true, ...annotationRegionsForUpload(candidate.annotation) };
}, [isHookGeneratedFile, deleteUnreferencedFiles]);
const chatAnnotation = useMemo<ImageLightboxAnnotationConfig>(() => ({
diff --git a/docs/dev-rules/protocol-compatibility.md b/docs/dev-rules/protocol-compatibility.md
index 8708443d0e..03bd6a261f 100644
--- a/docs/dev-rules/protocol-compatibility.md
+++ b/docs/dev-rules/protocol-compatibility.md
@@ -161,9 +161,10 @@ link-accept 双向声明,不改 relay)。Desktop 控制端在本机没有订
每张图至多 6 处)。只经既有 device-link 隧道与 IPC 透传,不新增 channel、relay 类型或
持久化 schema。消费端 `buildMakerUserMessage` 一律经 `sanitizeAnnotationRegions` 校验,
有合法区域时在原标注说明后另起一行按本条消息内图片顺序描述区域;仍然每条消息至多一条说明。
-旧主机忽略该字段,只注入原固定说明;新主机收到旧控制端或 mobile(不带该字段)的消息时,
-说明与旧版逐字节相同。remote 会话剥离 `annotationSourceUrl` / `annotationStrokes` 时保留
-区域字段。服务端无需改动。
+旧主机忽略该字段,只注入原固定说明;新主机收到旧控制端(不带该字段)的消息时,说明与
+旧版逐字节相同。remote 会话剥离 `annotationSourceUrl` / `annotationStrokes` 时保留区域字段。
+Mobile 以同一归纳算法在上传后的附件(含持久发件箱 `DurableUpload`,可选字段、旧记录缺省)
+上携带该字段;底图本身已是烧录图、旧红线位置不可知时不带区域。服务端无需改动。
## 事实来源