Skip to content

Open source CapCut alternative.

TypeScript

fix(site): card only media a chat tool made

David Leda5291818810
diff --git a/site/src/cut/components/AiPanel.tsx b/site/src/cut/components/AiPanel.tsx
index 3eb14953..527956b4 100644
--- a/site/src/cut/components/AiPanel.tsx
+++ b/site/src/cut/components/AiPanel.tsx
@@ -131,6 +131,7 @@ import { MentionedText, MentionTextarea, RefChips, RefThumb } from "./AssetRefs"
import { ComposerQueue, type QueuedMessage } from "./ComposerQueue";
import { DictationBody } from "./MicDictation";
import { RECORD_RUNNING_TTL_MS, ToolOutputAssets } from "./ChatAssets";
+import { attachedRefKeys, cardIsAttached } from "@/cut/lib/chatCards";
import { HostedErrorText } from "./hostedError";
import { useMicTranscription } from "@/cut/lib/micTranscribe";
@@ -1345,6 +1346,9 @@ function ChatSession({
() => visible ? recoverSceneCall(messages, threadId, savedScene) : messages,
[messages, threadId, savedScene, visible],
);
+ // What the user attached in this thread: an import of one of these needs no
+ // card of its own under the reply.
+ const attachedKeys = useMemo(() => attachedRefKeys(displayMessages), [displayMessages]);
// The scene card is part of the conversation, not a pinned banner: it
// renders right under the turn that planned the scene (the newest
@@ -2029,7 +2033,7 @@ function ChatSession({
)}
{displayMessages.map((m) => (
<Fragment key={m.id}>
- <MessageView message={m} />
+ <MessageView message={m} attachedKeys={attachedKeys} />
{m.id === sceneAnchorId && <SceneCard threadId={threadId} />}
</Fragment>
))}
@@ -2679,8 +2683,10 @@ function ThoughtBlock({ text }: { text: string }) {
const MessageView = memo(function MessageView({
message,
+ attachedKeys,
}: {
message: UIMessage;
+ attachedKeys: string;
}) {
// A continuation is the panel's own doing; the reply it brings reads as the
// ask's reply.
@@ -2727,7 +2733,8 @@ const MessageView = memo(function MessageView({
.filter((part) => part.type.startsWith("tool-") || part.type === "dynamic-tool")
.map((part) => part as unknown as { state: string; output?: unknown })
.filter((p) => p.state === "output-available")
- .map((p) => p.output);
+ .map((p) => p.output)
+ .filter((o) => !cardIsAttached(o, attachedKeys));
// The source's own words for anything the turn pulled off the web (a tweet's
// body, a video's title/description). It renders as a quote beside the media
// — straight from the tool output, so the model never has to retype it.
diff --git a/site/src/cut/components/ChatAssets.tsx b/site/src/cut/components/ChatAssets.tsx
index b53893e9..d8070c48 100644
--- a/site/src/cut/components/ChatAssets.tsx
+++ b/site/src/cut/components/ChatAssets.tsx
@@ -80,7 +80,9 @@ export function ChatProjectAsset({ assetId }: { assetId: string }) {
/** The assets a finished tool call produced, from its typed output fields:
* `assetId` for landed media, `jobId` for a video render still in flight,
- * `stillAssetId` for a staged render's opening frame (shown above its job). */
+ * `stillAssetId` for a staged render's opening frame (shown above its job).
+ * A tool that only reads or points at existing media names it `id` or
+ * `sourceId`, so it gets no card. */
export function ToolOutputAssets({ output }: { output: unknown }) {
if (!output || typeof output !== "object") return null;
const o = output as {
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 678144e1..491cecbd 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -1104,7 +1104,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
if (asset.block) throw new ToolError("This placeholder has no source file.");
const sizeBytes = asset.sizeBytes ?? await readMediaFileSize(asset.url);
return {
- assetId: asset.id, name: asset.name, fileName: asset.fileName, kind: asset.type,
+ id: asset.id, name: asset.name, fileName: asset.fileName, kind: asset.type,
sizeBytes, fileSize: formatBytes(sizeBytes), duration: round2(asset.duration), width: asset.width, height: asset.height,
};
},
@@ -2301,7 +2301,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
if (video.type !== "video") throw new ToolError(`"${video.name}" is not a video.`);
if (input.clear === true || input.audio_asset_id === null) {
unbindRecording(video.id);
- return { assetId: video.id, soundFrom: null, note: `"${video.name}" plays its own sound again.` };
+ return { id: video.id, soundFrom: null, note: `"${video.name}" plays its own sound again.` };
}
return syncVideoSound(s, video, input.audio_asset_id, clip);
},
@@ -3333,7 +3333,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
return {
id: next.id,
kind: "clip",
- assetId: asset.id,
+ sourceId: asset.id,
name: asset.name,
start: round2(next.start),
lenBefore: round2(before),
@@ -3352,7 +3352,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
return {
id: next.id,
kind: "audio",
- assetId: asset.id,
+ sourceId: asset.id,
name: asset.name,
start: round2(next.start),
lenBefore: round2(before),
@@ -3366,7 +3366,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
// A Lottie document animates and a still does not; the flag is read off
// the source, so it moves with it or the export plays the wrong one.
s.updateOverlay(overlay.id, { assetId: asset.id, lottie: isLottieAsset(asset) || undefined });
- return { id: overlay.id, kind: "sticker", assetId: asset.id, name: asset.name };
+ return { id: overlay.id, kind: "sticker", sourceId: asset.id, name: asset.name };
}
throw new ToolError(
`Nothing with id "${id}" carries a source. Clips, soundtrack clips and stickers do; text and shapes are edited with update_overlay.`
@@ -3955,6 +3955,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
duration: round2(vid || sfx ? asset.duration : IMAGE_CLIP_SECONDS),
addedToTimeline: addToTimeline,
clipId,
+ fromRef: { scope: "stock", id },
};
},
@@ -4319,6 +4320,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
...(place
? { addedToTimeline: true, clip: placeAssetOnTimeline(asset, input) }
: { addedToTimeline: false }),
+ fromRef: { scope: "library", id: lib.id },
};
},
@@ -5265,7 +5267,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
);
const next = useEditor.getState().assets.find((a) => a.id === asset.id)!;
return {
- assetId: next.id,
+ id: next.id,
profile: sourceProfileOf(next),
detected,
clips: useEditor.getState().clips.filter((c) => c.assetId === next.id).map((c) => c.id),
@@ -6272,8 +6274,10 @@ async function syncVideoSound(s: Editor, video: MediaAsset, audioId: unknown, cl
const bound = await bindRecording(video.id, rec.id, clip).catch((e: unknown) => {
throw new ToolError(e instanceof Error ? e.message : "Could not line the recording up.");
});
+ const { assetId, ...rest } = bound;
return {
- ...bound,
+ id: assetId,
+ ...rest,
note: `Every clip of "${video.name}" (${bound.clips} on the timeline) now plays "${rec.name}"; recording second = video second ${bound.offset < 0 ? "−" : "+"} ${Math.abs(bound.offset)}.${bound.refined ? "" : " Aligned to the nearest 20 ms; the waveforms had no sharp detail to place it closer."}`,
};
}
diff --git a/site/src/cut/lib/chatCards.test.ts b/site/src/cut/lib/chatCards.test.ts
new file mode 100644
index 00000000..b379cd79
--- /dev/null
+++ b/site/src/cut/lib/chatCards.test.ts
@@ -0,0 +1,54 @@
+import { beforeEach, expect, test } from "bun:test";
+import type { UIMessage } from "ai";
+import { runAiTool } from "./aiTools";
+import { attachedRefKeys, cardIsAttached } from "./chatCards";
+import { useEditor } from "./store";
+
+// A chat card shows media a tool made. Media the user already holds — the
+// source a swap points at, a stock sound they attached — gets no second card.
+
+beforeEach(() => {
+ useEditor.setState({
+ clips: [],
+ overlays: [],
+ transitions: [],
+ audioClips: [],
+ assets: [
+ { id: "tick", fileName: "tick.mp3", name: "Tick", type: "audio", duration: 1, url: "" },
+ { id: "shutter", fileName: "shutter.mp3", name: "Shutter", type: "audio", duration: 1.5, url: "" },
+ { id: "cam", fileName: "cam.mp4", name: "Cam", type: "video", duration: 10, sizeBytes: 1000, url: "" },
+ ],
+ });
+});
+
+test("a swap names its source without a card", async () => {
+ useEditor.getState().addAudioFromAsset("tick", 0);
+ const audio = useEditor.getState().audioClips[0]!;
+ const out = (await runAiTool("replace_item", { id: audio.id, asset_id: "shutter" })) as Record<string, unknown>;
+ expect(out.assetId).toBeUndefined();
+ expect(out.sourceId).toBe("shutter");
+});
+
+test("reading or retagging a source makes no card", async () => {
+ const info = (await runAiTool("get_asset_info", { asset_id: "cam" })) as Record<string, unknown>;
+ expect(info.assetId).toBeUndefined();
+ expect(info.id).toBe("cam");
+ const color = (await runAiTool("set_source_color", { assetId: "cam", profile: "hlg" })) as Record<string, unknown>;
+ expect(color.assetId).toBeUndefined();
+ expect(color.id).toBe("cam");
+});
+
+test("an import of an attached stock item gets no card", () => {
+ const messages = [
+ {
+ id: "u1",
+ role: "user",
+ parts: [{ type: "text", text: "Replace the ticks" }],
+ metadata: { attachments: [{ scope: "stock", id: "camera-shutter-2", name: "Camera Shutter 2", kind: "audio", url: "/stock/sfx/camera-shutter-2.mp3" }] },
+ },
+ ] as UIMessage[];
+ const attached = attachedRefKeys(messages);
+ expect(cardIsAttached({ assetId: "a1", fromRef: { scope: "stock", id: "camera-shutter-2" } }, attached)).toBe(true);
+ expect(cardIsAttached({ assetId: "a2", fromRef: { scope: "stock", id: "whoosh" } }, attached)).toBe(false);
+ expect(cardIsAttached({ assetId: "a3" }, attached)).toBe(false);
+});
diff --git a/site/src/cut/lib/chatCards.ts b/site/src/cut/lib/chatCards.ts
new file mode 100644
index 00000000..b2d1ec06
--- /dev/null
+++ b/site/src/cut/lib/chatCards.ts
@@ -0,0 +1,41 @@
+import type { UIMessage } from "ai";
+import { normalizeRef } from "./assetRef";
+
+// Tool outputs name the media a call made with `assetId`; the chat renders a
+// card for each. An import of something the user attached (a stock sound, a
+// Library clip) also says which ref it came from (`fromRef`), and the user's
+// own attachment card already shows it, so the import gets no second card.
+
+const KEY_SEP = "\n";
+
+const refKey = (scope: string, id: string) => `${scope}:${id}`;
+
+/** Every ref the user attached anywhere in the thread, as one string of
+ * `scope:id` keys — a string so memoized message views compare it by value. */
+export function attachedRefKeys(messages: readonly UIMessage[]): string {
+ const keys = new Set<string>();
+ for (const m of messages) {
+ if (m.role !== "user") {
+ continue;
+ }
+ const raw = (m.metadata as { attachments?: unknown[] } | undefined)?.attachments ?? [];
+ for (const r of raw.map(normalizeRef)) {
+ if (r) {
+ keys.add(refKey(r.scope, r.id));
+ }
+ }
+ }
+ return [...keys].sort().join(KEY_SEP);
+}
+
+/** Whether a tool output imported a ref the user attached in the thread. */
+export function cardIsAttached(output: unknown, attached: string): boolean {
+ if (!output || typeof output !== "object" || !attached) {
+ return false;
+ }
+ const from = (output as { fromRef?: { scope?: unknown; id?: unknown } }).fromRef;
+ if (typeof from?.scope !== "string" || typeof from.id !== "string") {
+ return false;
+ }
+ return attached.split(KEY_SEP).includes(refKey(from.scope, from.id));
+}