Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): let chat place audio on separate lanes [rebuild]

David Lea5c1851e477e
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.ts
index 332ea835..1299be0a 100644
--- a/site/src/cut/components/Inspector.tools.ts
+++ b/site/src/cut/components/Inspector.tools.ts
@@ -172,9 +172,10 @@ export const INSPECTOR_TOOLS = [
{
name: "update_audio",
description:
- "Update a soundtrack clip: volume (0..3), fadeIn/fadeOut seconds, start position, in/out trim, speed, reverse, hidden, or duck. `duck` is voiceover ducking — while this clip plays, ALL other audio (video-clip sound and other music) drops to that gain (0..1); pass 1 to clear ducking. Use it to make a voiceover sit over quieter music.",
+ "Update a soundtrack clip: volume (0..3), fadeIn/fadeOut seconds, start position, lane, in/out trim, speed, reverse, hidden, or duck. Audio overlaps across lanes; clips on the same lane slide to its next free slot. To put music under narration, move music to lane 1 and start 0 in the same call, with narration on lane 0. `duck` is voiceover ducking — while this clip plays, ALL other audio drops to that gain (0..1); pass 1 to clear ducking.",
inputSchema: obj({
id: str("Soundtrack clip id"),
+ lane: { type: "integer", minimum: 0, description: "Audio lane, 0-based. Separate lanes play together." },
volume: num("0..3 (1 = unchanged, above 1 boosts)"),
fadeIn: num("Fade-in seconds"),
fadeOut: num("Fade-out seconds"),
diff --git a/site/src/cut/components/Timeline.tools.ts b/site/src/cut/components/Timeline.tools.ts
index cbc67480..0d7b1971 100644
--- a/site/src/cut/components/Timeline.tools.ts
+++ b/site/src/cut/components/Timeline.tools.ts
@@ -50,6 +50,7 @@ export const TIMELINE_TOOLS = [
"Put a project asset on the timeline, the same way the user dragging it in would: a video or image lands on video track 0 (at `start`, inserted at `index`, or appended at the end; a taken spot slides it right), audio lands on the soundtrack (at `start`, default the playhead). Asset ids come from `media` in editor_state — imports, attachments, and chat media alike. Call it only when the user asked for the media in the cut (\"add my beach photo\", \"stitch these into a movie\"); otherwise media stays on its card or panel for them to drag.",
inputSchema: obj({
asset_id: str("Project asset id from `media` in editor_state"),
+ lane: { type: "integer", minimum: 0, description: "Audio assets only: soundtrack lane, default 0. Put music on lane 1 to overlap narration on lane 0." },
start: num("Timeline start s"),
index: num("Insert position on video track 0 (video/image only; 0 = first)"),
}, ["asset_id"]),
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 706cd819..909e7875 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -225,6 +225,7 @@ function tracksAfter() {
}));
const lanes = cur.audioClips.map((a) => ({
id: a.id,
+ lane: a.lane ?? 0,
start: round2(a.start),
len: round2(a.out - a.in),
}));
@@ -2076,6 +2077,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
const a = requireItem(s.audioClips, input.id, "soundtrack clip");
const len = retimeOf(a).len;
const patch: Partial<AudioClip> = {};
+ if (input.lane !== undefined) patch.lane = audioLaneInput(input.lane);
if (isNum(input.volume)) patch.volume = clamp(input.volume, 0, 3);
if (isNum(input.fadeIn)) patch.fadeIn = clamp(input.fadeIn, 0, len / 2);
if (isNum(input.fadeOut)) patch.fadeOut = clamp(input.fadeOut, 0, len / 2);
@@ -2103,6 +2105,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
return {
id: a.id,
...patch,
+ lane: landed.lane ?? 0,
...("start" in patch ? { start: round2(landed.start) } : {}),
};
},
@@ -4359,18 +4362,25 @@ const NOT_PLACED = { added: false, clipId: null, index: null };
/** Place a project asset on the timeline like a drag: video/image onto track 0
* (an `index` insert, a free-positioned `start`, or appended at the end),
* audio onto the soundtrack. Returns the created clip's summary. */
+function audioLaneInput(value: unknown): number {
+ if (!isNum(value) || !Number.isInteger(value) || value < 0)
+ throw new ToolError("Audio lane must be a non-negative integer.");
+ return value;
+}
+
function placeAssetOnTimeline(asset: MediaAsset, input: Record<string, unknown>) {
const s = useEditor.getState();
const start = isNum(input.start) ? Math.max(0, input.start) : undefined;
if (asset.type === "audio") {
- s.addAudioFromAsset(asset.id, start);
+ const lane = input.lane === undefined ? 0 : audioLaneInput(input.lane);
+ s.addAudioFromAsset(asset.id, start, { lane });
const sel = useEditor.getState().selection;
const c =
sel?.kind === "audio"
? useEditor.getState().audioClips.find((x) => x.id === sel.id)
: undefined;
if (!c) throw new ToolError("Could not create the soundtrack clip.");
- return { id: c.id, kind: asset.type, start: round2(c.start), len: round2(c.out - c.in) };
+ return { id: c.id, kind: asset.type, lane: c.lane ?? 0, start: round2(c.start), len: round2(c.out - c.in) };
}
let clipId: string | null;
if (isNum(input.index)) {
diff --git a/site/src/cut/lib/audioLaneTools.test.ts b/site/src/cut/lib/audioLaneTools.test.ts
new file mode 100644
index 00000000..a8e3fd9e
--- /dev/null
+++ b/site/src/cut/lib/audioLaneTools.test.ts
@@ -0,0 +1,35 @@
+import { beforeEach, expect, test } from "bun:test";
+import { runAiTool } from "./aiTools";
+import { useEditor } from "./store";
+
+beforeEach(() => {
+ useEditor.setState({
+ clips: [], overlays: [], transitions: [], audioClips: [],
+ assets: [{ id: "voice", fileName: "voice.m4a", name: "Voice", type: "audio", duration: 10, url: "" },
+ { id: "music", fileName: "music.m4a", name: "Music", type: "audio", duration: 20, url: "" }],
+ });
+});
+
+test("assistant can place narration and music together on separate lanes", async () => {
+ await runAiTool("add_clip", { asset_id: "voice", start: 0, lane: 0 });
+ await runAiTool("add_clip", { asset_id: "music", start: 0, lane: 1 });
+ expect(useEditor.getState().audioClips.map((a) => [a.start, a.lane ?? 0])).toEqual([[0, 0], [0, 1]]);
+});
+
+test("assistant can repair music appended after narration in one edit", async () => {
+ await runAiTool("add_clip", { asset_id: "voice", start: 0 });
+ await runAiTool("add_clip", { asset_id: "music", start: 0 });
+ const music = useEditor.getState().audioClips.find((a) => a.assetId === "music")!;
+ expect(music.start).toBe(10);
+ await runAiTool("update_audio", { id: music.id, lane: 1, start: 0, volume: 0.15 });
+ const repaired = useEditor.getState().audioClips.find((a) => a.id === music.id)!;
+ expect([repaired.start, repaired.lane, repaired.volume]).toEqual([0, 1, 0.15]);
+});
+
+test("assistant refuses invalid audio lane indices", async () => {
+ for (const lane of [-1, 1.5, Infinity]) {
+ const error = await runAiTool("add_clip", { asset_id: "music", lane }).catch((cause: unknown) => cause);
+ expect(error instanceof Error && error.message.includes("non-negative integer")).toBe(true);
+ }
+ expect(useEditor.getState().audioClips).toHaveLength(0);
+});