Skip to content

Open source CapCut alternative.

TypeScript

feat(site): timeline tool results carry the updated rows [rebuild]

David Led26f21b8bd19
diff --git a/site/src/cut/components/Timeline.tools.ts b/site/src/cut/components/Timeline.tools.ts
index d1048994..d18bc3b4 100644
--- a/site/src/cut/components/Timeline.tools.ts
+++ b/site/src/cut/components/Timeline.tools.ts
@@ -21,7 +21,7 @@ export const TIMELINE_TOOLS = [
{
name: "split_at",
description:
- "Split the video (or a selected soundtrack/overlay clip) at a time, like pressing S. Omit t to split at the playhead.",
+ "Split the video (or a selected soundtrack/overlay clip) at a time, like pressing S. Omit t to split at the playhead. Splits don't move times, so issue every planned split together in one round. The result carries the updated track-0 and soundtrack rows (ids, starts, lengths) — read the new ids from there.",
inputSchema: obj({ t: num("Timeline seconds to cut at (optional)") }),
},
{
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 0c6da6f5..4e02859b 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -112,6 +112,31 @@ import {
const round2 = (n: number) => Math.round(n * 100) / 100;
+/** The timeline rows a mutation touched, small enough to ride every result:
+ * ids, starts, and lengths for track 0 and the soundtrack, read fresh after
+ * the store settled. The model keeps editing from these — new ids after a
+ * split, closed-up starts after a delete — with no get_state poll between
+ * steps. */
+function tracksAfter() {
+ const cur = useEditor.getState();
+ const row = track0Clips(cur.clips).map((c) => ({
+ id: c.id,
+ start: round2(c.start),
+ len: round2((c.out - c.in) / (c.speed && c.speed > 0 ? c.speed : 1)),
+ }));
+ const lanes = cur.audioClips.map((a) => ({
+ id: a.id,
+ start: round2(a.start),
+ len: round2(a.out - a.in),
+ }));
+ return {
+ track0: row.slice(0, 60),
+ ...(row.length > 60 ? { track0Truncated: true } : {}),
+ soundtrack: lanes.slice(0, 60),
+ ...(lanes.length > 60 ? { soundtrackTruncated: true } : {}),
+ };
+}
+
/** The model crosses the transition and animation vocabularies — the retired
* edge styles (fadein/zoomin/…) especially. Style ids arrive as structured
* tool input, so resolve each known synonym to the nearest supported style. */
@@ -600,7 +625,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
const after = useEditor.getState();
const made = after.clips.length + after.audioClips.length - before;
if (made === 0) throw new ToolError("Nothing to split at that time.");
- return { split: true, videoClips: after.clips.length };
+ return { split: true, ...tracksAfter() };
},
move_clip: (s, input) => {
@@ -614,7 +639,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
if (!isNum(input.toIndex)) throw new ToolError("toIndex is required.");
const row = track0Clips(s.clips);
s.moveClip(clip.id, clamp(Math.round(input.toIndex), 0, row.length - 1));
- return { order: track0Clips(useEditor.getState().clips).map((c) => c.id) };
+ return { moved: clip.id, ...tracksAfter() };
},
place_clip: (s, input) => {
@@ -636,6 +661,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
...(Math.abs(at - Math.max(0, input.start)) > 0.005
? { note: "That spot was taken — slid right to the next free one." }
: {}),
+ ...tracksAfter(),
};
},
@@ -729,7 +755,12 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
// Trim through the store's resize path so track 0 keeps its no-overlap
// invariant: extending a clip pushes the following run right.
s.setClipTrim(clip.id, nextIn, nextOut);
- return { in: nextIn, out: nextOut, len: Math.round((nextOut - nextIn) * 100) / 100 };
+ return {
+ in: nextIn,
+ out: nextOut,
+ len: Math.round((nextOut - nextIn) * 100) / 100,
+ ...tracksAfter(),
+ };
},
refine_speech_cuts: async (s, input) => {
@@ -862,6 +893,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
};
}),
note: `Each moved edge sits ~${BREATH}s inside a real pause; clip spacing is preserved. Listen at any flagged edge.`,
+ ...tracksAfter(),
};
},
@@ -959,7 +991,10 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
kind === "overlayClip" ? "clip" : kind === "text" ? "overlay" : (kind as "clip" | "audio" | "overlay");
s.select({ kind: selKind, id });
s.deleteSelection();
- return { deleted: { kind: selKind, id } };
+ return {
+ deleted: { kind: selKind, id },
+ ...(selKind === "overlay" ? {} : tracksAfter()),
+ };
},
remove_gap: (s, input) => {
@@ -969,7 +1004,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
const gap = laneGapAt(s, lane, input.at);
if (!gap) throw new ToolError(`No track-${track} gap at ${input.at}s — pass a time inside the empty span.`);
s.removeLaneGap(lane, input.at);
- return { closed: { track, ...gap } };
+ return { closed: { track, ...gap }, ...tracksAfter() };
},
add_title: (s, input) => {
@@ -1734,12 +1769,18 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
const cur = useEditor.getState();
if (cur.subtitleStatus === "error")
throw new ToolError(cur.subtitleError ?? "Transcription failed.");
- const made = laneCues(cur.subtitles, lane).length;
- if (made > 0 && cur.timelineH < 276) cur.setTimelineH(276);
+ const made = laneCues(cur.subtitles, lane);
+ if (made.length > 0 && cur.timelineH < 276) cur.setTimelineH(276);
+ // The cues ride the result — the model's next move is usually editing
+ // them, and the ids are what update_cue/delete_cue/merge_cue take.
return {
status: cur.subtitleStatus,
track: lane,
- cues: made,
+ count: made.length,
+ cues: made
+ .slice(0, 60)
+ .map((q) => ({ id: q.id, start: round2(q.start), end: round2(q.end), text: q.text })),
+ ...(made.length > 60 ? { cuesTruncated: true } : {}),
note:
cur.subtitleStatus === "empty"
? "No speech found — no subtitles were added to the video."
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index b90b8c55..b1364012 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -96,7 +96,7 @@ The flow for "edit this for me":
2. Speech? subtitles_generate first — cue timings are timeline seconds and say what is said when.
3. watch_video each distinct source (or the ranges in question); note scene changes and what happens where.
4. Plan in thoughts: group the cues into sentences and thoughts, decide which thoughts stay whole, then detect_silence to find where the speaker actually pauses. Cue timings approximate the audio — a trailing word often spills past its cue — so aim every speech cut into a detected silence span, and aim loose: up to a second of spare air on each side beats a tight cut, because the refine step trues the edges after. A cue boundary with no silence span around it means the speech runs continuously there, so leave that joint alone.
-5. Execute: split_at (timeline s) to divide, delete_item to drop a segment, trim_clip (source s) to tighten edges, place_clip/move_clip to close gaps or reorder, set_speed to compress slow stretches.
+5. Execute: split_at (timeline s) to divide, delete_item to drop a segment, trim_clip (source s) to tighten edges, place_clip/move_clip to close gaps or reorder, set_speed to compress slow stretches. Batch each wave in one round: splits don't move times, so issue every planned split_at together, then every delete together. Every timeline mutation returns the updated track-0 and soundtrack rows (ids, starts, lengths) — read the new ids from the result and keep cutting; get_state between steps is a wasted round.
6. Refine: refine_speech_cuts with every recut clip id. It re-scans the audio at each edge and re-trims it to sit a ~0.25s breath inside the real pause, keeping the clips' spacing. It reads sound, not intent — leave out a clip whose edge you placed mid-speech on purpose.
7. Verify by ear: listen_audio at each edge the refine flagged, plus a spot-check or two (scope from/to to ~2s around the joint), and check the first and last words arrive whole and the hand-off into the next thought sounds natural. A clipped word → widen that trim deeper into the silence and listen again. After a heavy edit, listen once end-to-end. seek + capture_frame when the change is visual.