Open source CapCut alternative.
TypeScript
feat(site): timeline tool results carry the updated rows [rebuild]
David Led26f21b8bd19
diff --git a/site/src/cut/components/Timeline.tools.ts b/site/src/cut/components/Timeline.tools.tsindex d1048994..d18bc3b4 100644--- a/site/src/cut/components/Timeline.tools.ts+++ b/site/src/cut/components/Timeline.tools.ts@@ -21,7 +21,7 @@ export const TIMELINE_TOOLS = [{name: "split_at",description:- "Split the video (or a selected soundtrack/overlay clip) at a time, like pressing S. Omit t to split at the playhead.",+ "Split the video (or a selected soundtrack/overlay clip) at a time, like pressing S. Omit t to split at the playhead. Splits don't move times, so issue every planned split together in one round. The result carries the updated track-0 and soundtrack rows (ids, starts, lengths) — read the new ids from there.",inputSchema: obj({ t: num("Timeline seconds to cut at (optional)") }),},{diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 0c6da6f5..4e02859b 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -112,6 +112,31 @@ import {const round2 = (n: number) => Math.round(n * 100) / 100;+/** The timeline rows a mutation touched, small enough to ride every result:+ * ids, starts, and lengths for track 0 and the soundtrack, read fresh after+ * the store settled. The model keeps editing from these — new ids after a+ * split, closed-up starts after a delete — with no get_state poll between+ * steps. */+function tracksAfter() {+ const cur = useEditor.getState();+ const row = track0Clips(cur.clips).map((c) => ({+ id: c.id,+ start: round2(c.start),+ len: round2((c.out - c.in) / (c.speed && c.speed > 0 ? c.speed : 1)),+ }));+ const lanes = cur.audioClips.map((a) => ({+ id: a.id,+ start: round2(a.start),+ len: round2(a.out - a.in),+ }));+ return {+ track0: row.slice(0, 60),+ ...(row.length > 60 ? { track0Truncated: true } : {}),+ soundtrack: lanes.slice(0, 60),+ ...(lanes.length > 60 ? { soundtrackTruncated: true } : {}),+ };+}+/** The model crosses the transition and animation vocabularies — the retired* edge styles (fadein/zoomin/…) especially. Style ids arrive as structured* tool input, so resolve each known synonym to the nearest supported style. */@@ -600,7 +625,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {const after = useEditor.getState();const made = after.clips.length + after.audioClips.length - before;if (made === 0) throw new ToolError("Nothing to split at that time.");- return { split: true, videoClips: after.clips.length };+ return { split: true, ...tracksAfter() };},move_clip: (s, input) => {@@ -614,7 +639,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {if (!isNum(input.toIndex)) throw new ToolError("toIndex is required.");const row = track0Clips(s.clips);s.moveClip(clip.id, clamp(Math.round(input.toIndex), 0, row.length - 1));- return { order: track0Clips(useEditor.getState().clips).map((c) => c.id) };+ return { moved: clip.id, ...tracksAfter() };},place_clip: (s, input) => {@@ -636,6 +661,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {...(Math.abs(at - Math.max(0, input.start)) > 0.005? { note: "That spot was taken — slid right to the next free one." }: {}),+ ...tracksAfter(),};},@@ -729,7 +755,12 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {// Trim through the store's resize path so track 0 keeps its no-overlap// invariant: extending a clip pushes the following run right.s.setClipTrim(clip.id, nextIn, nextOut);- return { in: nextIn, out: nextOut, len: Math.round((nextOut - nextIn) * 100) / 100 };+ return {+ in: nextIn,+ out: nextOut,+ len: Math.round((nextOut - nextIn) * 100) / 100,+ ...tracksAfter(),+ };},refine_speech_cuts: async (s, input) => {@@ -862,6 +893,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {};}),note: `Each moved edge sits ~${BREATH}s inside a real pause; clip spacing is preserved. Listen at any flagged edge.`,+ ...tracksAfter(),};},@@ -959,7 +991,10 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {kind === "overlayClip" ? "clip" : kind === "text" ? "overlay" : (kind as "clip" | "audio" | "overlay");s.select({ kind: selKind, id });s.deleteSelection();- return { deleted: { kind: selKind, id } };+ return {+ deleted: { kind: selKind, id },+ ...(selKind === "overlay" ? {} : tracksAfter()),+ };},remove_gap: (s, input) => {@@ -969,7 +1004,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {const gap = laneGapAt(s, lane, input.at);if (!gap) throw new ToolError(`No track-${track} gap at ${input.at}s — pass a time inside the empty span.`);s.removeLaneGap(lane, input.at);- return { closed: { track, ...gap } };+ return { closed: { track, ...gap }, ...tracksAfter() };},add_title: (s, input) => {@@ -1734,12 +1769,18 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {const cur = useEditor.getState();if (cur.subtitleStatus === "error")throw new ToolError(cur.subtitleError ?? "Transcription failed.");- const made = laneCues(cur.subtitles, lane).length;- if (made > 0 && cur.timelineH < 276) cur.setTimelineH(276);+ const made = laneCues(cur.subtitles, lane);+ if (made.length > 0 && cur.timelineH < 276) cur.setTimelineH(276);+ // The cues ride the result — the model's next move is usually editing+ // them, and the ids are what update_cue/delete_cue/merge_cue take.return {status: cur.subtitleStatus,track: lane,- cues: made,+ count: made.length,+ cues: made+ .slice(0, 60)+ .map((q) => ({ id: q.id, start: round2(q.start), end: round2(q.end), text: q.text })),+ ...(made.length > 60 ? { cuesTruncated: true } : {}),note:cur.subtitleStatus === "empty"? "No speech found — no subtitles were added to the video."diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex b90b8c55..b1364012 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -96,7 +96,7 @@ The flow for "edit this for me":2. Speech? subtitles_generate first — cue timings are timeline seconds and say what is said when.3. watch_video each distinct source (or the ranges in question); note scene changes and what happens where.4. Plan in thoughts: group the cues into sentences and thoughts, decide which thoughts stay whole, then detect_silence to find where the speaker actually pauses. Cue timings approximate the audio — a trailing word often spills past its cue — so aim every speech cut into a detected silence span, and aim loose: up to a second of spare air on each side beats a tight cut, because the refine step trues the edges after. A cue boundary with no silence span around it means the speech runs continuously there, so leave that joint alone.-5. Execute: split_at (timeline s) to divide, delete_item to drop a segment, trim_clip (source s) to tighten edges, place_clip/move_clip to close gaps or reorder, set_speed to compress slow stretches.+5. Execute: split_at (timeline s) to divide, delete_item to drop a segment, trim_clip (source s) to tighten edges, place_clip/move_clip to close gaps or reorder, set_speed to compress slow stretches. Batch each wave in one round: splits don't move times, so issue every planned split_at together, then every delete together. Every timeline mutation returns the updated track-0 and soundtrack rows (ids, starts, lengths) — read the new ids from the result and keep cutting; get_state between steps is a wasted round.6. Refine: refine_speech_cuts with every recut clip id. It re-scans the audio at each edge and re-trims it to sit a ~0.25s breath inside the real pause, keeping the clips' spacing. It reads sound, not intent — leave out a clip whose edge you placed mid-speech on purpose.7. Verify by ear: listen_audio at each edge the refine flagged, plus a spot-check or two (scope from/to to ~2s around the joint), and check the first and last words arrive whole and the hand-off into the next thought sounds natural. A clipped word → widen that trim deeper into the silence and listen again. After a heavy edit, listen once end-to-end. seek + capture_frame when the change is visual.