Open source CapCut alternative.
TypeScript
feat(site): land scene-run clips through one placement primitive
David Le1950d065ec1c
diff --git a/site/src/cut/lib/genvideo/docWriter.ts b/site/src/cut/lib/genvideo/docWriter.tsindex 2f357a17..c759c370 100644--- a/site/src/cut/lib/genvideo/docWriter.ts+++ b/site/src/cut/lib/genvideo/docWriter.ts@@ -16,7 +16,7 @@import { apiJson, getBackend, type CutBackend } from "../backend";import { projectBackend } from "../residency";-import { RENDERS_CAP, storedAssets, useEditor } from "../store";+import { footprints, nextFreeStart, placeInRun, RENDERS_CAP, storedAssets, useEditor } from "../store";import { mediaUrl, SPEED_MIN } from "../types";import type { AudioClip, MediaAsset, ProjectDoc, RenderRecord, VideoClip } from "../types";import { fillSlot } from "./fillSlot";@@ -150,37 +150,55 @@ export async function findRunAsset(return stored ? { ...stored, url: mediaUrl(projectId, stored.fileName, backend) } : undefined;}-/** placeGenClip against a doc: fill [startSec, endSec) exactly on track 0,- * honoring the reviewer's source window. Muted only when asked (a provided-audio- * scene mutes its b-roll; a generated scene keeps the burned-in narration- * audible). Returns the new clip id. */+/** placeGenClip against a doc: fill a [startSec, endSec)-sized slot on track 0,+ * honoring the reviewer's source window, landing through the same `placeInRun`+ * primitive as the live store — after the run's earlier clips, never past its+ * later ones, slid clear of everything else. Muted only when asked (a+ * provided-audio scene mutes its b-roll; a generated scene keeps the burned-in+ * narration audible). Returns the new clip id. */export function docPlaceGenClip(doc: ProjectDoc,assetId: string,startSec: number,endSec: number,- srcInSec?: number,- muted = true+ opts?: { srcInSec?: number; muted?: boolean; anchorAfterIds?: string[]; followClipIds?: string[] }): string | null {const asset = doc.assets.find((a) => a.id === assetId);if (!asset || (asset.type !== "video" && asset.type !== "image")) return null;const slot = Math.max(MIN_LEN, endSec - startSec);const srcIn =asset.type === "video"- ? Math.min(Math.max(0, srcInSec ?? 0), Math.max(0, asset.duration - slot))+ ? Math.min(Math.max(0, opts?.srcInSec ?? 0), Math.max(0, asset.duration - slot)): 0;const { out, speed } = fillSlot(asset.type, Math.max(MIN_LEN, asset.duration - srcIn), slot, SPEED_MIN);+ const row = doc.clips.filter((c) => c.track === 0);+ const spans = row.map((c) => {+ const sp = c.speed && c.speed > 0 ? c.speed : 1;+ return { id: c.id, start: c.start, end: c.start + (c.out - c.in) / sp };+ });+ const anchorIds = new Set(opts?.anchorAfterIds ?? []);+ const prevEnd = spans.filter((s) => anchorIds.has(s.id)).reduce((m, s) => Math.max(m, s.end), -1);+ const len = speed !== undefined && speed > 0 ? out / speed : out;+ const { start, shifts } = placeInRun(+ spans,+ prevEnd >= 0 ? prevEnd : Math.max(0, startSec),+ Math.max(MIN_LEN, len),+ new Set(opts?.followClipIds ?? [])+ );+ const move = new Map(shifts.map((sh) => [sh.id, sh.start]));const clip: VideoClip = {id: uid(),assetId,track: 0,- start: Math.max(0, startSec),+ start,in: srcIn,out: srcIn + out,- muted,+ muted: opts?.muted ?? true,...(speed !== undefined ? { speed } : {}),};- doc.clips = [...doc.clips, clip].sort((a, b) => a.start - b.start);+ doc.clips = [...doc.clips.map((c) => (move.has(c.id) ? { ...c, start: move.get(c.id)! } : c)), clip].sort(+ (a, b) => a.start - b.start+ );return clip.id;}@@ -197,10 +215,13 @@ export function docPlaceGenAudio(if (!asset || asset.type !== "audio") return null;const out = Math.min(asset.duration, Math.max(MIN_LEN, durSec));const lane = opts?.lane ?? 0;+ // Same lane rule as the live store: slide to the next free slot, never land+ // on top of a sound already there.+ const taken = footprints((doc.audioClips ?? []).filter((a) => (a.lane ?? 0) === lane));const clip: AudioClip = {id: uid(),assetId,- start: Math.max(0, startSec),+ start: nextFreeStart(taken, Math.max(0, startSec), out),in: 0,out,volume: opts?.volume ?? 1,diff --git a/site/src/cut/lib/genvideo/editor.ts b/site/src/cut/lib/genvideo/editor.tsindex 89b14fa3..de566065 100644--- a/site/src/cut/lib/genvideo/editor.ts+++ b/site/src/cut/lib/genvideo/editor.ts@@ -28,18 +28,23 @@ export interface EditorBridge {/** Bring a generated media asset into the project pool; returns its media id. */importMedia(mediaId: string): Promise<string>;/**- * Place a video clip on the video track so it exactly fills- * [startFrame, endFrame). `srcInSec` starts the clip's source window there- * (the reviewer's chosen moment) instead of at 0. `muted` silences the clip's- * own audio — set for a provided-audio scene's b-roll, cleared for a generated- * scene whose shot carries its own burned-in narration (defaults to muted).+ * Place a video clip on the video track filling a [startFrame, endFrame)-+ * sized slot. Where it lands is the editor's call: the planned frames are the+ * target, and the anchors keep a moving timeline honest — the clip lands+ * after the furthest of `anchorAfterIds` (clips the run's earlier shots still+ * hold), never slides past any of `followClipIds` (clips its later shots+ * hold; those push right instead), and clears whatever else occupies the row.+ * `srcInSec` starts the clip's source window there (the reviewer's chosen+ * moment) instead of at 0. `muted` silences the clip's own audio — set for a+ * provided-audio scene's b-roll, cleared for a generated scene whose shot+ * carries its own burned-in narration (defaults to muted).* Returns the new timeline clip id.*/placeClip(mediaId: string,startFrame: number,endFrame: number,- opts?: { srcInSec?: number; muted?: boolean }+ opts?: { srcInSec?: number; muted?: boolean; anchorAfterIds?: string[]; followClipIds?: string[] }): Promise<string>;/** Swap the media under an existing clip (regeneration), keeping its slot. */replaceClipMedia(clipId: string, mediaId: string): Promise<void>;@@ -105,7 +110,7 @@ export class FakeEditor implements EditorBridge {mediaId: string,startFrame: number,endFrame: number,- opts?: { srcInSec?: number; muted?: boolean }+ opts?: { srcInSec?: number; muted?: boolean; anchorAfterIds?: string[]; followClipIds?: string[] }): Promise<string> {const clipId = `clip:${this.seq++}`;this.placed.push({diff --git a/site/src/cut/lib/genvideo/editorBridge.ts b/site/src/cut/lib/genvideo/editorBridge.tsindex c87aae2e..d3b5cd6b 100644--- a/site/src/cut/lib/genvideo/editorBridge.ts+++ b/site/src/cut/lib/genvideo/editorBridge.ts@@ -7,8 +7,9 @@* seconds, and every method here divides or multiplies by GEN_FPS exactly once.** Placement forwards to the store's id-returning gen actions (placeGenClip /- * placeGenAudio / removeClipById / removeAudioById), which place at an exact- * time without sliding and never touch the user's selection — so a run+ * placeGenAudio / removeClipById / removeAudioById), which land through the+ * store's run placement primitive — anchored to the run's own clips, slid+ * clear of everything else — and never touch the user's selection, so a run* populates the track in the background while the user keeps editing. A* provided-audio scene mutes its b-roll under the user's spine; a generated* scene leaves the shot's burned-in narration audible. `importMedia` is@@ -101,7 +102,7 @@ export class StoreEditorBridge implements EditorBridge {mediaId: string,startFrame: number,endFrame: number,- opts?: { srcInSec?: number; muted?: boolean }+ opts?: { srcInSec?: number; muted?: boolean; anchorAfterIds?: string[]; followClipIds?: string[] }): Promise<string> {if ((await this.mode()) === "store") {const id = useEditor.getState().placeGenClip(mediaId, toSec(startFrame), toSec(endFrame), opts);@@ -110,7 +111,7 @@ export class StoreEditorBridge implements EditorBridge {}let placed: string | null = null;await withProjectDoc(this.projectId, (doc) => {- placed = docPlaceGenClip(doc, mediaId, toSec(startFrame), toSec(endFrame), opts?.srcInSec, opts?.muted ?? true);+ placed = docPlaceGenClip(doc, mediaId, toSec(startFrame), toSec(endFrame), opts);});if (!placed) throw new Error(`placeClip: no placeable asset ${mediaId}`);return placed;diff --git a/site/src/cut/lib/genvideo/orchestrator.ts b/site/src/cut/lib/genvideo/orchestrator.tsindex 8c868a63..50ed5daf 100644--- a/site/src/cut/lib/genvideo/orchestrator.ts+++ b/site/src/cut/lib/genvideo/orchestrator.ts@@ -530,6 +530,7 @@ export class VideoOrchestrator {// Generated mode: the clip carries its own burned-in narration, so it// plays. Provided mode: mute the b-roll under the user's audio spine.muted: this.project.audioMode === "provided",+ ...this.placementAnchors(shot),});this.updateShot(shot, { status: "placed", error: undefined });this.emitProgress();@@ -598,7 +599,7 @@ export class VideoOrchestrator {// `still` is already an imported project media id; placing it directly// avoids re-importing an id the pool already holds.shot.clip = still;- shot.timelineClipId = await this.deps.editor.placeClip(still, shot.startFrame, shot.endFrame);+ shot.timelineClipId = await this.deps.editor.placeClip(still, shot.startFrame, shot.endFrame, this.placementAnchors(shot));} else {// Nothing to hold — keep whatever was there rather than opening a hole.this.deps.emit({ type: "error", message: `Shot ${shot.id} could not be generated or filled.` });@@ -634,6 +635,32 @@ export class VideoOrchestrator {return undefined;}+ /** The run's ordering anchors for one shot's landing: every clip an earlier+ * shot still holds on the track (the take lands after the furthest of them)+ * and every clip a later shot holds (the take never slides past those — they+ * push right instead). Renders finish out of order, so this — not the plan's+ * absolute frames — is what keeps shots assembling in story order on a+ * timeline the user may be editing while the run works. */+ private placementAnchors(shot: Shot): { anchorAfterIds?: string[]; followClipIds?: string[] } {+ const shots = this.project.shots;+ const idx = shots.findIndex((s) => s.id === shot.id);+ if (idx < 0) return {};+ const held = (s: Shot) =>+ [s.timelineClipId, ...(s.replacesClipIds ?? [])].filter((id): id is string => !!id);+ const anchorAfterIds = shots.slice(0, idx).flatMap(held);+ // A re-cut's replaced clip can span several fresh shots and so be held on+ // both sides; the earlier side wins — it anchors, it must not also push.+ const anchored = new Set(anchorAfterIds);+ const followClipIds = shots+ .slice(idx + 1)+ .flatMap(held)+ .filter((id) => !anchored.has(id));+ return {+ ...(anchorAfterIds.length ? { anchorAfterIds } : {}),+ ...(followClipIds.length ? { followClipIds } : {}),+ };+ }+private async clearVideoPlacement(shot: Shot): Promise<void> {if (shot.timelineClipId) {await this.deps.editor.removeClip(shot.timelineClipId);diff --git a/site/src/cut/lib/store.test.ts b/site/src/cut/lib/store.test.tsindex af30306e..1564221d 100644--- a/site/src/cut/lib/store.test.ts+++ b/site/src/cut/lib/store.test.ts@@ -1,5 +1,5 @@import { beforeEach, describe, expect, test } from "bun:test";-import { clipLen, useEditor } from "./store";+import { clipLen, placeInRun, useEditor } from "./store";import { emptySubtitles } from "./types";import type { AudioClip, MediaAsset, SubtitleCue, TextOverlay, VideoClip } from "./types";@@ -534,3 +534,67 @@ describe("last pick wins per edge", () => {expect(clipById(b.id).animOut).toEqual({ style: "fade", seconds: 0.5 });});});++describe("generated-scene placement (placeInRun)", () => {+ const span = (id: string, start: number, end: number) => ({ id, start, end });++ test("with no run neighbors it slides right past residents", () => {+ const { start, shifts } = placeInRun([span("u1", 0, 4)], 2, 3, new Set());+ expect(start).toBeCloseTo(4);+ expect(shifts).toEqual([]);+ });++ test("never slides past a later shot's clip — pushes it right instead", () => {+ const row = [span("u1", 0, 4), span("s2", 4, 8), span("u2", 8, 10)];+ const { start, shifts } = placeInRun(row, 0, 4, new Set(["s2"]));+ expect(start).toBeCloseTo(4); // slid clear of the resident, not past s2+ // s2 and everything after it open the slot as one run.+ expect(shifts).toEqual([+ { id: "s2", start: 8 },+ { id: "u2", start: 12 },+ ]);+ });++ test("out-of-order landings assemble gapless in shot order on an empty track", () => {+ const row: { id: string; start: number; end: number }[] = [];+ const p3 = placeInRun(row, 15, 5, new Set());+ row.push(span("s3", p3.start, p3.start + 5));+ const p1 = placeInRun(row, 0, 8, new Set(["s3"]));+ expect(p1.start).toBeCloseTo(0);+ expect(p1.shifts).toEqual([]);+ row.push(span("s1", 0, 8));+ const p2 = placeInRun(row, 8, 7, new Set(["s3"])); // want = live end of s1+ expect(p2.start).toBeCloseTo(8);+ expect(p2.shifts).toEqual([]);+ });++ test("placeGenClip anchors after the run's earlier clip when the plan's frames are stale", () => {+ const av = asset(4);+ const user = vclip({ start: 0, out: 4, assetId: "x" });+ const s1 = vclip({ start: 4, out: 4, assetId: "x" });+ useEditor.setState({ assets: [av], clips: [user, s1] });+ // Planned [4, 8) went stale when the user's clip pushed the run to 4..8.+ const id = s().placeGenClip(av.id, 4, 8, { anchorAfterIds: [s1.id] })!;+ expect(clipById(id).start).toBeCloseTo(8);+ expectLaneSound(videoLane(0));+ });++ test("placeGenClip pushes a later shot's clip instead of landing on or past it", () => {+ const av = asset(4);+ const user = vclip({ start: 0, out: 4, assetId: "x" });+ const s2 = vclip({ start: 4, out: 4, assetId: "x" });+ useEditor.setState({ assets: [av], clips: [user, s2] });+ const id = s().placeGenClip(av.id, 0, 4, { followClipIds: [s2.id] })!;+ expect(clipById(id).start).toBeCloseTo(4); // after the resident+ expect(clipById(s2.id).start).toBeCloseTo(8); // pushed, still after the new take+ expectLaneSound(videoLane(0));+ });++ test("placeGenAudio slides to its lane's next free slot", () => {+ const aa = asset(4, "audio");+ const resident = aclip({ start: 0, out: 4, assetId: "x" });+ useEditor.setState({ assets: [aa], audioClips: [resident] });+ const id = s().placeGenAudio(aa.id, 1, 4)!;+ expect(audioById(id).start).toBeCloseTo(4);+ });+});diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.tsindex de726e2f..ec7ff39f 100644--- a/site/src/cut/lib/store.ts+++ b/site/src/cut/lib/store.ts@@ -222,13 +222,16 @@ export interface EditorState {/** Set (or clear) the persisted brief-to-video run. Replaces the object by* reference so autosave detects the change. */setGenvideo: (project: VideoProject | undefined) => void;- /** Brief-to-video placement: place a generated clip so it fills exactly- * [startSec, endSec) on track 0 — exact start (no slide), muted, time-stretched- * or trimmed to the slot — and return its id. Leaves selection untouched (the- * run is a background process). */- placeGenClip: (assetId: string, startSec: number, endSec: number, opts?: { srcInSec?: number; muted?: boolean }) => string | null;- /** Brief-to-video placement: place a generated audio clip at startSec spanning- * up to durSec on the soundtrack (duck/lane/volume optional), returning its id. */+ /** Brief-to-video placement: place a generated clip filling a [startSec,+ * endSec)-sized slot on track 0 — muted, time-stretched or trimmed to the+ * slot — and return its id. Where it lands goes through `placeInRun`:+ * anchored after the run's earlier clips, never past its later ones, slid+ * clear of everything else. Leaves selection untouched (the run is a+ * background process). */+ placeGenClip: (assetId: string, startSec: number, endSec: number, opts?: { srcInSec?: number; muted?: boolean; anchorAfterIds?: string[]; followClipIds?: string[] }) => string | null;+ /** Brief-to-video placement: place a generated audio clip at the next free+ * slot at/after startSec on its soundtrack lane, spanning up to durSec+ * (duck/lane/volume optional), returning its id. */placeGenAudio: (assetId: string, startSec: number, durSec: number, opts?: { duck?: number; lane?: number; volume?: number }) => string | null;/** Remove a video clip by id (a background gen swap; leaves its slot empty). */removeClipById: (id: string) => void;@@ -641,6 +644,42 @@ export function footprints(items: (VideoClip | AudioClip)[]): { start: number; ereturn items.map((c) => ({ start: c.start, end: c.start + clipLen(c) }));}+/** Where a scene run's clip lands on a lane that may have moved since the+ * plan. `want` is the target spot (the plan's, or the live end of the run's+ * previous shot). Free residents slide the landing right exactly like+ * `nextFreeStart`; clips in `follow` — the run's own later shots — are never+ * slid past, because that would reorder the story: the landing pushes them+ * (and everything at or after them) right as one run instead. So takes that+ * render out of order still assemble in shot order, whatever the user moved+ * meanwhile. The one placement primitive for generated-scene clips, shared by+ * the live store and the background doc writer. */+export function placeInRun(+ row: { id: string; start: number; end: number }[],+ want: number,+ len: number,+ follow: ReadonlySet<string>+): { start: number; shifts: { id: string; start: number }[] } {+ const sorted = [...row].sort((a, b) => a.start - b.start);+ const followStart = sorted+ .filter((s) => follow.has(s.id))+ .reduce((m, s) => Math.min(m, s.start), Infinity);+ let at = Math.max(0, want);+ for (const sp of sorted) {+ if (sp.start >= followStart - 1e-6) break; // the push below clears these+ if (sp.end <= at + 1e-3) continue;+ if (sp.start >= at + len - 1e-3) break;+ at = sp.end;+ }+ const delta = at + len - followStart;+ const shifts =+ Number.isFinite(followStart) && delta > 1e-9+ ? sorted+ .filter((s) => s.start >= followStart - 1e-6)+ .map((s) => ({ id: s.id, start: s.start + delta }))+ : [];+ return { start: at, shifts };+}+/** Where a `len`-long clip dropped at pointer-time `t` lands on its row,* and how the clips after it slide to open room. The drop-at-pointer companion* to `nextFreeStart` (which only ever appends): clips whose center sits left of@@ -1171,11 +1210,28 @@ export const useEditor = create<EditorState>((baseSet, get) => {slot,SPEED_MIN);+ // Land through the run placement primitive against the live row: after+ // the furthest clip the run's earlier shots still hold, never past a+ // later shot's clip, slid clear of everything else on the track.+ const row = track0Clips(get().clips);+ const spans = row.map((c) => ({ id: c.id, start: c.start, end: c.start + clipLen(c) }));+ const anchorIds = new Set(opts?.anchorAfterIds ?? []);+ const prevEnd = spans+ .filter((s) => anchorIds.has(s.id))+ .reduce((m, s) => Math.max(m, s.end), -1);+ const len = speed !== undefined && speed > 0 ? out / speed : out;+ const { start, shifts } = placeInRun(+ spans,+ prevEnd >= 0 ? prevEnd : Math.max(0, startSec),+ Math.max(MIN_LEN, len),+ new Set(opts?.followClipIds ?? [])+ );+ const move = new Map(shifts.map((sh) => [sh.id, sh.start]));const clip: VideoClip = {id: uid(),assetId,track: 0,- start: Math.max(0, startSec),+ start,in: srcIn,out: srcIn + out,// Muted only when the caller asks — a provided-audio scene mutes its@@ -1188,7 +1244,11 @@ export const useEditor = create<EditorState>((baseSet, get) => {// the orchestrator manages this clip (it swaps clips idempotently), so a// mid-render Cmd+Z must not pull a shot out from under the run.genClipIds.add(clip.id);- set((s) => ({ clips: [...s.clips, clip].sort((a, b) => a.start - b.start) }));+ set((s) => ({+ clips: [...s.clips.map((c) => (move.has(c.id) ? { ...c, start: move.get(c.id)! } : c)), clip].sort(+ (a, b) => a.start - b.start+ ),+ }));return clip.id;},@@ -1197,10 +1257,13 @@ export const useEditor = create<EditorState>((baseSet, get) => {if (!asset || asset.type !== "audio") return null;const out = Math.min(asset.duration, Math.max(MIN_LEN, durSec));const lane = opts?.lane ?? 0;+ // Slide to the lane's next free slot so a background run never lands a+ // bed or voiceover on top of a sound the user placed meanwhile.+ const taken = footprints(get().audioClips.filter((a) => (a.lane ?? 0) === lane));const clip: AudioClip = {id: uid(),assetId,- start: Math.max(0, startSec),+ start: nextFreeStart(taken, Math.max(0, startSec), out),in: 0,out,volume: opts?.volume ?? 1,@@ -1357,8 +1420,18 @@ export const useEditor = create<EditorState>((baseSet, get) => {const st = get();const gone = st.assets.find((a) => a.id === id);if (!gone) return;- // The media file dies with its last referencing asset — history- // snapshots don't cover the asset list, so nothing can resurrect it.+ // The media file dies with its last referencing asset, so no undo may+ // bring a clip of it back to point at bytes that no longer exist: take+ // this asset's clips out of every snapshot that holds them. Deleting an+ // asset is not itself undoable, and the asset list is not in a snapshot,+ // so nothing can resurrect either the clips or the file.+ const scrub = (snap: DocSnapshot) => {+ snap.clips = snap.clips.filter((c) => c.assetId !== id);+ snap.audioClips = snap.audioClips.filter((c) => c.assetId !== id);+ };+ for (const snap of history) scrub(snap);+ for (const snap of future) scrub(snap);+ if (pending) scrub(pending.snap);const dropFile = () => {const s = get();if (!s.projectId || s.assets.some((a) => a.fileName === gone.fileName)) return;@@ -1379,7 +1452,6 @@ export const useEditor = create<EditorState>((baseSet, get) => {dropFile();return;}- push();// Cascade removes this asset's clips; every clip is free-positioned, so// the rest of the timeline (and its annotations) stays where it is.set((s) => {