Skip to content

Open source CapCut alternative.

TypeScript

feat(site): land scene-run clips through one placement primitive

David Le1950d065ec1c
diff --git a/site/src/cut/lib/genvideo/docWriter.ts b/site/src/cut/lib/genvideo/docWriter.ts
index 2f357a17..c759c370 100644
--- a/site/src/cut/lib/genvideo/docWriter.ts
+++ b/site/src/cut/lib/genvideo/docWriter.ts
@@ -16,7 +16,7 @@
import { apiJson, getBackend, type CutBackend } from "../backend";
import { projectBackend } from "../residency";
-import { RENDERS_CAP, storedAssets, useEditor } from "../store";
+import { footprints, nextFreeStart, placeInRun, RENDERS_CAP, storedAssets, useEditor } from "../store";
import { mediaUrl, SPEED_MIN } from "../types";
import type { AudioClip, MediaAsset, ProjectDoc, RenderRecord, VideoClip } from "../types";
import { fillSlot } from "./fillSlot";
@@ -150,37 +150,55 @@ export async function findRunAsset(
return stored ? { ...stored, url: mediaUrl(projectId, stored.fileName, backend) } : undefined;
}
-/** placeGenClip against a doc: fill [startSec, endSec) exactly on track 0,
- * honoring the reviewer's source window. Muted only when asked (a provided-audio
- * scene mutes its b-roll; a generated scene keeps the burned-in narration
- * audible). Returns the new clip id. */
+/** placeGenClip against a doc: fill a [startSec, endSec)-sized slot on track 0,
+ * honoring the reviewer's source window, landing through the same `placeInRun`
+ * primitive as the live store — after the run's earlier clips, never past its
+ * later ones, slid clear of everything else. Muted only when asked (a
+ * provided-audio scene mutes its b-roll; a generated scene keeps the burned-in
+ * narration audible). Returns the new clip id. */
export function docPlaceGenClip(
doc: ProjectDoc,
assetId: string,
startSec: number,
endSec: number,
- srcInSec?: number,
- muted = true
+ opts?: { srcInSec?: number; muted?: boolean; anchorAfterIds?: string[]; followClipIds?: string[] }
): string | null {
const asset = doc.assets.find((a) => a.id === assetId);
if (!asset || (asset.type !== "video" && asset.type !== "image")) return null;
const slot = Math.max(MIN_LEN, endSec - startSec);
const srcIn =
asset.type === "video"
- ? Math.min(Math.max(0, srcInSec ?? 0), Math.max(0, asset.duration - slot))
+ ? Math.min(Math.max(0, opts?.srcInSec ?? 0), Math.max(0, asset.duration - slot))
: 0;
const { out, speed } = fillSlot(asset.type, Math.max(MIN_LEN, asset.duration - srcIn), slot, SPEED_MIN);
+ const row = doc.clips.filter((c) => c.track === 0);
+ const spans = row.map((c) => {
+ const sp = c.speed && c.speed > 0 ? c.speed : 1;
+ return { id: c.id, start: c.start, end: c.start + (c.out - c.in) / sp };
+ });
+ const anchorIds = new Set(opts?.anchorAfterIds ?? []);
+ const prevEnd = spans.filter((s) => anchorIds.has(s.id)).reduce((m, s) => Math.max(m, s.end), -1);
+ const len = speed !== undefined && speed > 0 ? out / speed : out;
+ const { start, shifts } = placeInRun(
+ spans,
+ prevEnd >= 0 ? prevEnd : Math.max(0, startSec),
+ Math.max(MIN_LEN, len),
+ new Set(opts?.followClipIds ?? [])
+ );
+ const move = new Map(shifts.map((sh) => [sh.id, sh.start]));
const clip: VideoClip = {
id: uid(),
assetId,
track: 0,
- start: Math.max(0, startSec),
+ start,
in: srcIn,
out: srcIn + out,
- muted,
+ muted: opts?.muted ?? true,
...(speed !== undefined ? { speed } : {}),
};
- doc.clips = [...doc.clips, clip].sort((a, b) => a.start - b.start);
+ doc.clips = [...doc.clips.map((c) => (move.has(c.id) ? { ...c, start: move.get(c.id)! } : c)), clip].sort(
+ (a, b) => a.start - b.start
+ );
return clip.id;
}
@@ -197,10 +215,13 @@ export function docPlaceGenAudio(
if (!asset || asset.type !== "audio") return null;
const out = Math.min(asset.duration, Math.max(MIN_LEN, durSec));
const lane = opts?.lane ?? 0;
+ // Same lane rule as the live store: slide to the next free slot, never land
+ // on top of a sound already there.
+ const taken = footprints((doc.audioClips ?? []).filter((a) => (a.lane ?? 0) === lane));
const clip: AudioClip = {
id: uid(),
assetId,
- start: Math.max(0, startSec),
+ start: nextFreeStart(taken, Math.max(0, startSec), out),
in: 0,
out,
volume: opts?.volume ?? 1,
diff --git a/site/src/cut/lib/genvideo/editor.ts b/site/src/cut/lib/genvideo/editor.ts
index 89b14fa3..de566065 100644
--- a/site/src/cut/lib/genvideo/editor.ts
+++ b/site/src/cut/lib/genvideo/editor.ts
@@ -28,18 +28,23 @@ export interface EditorBridge {
/** Bring a generated media asset into the project pool; returns its media id. */
importMedia(mediaId: string): Promise<string>;
/**
- * Place a video clip on the video track so it exactly fills
- * [startFrame, endFrame). `srcInSec` starts the clip's source window there
- * (the reviewer's chosen moment) instead of at 0. `muted` silences the clip's
- * own audio — set for a provided-audio scene's b-roll, cleared for a generated
- * scene whose shot carries its own burned-in narration (defaults to muted).
+ * Place a video clip on the video track filling a [startFrame, endFrame)-
+ * sized slot. Where it lands is the editor's call: the planned frames are the
+ * target, and the anchors keep a moving timeline honest — the clip lands
+ * after the furthest of `anchorAfterIds` (clips the run's earlier shots still
+ * hold), never slides past any of `followClipIds` (clips its later shots
+ * hold; those push right instead), and clears whatever else occupies the row.
+ * `srcInSec` starts the clip's source window there (the reviewer's chosen
+ * moment) instead of at 0. `muted` silences the clip's own audio — set for a
+ * provided-audio scene's b-roll, cleared for a generated scene whose shot
+ * carries its own burned-in narration (defaults to muted).
* Returns the new timeline clip id.
*/
placeClip(
mediaId: string,
startFrame: number,
endFrame: number,
- opts?: { srcInSec?: number; muted?: boolean }
+ opts?: { srcInSec?: number; muted?: boolean; anchorAfterIds?: string[]; followClipIds?: string[] }
): Promise<string>;
/** Swap the media under an existing clip (regeneration), keeping its slot. */
replaceClipMedia(clipId: string, mediaId: string): Promise<void>;
@@ -105,7 +110,7 @@ export class FakeEditor implements EditorBridge {
mediaId: string,
startFrame: number,
endFrame: number,
- opts?: { srcInSec?: number; muted?: boolean }
+ opts?: { srcInSec?: number; muted?: boolean; anchorAfterIds?: string[]; followClipIds?: string[] }
): Promise<string> {
const clipId = `clip:${this.seq++}`;
this.placed.push({
diff --git a/site/src/cut/lib/genvideo/editorBridge.ts b/site/src/cut/lib/genvideo/editorBridge.ts
index c87aae2e..d3b5cd6b 100644
--- a/site/src/cut/lib/genvideo/editorBridge.ts
+++ b/site/src/cut/lib/genvideo/editorBridge.ts
@@ -7,8 +7,9 @@
* seconds, and every method here divides or multiplies by GEN_FPS exactly once.
*
* Placement forwards to the store's id-returning gen actions (placeGenClip /
- * placeGenAudio / removeClipById / removeAudioById), which place at an exact
- * time without sliding and never touch the user's selection — so a run
+ * placeGenAudio / removeClipById / removeAudioById), which land through the
+ * store's run placement primitive — anchored to the run's own clips, slid
+ * clear of everything else — and never touch the user's selection, so a run
* populates the track in the background while the user keeps editing. A
* provided-audio scene mutes its b-roll under the user's spine; a generated
* scene leaves the shot's burned-in narration audible. `importMedia` is
@@ -101,7 +102,7 @@ export class StoreEditorBridge implements EditorBridge {
mediaId: string,
startFrame: number,
endFrame: number,
- opts?: { srcInSec?: number; muted?: boolean }
+ opts?: { srcInSec?: number; muted?: boolean; anchorAfterIds?: string[]; followClipIds?: string[] }
): Promise<string> {
if ((await this.mode()) === "store") {
const id = useEditor.getState().placeGenClip(mediaId, toSec(startFrame), toSec(endFrame), opts);
@@ -110,7 +111,7 @@ export class StoreEditorBridge implements EditorBridge {
}
let placed: string | null = null;
await withProjectDoc(this.projectId, (doc) => {
- placed = docPlaceGenClip(doc, mediaId, toSec(startFrame), toSec(endFrame), opts?.srcInSec, opts?.muted ?? true);
+ placed = docPlaceGenClip(doc, mediaId, toSec(startFrame), toSec(endFrame), opts);
});
if (!placed) throw new Error(`placeClip: no placeable asset ${mediaId}`);
return placed;
diff --git a/site/src/cut/lib/genvideo/orchestrator.ts b/site/src/cut/lib/genvideo/orchestrator.ts
index 8c868a63..50ed5daf 100644
--- a/site/src/cut/lib/genvideo/orchestrator.ts
+++ b/site/src/cut/lib/genvideo/orchestrator.ts
@@ -530,6 +530,7 @@ export class VideoOrchestrator {
// Generated mode: the clip carries its own burned-in narration, so it
// plays. Provided mode: mute the b-roll under the user's audio spine.
muted: this.project.audioMode === "provided",
+ ...this.placementAnchors(shot),
});
this.updateShot(shot, { status: "placed", error: undefined });
this.emitProgress();
@@ -598,7 +599,7 @@ export class VideoOrchestrator {
// `still` is already an imported project media id; placing it directly
// avoids re-importing an id the pool already holds.
shot.clip = still;
- shot.timelineClipId = await this.deps.editor.placeClip(still, shot.startFrame, shot.endFrame);
+ shot.timelineClipId = await this.deps.editor.placeClip(still, shot.startFrame, shot.endFrame, this.placementAnchors(shot));
} else {
// Nothing to hold — keep whatever was there rather than opening a hole.
this.deps.emit({ type: "error", message: `Shot ${shot.id} could not be generated or filled.` });
@@ -634,6 +635,32 @@ export class VideoOrchestrator {
return undefined;
}
+ /** The run's ordering anchors for one shot's landing: every clip an earlier
+ * shot still holds on the track (the take lands after the furthest of them)
+ * and every clip a later shot holds (the take never slides past those — they
+ * push right instead). Renders finish out of order, so this — not the plan's
+ * absolute frames — is what keeps shots assembling in story order on a
+ * timeline the user may be editing while the run works. */
+ private placementAnchors(shot: Shot): { anchorAfterIds?: string[]; followClipIds?: string[] } {
+ const shots = this.project.shots;
+ const idx = shots.findIndex((s) => s.id === shot.id);
+ if (idx < 0) return {};
+ const held = (s: Shot) =>
+ [s.timelineClipId, ...(s.replacesClipIds ?? [])].filter((id): id is string => !!id);
+ const anchorAfterIds = shots.slice(0, idx).flatMap(held);
+ // A re-cut's replaced clip can span several fresh shots and so be held on
+ // both sides; the earlier side wins — it anchors, it must not also push.
+ const anchored = new Set(anchorAfterIds);
+ const followClipIds = shots
+ .slice(idx + 1)
+ .flatMap(held)
+ .filter((id) => !anchored.has(id));
+ return {
+ ...(anchorAfterIds.length ? { anchorAfterIds } : {}),
+ ...(followClipIds.length ? { followClipIds } : {}),
+ };
+ }
+
private async clearVideoPlacement(shot: Shot): Promise<void> {
if (shot.timelineClipId) {
await this.deps.editor.removeClip(shot.timelineClipId);
diff --git a/site/src/cut/lib/store.test.ts b/site/src/cut/lib/store.test.ts
index af30306e..1564221d 100644
--- a/site/src/cut/lib/store.test.ts
+++ b/site/src/cut/lib/store.test.ts
@@ -1,5 +1,5 @@
import { beforeEach, describe, expect, test } from "bun:test";
-import { clipLen, useEditor } from "./store";
+import { clipLen, placeInRun, useEditor } from "./store";
import { emptySubtitles } from "./types";
import type { AudioClip, MediaAsset, SubtitleCue, TextOverlay, VideoClip } from "./types";
@@ -534,3 +534,67 @@ describe("last pick wins per edge", () => {
expect(clipById(b.id).animOut).toEqual({ style: "fade", seconds: 0.5 });
});
});
+
+describe("generated-scene placement (placeInRun)", () => {
+ const span = (id: string, start: number, end: number) => ({ id, start, end });
+
+ test("with no run neighbors it slides right past residents", () => {
+ const { start, shifts } = placeInRun([span("u1", 0, 4)], 2, 3, new Set());
+ expect(start).toBeCloseTo(4);
+ expect(shifts).toEqual([]);
+ });
+
+ test("never slides past a later shot's clip — pushes it right instead", () => {
+ const row = [span("u1", 0, 4), span("s2", 4, 8), span("u2", 8, 10)];
+ const { start, shifts } = placeInRun(row, 0, 4, new Set(["s2"]));
+ expect(start).toBeCloseTo(4); // slid clear of the resident, not past s2
+ // s2 and everything after it open the slot as one run.
+ expect(shifts).toEqual([
+ { id: "s2", start: 8 },
+ { id: "u2", start: 12 },
+ ]);
+ });
+
+ test("out-of-order landings assemble gapless in shot order on an empty track", () => {
+ const row: { id: string; start: number; end: number }[] = [];
+ const p3 = placeInRun(row, 15, 5, new Set());
+ row.push(span("s3", p3.start, p3.start + 5));
+ const p1 = placeInRun(row, 0, 8, new Set(["s3"]));
+ expect(p1.start).toBeCloseTo(0);
+ expect(p1.shifts).toEqual([]);
+ row.push(span("s1", 0, 8));
+ const p2 = placeInRun(row, 8, 7, new Set(["s3"])); // want = live end of s1
+ expect(p2.start).toBeCloseTo(8);
+ expect(p2.shifts).toEqual([]);
+ });
+
+ test("placeGenClip anchors after the run's earlier clip when the plan's frames are stale", () => {
+ const av = asset(4);
+ const user = vclip({ start: 0, out: 4, assetId: "x" });
+ const s1 = vclip({ start: 4, out: 4, assetId: "x" });
+ useEditor.setState({ assets: [av], clips: [user, s1] });
+ // Planned [4, 8) went stale when the user's clip pushed the run to 4..8.
+ const id = s().placeGenClip(av.id, 4, 8, { anchorAfterIds: [s1.id] })!;
+ expect(clipById(id).start).toBeCloseTo(8);
+ expectLaneSound(videoLane(0));
+ });
+
+ test("placeGenClip pushes a later shot's clip instead of landing on or past it", () => {
+ const av = asset(4);
+ const user = vclip({ start: 0, out: 4, assetId: "x" });
+ const s2 = vclip({ start: 4, out: 4, assetId: "x" });
+ useEditor.setState({ assets: [av], clips: [user, s2] });
+ const id = s().placeGenClip(av.id, 0, 4, { followClipIds: [s2.id] })!;
+ expect(clipById(id).start).toBeCloseTo(4); // after the resident
+ expect(clipById(s2.id).start).toBeCloseTo(8); // pushed, still after the new take
+ expectLaneSound(videoLane(0));
+ });
+
+ test("placeGenAudio slides to its lane's next free slot", () => {
+ const aa = asset(4, "audio");
+ const resident = aclip({ start: 0, out: 4, assetId: "x" });
+ useEditor.setState({ assets: [aa], audioClips: [resident] });
+ const id = s().placeGenAudio(aa.id, 1, 4)!;
+ expect(audioById(id).start).toBeCloseTo(4);
+ });
+});
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index de726e2f..ec7ff39f 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -222,13 +222,16 @@ export interface EditorState {
/** Set (or clear) the persisted brief-to-video run. Replaces the object by
* reference so autosave detects the change. */
setGenvideo: (project: VideoProject | undefined) => void;
- /** Brief-to-video placement: place a generated clip so it fills exactly
- * [startSec, endSec) on track 0 — exact start (no slide), muted, time-stretched
- * or trimmed to the slot — and return its id. Leaves selection untouched (the
- * run is a background process). */
- placeGenClip: (assetId: string, startSec: number, endSec: number, opts?: { srcInSec?: number; muted?: boolean }) => string | null;
- /** Brief-to-video placement: place a generated audio clip at startSec spanning
- * up to durSec on the soundtrack (duck/lane/volume optional), returning its id. */
+ /** Brief-to-video placement: place a generated clip filling a [startSec,
+ * endSec)-sized slot on track 0 — muted, time-stretched or trimmed to the
+ * slot — and return its id. Where it lands goes through `placeInRun`:
+ * anchored after the run's earlier clips, never past its later ones, slid
+ * clear of everything else. Leaves selection untouched (the run is a
+ * background process). */
+ placeGenClip: (assetId: string, startSec: number, endSec: number, opts?: { srcInSec?: number; muted?: boolean; anchorAfterIds?: string[]; followClipIds?: string[] }) => string | null;
+ /** Brief-to-video placement: place a generated audio clip at the next free
+ * slot at/after startSec on its soundtrack lane, spanning up to durSec
+ * (duck/lane/volume optional), returning its id. */
placeGenAudio: (assetId: string, startSec: number, durSec: number, opts?: { duck?: number; lane?: number; volume?: number }) => string | null;
/** Remove a video clip by id (a background gen swap; leaves its slot empty). */
removeClipById: (id: string) => void;
@@ -641,6 +644,42 @@ export function footprints(items: (VideoClip | AudioClip)[]): { start: number; e
return items.map((c) => ({ start: c.start, end: c.start + clipLen(c) }));
}
+/** Where a scene run's clip lands on a lane that may have moved since the
+ * plan. `want` is the target spot (the plan's, or the live end of the run's
+ * previous shot). Free residents slide the landing right exactly like
+ * `nextFreeStart`; clips in `follow` — the run's own later shots — are never
+ * slid past, because that would reorder the story: the landing pushes them
+ * (and everything at or after them) right as one run instead. So takes that
+ * render out of order still assemble in shot order, whatever the user moved
+ * meanwhile. The one placement primitive for generated-scene clips, shared by
+ * the live store and the background doc writer. */
+export function placeInRun(
+ row: { id: string; start: number; end: number }[],
+ want: number,
+ len: number,
+ follow: ReadonlySet<string>
+): { start: number; shifts: { id: string; start: number }[] } {
+ const sorted = [...row].sort((a, b) => a.start - b.start);
+ const followStart = sorted
+ .filter((s) => follow.has(s.id))
+ .reduce((m, s) => Math.min(m, s.start), Infinity);
+ let at = Math.max(0, want);
+ for (const sp of sorted) {
+ if (sp.start >= followStart - 1e-6) break; // the push below clears these
+ if (sp.end <= at + 1e-3) continue;
+ if (sp.start >= at + len - 1e-3) break;
+ at = sp.end;
+ }
+ const delta = at + len - followStart;
+ const shifts =
+ Number.isFinite(followStart) && delta > 1e-9
+ ? sorted
+ .filter((s) => s.start >= followStart - 1e-6)
+ .map((s) => ({ id: s.id, start: s.start + delta }))
+ : [];
+ return { start: at, shifts };
+}
+
/** Where a `len`-long clip dropped at pointer-time `t` lands on its row,
* and how the clips after it slide to open room. The drop-at-pointer companion
* to `nextFreeStart` (which only ever appends): clips whose center sits left of
@@ -1171,11 +1210,28 @@ export const useEditor = create<EditorState>((baseSet, get) => {
slot,
SPEED_MIN
);
+ // Land through the run placement primitive against the live row: after
+ // the furthest clip the run's earlier shots still hold, never past a
+ // later shot's clip, slid clear of everything else on the track.
+ const row = track0Clips(get().clips);
+ const spans = row.map((c) => ({ id: c.id, start: c.start, end: c.start + clipLen(c) }));
+ const anchorIds = new Set(opts?.anchorAfterIds ?? []);
+ const prevEnd = spans
+ .filter((s) => anchorIds.has(s.id))
+ .reduce((m, s) => Math.max(m, s.end), -1);
+ const len = speed !== undefined && speed > 0 ? out / speed : out;
+ const { start, shifts } = placeInRun(
+ spans,
+ prevEnd >= 0 ? prevEnd : Math.max(0, startSec),
+ Math.max(MIN_LEN, len),
+ new Set(opts?.followClipIds ?? [])
+ );
+ const move = new Map(shifts.map((sh) => [sh.id, sh.start]));
const clip: VideoClip = {
id: uid(),
assetId,
track: 0,
- start: Math.max(0, startSec),
+ start,
in: srcIn,
out: srcIn + out,
// Muted only when the caller asks — a provided-audio scene mutes its
@@ -1188,7 +1244,11 @@ export const useEditor = create<EditorState>((baseSet, get) => {
// the orchestrator manages this clip (it swaps clips idempotently), so a
// mid-render Cmd+Z must not pull a shot out from under the run.
genClipIds.add(clip.id);
- set((s) => ({ clips: [...s.clips, clip].sort((a, b) => a.start - b.start) }));
+ set((s) => ({
+ clips: [...s.clips.map((c) => (move.has(c.id) ? { ...c, start: move.get(c.id)! } : c)), clip].sort(
+ (a, b) => a.start - b.start
+ ),
+ }));
return clip.id;
},
@@ -1197,10 +1257,13 @@ export const useEditor = create<EditorState>((baseSet, get) => {
if (!asset || asset.type !== "audio") return null;
const out = Math.min(asset.duration, Math.max(MIN_LEN, durSec));
const lane = opts?.lane ?? 0;
+ // Slide to the lane's next free slot so a background run never lands a
+ // bed or voiceover on top of a sound the user placed meanwhile.
+ const taken = footprints(get().audioClips.filter((a) => (a.lane ?? 0) === lane));
const clip: AudioClip = {
id: uid(),
assetId,
- start: Math.max(0, startSec),
+ start: nextFreeStart(taken, Math.max(0, startSec), out),
in: 0,
out,
volume: opts?.volume ?? 1,
@@ -1357,8 +1420,18 @@ export const useEditor = create<EditorState>((baseSet, get) => {
const st = get();
const gone = st.assets.find((a) => a.id === id);
if (!gone) return;
- // The media file dies with its last referencing asset — history
- // snapshots don't cover the asset list, so nothing can resurrect it.
+ // The media file dies with its last referencing asset, so no undo may
+ // bring a clip of it back to point at bytes that no longer exist: take
+ // this asset's clips out of every snapshot that holds them. Deleting an
+ // asset is not itself undoable, and the asset list is not in a snapshot,
+ // so nothing can resurrect either the clips or the file.
+ const scrub = (snap: DocSnapshot) => {
+ snap.clips = snap.clips.filter((c) => c.assetId !== id);
+ snap.audioClips = snap.audioClips.filter((c) => c.assetId !== id);
+ };
+ for (const snap of history) scrub(snap);
+ for (const snap of future) scrub(snap);
+ if (pending) scrub(pending.snap);
const dropFile = () => {
const s = get();
if (!s.projectId || s.assets.some((a) => a.fileName === gone.fileName)) return;
@@ -1379,7 +1452,6 @@ export const useEditor = create<EditorState>((baseSet, get) => {
dropFile();
return;
}
- push();
// Cascade removes this asset's clips; every clip is free-positioned, so
// the rest of the timeline (and its annotations) stays where it is.
set((s) => {