Skip to content

Open source CapCut alternative.

TypeScript

feat(site): transition bars ride retimes and report parked leftovers [rebuild]

David Le6cce6b7ea8b4
diff --git a/site/src/cut/components/TransitionsPanel.tools.ts b/site/src/cut/components/TransitionsPanel.tools.ts
index 93aa7599..f7342652 100644
--- a/site/src/cut/components/TransitionsPanel.tools.ts
+++ b/site/src/cut/components/TransitionsPanel.tools.ts
@@ -25,6 +25,14 @@ export const TRANSITIONS_TOOLS = [
},
}, ["clipId", "seconds"]),
},
+ {
+ name: "remove_transition",
+ description:
+ "Delete a transition bar by its own id — the handle for a parked bar, one lining up with no cut or clip edge and playing nothing. editor_state's `transitions` lists every bar and marks the parked ones, and each timeline mutation reports the bars its edit left parked. Use this to clear leftovers after moving or retiming clips; a bar that still plays a cut is better changed with set_transition (0 clears it).",
+ inputSchema: obj({
+ transitionId: str("Transition bar id, from editor_state `transitions`"),
+ }, ["transitionId"]),
+ },
{
name: "set_animation",
description:
diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.ts
index 1d98dbc7..8ac0716b 100644
--- a/site/src/cut/lib/aiContext.ts
+++ b/site/src/cut/lib/aiContext.ts
@@ -3,7 +3,7 @@
import { hasOverlayAnim } from "@donkeycut/effects-kit";
import { chatOwner } from "./chatAssets";
import { useGenerate } from "./generate";
-import { getClipSpans, overlayLayers, totalDuration, useEditor } from "./store";
+import { getClipSpans, overlayLayers, resolveTransitions, totalDuration, useEditor } from "./store";
import { laneCues, subtitleLaneCount } from "./subtitles";
import { watchSweepActive } from "./watch/sweep";
import { frameOf, rectOf, regionLabel, type ClipSpan, type Overlay, type VideoClip } from "./types";
@@ -243,6 +243,29 @@ export function buildAiContext(opts?: { fullCues?: boolean; chatId?: string | nu
...clipEffects(sp.clip),
}));
}),
+ // Every transition bar in the doc. A bar is its own object on the
+ // transitions row: it plays whatever cut or clip edge its window lines up
+ // with, and one lining up with nothing sits parked, playing nothing. The
+ // clip-level transitionToNext above only shows bars that landed on a cut,
+ // so this is where a parked leftover — the debris a move or a retime
+ // leaves — is visible and addressable by id.
+ transitions: (() => {
+ const roles = resolveTransitions(s.clips, s.transitions);
+ return [...s.transitions]
+ .sort((a, b) => a.start - b.start)
+ .map((t) => {
+ const plays = roles.get(t.id) ?? [];
+ return {
+ id: t.id,
+ start: r(t.start),
+ seconds: r(t.seconds),
+ style: t.style,
+ ...(plays.length > 0
+ ? { plays: plays.map((p) => ({ at: p.kind, clipId: p.clipId })) }
+ : { parked: true }),
+ };
+ });
+ })(),
soundtrack: s.audioClips.map((a) => ({ id: a.id, ...describeAudio(a, assetById) })),
// Every overlay element on the title lanes: titles, shapes, stickers —
// each entry carries its `kind`.
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index fdb380ce..4a017822 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -71,7 +71,7 @@ import { blobToInlineAudio, refToInlineAudio, visualRefs, type InlineImage } fro
import { characterPrompt, stockAspectDims, stockTitle } from "./stock";
import { STOCK_IMAGES } from "./stockManifest";
import { STOCK_VIDEOS } from "./stockVideoManifest";
-import { applyOverlayPatchSettled, track0Clips, laneGapAt, getClipSpans, nextFreeStart, overlayLayers, totalDuration, useEditor } from "./store";
+import { applyOverlayPatchSettled, track0Clips, laneGapAt, getClipSpans, nextFreeStart, overlayLayers, parkedTransitions, totalDuration, useEditor } from "./store";
import { buildAiContext } from "./aiContext";
import { sampleClipFrameData } from "./previewCanvas";
import { laneCues, subtitleLaneCount } from "./subtitles";
@@ -122,7 +122,12 @@ const round2 = (n: number) => Math.round(n * 100) / 100;
* ids, starts, and lengths for track 0 and the soundtrack, read fresh after
* the store settled. The model keeps editing from these — new ids after a
* split, closed-up starts after a delete — with no get_state poll between
- * steps. */
+ * steps.
+ *
+ * `parkedTransitions` rides along: a bar the edit left lining up with
+ * nothing. The model sees its own debris the moment it makes it, in the same
+ * payload it already reads, so a stranded blend gets cleared or reattached
+ * inside the turn instead of sitting on the row. */
function tracksAfter() {
const cur = useEditor.getState();
const row = track0Clips(cur.clips).map((c) => ({
@@ -135,11 +140,24 @@ function tracksAfter() {
start: round2(a.start),
len: round2(a.out - a.in),
}));
+ const parked = parkedTransitions(cur.clips, cur.transitions);
return {
track0: row.slice(0, 60),
...(row.length > 60 ? { track0Truncated: true } : {}),
soundtrack: lanes.slice(0, 60),
...(lanes.length > 60 ? { soundtrackTruncated: true } : {}),
+ ...(parked.length > 0
+ ? {
+ parkedTransitions: parked.map((t) => ({
+ id: t.id,
+ start: round2(t.start),
+ seconds: round2(t.seconds),
+ style: t.style,
+ })),
+ parkedTransitionsNote:
+ "These transition bars line up with no cut or clip edge and play nothing. Clear each one with remove_transition, or reattach it with set_transition/set_animation on the edge it belongs to. If it isn't clear which the user wanted, say what's there and ask.",
+ }
+ : {}),
};
}
@@ -472,6 +490,14 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
return { id: next.id, transition: next.transition ?? 0, style: next.transitionStyle ?? "crossfade" };
},
+ remove_transition: (s, input) => {
+ const id = String(input.transitionId ?? "");
+ if (!s.transitions.some((t) => t.id === id))
+ throw new ToolError(`No transition bar with id ${id}.`);
+ s.removeTransition(id);
+ return { removed: id, ...tracksAfter() };
+ },
+
set_animation: (s, input) => {
const clip = requireItem(s.clips, input.clipId, "video clip");
const which = input.which === "in" || input.which === "out" ? input.which : null;
@@ -777,8 +803,13 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
end: c.start + (c.out - c.in) / (c.speed && c.speed > 0 ? c.speed : 1),
}));
const at = nextFreeStart(taken, Math.max(0, input.start), len);
+ // Closing a gap is this tool's main job, so the clip's blends travel
+ // with it: a bar playing its head or its cut lands on the edge's new
+ // time instead of staying behind on the row.
+ const before = s.clips;
s.updateClip(clip.id, { start: at });
useEditor.getState().sortClips();
+ useEditor.getState().reanchorBars(before);
return {
id: clip.id,
start: round2(at),
@@ -977,6 +1008,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
// change — butted joints stay butted, deliberate beats keep their width,
// and a tightened clip closes up instead of opening a gap.
if (plans.size > 0) {
+ const before = s.clips;
s.pushHistory();
const patches: { id: string; patch: Partial<VideoClip> }[] = [];
for (const row of rows.values()) {
@@ -995,6 +1027,9 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
}
useEditor.getState().updateClipsTransient(patches);
useEditor.getState().sortClips();
+ // Every edge here moved because the audio said so, and the run behind
+ // each one shifted with it — a retime, so the bars ride along.
+ useEditor.getState().reanchorBars(before);
}
return {
diff --git a/site/src/cut/lib/store.test.ts b/site/src/cut/lib/store.test.ts
index a7d6acd3..98be96b8 100644
--- a/site/src/cut/lib/store.test.ts
+++ b/site/src/cut/lib/store.test.ts
@@ -1,6 +1,6 @@
import { beforeEach, describe, expect, test } from "bun:test";
import { groupRemap } from "@donkeycut/effects-kit";
-import { adoptTransitionFields, clipLen, deriveTransitionFields, docOverlays, getClipSpans, liftClipLooks, moveOverlayGroup, overlayLaneOrder, normalizeElementLanes, placeInRun, projectDuration, separateOverlaps, serializeDoc, useEditor } from "./store";
+import { adoptTransitionFields, clipLen, deriveTransitionFields, docOverlays, getClipSpans, liftClipLooks, moveOverlayGroup, overlayLaneOrder, normalizeElementLanes, parkedTransitions, placeInRun, projectDuration, reanchorTransitions, separateOverlaps, serializeDoc, useEditor } from "./store";
import { emptySubtitles } from "./types";
import type { AudioClip, MediaAsset, SubtitleCue, TextOverlay, VideoClip } from "./types";
@@ -197,12 +197,14 @@ describe("committed video updates (AI chat / inspector)", () => {
});
test("retracking onto an occupied track slides clear", () => {
- const c1 = vclip({ track: 1, start: 0, out: 2 });
- const c2 = vclip({ track: 2, start: 0.5, out: 2 });
+ // Grounded on write: the stack always sits on track 0, so the pair is
+ // written as the rows they would ground to.
+ const c1 = vclip({ track: 0, start: 0, out: 2 });
+ const c2 = vclip({ track: 1, start: 0.5, out: 2 });
useEditor.setState({ clips: [c1, c2] });
- s().updateClip(c2.id, { track: 1 });
+ s().updateClip(c2.id, { track: 0 });
expect(clipById(c2.id).start).toBeCloseTo(2);
- expectLaneSound(videoLane(1));
+ expectLaneSound(videoLane(0));
});
test("extending a clip pushes the same-track run right", () => {
@@ -879,6 +881,50 @@ describe("bars and clip edges", () => {
expect(clipById(a.id).transition).toBeUndefined();
});
+ test("a trim carries the bar and the clip fields the export renders from", () => {
+ // The preview and the export read clip.transition, not the bar. A resize
+ // moves the clip's edge and the run behind it, so both have to land
+ // together: a bar left behind, or a field left stale, is a blend playing
+ // at a joint that no longer has one.
+ const av = asset(8);
+ const a = vclip({ track: 0, start: 0, out: 4, assetId: av.id });
+ const b = vclip({ track: 0, start: 4, out: 4, assetId: av.id });
+ useEditor.setState({ assets: [av], clips: [a, b] });
+ s().setClipTransition(a.id, 0.8, "wipeleft");
+ s().setClipTrim(a.id, 0, 3);
+ // The pair stays in contact, so the cut moved to 3s with the bar on it.
+ expect(clipById(b.id).start).toBeCloseTo(3);
+ expect(s().transitions[0].start).toBeCloseTo(2.2);
+ expect(parkedTransitions(s().clips, s().transitions)).toEqual([]);
+ // And the render fields still describe that blend.
+ expect(clipById(a.id).transition).toBeCloseTo(0.8);
+ expect(clipById(a.id).transitionStyle).toBe("wipeleft");
+ });
+
+ test("a bar lining up with nothing reads as parked, and reanchorBars brings it back", () => {
+ const av = asset(8);
+ const a = vclip({ track: 0, start: 0, out: 4, assetId: av.id });
+ const b = vclip({ track: 0, start: 6, out: 4, assetId: av.id });
+ useEditor.setState({ assets: [av], clips: [a, b] });
+ // A blend on a's open tail, the bar ending on 4s.
+ s().setClipAnim(a.id, "out", { style: "fade", seconds: 0.8 });
+ expect(parkedTransitions(s().clips, s().transitions)).toEqual([]);
+ // The clip slides — the move an assistant makes with place_clip — and the
+ // bar stays where it was, now lining up with nothing.
+ const before = s().clips;
+ s().updateClip(a.id, { start: 1 });
+ const parked = parkedTransitions(s().clips, s().transitions);
+ expect(parked.length).toBe(1);
+ expect(parked[0].style).toBe("crossfade");
+ expect(clipById(a.id).animOut).toBeUndefined();
+ // Carrying the bars through that move lands it back on the edge it
+ // played, and the render fields follow.
+ s().reanchorBars(before);
+ expect(s().transitions[0].start).toBeCloseTo(4.2);
+ expect(parkedTransitions(s().clips, s().transitions)).toEqual([]);
+ expect(clipById(a.id).animOut).toEqual({ style: "fade", seconds: 0.8 });
+ });
+
test("an animation on a far edge leaves an unrelated transition alone", () => {
const av = asset(4);
const a = vclip({ track: 0, start: 0, out: 4, assetId: av.id });
diff --git a/site/src/cut/lib/store.ts b/site/src/cut/lib/store.ts
index f1a1cfbf..23edbe9d 100644
--- a/site/src/cut/lib/store.ts
+++ b/site/src/cut/lib/store.ts
@@ -315,6 +315,11 @@ export interface EditorState {
/** The same, with no undo checkpoint — for mid-gesture updates. */
updateTransitionTransient: (id: string, patch: Partial<Omit<TimelineTransition, "id">>) => void;
removeTransition: (id: string) => void;
+ /** Carry the bars through a retime the caller just wrote: given the clip row
+ * as it stood before, every bar keeps playing the boundary it played, at
+ * wherever that boundary moved to. No undo checkpoint — the caller's own
+ * push covers the whole edit. For the retiming paths outside this store. */
+ reanchorBars: (before: VideoClip[]) => void;
/** Set (or clear with null) a clip's preset filter look; amount 0..1. */
updateAudio: (id: string, patch: Partial<AudioClip>) => void;
/** Hide or show every clip on one video track, in one undo step. Showing a
@@ -984,46 +989,63 @@ const DOC_KEYS = [
] as const;
let hydrating = false;
-export const useEditor = create<EditorState>((baseSet, get) => {
- // Every write that touches clips grounds the stack, so the spine invariant
- // holds by construction — across deletes, drops, undo, and doc loads.
+/**
+ * The invariants every write to the store passes through, wherever it comes
+ * from: a read-only project takes no doc changes, the track stack stays
+ * grounded, the per-clip transition/anim fields stay caches of the bars, and
+ * the playhead stays inside the timeline.
+ *
+ * The clip fields are what the preview and the export actually render from,
+ * so a write that moved clips or bars without re-deriving them would ship a
+ * blend at a joint that no longer has one. This runs inside `setState`
+ * itself, so no caller — action, module helper, or anything outside this
+ * file — can write past it.
+ */
+function normalizeWrite(prev: EditorState, incoming: Partial<EditorState>): Partial<EditorState> {
+ let next = incoming;
+ if (prev.readOnly && !hydrating) {
+ next = { ...next };
+ for (const k of DOC_KEYS) delete (next as Record<string, unknown>)[k];
+ }
+ if (next.clips) next = { ...next, clips: groundTracks(next.clips) };
+ if (next.clips || next.transitions) {
+ const clips = next.clips ?? prev.clips;
+ const derived = deriveTransitionFields(clips, next.transitions ?? prev.transitions);
+ if (derived !== clips) next = { ...next, clips: derived };
+ }
+ // The playhead cannot outlive the timeline. Deleting the last of a long
+ // row shortens the project under a playhead standing past the new end,
+ // which leaves the readout ahead of the total and — since the playhead is
+ // placed with a transform, and transforms count toward scrollable
+ // overflow — stretches the scroll area into empty space. Clamping here
+ // covers every edit that shortens anything, present and future.
+ if (next.clips || next.audioClips || next.overlays) {
+ const total = projectDuration({
+ clips: next.clips ?? prev.clips,
+ audioClips: next.audioClips ?? prev.audioClips,
+ overlays: next.overlays ?? prev.overlays,
+ });
+ if ((next.currentTime ?? prev.currentTime) > total) next = { ...next, currentTime: total };
+ }
+ return next;
+}
+
+export const useEditor = create<EditorState>((baseSet, get, api) => {
+ // Normalizing is bolted onto setState itself, not onto a local helper the
+ // actions happen to use: the store's own module-level helpers write through
+ // `useEditor.setState`, and one of those skipping the derive is how a clip
+ // kept rendering a transition its bar had already left.
const set = (
partial:
| Partial<EditorState>
| ((s: EditorState) => Partial<EditorState>),
replace?: boolean
) =>
- baseSet((prev) => {
- let next = typeof partial === "function" ? partial(prev) : partial;
- if (prev.readOnly && !hydrating) {
- next = { ...next };
- for (const k of DOC_KEYS) delete (next as Record<string, unknown>)[k];
- }
- if (next.clips) next = { ...next, clips: groundTracks(next.clips) };
- // The per-clip transition/anim fields are caches of the transition bars:
- // any write that moves a clip or a bar re-derives them here, so the two
- // can never disagree.
- if (next.clips || next.transitions) {
- const clips = next.clips ?? prev.clips;
- const derived = deriveTransitionFields(clips, next.transitions ?? prev.transitions);
- if (derived !== clips) next = { ...next, clips: derived };
- }
- // The playhead cannot outlive the timeline. Deleting the last of a long
- // row shortens the project under a playhead standing past the new end,
- // which leaves the readout ahead of the total and — since the playhead is
- // placed with a transform, and transforms count toward scrollable
- // overflow — stretches the scroll area into empty space. Clamping here
- // covers every edit that shortens anything, present and future.
- if (next.clips || next.audioClips || next.overlays) {
- const total = projectDuration({
- clips: next.clips ?? prev.clips,
- audioClips: next.audioClips ?? prev.audioClips,
- overlays: next.overlays ?? prev.overlays,
- });
- if ((next.currentTime ?? prev.currentTime) > total) next = { ...next, currentTime: total };
- }
- return next;
- }, replace as false | undefined);
+ baseSet(
+ (prev) => normalizeWrite(prev, typeof partial === "function" ? partial(prev) : partial),
+ replace as false | undefined
+ );
+ api.setState = set as typeof api.setState;
const snapshot = (): DocSnapshot => {
const { clips, transitions, audioClips, overlays, subtitles } = get();
@@ -2068,6 +2090,13 @@ export const useEditor = create<EditorState>((baseSet, get) => {
}));
},
+ reanchorBars: (before) => {
+ const s = get();
+ if (s.transitions.length === 0) return;
+ const next = reanchorTransitions(before, s.clips, s.transitions);
+ if (next !== s.transitions) set({ transitions: next });
+ },
+
addTransition: (bar) => {
const id = uid();
push();
@@ -4239,6 +4268,21 @@ export function resolveTransitions(
return roles;
}
+/**
+ * The bars lining up with nothing: on the row, playing no cut and no clip
+ * edge. A user who drags a clip away sees the bar it left and can drag it
+ * back; an assistant edit has no such eye, so the tool layer reports these
+ * after every timeline mutation and the model clears or reattaches them.
+ */
+export function parkedTransitions(
+ clips: VideoClip[],
+ transitions: TimelineTransition[]
+): TimelineTransition[] {
+ if (transitions.length === 0) return [];
+ const roles = resolveTransitions(clips, transitions);
+ return transitions.filter((t) => !(roles.get(t.id) ?? []).length);
+}
+
/**
* Carry the bars through a retime: every bar keeps playing the boundary it
* played, at wherever that boundary moved to.
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index 0cbe512f..79172d21 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -73,7 +73,7 @@ Times are in seconds on the shared timeline. The playhead is currentTime; a skim
- A clip's timeline length is (out-in)/speed; total duration runs to the last clip's end, gaps included. Transitions blend at cuts and never change layout or length.
- trim_clip changes in/out inside the source media. in >= 0, out <= source duration, out-in >= 0.1.
- set_speed sets a clip's playback rate; it changes the clip's timeline length, and later titles/captions ripple to stay in sync.
-- set_transition joins a clip into the next one (0–2s, ${TRANSITION_STYLE_IDS.length} styles); set_animation animates one clip's own entrance/exit — read the transitions-and-fades skill before styling cuts. Splitting or deleting clears the affected transition. Grading a clip's picture is add_effect: an effect dropped over a clip opens covering it.
+- set_transition joins a clip into the next one (0–2s, ${TRANSITION_STYLE_IDS.length} styles); set_animation animates one clip's own entrance/exit — read the transitions-and-fades skill before styling cuts. Splitting or deleting clears the affected transition. A transition is a bar on its own row at an absolute time, so an edit can leave one lining up with nothing: every timeline mutation reports those as \`parkedTransitions\`, and clearing them (remove_transition) is part of finishing the edit. Grading a clip's picture is add_effect: an effect dropped over a clip opens covering it.
- split_at cuts the track-0 clip under that time into two clips at the exact frame. With a soundtrack or overlay clip selected it splits that instead.
- The user can multi-select (⌘/⇧-click) and delete several items at once; a hover chip on each video clip toggles its own audio.
- Anything can be hidden instead of deleted: it stays on the timeline (grayed) but drops out of playback and export — a hidden video clip plays black and silent. Per item: set_clip_hidden, or the \`hidden\` field on update_overlay_video/update_overlay/update_audio. Whole rows: set_track_hidden (video/soundtrack/text/subtitles) and set_track_muted (video) — the eye/speaker toggles on the track headers.
@@ -117,8 +117,11 @@ Route the ask to the right feature:
set_transition styles (clipId = the leading clip; 0 clears, max 2s): crossfade blends; crosszoom adds a zoom punch; dipblack/dipwhite dip through a solid color; blur defocuses through the cut; pushleft/right/up/down shove the old frame out; wipeleft/right/up/down reveal with a hard traveling edge; circleopen/close and splitopen/close are shape reveals. Directional names describe the motion.
set_animation styles: fade (audio follows), zoom, pop, slideleft/right/up/down — the slide names are the motion direction (slideleft enters from the right edge moving left, or exits off the left). Each edge holds one effect and the last pick wins: set_animation on a transitioned edge replaces that transition, set_transition clears the animations adjacent to its joint.
+
+Bars and the debris they leave: a transition lives on the transitions row as its own object at an absolute time, listed in editor_state \`transitions\` with the boundary it \`plays\`. It plays whatever cut or clip edge its window lines up with; a bar lining up with nothing reads \`parked: true\` and plays nothing — the badge is on the row, the blend is gone. Moving, retiming, or replacing clips is what parks a bar, so every timeline mutation returns \`parkedTransitions\` when its edit left one. Own that debris in the same turn you make it: a bar you parked and know the intent of, reattach with set_transition/set_animation on the edge it belongs to, or clear with remove_transition. When you can't tell what a parked bar was for — it predates your edits, or the user's ask doesn't say where the blend should end up — do not guess and do not leave it silent: name what's there ("there's a 0.5s dip to black at 29.4s that isn't on a cut any more") and ask whether to remove it or move it to a specific joint. The same holds for anything else on the timeline that reads as leftover from your own edits: say what you see and confirm before deleting the user's work.
Picking for a vibe: "smooth" → crossfade; "punchy/energetic" → crosszoom or a push; "dramatic scene change" → dipblack 0.5–0.8s; "dreamy" → blur (or the dreamy look); "retro" → vhs or vintage look. Between clips 0.4–0.8s reads well; 1s+ is slow and cuts total duration.
-UI: select a clip → Inspector "Effects" opens the panel (Transition / In / Out / Looks tabs); a blue badge marks each styled joint on the timeline. Preview and export render the same treatment.`,
+UI: select a clip → Inspector "Effects" opens the panel (Transition / In / Out / Looks tabs). Preview and export render the same treatment.
+What the user sees on the transitions row: one badge per bar, and nothing else — the row holds no buttons, no placeholders, and no add-affordances. A playing bar is blue; a parked one is the same badge in grey. So a user reporting "greyed out transitions", "grey circles", or "dead/lingering transitions" on that row is describing parked bars, which are real objects saved in their project — never dismiss them as editor chrome or claim they won't export, because the badge is the transition itself. Read \`transitions\` in editor_state and answer from the parked entries there. Grey elsewhere means something different: a clip, element, or track shown grayed is hidden (the \`hidden\` flag), which does drop out of playback and export.`,
"graphics": `# Graphics: titles, shapes & stickers
The title lanes hold three overlay element kinds, all sharing timing (start/end), center position (x/y fractions 0..1), rotation (degrees), opacity, and a lane; \`overlays\` in editor_state lists them with their \`kind\`. All burn into the export exactly as previewed.