Open source CapCut alternative.
TypeScript
feat(cut): time font cycle faces to beats or switches [rebuild]
David Le19b4af87dbbc
diff --git a/site/src/cut/components/AnimationTiles.tools.ts b/site/src/cut/components/AnimationTiles.tools.tsindex 68d2169f..29e9127d 100644--- a/site/src/cut/components/AnimationTiles.tools.ts+++ b/site/src/cut/components/AnimationTiles.tools.ts@@ -70,7 +70,7 @@ export const OVERLAY_ANIMATION_TOOLS = [{name: "set_overlay_animation",description:- `Animate an overlay element (title, shape, or sticker): preset In/Out ramps, a Loop that runs its whole duration, and a Move that says what the element does WHILE it holds. Omitted slots keep their setting; pass "none" to clear one. In/Out styles: ${OVERLAY_ANIM_STYLE_IDS.join(", ")} — slide names are the motion direction; ${TEXT_ONLY_ANIM_STYLE_IDS.join(", ")} animate titles only; ${GLYPH_ANIM_STYLE_IDS.join(", ")} move a title's letters one at a time, and move any other kind as one piece; in_stagger/out_stagger set how much of their ramp the hand-off from the first letter to the last takes, so a 1.6s drop at stagger 0.8 lands a letter every ~0.27s, each moving for ~0.3s. ${edgeNotes()} ${DIVE_ANIM_STYLE_IDS.join(", ")} as the exit is the zoom-through-a-letter transition: the shot after the element shows once the ink has filled the frame, so it needs no freeze frames, image stills or per-letter keyframes. Loop styles: ${OVERLAY_LOOP_STYLE_IDS.join(", ")} — ${GLYPH_LOOP_STYLE_IDS.join(", ")} carry a title's letters on their own delays, and carry any other kind as one piece. A move is a slot like the others and never touches the element's keyframe track; set_overlay_keyframes is the precise way to move an element and the default reach, so name a move when it is exactly what you mean — the two compose together. Word effects — words_style, titles only — play the line word by word, timed against the cut's transcript when there is one and spread across the element's own span when there is not: an emphasis travels along a line that is fully up, and a build assembles the line as it is spoken. It is the slot for "make each word pop as I say it" and for "have the words appear one by one". A typewriter entrance on a title can carry a caret: a thin bar at the end of the typed text, solid while it types, then blinking every ${CARET_BLINK_SECONDS}s — the typed comment or search box of a call-to-action card. A hit plays one motion once at a moment inside the element (hit_at seconds from its start), between its entrance and exit: ${hitNotes()}. Use press for a button being pressed, a tap on a UI element, a like or post button; pair it with a click from stock_search kind "sound" placed at the same moment when the tap should be heard. A font cycle flickers a title through typefaces — a new face font_cycle_rate times a second, from font_cycle_at for font_cycle_seconds — and then the title settles back into its own face: the typeface-shuffle beat of a teaser title. Give each face a scale (and tracking) so it fills about the room the title's own face does; a wide tracked title cycling through condensed or hand-drawn faces wants tracking 0 and a scale above 1 on them. A face's hold sets how long it shows, so changes that land on the music's beats are one face per beat with its measured hold; faces repeat in the list to revisit a typeface.`,+ `Animate an overlay element (title, shape, or sticker): preset In/Out ramps, a Loop that runs its whole duration, and a Move that says what the element does WHILE it holds. Omitted slots keep their setting; pass "none" to clear one. In/Out styles: ${OVERLAY_ANIM_STYLE_IDS.join(", ")} — slide names are the motion direction; ${TEXT_ONLY_ANIM_STYLE_IDS.join(", ")} animate titles only; ${GLYPH_ANIM_STYLE_IDS.join(", ")} move a title's letters one at a time, and move any other kind as one piece; in_stagger/out_stagger set how much of their ramp the hand-off from the first letter to the last takes, so a 1.6s drop at stagger 0.8 lands a letter every ~0.27s, each moving for ~0.3s. ${edgeNotes()} ${DIVE_ANIM_STYLE_IDS.join(", ")} as the exit is the zoom-through-a-letter transition: the shot after the element shows once the ink has filled the frame, so it needs no freeze frames, image stills or per-letter keyframes. Loop styles: ${OVERLAY_LOOP_STYLE_IDS.join(", ")} — ${GLYPH_LOOP_STYLE_IDS.join(", ")} carry a title's letters on their own delays, and carry any other kind as one piece. A move is a slot like the others and never touches the element's keyframe track; set_overlay_keyframes is the precise way to move an element and the default reach, so name a move when it is exactly what you mean — the two compose together. Word effects — words_style, titles only — play the line word by word, timed against the cut's transcript when there is one and spread across the element's own span when there is not: an emphasis travels along a line that is fully up, and a build assembles the line as it is spoken. It is the slot for "make each word pop as I say it" and for "have the words appear one by one". A typewriter entrance on a title can carry a caret: a thin bar at the end of the typed text, solid while it types, then blinking every ${CARET_BLINK_SECONDS}s — the typed comment or search box of a call-to-action card. A hit plays one motion once at a moment inside the element (hit_at seconds from its start), between its entrance and exit: ${hitNotes()}. Use press for a button being pressed, a tap on a UI element, a like or post button; pair it with a click from stock_search kind "sound" placed at the same moment when the tap should be heard. A font cycle flickers a title through typefaces — a new face font_cycle_rate times a second, from font_cycle_at for font_cycle_seconds — and then the title settles back into its own face: the typeface-shuffle beat of a teaser title. Give each face a scale (and tracking) so it fills about the room the title's own face does; a wide tracked title cycling through condensed or hand-drawn faces wants tracking 0 and a scale above 1 on them. Time the faces to the music with font_cycle_beats — a new face on each beat of that clip's detect_beats grid, font_cycle_beat_step 2 for every other beat or 0.5 for half beats — or to a reference's own face changes with font_cycle_switches, the timeline seconds you read them at; either sets every hold and repeats the faces in order to fill the run. A face's own hold times one face by hand; faces repeat in the list to revisit a typeface.`,inputSchema: obj({id: str("Overlay element id"),move: {@@ -150,6 +150,16 @@ export const OVERLAY_ANIMATION_TOOLS = [font_cycle_rate: num(`Face changes a second, ${FONT_CYCLE_RATE_MIN}..${FONT_CYCLE_RATE_MAX} (default ${FONT_CYCLE_RATE}: a new face every other frame at 30 fps)`),+ font_cycle_switches: {+ type: "array",+ items: { type: "number" },+ description:+ "Timeline seconds where the face changes, in order: the first shows the first face, each next one the next face, and the last returns the title to its own face (the title's end keeps the last face up). Sets every face's hold",+ },+ font_cycle_beats: str(+ "Clip id (soundtrack or video) whose beat grid times the faces: a new face on each beat inside the run, the last held to the run's end. Call detect_beats on that clip first"+ ),+ font_cycle_beat_step: num("Beats per face with font_cycle_beats: 1 (default), 2 every other beat, 0.5 two faces a beat"),}, ["id"]),},{diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 4b863d46..6c611ad0 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -823,7 +823,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {}applyCaretInput(o, anim, input);applyHitInput(o, anim, input);- applyFontCycleInput(o, anim, input);+ applyFontCycleInput(s, o, anim, input);s.updateOverlay(o.id, { anim: hasOverlayAnim(anim) ? anim : undefined });const next = useEditor.getState().overlays.find((x) => x.id === o.id)!;// The loop's real cycle, so a rate set to match a measured one can be@@ -6708,8 +6708,11 @@ function applyHitInput(/** set_overlay_animation's font-cycle fields, written onto `anim`. A new* cycle runs from the element's start to its end at the default rate; an* empty face list removes it. Only font ids the registry knows are kept, so- * the cycle never shows a face the user could not pick. */+ * the cycle never shows a face the user could not pick. Switch times or a+ * clip's beat grid time the faces instead: each face holds from one switch+ * to the next. */function applyFontCycleInput(+ s: Editor,o: { kind?: string; start: number; end: number },anim: OverlayAnim,input: Record<string, unknown>@@ -6719,7 +6722,9 @@ function applyFontCycleInput(raw !== undefined ||isNum(input.font_cycle_at) ||isNum(input.font_cycle_seconds) ||- isNum(input.font_cycle_rate);+ isNum(input.font_cycle_rate) ||+ input.font_cycle_switches !== undefined ||+ input.font_cycle_beats !== undefined;if (!named) return;// An empty list clears the cycle.@@ -6743,9 +6748,119 @@ function applyFontCycleInput(: (anim.fonts?.rate ?? FONT_CYCLE_RATE);// To the microsecond, so a run that starts on a frame stays on it.const round = (v: number) => onFrame(Math.round(v * 1e6) / 1e6);++ // Switch times (passed, or the beats of the music's clip) give each face+ // its own hold, the faces repeating in order to fill them. Example: faces+ // [a, b] over switches 2.5, 3, 4 on a title at 2 → a at 0.5 for 0.5s,+ // then b for 1s, and the title's own face after 4.+ const edges = cycleSwitchesOf(s, o, input, at, seconds);+ if (edges) {+ const timed = edges.slice(1).map((end, i) => ({+ ...faces[i % faces.length],+ hold: round(end - edges[i]),+ }));+ anim.fonts = { faces: timed, at: round(edges[0]), seconds: round(edges[edges.length - 1] - edges[0]), rate };+ return;+ }anim.fonts = { faces, at: round(at), seconds: round(seconds), rate };}+/** The face-change times of a timed font cycle, in seconds from the element's+ * start, first to last: font_cycle_switches as passed, or the stored beat+ * grid of the font_cycle_beats clip inside the cycle's run [at, at+seconds].+ * Null when neither is named. */+function cycleSwitchesOf(+ s: Editor,+ o: { start: number; end: number },+ input: Record<string, unknown>,+ at: number,+ seconds: number+): number[] | null {+ const dur = o.end - o.start;+ let edges: number[];+ if (input.font_cycle_switches !== undefined) {+ if (!Array.isArray(input.font_cycle_switches)) throw new ToolError("font_cycle_switches must be a list of timeline seconds.");+ edges = input.font_cycle_switches.filter(isNum).map((t) => t - o.start);+ } else if (input.font_cycle_beats !== undefined) {+ edges = beatSwitchesOf(s, String(input.font_cycle_beats), input, o.start, at, at + seconds);+ } else {+ return null;+ }++ // Inside the element, in order; a switch under a face's shortest hold+ // after the one before it merges into it, the run's end kept.+ const sorted = edges+ .filter((t) => t >= -1e-6 && t <= dur + 1e-6)+ .map((t) => Math.round(clamp(t, 0, dur) * 1e6) / 1e6)+ .sort((a, b) => a - b);+ edges = [];+ for (const t of sorted) {+ if (edges.length > 0 && t - edges[edges.length - 1] < CYCLE_HOLD_MIN) {+ if (t === sorted[sorted.length - 1] && edges.length > 1) {+ edges[edges.length - 1] = t;+ }+ continue;+ }+ edges.push(t);+ }+ if (edges.length < 2)+ throw new ToolError("A timed font cycle needs at least two switch times inside the title: the first face's start and the last face's end.");+ if (edges.length - 1 > FONT_CYCLE_FACES_MAX)+ throw new ToolError(+ `That run changes face ${edges.length - 1} times; one cycle holds ${FONT_CYCLE_FACES_MAX}. Pass font_cycle_beat_step 2 (every other beat), or shorten the run with font_cycle_seconds.`+ );+ const longest = Math.max(...edges.slice(1).map((t, i) => t - edges[i]));+ if (longest > CYCLE_HOLD_MAX)+ throw new ToolError(`One face would hold ${round2(longest)}s; a face holds at most ${CYCLE_HOLD_MAX}s. Add a switch inside that gap.`);+ return edges;+}++/** Face changes on a clip's beat grid, seconds from the element's start: the+ * beats (every `step`th, or each split into 1/step parts) that land inside+ * the run, closed by the next one or the run's end. */+function beatSwitchesOf(+ s: Editor,+ clipId: string,+ input: Record<string, unknown>,+ elementStart: number,+ from: number,+ to: number+): number[] {+ const clip = s.clips.find((c) => c.id === clipId) ?? s.audioClips.find((c) => c.id === clipId);+ if (!clip) throw new ToolError(`No clip with id ${clipId}. font_cycle_beats takes the clip that plays the music.`);+ const asset = s.assets.find((a) => a.id === clip.assetId);+ const grid = asset?.beats?.beats ?? [];+ if (grid.length < 2)+ throw new ToolError(`"${asset?.name ?? clipId}" has no beat grid yet. Call detect_beats with clip_id ${clipId} first, then time the cycle to it.`);++ // The beats the clip plays, on the element's clock, in play order.+ const retime = retimeOf(clip);+ const beats = grid+ .filter((b) => b >= clip.in && b <= clip.out)+ .map((b) => clip.start + retime.tAt(b) - elementStart)+ .sort((a, b) => a - b);++ // Every Nth beat counted from the run's first, or each beat split evenly.+ const step = isNum(input.font_cycle_beat_step) && input.font_cycle_beat_step > 0 ? input.font_cycle_beat_step : 1;+ const whole = step >= 1 ? Math.round(step) : 1;+ const parts = step < 1 ? Math.round(1 / step) : 1;+ if (Math.abs((step >= 1 ? whole : 1 / parts) - step) > 1e-3)+ throw new ToolError("font_cycle_beat_step is a whole number of beats (1, 2, 4) or a split of one (0.5, 0.25).");+ const first = beats.findIndex((t) => t >= from - 1e-6);+ const picked = (first < 0 ? [] : beats.slice(first)).filter((_, i) => i % whole === 0);+ const points = picked.flatMap((b, i) =>+ i + 1 < picked.length ? Array.from({ length: parts }, (_, k) => b + ((picked[i + 1] - b) * k) / parts) : [b]+ );++ // The switches inside the run, closed by the next one or the run's end.+ const inside = points.filter((t) => t >= from - 1e-6 && t < to - 1e-6);+ if (inside.length === 0) throw new ToolError("No beat of that clip lands inside the cycle's run.");+ const after = points.find((t) => t >= to - 1e-6);+ const spacing = inside.length > 1 ? (inside[inside.length - 1] - inside[0]) / (inside.length - 1) : to - inside[0];+ const close = after !== undefined ? to : Math.min(to, inside[inside.length - 1] + spacing);+ return [...inside, close];+}+/** A font id the registry knows, or a refusal naming the one it does not. */function knownFont(v: unknown): FontId {if (typeof v !== "string" || !allFonts().some((f) => f.id === v)) {diff --git a/site/src/cut/lib/fontCycleTool.test.ts b/site/src/cut/lib/fontCycleTool.test.tsindex 3fd8146c..9bcd6651 100644--- a/site/src/cut/lib/fontCycleTool.test.ts+++ b/site/src/cut/lib/fontCycleTool.test.ts@@ -2,6 +2,7 @@ import { beforeEach, describe, expect, test } from "bun:test";import { runAiTool } from "./aiTools";import { useEditor } from "./store";import { emptySubtitles, isTextOverlay } from "./types";+import { fontCycleCuts } from "@donkeycut/effects-kit";/** A title's font cycle and its heavy weights through the assistant's tools. */@@ -97,3 +98,123 @@ describe("set_overlay_animation font cycle", () => {expect(titles()[0].weight).toBe(900);});});++describe("set_overlay_animation font cycle timed to switches and beats", () => {+ beforeEach(reset);++ /** A title from `start` to `end` on the timeline. */+ const titleAt = async (start: number, end: number) => {+ await runAiTool("add_text_sequence", { look: "serif-mood", lines: [{ text: "TRAVEL", start, end }] });+ return titles()[0];+ };+ /** A song on the soundtrack at timeline `start`, its grid already detected. */+ const song = (beats: number[], start = 0, inAt = 0) =>+ useEditor.setState({+ assets: [{ id: "song", fileName: "song.mp3", name: "Song", type: "audio", duration: 30, url: "", beats: { beats, bpm: 120 } }],+ audioClips: [{ id: "a1", assetId: "song", start, in: inAt, out: 30, volume: 1 }],+ } as never);+ const faceTimes = () => {+ const c = titles()[0].anim!.fonts!;+ let t = c.at;+ return c.faces.map((f) => {+ const at = t;+ t += f.hold!;+ return [f.font, +at.toFixed(4)];+ });+ };++ test("switch times set every face's hold, faces repeating in order", async () => {+ const o = await titleAt(2, 6);+ await runAiTool("set_overlay_animation", {+ id: o.id,+ font_cycle: [{ font: "impact" }, { font: "serif", hold: 3 }],+ font_cycle_switches: [2.5, 3, 3.25, 4, 5],+ });+ const c = titles()[0].anim!.fonts!;+ expect(c.at).toBe(0.5);+ expect(c.seconds).toBe(2.5);+ expect(c.faces.map((f) => [f.font, f.hold])).toEqual([+ ["impact", 0.5],+ ["serif", 0.25],+ ["impact", 0.75],+ ["serif", 1],+ ]);+ // Every renderer cuts its pictures at the face changes: the times passed,+ // from the title's start.+ expect(fontCycleCuts(titles()[0], 4)).toEqual([0.5, 1, 1.25, 2, 3]);+ });++ test("switches outside the title are dropped, and two are needed", async () => {+ const o = await titleAt(2, 6);+ await runAiTool("set_overlay_animation", {+ id: o.id,+ font_cycle: [{ font: "impact" }],+ font_cycle_switches: [1, 2, 3, 9],+ });+ expect(titles()[0].anim!.fonts).toMatchObject({ at: 0, seconds: 1 });+ expect(await failure({ id: o.id, font_cycle_switches: [3] })).toContain("two");+ });++ test("the music's beats set the switches through the clip that plays it", async () => {+ // Beats at 0.5s steps in the source; the song sits 1s into the timeline.+ song([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5], 1);+ const o = await titleAt(2, 4.2);+ await runAiTool("set_overlay_animation", {+ id: o.id,+ font_cycle: [{ font: "impact" }, { font: "serif" }, { font: "mono" }],+ font_cycle_beats: "a1",+ });+ expect(faceTimes()).toEqual([+ ["impact", 0],+ ["serif", 0.5],+ ["mono", 1],+ ["impact", 1.5],+ ["serif", 2],+ ]);+ // The last face holds to the title's end: 0.2s past the last beat.+ expect(titles()[0].anim!.fonts!.seconds).toBe(2.2);+ });++ test("a beat step takes every other beat, or splits each beat", async () => {+ song([0, 0.5, 1, 1.5, 2, 2.5, 3]);+ const o = await titleAt(0, 2);+ await runAiTool("set_overlay_animation", {+ id: o.id,+ font_cycle: [{ font: "impact" }, { font: "serif" }],+ font_cycle_beats: "a1",+ font_cycle_beat_step: 2,+ });+ expect(faceTimes()).toEqual([+ ["impact", 0],+ ["serif", 1],+ ]);+ await runAiTool("set_overlay_animation", { id: o.id, font_cycle_beats: "a1", font_cycle_beat_step: 0.5, font_cycle_seconds: 1 });+ expect(faceTimes()).toEqual([+ ["impact", 0],+ ["serif", 0.25],+ ["impact", 0.5],+ ["serif", 0.75],+ ]);+ // Every other beat counts from the run's first beat.+ await runAiTool("set_overlay_animation", { id: o.id, font_cycle_beats: "a1", font_cycle_beat_step: 2, font_cycle_at: 0.5, font_cycle_seconds: 1.5 });+ expect(faceTimes()).toEqual([+ ["impact", 0.5],+ ["serif", 1.5],+ ]);+ });++ test("a clip with no beat grid sends the model to detect_beats", async () => {+ useEditor.setState({+ assets: [{ id: "song", fileName: "song.mp3", name: "Song", type: "audio", duration: 30, url: "" }],+ audioClips: [{ id: "a1", assetId: "song", start: 0, in: 0, out: 30, volume: 1 }],+ } as never);+ const o = await titleAt(0, 2);+ expect(await failure({ id: o.id, font_cycle: [{ font: "impact" }], font_cycle_beats: "a1" })).toContain("detect_beats");+ });++ test("more switches than a cycle holds is refused with what to do", async () => {+ song(Array.from({ length: 60 }, (_, i) => i * 0.25));+ const o = await titleAt(0, 14);+ expect(await failure({ id: o.id, font_cycle: [{ font: "impact" }], font_cycle_beats: "a1" })).toContain("font_cycle_beat_step");+ });+});