Open source CapCut alternative.
TypeScript
feat(site): the assistant places audio effects
David Le2f462777b23a
diff --git a/site/scripts/lib/cut-eval/cases.ts b/site/scripts/lib/cut-eval/cases.tsindex 87c413de..aa346f2d 100644--- a/site/scripts/lib/cut-eval/cases.ts+++ b/site/scripts/lib/cut-eval/cases.ts@@ -428,6 +428,32 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCasereturn { id: "fx1", effect: "vhs", start, end, amount: Number(args.amount) || 0.5 };},},+ {+ // A treatment on the SOUND is the same effect element as a treatment on+ // the picture: "sound like a phone call" over a clip is one add_effect+ // with the telephone id spanning that clip. Reaching for the per-clip+ // audio knobs (volume, fades, duck) instead misses the ask.+ name: "audio-effect-single-tool",+ bucket: "single-tool",+ input: () => [+ userTurn("make the second clip sound like it's coming through a phone", {+ state: TWO_CLIP_STATE,+ }),+ ],+ reply: /phone|telephone|effect/i,+ requiredTools: ["add_effect"],+ maxToolCalls: 3,+ state: TWO_CLIP_STATE,+ simulate: () => (name, args) => {+ if (name !== "add_effect") return undefined;+ if (args.effect !== "telephone") throw new Error(`effect ${args.effect} — expected telephone`);+ const start = Number(args.start ?? 0);+ const end = Number(args.end ?? start + 3);+ if (start > 6.5 || end < 11.5)+ throw new Error(`effect spans ${start}..${end} — the second clip runs 6..12.5`);+ return { id: "fx1", effect: "telephone", start, end, amount: Number(args.amount) || 0.5 };+ },+ },{// Last pick wins per edge: swapping a crossfading joint for an entrance// animation is a set_animation — the override clears the transitiondiff --git a/site/src/cut/components/EffectsPanel.tools.ts b/site/src/cut/components/EffectsPanel.tools.tsindex 72e69b8d..36a14b40 100644--- a/site/src/cut/components/EffectsPanel.tools.ts+++ b/site/src/cut/components/EffectsPanel.tools.ts@@ -6,7 +6,7 @@* removed or renamed tool breaks the build until every side catches up.*/-import { EFFECT_IDS, LOOK_EFFECTS } from "@donkeycut/effects-kit";+import { ALL_EFFECT_IDS, AUDIO_EFFECT_IDS, EFFECT_IDS, LOOK_EFFECTS } from "@donkeycut/effects-kit";import { num, obj, type AiToolDef } from "@/cut/lib/aiToolDef";/** The effects with a treatment recipe of their own; the rest of the list is@@ -19,12 +19,12 @@ export const EFFECTS_TOOLS = [{name: "add_effect",description:- `Add a time-ranged visual effect element over the video: a zoom into part of the frame, the treatments (${EFFECT_TREATMENTS.join(", ")}) or a graded look (${LOOK_EFFECTS.join(", ")}). It filters everything under it for its window (start..end) and stacks with other effects. Tasteful default: one effect at a time, amount under ~0.5.`,+ `Add a time-ranged effect element over the cut: a zoom into part of the frame, a picture treatment (${EFFECT_TREATMENTS.join(", ")}), a graded look (${LOOK_EFFECTS.join(", ")}), or an audio treatment (${AUDIO_EFFECT_IDS.join(", ")}) over the sound. A picture effect filters everything visible under its window (start..end); an audio effect treats everything audible under it — clip sound, upper tracks and soundtrack together. They stack. Tasteful default: one effect at a time, amount under ~0.5.`,inputSchema: obj({- effect: { type: "string", enum: [...EFFECT_IDS], description: "Effect id" },+ effect: { type: "string", enum: [...ALL_EFFECT_IDS], description: "Effect id" },start: num("Start time s (default: playhead)"),end: num("End time s (default: start+3)"),- amount: num("Strength 0.05..1 (default 0.5). For zoom it is the depth: 0.25 shallow, 0.5 moderate, 1 deep"),+ amount: num("Strength 0.05..1 (default 0.5). For zoom it is the depth: 0.25 shallow, 0.5 moderate, 1 deep; for an audio effect it is how far the treatment goes"),focus_x: num("Zoom only: the x of the point to zoom into, 0..1 (default 0.5)"),focus_y: num("Zoom only: the y of the point to zoom into, 0..1 (default 0.5)"),lane: num("Element row (0 = the front row, drawn over every higher row). Elements on one row never overlap — a title over a shape needs a lower row than the shape."),diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex befbf61f..f4b88019 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -1,9 +1,9 @@"use client";import {+ ALL_EFFECT_IDS,autoGradeFromImageData,colorStatsFromImageData,- EFFECT_IDS,GRADE_BASIC_FIELDS,GRADE_PRESETS,HSL_BANDS,@@ -388,8 +388,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {add_effect: (s, input) => {const effect = String(input.effect ?? "");- if (!(EFFECT_IDS as string[]).includes(effect))- throw new ToolError(`effect must be one of ${EFFECT_IDS.join(", ")}.`);+ if (!(ALL_EFFECT_IDS as string[]).includes(effect))+ throw new ToolError(`effect must be one of ${ALL_EFFECT_IDS.join(", ")}.`);if (isNum(input.start)) s.seek(input.start);s.addEffect(effect as EffectId, aimedLane(input));const sel = useEditor.getState().selection;diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex 4b8ba93b..503a9a66 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -31,6 +31,7 @@ import { TIMELINE_TOOLS } from "@/cut/components/Timeline.tools";import { TOP_BAR_TOOLS } from "@/cut/components/TopBar.tools";import { TRANSITIONS_TOOLS } from "@/cut/components/TransitionsPanel.tools";import {+ AUDIO_EFFECT_IDS,gradePresetCatalogText,HSL_BANDS,LOOK_EFFECTS,@@ -83,7 +84,7 @@ Times are in seconds on the shared timeline. The playhead is currentTime; a skim- A clip's timeline length is (out-in)/speed; total duration runs to the last clip's end, gaps included. Transitions blend at cuts and never change layout or length.- trim_clip changes in/out inside the source media. in >= 0, out <= source duration, out-in >= 0.1.- set_speed sets a clip's playback rate; it changes the clip's timeline length, and later titles/captions ripple to stay in sync.-- set_transition joins a clip into the next one (0–2s, ${TRANSITION_STYLE_IDS.length} styles); set_animation animates one clip's own entrance/exit — read the transitions-and-fades skill before styling cuts. Splitting or deleting clears the affected transition. A transition is a bar on its own row at an absolute time, so an edit can leave one lining up with nothing: every timeline mutation reports those as \`parkedTransitions\`, and clearing them (remove_transition) is part of finishing the edit. Grading a clip's picture is the color tools (set_color_preset, set_color_grade, set_color_curves, set_color_wheels, set_color_hsl — read the color-grading skill); a stylized post treatment (grain, VHS, halation…) is add_effect, dropped over the stretch of timeline it covers.+- set_transition joins a clip into the next one (0–2s, ${TRANSITION_STYLE_IDS.length} styles); set_animation animates one clip's own entrance/exit — read the transitions-and-fades skill before styling cuts. Splitting or deleting clears the affected transition. A transition is a bar on its own row at an absolute time, so an edit can leave one lining up with nothing: every timeline mutation reports those as \`parkedTransitions\`, and clearing them (remove_transition) is part of finishing the edit. Grading a clip's picture is the color tools (set_color_preset, set_color_grade, set_color_curves, set_color_wheels, set_color_hsl — read the color-grading skill); a stylized post treatment (grain, VHS, halation…) is add_effect, dropped over the stretch of timeline it covers — as is a treatment on the sound (echo, reverb, telephone…).- split_at cuts the track-0 clip under that time into two clips at the exact frame. With a soundtrack or overlay clip selected it splits that instead.- The user can multi-select (⌘/⇧-click, or sweep a rubber-band over empty track space) across every kind at once — clips, audio, titles, cues. Deleting or copying acts on the whole selection, and dragging any member moves the set as one: every item shifts by the same time, relative spacing kept, while unselected items slide out of the way and flow back. A selection of one kind on the multi-row tracks (audio, titles) also retracks together on a vertical drag, one row past the end opening a new track. A hover chip on each video clip toggles its own audio.- Anything can be hidden instead of deleted: it stays on the timeline (grayed) but drops out of playback and export — a hidden video clip plays black and silent. Per item: set_clip_hidden, or the \`hidden\` field on update_overlay_video/update_overlay/update_audio. Whole rows: set_track_hidden (video/soundtrack/text/subtitles) and set_track_muted (video) — the eye/speaker toggles on the track headers.@@ -171,7 +172,7 @@ Titles (add_title / update_overlay): text, size (frame px; the design short sideShapes (add_shape): rect, ellipse, line, arrow. w/h are frame fractions; rect/ellipse take fill + fill_opacity, corner radius (rect), and an outline (stroke_color/stroke_width); line/arrow draw in \`fill\` with h as their thickness, and rotation gives them their direction (0° points right, 90° points down).Stickers (add_sticker): a project image asset (generated cutouts in the Elements panel carry origin "sticker"); w is a frame-width fraction and height follows the source's aspect. Lottie JSON uploads become animated stickers that loop for the element's duration.Custom stickers (create_sticker): one call generates the image, removes the background, adds the die-cut outline, and places it — use it when the user asks for a sticker of something they don't have (it spends credits like image generation).-Effects (add_effect): time-ranged treatments over the finished picture — the footage and everything laid over it. Two families, one list: the treatments (${EFFECT_TREATMENTS.join(", ")}) and the graded looks (${LOOK_EFFECTS.join(", ")}). Placed like any element and tuned with \`amount\` (update_overlay); an effect added over a clip opens covering that clip, and trims like anything else. One effect at a time reads best, amounts under ~0.5. Effects are stylized post treatments on a stretch of timeline; pure color — exposure, warmth, a cinematic or film palette on one clip — is the per-clip grading tools instead (color-grading skill), whose preset catalog is separate from the look effects.+Effects (add_effect): time-ranged treatments over the finished cut. Three families, one list: the picture treatments (${EFFECT_TREATMENTS.join(", ")}), the graded looks (${LOOK_EFFECTS.join(", ")}), and the audio treatments (${AUDIO_EFFECT_IDS.join(", ")}). A picture effect treats the footage and everything laid over it; an audio effect treats everything audible under its window at once — clip sound, upper-track sound and the soundtrack — so "make this bit sound like a phone call", "put an echo on that line", or "muffle the music here" is an effect element over that stretch, and a per-clip volume, fade or duck is the audio tools instead. A cut's own audio crossfade at a join belongs to the transition on that cut, not here. Placed like any element and tuned with \`amount\` (update_overlay); an effect added over a clip opens covering that clip, and trims like anything else. One effect at a time reads best, amounts under ~0.5. Effects are stylized post treatments on a stretch of timeline; pure color — exposure, warmth, a cinematic or film palette on one clip — is the per-clip grading tools instead (color-grading skill), whose preset catalog is separate from the look effects.Advanced text (update_overlay): italic, align (multi-line), letter_spacing (em), line_height, stretch_x/stretch_y (glyph stretch multipliers, the side grips' axis-only resize), stroke_color/stroke_width (em outline behind the fill), and richer shadows. Bundled Google families join the system set — ${GOOGLE_FONT_IDS.join(", ")} — and so do the user's own: a .ttf/.otf/.woff/.woff2 file dropped on the Library is offered in every project under a font id like "font:<id>". Editor_state lists the ones this account has under \`fonts\` — use an id from there instead of guessing at a family name, and when someone asks for a typeface that is not in the list, tell them the Library takes font files.Animation (set_overlay_animation): every element takes preset In/Out ramps (${OVERLAY_ANIM_STYLE_IDS.join(", ")} — typewriter for titles only; the per-letter styles sweep a title character by character), a Loop (${OVERLAY_LOOP_STYLE_IDS.join(", ")}) that runs its whole duration, and a Move that says what it does WHILE it holds. Entrances 0.3–0.6s read well; one loop per scene, sparingly; a move on the lines that matter. All four are slots on the element, applied internally; the keyframe track (set_overlay_keyframes) is the separate power tier for motion the names do not cover, and slots compose over it.Keyframes (set_overlay_keyframes): for motion no preset covers — a path across the frame, a slow push in, a drift that holds first. A key is a whole pose (position, scale, rotation, opacity) at a time in seconds from the element's start; the pose moves linearly between keys and holds outside them, and presets still compose on top. Two or three keys carry almost everything; reach for a preset first and keyframes when the user describes a specific path or timing. In the UI the diamond in the inspector adds a key at the playhead, and dragging the element then records into it. set_clip_keyframes is the same track on a video clip (any track): pans, push-ins, picture-in-picture flights, spins, and fades on the footage itself.@@ -299,6 +300,7 @@ ${textMoveCatalog()}Lyrics and scripts the user already has: sync_lyrics times their exact words against the audio (transcribing first when needed) and writes them to a caption track with per-word timings; set_caption_look then dresses the track — preset, size, font, karaoke word highlight, accent color — from a text-videos look id or field by field. The text-videos skill has the whole flow, including turning the same timed lines into full-frame cards.Soundtrack clips: volume 0..1.5, fadeIn/fadeOut seconds (max half the clip), start = timeline position, in/out = trim inside the source; clips can spread across several soundtrack lanes (the \`lane\` field), new sounds slide to free space in their lane. Fades render with ffmpeg afade on export.Ducking: a soundtrack clip's \`duck\` (0..1, via update_audio) lowers ALL other audio — video-clip sound and other music — to that gain while the clip plays; 1 clears it. Voiceovers set this so narration sits over quieter music. It applies in both the preview and the export.+Audio effects (add_effect): the treatments over the sound — ${AUDIO_EFFECT_IDS.join(", ")} — placed on the timeline as elements, each treating everything audible under its window at once. Reach for one when the ask is how a stretch SOUNDS ("make this line sound like a phone call", "echo on that word", "muffle the music behind the voiceover"); \`amount\` says how far the treatment goes, and the effect trims and moves like any element. Loudness, fades and ducking on one clip stay with the audio tools; the crossfade across a cut stays with the transition on that cut.Voiceover (Donkey's hosted speech model — signed in, spends credits, like image/video generation):- voiceover_generate(script, voice?, direction?, duck?, add_to_timeline?, start?, language?): synthesizes the script into a playable chat card; add_to_timeline:true (or a start) also drops it on the soundtrack at the playhead (or start) when the user asked for it in the cut. Defaults to a 0.4 duck when placed so it sits over other audio. Voices are Gemini's prebuilt set — list_voices returns them (id + one-word character like Warm, Upbeat, Gravelly); omit voice for a good default. \`direction\` steers delivery in natural language ("Say warmly, like an old friend") and may ask for another language — "say it in Spanish" translates the script before synthesis; the script itself can carry inline tags like [whispers], [excited], [laughs]. \`language\` only pins pronunciation of the text as written.- read_subtitles_aloud(voice?, direction?, duck?, track?): speaks one subtitle track's cues, each line at its own cue time — captions become narration. Needs cues first. The Inspector offers the same per-clip: "Generate audio for clip" transcribes just that clip when it has no cues yet, then voices it.