Open source CapCut alternative.
TypeScript
feat(site): the assistant shapes a clip's sound [rebuild]
David Le59ffe3e255f7
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.tsindex 187e82d6..fe5f9ee2 100644--- a/site/src/cut/components/Inspector.tools.ts+++ b/site/src/cut/components/Inspector.tools.ts@@ -13,6 +13,11 @@ import {GRADE_PRESET_IDS,gradePresetCatalogText,HSL_BANDS,+ SOUND_COMPRESSOR_RANGE,+ SOUND_EQ_BANDS,+ SOUND_EQ_DB_RANGE,+ SOUND_LIMITER_RANGE,+ SOUND_PRESETS,} from "@donkeycut/effects-kit";import { bool, num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";@@ -175,6 +180,52 @@ export const INSPECTOR_TOOLS = [description: "Set the gain on a video clip's own audio (soundtrack clips use update_audio).",inputSchema: obj({ clipId: str("Video clip id"), volume: num("0..3 (1 = unchanged, up to 3 boosts the clip's own sound)") }, ["clipId", "volume"]),},+ {+ name: "set_clip_sound",+ description:+ `Shape a clip's own sound — video clip or soundtrack clip — with the Sound quality section's three stages: a ${SOUND_EQ_BANDS.length}-band equalizer, a compressor, and a limiter. Each stage runs on that clip alone, before its fades and any ducking, in the preview and every export. Pass \`preset\` to apply a shipped treatment (${SOUND_PRESETS.map((p) => `"${p.name}"`).join(", ")}) or one the user saved, by name or id; or set the stages directly, leaving out the ones that stay as they are. \`clear\` names stages to remove, and the "Off" preset removes all three. Read the clip's current treatment from editor_state before a partial change.`,+ inputSchema: obj(+ {+ clipId: str("Video clip id or soundtrack clip id"),+ preset: str(+ `A preset by name or id: shipped are ${SOUND_PRESETS.map((p) => `"${p.name}" (${p.id})`).join(", ")}, plus whatever the user has saved. "Clear voice" is the podcast/voice treatment: warmth below 200 Hz, mud pulled out at 400 Hz, presence at 3–5 kHz, 3:1 compression at −18 dB and a −1 dB ceiling. "Off" leaves the clip's sound as it came.`+ ),+ clear: {+ type: "array",+ description: "Stages to remove from the clip's treatment.",+ items: { type: "string", enum: ["eq", "compressor", "limiter"] },+ },+ eq: {+ type: "array",+ description: `Gain in dB per band, −${SOUND_EQ_DB_RANGE}..+${SOUND_EQ_DB_RANGE}, in this order: ${SOUND_EQ_BANDS.map((b) => b.label).join(", ")} (the first band is a low shelf, the last a high shelf, the rest peaks). Fewer values leave the remaining bands flat.`,+ items: { type: "number" },+ },+ compressor: obj(+ {+ threshold: num(`dB, ${SOUND_COMPRESSOR_RANGE.threshold.min}..${SOUND_COMPRESSOR_RANGE.threshold.max}`),+ ratio: num(`${SOUND_COMPRESSOR_RANGE.ratio.min}..${SOUND_COMPRESSOR_RANGE.ratio.max} (:1)`),+ attack: num(`ms, ${SOUND_COMPRESSOR_RANGE.attack.min}..${SOUND_COMPRESSOR_RANGE.attack.max}`),+ release: num(`ms, ${SOUND_COMPRESSOR_RANGE.release.min}..${SOUND_COMPRESSOR_RANGE.release.max}`),+ },+ ["threshold", "ratio", "attack", "release"]+ ),+ limiter: obj(+ { ceiling: num(`dBFS ceiling, ${SOUND_LIMITER_RANGE.ceiling.min}..${SOUND_LIMITER_RANGE.ceiling.max}`) },+ ["ceiling"]+ ),+ },+ ["clipId"]+ ),+ },+ {+ name: "save_sound_preset",+ description:+ "Save a clip's current sound treatment (equalizer, compressor, limiter) to the user's Library under a name, so set_clip_sound can apply it to other clips by that name and the Sound quality section offers it in its preset dropdown.",+ inputSchema: obj(+ { clipId: str("Video clip id or soundtrack clip id whose treatment to save"), name: str("Preset name") },+ ["clipId", "name"]+ ),+ },{name: "detach_audio",description:diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.tsindex acb00d02..598b2c7a 100644--- a/site/src/cut/lib/aiContext.ts+++ b/site/src/cut/lib/aiContext.ts@@ -1,6 +1,6 @@"use client";-import { hasOverlayAnim } from "@donkeycut/effects-kit";+import { hasOverlayAnim, type ClipSound } from "@donkeycut/effects-kit";import { chatOwner } from "./chatAssets";import { useGenerate } from "./generate";import { useMatteBakes } from "./removal/bakeJobs";@@ -333,6 +333,9 @@ export function buildAiContext(opts?: { fullCues?: boolean; chatId?: string | nusourceDuration: r(sp.asset.type === "image" ? sp.len : sp.asset.duration),muted: sp.clip.muted,...(sp.clip.hidden ? { hidden: true } : {}),+ // The clip's own equalizer/compressor/limiter, set through+ // set_clip_sound; absent when its sound is untouched.+ ...(sp.clip.sound ? { sound: sp.clip.sound } : {}),framing: sp.clip.fit ?? "fit",speed: r(sp.clip.speed ?? 1),// The generated scene shot this clip came from — sceneShot is the@@ -450,7 +453,7 @@ export function buildAiContext(opts?: { fullCues?: boolean; chatId?: string | nu}function describeAudio(- a: { assetId: string; start: number; in: number; out: number; volume: number; fadeIn?: number; fadeOut?: number; speed?: number; duck?: number; lane?: number; hidden?: boolean },+ a: { assetId: string; start: number; in: number; out: number; volume: number; fadeIn?: number; fadeOut?: number; speed?: number; sound?: ClipSound; duck?: number; lane?: number; hidden?: boolean },assets: Map<string, { name: string }>) {const speed = a.speed && a.speed > 0 ? a.speed : 1;@@ -463,6 +466,7 @@ function describeAudio(volume: r(a.volume),fadeIn: r(a.fadeIn ?? 0),fadeOut: r(a.fadeOut ?? 0),+ ...(a.sound ? { sound: a.sound } : {}),...(speed !== 1 ? { speed: r(speed) } : {}),...(a.lane ? { lane: a.lane } : {}),...(a.hidden ? { hidden: true } : {}),@@ -483,6 +487,7 @@ function describeOverlayClip(c: VideoClip, assets: Map<string, { name: string }>out: r(c.out),muted: c.muted,...(c.hidden ? { hidden: true } : {}),+ ...(c.sound ? { sound: c.sound } : {}),// The frame region this layer occupies: Full covers the frame; Top/Bottom/// Left/Right split it; PiP floats inside it.layout: regionLabel(rect),diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 98f3775b..3190fff4 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -9,7 +9,10 @@ import {HSL_BANDS,matchGrade,normalizeGrade,+ normalizeSound,semanticMasterCurve,+ SOUND_PRESETS,+ type ClipSound,type ColorStats,type CurvePoint,type EffectId,@@ -106,6 +109,8 @@ import { characterPrompt, stockAspectDims, stockTitle } from "./stock";import { STOCK_IMAGES } from "./stockManifest";import { STOCK_VIDEOS } from "./stockVideoManifest";import { storedMediaUrl } from "./mediaSync";+import { isSoundPresetTemplate, listSoundPresets, saveSoundPreset } from "./soundPresets";+import { isStylePresetTemplate } from "./stylePresets";import { applyOverlayPatchSettled, track0Clips, laneGapAt, getClipSpans, nextFreeStart, overlayLaneOrder, overlayLayers, parkedTransitions, projectDuration, resolveTransitions, totalDuration, useEditor } from "./store";import { playheadAt } from "./playhead";import { renderProjectFrame } from "./exportRender";@@ -1481,6 +1486,79 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {return { id: clip.id, hidden: Boolean(input.hidden) };},+ set_clip_sound: async (s, input) => {+ const target = soundTarget(s, input.clipId);+ let next: ClipSound | undefined;+ if (typeof input.preset === "string" && input.preset.trim()) {+ const wanted = input.preset.trim().toLowerCase();+ // The dropdown labels the shipped presets by name, so a name is what+ // the user says; the ids are what the catalog lists.+ const shipped = SOUND_PRESETS.find(+ (p) => p.id === wanted || p.name.toLowerCase() === wanted+ );+ if (shipped) {+ next = shipped.sound;+ } else {+ const saved = (await listSoundPresets()).find(+ (p) => p.name.toLowerCase() === wanted || p.id === input.preset+ );+ if (!saved)+ throw new ToolError(+ `No sound preset called "${String(input.preset)}". Shipped: ${SOUND_PRESETS.map((p) => p.name).join(", ")}.`+ );+ next = saved.sound;+ }+ } else {+ const patch: ClipSound = { ...(target.current ?? {}) };+ if ("eq" in input) {+ if (input.eq === null) delete patch.eq;+ else if (Array.isArray(input.eq)) patch.eq = input.eq.map((v) => (isNum(v) ? v : 0));+ else throw new ToolError("eq must be an array of dB values, one per band.");+ }+ if ("compressor" in input) {+ const c = input.compressor as Record<string, unknown> | null;+ if (c === null) delete patch.compressor;+ else if (c && isNum(c.threshold) && isNum(c.ratio) && isNum(c.attack) && isNum(c.release))+ patch.compressor = { threshold: c.threshold, ratio: c.ratio, attack: c.attack, release: c.release };+ else throw new ToolError("compressor needs threshold, ratio, attack and release.");+ }+ if ("limiter" in input) {+ const l = input.limiter as Record<string, unknown> | null;+ if (l === null) delete patch.limiter;+ else if (l && isNum(l.ceiling)) patch.limiter = { ceiling: l.ceiling };+ else throw new ToolError("limiter needs a ceiling.");+ }+ for (const stage of Array.isArray(input.clear) ? input.clear : []) {+ if (stage === "eq") delete patch.eq;+ else if (stage === "compressor") delete patch.compressor;+ else if (stage === "limiter") delete patch.limiter;+ else throw new ToolError(`clear takes eq, compressor or limiter; got "${String(stage)}".`);+ }+ if (+ !("eq" in input) &&+ !("compressor" in input) &&+ !("limiter" in input) &&+ !Array.isArray(input.clear)+ )+ throw new ToolError("Pass a preset, a clear list, or eq / compressor / limiter.");+ next = patch;+ }+ const sound = normalizeSound(next);+ target.write(sound);+ return { id: target.id, kind: target.kind, sound: soundTarget(useEditor.getState(), target.id).current ?? null };+ },++ save_sound_preset: async (s, input) => {+ const target = soundTarget(s, input.clipId);+ const name = typeof input.name === "string" ? input.name.trim() : "";+ if (!name) throw new ToolError("name is required.");+ if (!target.current) throw new ToolError("That clip's sound is flat; shape it first.");+ if (!s.projectId) throw new ToolError("No open project.");+ const saved = await saveSoundPreset(s.projectId, name, target.current);+ if (!saved) throw new ToolError("The preset could not be saved.");+ return { id: saved.id, name: saved.name, residency: saved.residency, sound: saved.sound };+ },+reorder_track: (s, input) => {if (!isNum(input.from) || !isNum(input.to) || input.from < 0 || input.to < 0)throw new ToolError("from and to must be 0 or higher.");@@ -2781,7 +2859,9 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {// "inspiration" = saved from the phone's Ideas tab....(a.origin ? { origin: a.origin } : {}),})),- templates: lib.templates.map((t) => ({+ templates: lib.templates+ .filter((t) => !isSoundPresetTemplate(t) && !isStylePresetTemplate(t))+ .map((t) => ({id: t.id,name: t.name,duration: round2(t.duration),@@ -4047,6 +4127,32 @@ async function listenToSource(return blobToInlineAudio(await res.blob());}+/** The clip a sound tool names — a video clip or a soundtrack clip, whichever+ * carries the id — with its current treatment and the store write that+ * lands a new one. */+function soundTarget(s: Editor, id: unknown) {+ const key = typeof id === "string" ? id.trim() : "";+ const clip = s.clips.find((c) => c.id === key);+ if (clip) {+ return {+ id: clip.id,+ kind: "clip" as const,+ current: clip.sound,+ write: (sound: ClipSound | undefined) => s.updateClip(clip.id, { sound }),+ };+ }+ const audio = s.audioClips.find((a) => a.id === key);+ if (audio) {+ return {+ id: audio.id,+ kind: "audio" as const,+ current: audio.sound,+ write: (sound: ClipSound | undefined) => s.updateAudio(audio.id, { sound }),+ };+ }+ throw new ToolError(`No video or soundtrack clip with id "${key}".`);+}+function requireItem<T extends { id: string }>(pool: T[], id: unknown, label: string): T {const item = pool.find((x) => x.id === String(id ?? ""));if (!item) throw new ToolError(`No ${label} with id ${String(id)}. Call get_state for current ids.`);diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex eebbee6d..fb48ee80 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -45,6 +45,7 @@ import {LOOK_EFFECTS,OVERLAY_ANIM_STYLE_IDS,OVERLAY_LOOP_STYLE_IDS,+ SOUND_EQ_BANDS,} from "@donkeycut/effects-kit";import type { AiToolDef } from "../../lib/aiToolDef";import { GOOGLE_FONT_IDS } from "../../lib/fontCatalog";@@ -323,6 +324,7 @@ ${textMoveCatalog()}Lyrics and scripts the user already has: sync_lyrics times their exact words against the audio (transcribing first when needed) and writes them to a caption track with per-word timings; set_caption_look then dresses the track — preset, size, font, word effect, accent color — from a text-videos look id or field by field. The text-videos skill has the whole flow, including turning the same timed lines into full-frame cards.Soundtrack clips: volume 0..3 (1 = unchanged, above 1 boosts), fadeIn/fadeOut seconds (max half the clip), start = timeline position, in/out = trim inside the source; clips can spread across several soundtrack lanes (the \`lane\` field), new sounds slide to free space in their lane. Fades render with ffmpeg afade on export.Ducking: a soundtrack clip's \`duck\` (0..1, via update_audio) lowers ALL other audio — video-clip sound and other music — to that gain while the clip plays; 1 clears it. Voiceovers set this so narration sits over quieter music. It applies in both the preview and the export.+Sound quality (set_clip_sound): a clip's own treatment — video clip or soundtrack clip — in three stages the Inspector's Audio tab shows as Sound quality: a ${SOUND_EQ_BANDS.length}-band equalizer (${SOUND_EQ_BANDS.map((b) => b.label).join(", ")}; dB per band), a compressor (threshold dB, ratio, attack ms, release ms) and a limiter (dBFS ceiling). It runs on that clip alone, before its fades and any ducking, in the preview and every export. "Make the voice clearer / more podcast-like / less muddy / more even in level / stop it clipping" is this tool: \`preset: "clear-voice"\` is the shipped voice treatment (warmth under 200 Hz, mud out at 400 Hz, presence at 3–5 kHz, 3:1 at −18 dB, −1 dB ceiling); \`preset: "flat"\` clears it; the user's own saved presets apply by name; or set the stages directly, null clearing one. save_sound_preset stores a clip's treatment in the Library under a name for reuse. A treatment shapes one clip's sound; an effect element treats a stretch of the mix.Cutting to music: detect_beats reads a track's beat grid once — bpm plus every beat, with timeline times through the clip you pass — and the grid stays on the asset as yellow dots the user can drag, add (⌥-click the bar), or delete, with every drag and trim snapping to them. Land cuts, clip starts, and text entrances on those times: split_at at the timeline beats, or place each item to start on one.Audio effects (add_effect): the treatments over the sound — ${AUDIO_EFFECT_IDS.join(", ")} — placed on the timeline as elements, each treating everything audible under its window at once. Reach for one when the ask is how a stretch SOUNDS ("make this line sound like a phone call", "echo on that word", "muffle the music behind the voiceover"); \`amount\` says how far the treatment goes, and the effect trims and moves like any element. Loudness, fades and ducking on one clip stay with the audio tools; a dissolve across a cut stays with the transition on that cut (the audiocross style).Voiceover (Donkey's hosted speech model — signed in, spends credits, like image/video generation):