Skip to content

Open source CapCut alternative.

TypeScript

feat(site): the assistant shapes a clip's sound [rebuild]

David Le59ffe3e255f7
diff --git a/site/src/cut/components/Inspector.tools.ts b/site/src/cut/components/Inspector.tools.ts
index 187e82d6..fe5f9ee2 100644
--- a/site/src/cut/components/Inspector.tools.ts
+++ b/site/src/cut/components/Inspector.tools.ts
@@ -13,6 +13,11 @@ import {
GRADE_PRESET_IDS,
gradePresetCatalogText,
HSL_BANDS,
+ SOUND_COMPRESSOR_RANGE,
+ SOUND_EQ_BANDS,
+ SOUND_EQ_DB_RANGE,
+ SOUND_LIMITER_RANGE,
+ SOUND_PRESETS,
} from "@donkeycut/effects-kit";
import { bool, num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";
@@ -175,6 +180,52 @@ export const INSPECTOR_TOOLS = [
description: "Set the gain on a video clip's own audio (soundtrack clips use update_audio).",
inputSchema: obj({ clipId: str("Video clip id"), volume: num("0..3 (1 = unchanged, up to 3 boosts the clip's own sound)") }, ["clipId", "volume"]),
},
+ {
+ name: "set_clip_sound",
+ description:
+ `Shape a clip's own sound — video clip or soundtrack clip — with the Sound quality section's three stages: a ${SOUND_EQ_BANDS.length}-band equalizer, a compressor, and a limiter. Each stage runs on that clip alone, before its fades and any ducking, in the preview and every export. Pass \`preset\` to apply a shipped treatment (${SOUND_PRESETS.map((p) => `"${p.name}"`).join(", ")}) or one the user saved, by name or id; or set the stages directly, leaving out the ones that stay as they are. \`clear\` names stages to remove, and the "Off" preset removes all three. Read the clip's current treatment from editor_state before a partial change.`,
+ inputSchema: obj(
+ {
+ clipId: str("Video clip id or soundtrack clip id"),
+ preset: str(
+ `A preset by name or id: shipped are ${SOUND_PRESETS.map((p) => `"${p.name}" (${p.id})`).join(", ")}, plus whatever the user has saved. "Clear voice" is the podcast/voice treatment: warmth below 200 Hz, mud pulled out at 400 Hz, presence at 3–5 kHz, 3:1 compression at −18 dB and a −1 dB ceiling. "Off" leaves the clip's sound as it came.`
+ ),
+ clear: {
+ type: "array",
+ description: "Stages to remove from the clip's treatment.",
+ items: { type: "string", enum: ["eq", "compressor", "limiter"] },
+ },
+ eq: {
+ type: "array",
+ description: `Gain in dB per band, −${SOUND_EQ_DB_RANGE}..+${SOUND_EQ_DB_RANGE}, in this order: ${SOUND_EQ_BANDS.map((b) => b.label).join(", ")} (the first band is a low shelf, the last a high shelf, the rest peaks). Fewer values leave the remaining bands flat.`,
+ items: { type: "number" },
+ },
+ compressor: obj(
+ {
+ threshold: num(`dB, ${SOUND_COMPRESSOR_RANGE.threshold.min}..${SOUND_COMPRESSOR_RANGE.threshold.max}`),
+ ratio: num(`${SOUND_COMPRESSOR_RANGE.ratio.min}..${SOUND_COMPRESSOR_RANGE.ratio.max} (:1)`),
+ attack: num(`ms, ${SOUND_COMPRESSOR_RANGE.attack.min}..${SOUND_COMPRESSOR_RANGE.attack.max}`),
+ release: num(`ms, ${SOUND_COMPRESSOR_RANGE.release.min}..${SOUND_COMPRESSOR_RANGE.release.max}`),
+ },
+ ["threshold", "ratio", "attack", "release"]
+ ),
+ limiter: obj(
+ { ceiling: num(`dBFS ceiling, ${SOUND_LIMITER_RANGE.ceiling.min}..${SOUND_LIMITER_RANGE.ceiling.max}`) },
+ ["ceiling"]
+ ),
+ },
+ ["clipId"]
+ ),
+ },
+ {
+ name: "save_sound_preset",
+ description:
+ "Save a clip's current sound treatment (equalizer, compressor, limiter) to the user's Library under a name, so set_clip_sound can apply it to other clips by that name and the Sound quality section offers it in its preset dropdown.",
+ inputSchema: obj(
+ { clipId: str("Video clip id or soundtrack clip id whose treatment to save"), name: str("Preset name") },
+ ["clipId", "name"]
+ ),
+ },
{
name: "detach_audio",
description:
diff --git a/site/src/cut/lib/aiContext.ts b/site/src/cut/lib/aiContext.ts
index acb00d02..598b2c7a 100644
--- a/site/src/cut/lib/aiContext.ts
+++ b/site/src/cut/lib/aiContext.ts
@@ -1,6 +1,6 @@
"use client";
-import { hasOverlayAnim } from "@donkeycut/effects-kit";
+import { hasOverlayAnim, type ClipSound } from "@donkeycut/effects-kit";
import { chatOwner } from "./chatAssets";
import { useGenerate } from "./generate";
import { useMatteBakes } from "./removal/bakeJobs";
@@ -333,6 +333,9 @@ export function buildAiContext(opts?: { fullCues?: boolean; chatId?: string | nu
sourceDuration: r(sp.asset.type === "image" ? sp.len : sp.asset.duration),
muted: sp.clip.muted,
...(sp.clip.hidden ? { hidden: true } : {}),
+ // The clip's own equalizer/compressor/limiter, set through
+ // set_clip_sound; absent when its sound is untouched.
+ ...(sp.clip.sound ? { sound: sp.clip.sound } : {}),
framing: sp.clip.fit ?? "fit",
speed: r(sp.clip.speed ?? 1),
// The generated scene shot this clip came from — sceneShot is the
@@ -450,7 +453,7 @@ export function buildAiContext(opts?: { fullCues?: boolean; chatId?: string | nu
}
function describeAudio(
- a: { assetId: string; start: number; in: number; out: number; volume: number; fadeIn?: number; fadeOut?: number; speed?: number; duck?: number; lane?: number; hidden?: boolean },
+ a: { assetId: string; start: number; in: number; out: number; volume: number; fadeIn?: number; fadeOut?: number; speed?: number; sound?: ClipSound; duck?: number; lane?: number; hidden?: boolean },
assets: Map<string, { name: string }>
) {
const speed = a.speed && a.speed > 0 ? a.speed : 1;
@@ -463,6 +466,7 @@ function describeAudio(
volume: r(a.volume),
fadeIn: r(a.fadeIn ?? 0),
fadeOut: r(a.fadeOut ?? 0),
+ ...(a.sound ? { sound: a.sound } : {}),
...(speed !== 1 ? { speed: r(speed) } : {}),
...(a.lane ? { lane: a.lane } : {}),
...(a.hidden ? { hidden: true } : {}),
@@ -483,6 +487,7 @@ function describeOverlayClip(c: VideoClip, assets: Map<string, { name: string }>
out: r(c.out),
muted: c.muted,
...(c.hidden ? { hidden: true } : {}),
+ ...(c.sound ? { sound: c.sound } : {}),
// The frame region this layer occupies: Full covers the frame; Top/Bottom/
// Left/Right split it; PiP floats inside it.
layout: regionLabel(rect),
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 98f3775b..3190fff4 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -9,7 +9,10 @@ import {
HSL_BANDS,
matchGrade,
normalizeGrade,
+ normalizeSound,
semanticMasterCurve,
+ SOUND_PRESETS,
+ type ClipSound,
type ColorStats,
type CurvePoint,
type EffectId,
@@ -106,6 +109,8 @@ import { characterPrompt, stockAspectDims, stockTitle } from "./stock";
import { STOCK_IMAGES } from "./stockManifest";
import { STOCK_VIDEOS } from "./stockVideoManifest";
import { storedMediaUrl } from "./mediaSync";
+import { isSoundPresetTemplate, listSoundPresets, saveSoundPreset } from "./soundPresets";
+import { isStylePresetTemplate } from "./stylePresets";
import { applyOverlayPatchSettled, track0Clips, laneGapAt, getClipSpans, nextFreeStart, overlayLaneOrder, overlayLayers, parkedTransitions, projectDuration, resolveTransitions, totalDuration, useEditor } from "./store";
import { playheadAt } from "./playhead";
import { renderProjectFrame } from "./exportRender";
@@ -1481,6 +1486,79 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
return { id: clip.id, hidden: Boolean(input.hidden) };
},
+ set_clip_sound: async (s, input) => {
+ const target = soundTarget(s, input.clipId);
+ let next: ClipSound | undefined;
+ if (typeof input.preset === "string" && input.preset.trim()) {
+ const wanted = input.preset.trim().toLowerCase();
+ // The dropdown labels the shipped presets by name, so a name is what
+ // the user says; the ids are what the catalog lists.
+ const shipped = SOUND_PRESETS.find(
+ (p) => p.id === wanted || p.name.toLowerCase() === wanted
+ );
+ if (shipped) {
+ next = shipped.sound;
+ } else {
+ const saved = (await listSoundPresets()).find(
+ (p) => p.name.toLowerCase() === wanted || p.id === input.preset
+ );
+ if (!saved)
+ throw new ToolError(
+ `No sound preset called "${String(input.preset)}". Shipped: ${SOUND_PRESETS.map((p) => p.name).join(", ")}.`
+ );
+ next = saved.sound;
+ }
+ } else {
+ const patch: ClipSound = { ...(target.current ?? {}) };
+ if ("eq" in input) {
+ if (input.eq === null) delete patch.eq;
+ else if (Array.isArray(input.eq)) patch.eq = input.eq.map((v) => (isNum(v) ? v : 0));
+ else throw new ToolError("eq must be an array of dB values, one per band.");
+ }
+ if ("compressor" in input) {
+ const c = input.compressor as Record<string, unknown> | null;
+ if (c === null) delete patch.compressor;
+ else if (c && isNum(c.threshold) && isNum(c.ratio) && isNum(c.attack) && isNum(c.release))
+ patch.compressor = { threshold: c.threshold, ratio: c.ratio, attack: c.attack, release: c.release };
+ else throw new ToolError("compressor needs threshold, ratio, attack and release.");
+ }
+ if ("limiter" in input) {
+ const l = input.limiter as Record<string, unknown> | null;
+ if (l === null) delete patch.limiter;
+ else if (l && isNum(l.ceiling)) patch.limiter = { ceiling: l.ceiling };
+ else throw new ToolError("limiter needs a ceiling.");
+ }
+ for (const stage of Array.isArray(input.clear) ? input.clear : []) {
+ if (stage === "eq") delete patch.eq;
+ else if (stage === "compressor") delete patch.compressor;
+ else if (stage === "limiter") delete patch.limiter;
+ else throw new ToolError(`clear takes eq, compressor or limiter; got "${String(stage)}".`);
+ }
+ if (
+ !("eq" in input) &&
+ !("compressor" in input) &&
+ !("limiter" in input) &&
+ !Array.isArray(input.clear)
+ )
+ throw new ToolError("Pass a preset, a clear list, or eq / compressor / limiter.");
+ next = patch;
+ }
+ const sound = normalizeSound(next);
+ target.write(sound);
+ return { id: target.id, kind: target.kind, sound: soundTarget(useEditor.getState(), target.id).current ?? null };
+ },
+
+ save_sound_preset: async (s, input) => {
+ const target = soundTarget(s, input.clipId);
+ const name = typeof input.name === "string" ? input.name.trim() : "";
+ if (!name) throw new ToolError("name is required.");
+ if (!target.current) throw new ToolError("That clip's sound is flat; shape it first.");
+ if (!s.projectId) throw new ToolError("No open project.");
+ const saved = await saveSoundPreset(s.projectId, name, target.current);
+ if (!saved) throw new ToolError("The preset could not be saved.");
+ return { id: saved.id, name: saved.name, residency: saved.residency, sound: saved.sound };
+ },
+
reorder_track: (s, input) => {
if (!isNum(input.from) || !isNum(input.to) || input.from < 0 || input.to < 0)
throw new ToolError("from and to must be 0 or higher.");
@@ -2781,7 +2859,9 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
// "inspiration" = saved from the phone's Ideas tab.
...(a.origin ? { origin: a.origin } : {}),
})),
- templates: lib.templates.map((t) => ({
+ templates: lib.templates
+ .filter((t) => !isSoundPresetTemplate(t) && !isStylePresetTemplate(t))
+ .map((t) => ({
id: t.id,
name: t.name,
duration: round2(t.duration),
@@ -4047,6 +4127,32 @@ async function listenToSource(
return blobToInlineAudio(await res.blob());
}
+/** The clip a sound tool names — a video clip or a soundtrack clip, whichever
+ * carries the id — with its current treatment and the store write that
+ * lands a new one. */
+function soundTarget(s: Editor, id: unknown) {
+ const key = typeof id === "string" ? id.trim() : "";
+ const clip = s.clips.find((c) => c.id === key);
+ if (clip) {
+ return {
+ id: clip.id,
+ kind: "clip" as const,
+ current: clip.sound,
+ write: (sound: ClipSound | undefined) => s.updateClip(clip.id, { sound }),
+ };
+ }
+ const audio = s.audioClips.find((a) => a.id === key);
+ if (audio) {
+ return {
+ id: audio.id,
+ kind: "audio" as const,
+ current: audio.sound,
+ write: (sound: ClipSound | undefined) => s.updateAudio(audio.id, { sound }),
+ };
+ }
+ throw new ToolError(`No video or soundtrack clip with id "${key}".`);
+}
+
function requireItem<T extends { id: string }>(pool: T[], id: unknown, label: string): T {
const item = pool.find((x) => x.id === String(id ?? ""));
if (!item) throw new ToolError(`No ${label} with id ${String(id)}. Call get_state for current ids.`);
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index eebbee6d..fb48ee80 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -45,6 +45,7 @@ import {
LOOK_EFFECTS,
OVERLAY_ANIM_STYLE_IDS,
OVERLAY_LOOP_STYLE_IDS,
+ SOUND_EQ_BANDS,
} from "@donkeycut/effects-kit";
import type { AiToolDef } from "../../lib/aiToolDef";
import { GOOGLE_FONT_IDS } from "../../lib/fontCatalog";
@@ -323,6 +324,7 @@ ${textMoveCatalog()}
Lyrics and scripts the user already has: sync_lyrics times their exact words against the audio (transcribing first when needed) and writes them to a caption track with per-word timings; set_caption_look then dresses the track — preset, size, font, word effect, accent color — from a text-videos look id or field by field. The text-videos skill has the whole flow, including turning the same timed lines into full-frame cards.
Soundtrack clips: volume 0..3 (1 = unchanged, above 1 boosts), fadeIn/fadeOut seconds (max half the clip), start = timeline position, in/out = trim inside the source; clips can spread across several soundtrack lanes (the \`lane\` field), new sounds slide to free space in their lane. Fades render with ffmpeg afade on export.
Ducking: a soundtrack clip's \`duck\` (0..1, via update_audio) lowers ALL other audio — video-clip sound and other music — to that gain while the clip plays; 1 clears it. Voiceovers set this so narration sits over quieter music. It applies in both the preview and the export.
+Sound quality (set_clip_sound): a clip's own treatment — video clip or soundtrack clip — in three stages the Inspector's Audio tab shows as Sound quality: a ${SOUND_EQ_BANDS.length}-band equalizer (${SOUND_EQ_BANDS.map((b) => b.label).join(", ")}; dB per band), a compressor (threshold dB, ratio, attack ms, release ms) and a limiter (dBFS ceiling). It runs on that clip alone, before its fades and any ducking, in the preview and every export. "Make the voice clearer / more podcast-like / less muddy / more even in level / stop it clipping" is this tool: \`preset: "clear-voice"\` is the shipped voice treatment (warmth under 200 Hz, mud out at 400 Hz, presence at 3–5 kHz, 3:1 at −18 dB, −1 dB ceiling); \`preset: "flat"\` clears it; the user's own saved presets apply by name; or set the stages directly, null clearing one. save_sound_preset stores a clip's treatment in the Library under a name for reuse. A treatment shapes one clip's sound; an effect element treats a stretch of the mix.
Cutting to music: detect_beats reads a track's beat grid once — bpm plus every beat, with timeline times through the clip you pass — and the grid stays on the asset as yellow dots the user can drag, add (⌥-click the bar), or delete, with every drag and trim snapping to them. Land cuts, clip starts, and text entrances on those times: split_at at the timeline beats, or place each item to start on one.
Audio effects (add_effect): the treatments over the sound — ${AUDIO_EFFECT_IDS.join(", ")} — placed on the timeline as elements, each treating everything audible under its window at once. Reach for one when the ask is how a stretch SOUNDS ("make this line sound like a phone call", "echo on that word", "muffle the music behind the voiceover"); \`amount\` says how far the treatment goes, and the effect trims and moves like any element. Loudness, fades and ducking on one clip stay with the audio tools; a dissolve across a cut stays with the transition on that cut (the audiocross style).
Voiceover (Donkey's hosted speech model — signed in, spends credits, like image/video generation):