Open source CapCut alternative.
TypeScript
feat(site): the chat measures clip levels before it matches volumes [rebuild]
David Lee4679915d394
diff --git a/site/scripts/lib/cut-eval/fixtures.ts b/site/scripts/lib/cut-eval/fixtures.tsindex 90144e52..6c59d857 100644--- a/site/scripts/lib/cut-eval/fixtures.ts+++ b/site/scripts/lib/cut-eval/fixtures.ts@@ -24,6 +24,7 @@ export const SAFE_TOOLS = new Set(["detect_silence","detect_beats","listen_audio",+ "measure_level",// The model's own memory of a source and the user's synced notes: reading// notes and writing down a look are bookkeeping, never a project edit."note_source",@@ -630,6 +631,7 @@ export function serveSafeTool(name: string, state: unknown): unknown {if (name === "library_list") return { folders: [], assets: [], templates: [] };if (name === "notes_list") return { notes: [], folders: [], labels: [] };if (name === "detect_silence") return { silences: [] };+ if (name === "measure_level") return { targetId: "", clips: [] };if (name === "detect_beats") return { bpm: 0, beats: [] };return { ok: true };}diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.tsindex 7fb127ac..88a7f3e3 100644--- a/site/src/cut/components/AiPanel.tools.ts+++ b/site/src/cut/components/AiPanel.tools.ts@@ -65,6 +65,18 @@ export const AI_PANEL_TOOLS = [min_silence: num("Shortest silent stretch to report, seconds (default 0.35)"),}),},+ {+ name: "measure_level",+ description:+ "Measure how loud clips actually play — the level of each clip's audible sound over its trimmed range, in dBFS, with its current volume applied — so volumes are set from numbers, never by ear. Returns per clip: sourceDb (the media), volume, levelDb (what plays), peakDb, and for every clip except the target, volumeToMatch: the volume that lands it at the target's level. \"Make the voiceover as loud as the clip\", \"match the levels\", \"balance the music under the speech\": call this with every clip involved and target_id = the one to match, then set_clip_volume / update_audio with volumeToMatch (for a bed under speech, a fraction of it). A shortfallDb means the volume ceiling (3) cannot reach the target; lower the target instead. A clip marked muted plays nothing until set_clip_muted / update_audio unmutes it; its level is what it would play unmuted.",+ inputSchema: obj(+ {+ ids: { type: "array", items: { type: "string" }, description: "Clip ids to measure — video clips (their own sound) and soundtrack clips alike" },+ target_id: str("The clip whose level the others should match (default: the first id)"),+ },+ ["ids"]+ ),+ },{name: "detect_beats",description:diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex e1542c08..61cf7462 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -97,6 +97,7 @@ import {beatlessFor,detectAssetBeats,detectSilenceClientSide,+ measureSourceLevel,enrichAsset,ensurePeaks,importImage,@@ -1108,6 +1109,54 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {};},+ measure_level: async (s, input) => {+ const ids = Array.isArray(input.ids) ? input.ids.map(String) : [];+ if (ids.length === 0) throw new ToolError("ids is required: the clip ids to measure.");+ const targetId = input.target_id === undefined || input.target_id === null ? ids[0] : String(input.target_id);+ if (!ids.includes(targetId)) ids.push(targetId);+ const rows = new Map<+ string,+ { id: string; name: string; kind: "video" | "soundtrack"; sourceDb: number; peakDb: number; audibleSeconds: number; volume: number; levelDb: number; muted?: boolean }+ >();+ await eachScanned(ids, async (id) => {+ const { asset, clip, from, to } = resolveWatchRange(s, { clip_id: id });+ if (asset.type === "image") throw new ToolError(`"${asset.name}" is an image — it has no sound.`);+ const level = await measureSourceLevel(asset.url, { from, ...(to !== undefined ? { to } : {}) }).catch((e) => {+ throw new ToolError(e instanceof Error ? e.message : "Could not read the audio.");+ });+ const soundtrack = s.audioClips.some((a) => a.id === clip!.id);+ const { volume = 1, muted = false } = clip as { volume?: number; muted?: boolean };+ rows.set(id, {+ id,+ name: asset.name,+ kind: soundtrack ? "soundtrack" : "video",+ sourceDb: level.rmsDb,+ peakDb: level.peakDb,+ audibleSeconds: level.audibleSeconds,+ volume: round2(volume),+ // A muted clip plays nothing; its level is what it would play unmuted.+ levelDb: round2(level.rmsDb + gainDb(volume)),+ ...(muted ? { muted: true } : {}),+ });+ });+ const target = rows.get(targetId)!;+ // The volume that would land each clip at the target's level, from the+ // clip's own level under its current volume. The volume range is 0..3, so+ // a clip too quiet to reach the target says how far it gets.+ const clips = ids.map((id) => {+ const row = rows.get(id)!;+ if (id === targetId) return row;+ const wanted = row.volume * 10 ** ((target.levelDb - row.levelDb) / 20);+ const volumeToMatch = round2(clamp(wanted, 0, 3));+ return {+ ...row,+ volumeToMatch,+ ...(wanted > 3 ? { shortfallDb: round2(20 * Math.log10(wanted / 3)) } : {}),+ };+ });+ return { targetId, clips };+ },+listen_audio: async (s, input) => {const { projectId, asset, clip, from, to } = resolveWatchRange(s, input);if (asset.type === "image")@@ -3921,6 +3970,7 @@ export const MEDIA_RUNTIME_TOOLS: ReadonlySet<string> = new Set(["convert_media","watch_video","listen_audio",+ "measure_level","detect_silence","detect_beats","refine_speech_cuts",@@ -4168,6 +4218,9 @@ function notesIn(.map((n) => ({ from: round2(n.from), to: round2(n.to), text: n.text }));}+/** A clip volume as decibels of gain; silence reads as the quiet floor. */+const gainDb = (volume: number) => (volume > 0 ? 20 * Math.log10(volume) : -100);+function resolveWatchTarget(s: ReturnType<typeof useEditor.getState>,input: Record<string, unknown>diff --git a/site/src/cut/lib/audioScan.test.ts b/site/src/cut/lib/audioScan.test.tsindex 27204d61..2309ecc3 100644--- a/site/src/cut/lib/audioScan.test.ts+++ b/site/src/cut/lib/audioScan.test.ts@@ -1,5 +1,5 @@import { describe, expect, test } from "bun:test";-import { scanBeats, scanSilence, scanSpeech, type PcmChunk } from "./audioScan";+import { scanBeats, scanLevel, scanSilence, scanSpeech, type PcmChunk } from "./audioScan";const RATE = 8000;@@ -157,6 +157,30 @@ function expectWords(});}+describe("scanLevel", () => {+ test("reads the level of what plays, leaving the pauses out", async () => {+ // Half-scale tone with a second of silence in the middle: the mean is+ // the tone's own level, and the silence does not drag it down.+ const half = 20 * Math.log10(0.5);+ const level = await scanLevel(chunks([0.5, 0, 0.5]), { from: 0 });+ expect(level.rmsDb).toBeCloseTo(half, 1);+ expect(level.peakDb).toBeCloseTo(half, 1);+ expect(level.audibleSeconds).toBeCloseTo(2, 1);+ });++ test("a quieter take reads quieter by its gain", async () => {+ const loud = await scanLevel(chunks([1]), { from: 0 });+ const soft = await scanLevel(chunks([0.25]), { from: 0 });+ expect(loud.rmsDb - soft.rmsDb).toBeCloseTo(20 * Math.log10(4), 1);+ });++ test("nothing audible reads at the floor of what was heard", async () => {+ const level = await scanLevel(chunks([0, 0]), { from: 0 });+ expect(level.audibleSeconds).toBe(0);+ expect(level.rmsDb).toBeLessThan(-90);+ });+});+describe("scanSpeech", () => {test("finds the words in a quiet room", async () => {const scan = await scanSpeech(take(SPOKEN, { floorDb: -60, duration: 5 }), { from: 0 });diff --git a/site/src/cut/lib/audioScan.ts b/site/src/cut/lib/audioScan.tsindex f39fa1d7..7e3b9541 100644--- a/site/src/cut/lib/audioScan.ts+++ b/site/src/cut/lib/audioScan.ts@@ -150,6 +150,57 @@ export function silencesFrom(return silences;}+/** Frames under this are the room, or nothing at all: they carry no program+ * and would drag a level reading down with the pauses between words. */+const LEVEL_FLOOR_DB = -45;++/** How loud a span plays: its program level and its peak, in dBFS. */+export interface LevelScan {+ /** Power-mean level of the audible frames — what the ear averages. */+ rmsDb: number;+ /** The loudest 20ms frame. */+ peakDb: number;+ /** Seconds of the span that read as audible. */+ audibleSeconds: number;+}++/**+ * The level of an envelope. Pauses and room tone are left out of the mean so+ * a recording with long gaps reads at the level of what it says, the way two+ * clips are compared by ear; a span with nothing audible reads at its floor.+ */+export function levelFrom(env: Envelope): LevelScan {+ let power = 0;+ let audible = 0;+ let peak = QUIET;+ let heard = 0;+ let allPower = 0;+ for (const level of env.db) {+ if (!Number.isFinite(level)) continue;+ heard++;+ const p = 10 ** (level / 10);+ allPower += p;+ if (level > peak) peak = level;+ if (level < LEVEL_FLOOR_DB) continue;+ power += p;+ audible++;+ }+ const mean = audible > 0 ? power / audible : heard > 0 ? allPower / heard : 0;+ return {+ rmsDb: round2(Math.max(QUIET, 10 * Math.log10(mean || 1e-10))),+ peakDb: round2(peak),+ audibleSeconds: round2(audible * env.hop),+ };+}++/** The level of `chunks`, by RMS over 20ms windows. */+export async function scanLevel(+ chunks: AsyncIterable<PcmChunk>,+ opts: { from: number; to?: number }+): Promise<LevelScan> {+ return levelFrom(await scanEnvelope(chunks, opts));+}+/** Silent spans in `chunks`, by RMS over 20ms windows. */export async function scanSilence(chunks: AsyncIterable<PcmChunk>,diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 0299c73a..417b01fc 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -1,6 +1,6 @@"use client";-import { scanBeats, scanSilence, scanSpeech, type BeatScan, type PcmChunk, type SpeechScan } from "./audioScan";+import { scanBeats, scanLevel, scanSilence, scanSpeech, type BeatScan, type LevelScan, type PcmChunk, type SpeechScan } from "./audioScan";import { apiFetch, apiJson, getBackend, type CutBackend } from "./backend";import { registerBlobFile } from "./backend/browser/registry";import { cloudBackend } from "./backend/cloud";@@ -1261,6 +1261,17 @@ export async function scanSourceSpeech(return scanSpeech(sourcePcm(sourceUrl, from, opts.to), { ...opts, from });}+/** How loud a span of a source plays. Same residency story as the speech+ * scan: the page decodes its own media, the headless runner installs the+ * same decoders. */+export async function measureSourceLevel(+ sourceUrl: string,+ opts: { from: number; to?: number }+): Promise<LevelScan> {+ const from = Math.max(0, opts.from);+ return scanLevel(sourcePcm(sourceUrl, from, opts.to), { ...opts, from });+}+/** The source's musical beat grid, end to end. The grid is a property of the* whole file — tempo tracking needs the run of it — so a scan always reads* everything; callers window the result. Same residency story as the speechdiff --git a/site/src/cut/lib/pi/mutationLedger.ts b/site/src/cut/lib/pi/mutationLedger.tsindex cbc5e242..d2625805 100644--- a/site/src/cut/lib/pi/mutationLedger.ts+++ b/site/src/cut/lib/pi/mutationLedger.ts@@ -16,6 +16,7 @@ const READ_ONLY = new Set(["detect_silence","detect_beats","listen_audio",+ "measure_level","read_color_stats","library_list","library_list_folder",diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.tsindex 3e8441aa..75c3792a 100644--- a/site/src/cut/server/ai/catalog.ts+++ b/site/src/cut/server/ai/catalog.ts@@ -331,6 +331,7 @@ ${textMoveCatalog()}"audio-and-subtitles": `# Audio, voiceover & subtitlesLyrics and scripts the user already has: sync_lyrics times their exact words against the audio (transcribing first when needed) and writes them to a caption track with per-word timings; set_caption_look then dresses the track — preset, size, font, word effect, accent color — from a text-videos look id or field by field. The text-videos skill has the whole flow, including turning the same timed lines into full-frame cards.Soundtrack clips: volume 0..3 (1 = unchanged, above 1 boosts), fadeIn/fadeOut seconds (max half the clip), start = timeline position, in/out = trim inside the source; clips can spread across several soundtrack lanes (the \`lane\` field), new sounds slide to free space in their lane. Fades render with ffmpeg afade on export.+Matching levels: "as loud as", "match", "balance", "sit under" are measurements, never guesses. measure_level with every clip involved and target_id = the one to match returns each clip's playing level and the volumeToMatch that lands it on the target; set that with set_clip_volume / update_audio (a fraction of it for a bed under speech). Listening tells you how a sound feels, never how many dB it is.Ducking: a soundtrack clip's \`duck\` (0..1, via update_audio) lowers ALL other audio — video-clip sound and other music — to that gain while the clip plays; 1 clears it. Voiceovers set this so narration sits over quieter music. It applies in both the preview and the export.Sound quality (set_clip_sound): a clip's own treatment — video clip or soundtrack clip — in three stages the Inspector's Audio tab shows as Sound quality: a ${SOUND_EQ_BANDS.length}-band equalizer (${SOUND_EQ_BANDS.map((b) => b.label).join(", ")}; dB per band), a compressor (threshold dB, ratio, attack ms, release ms) and a limiter (dBFS ceiling). It runs on that clip alone, before its fades and any ducking, in the preview and every export. "Make the voice clearer / more podcast-like / less muddy / more even in level / stop it clipping" is this tool: pick the shipped preset whose character fits the ask — ${SOUND_PRESETS.filter((p) => p.sound).map((p) => `\`${p.id}\` (${p.character})`).join(", ")} — with \`studio\` as the default for a talking-head voice; \`preset: "flat"\` clears it; the user's own saved presets apply by name; or set the stages directly, null clearing one. save_sound_preset stores a clip's treatment in the Library under a name for reuse. A treatment shapes one clip's sound; an effect element treats a stretch of the mix.Cutting to music: detect_beats reads a track's beat grid once — bpm plus every beat, with timeline times through the clip you pass — and the grid stays on the asset as yellow dots the user can drag, add (⌥-click the bar), or delete, with every drag and trim snapping to them. Land cuts, clip starts, and text entrances on those times: split_at at the timeline beats, or place each item to start on one.diff --git a/site/src/cut/server/ai/headlessTools.ts b/site/src/cut/server/ai/headlessTools.tsindex 3ac8e670..9ccf1d04 100644--- a/site/src/cut/server/ai/headlessTools.ts+++ b/site/src/cut/server/ai/headlessTools.ts@@ -22,6 +22,7 @@ import { uniqueName } from "../util";const PAGE_MEDIA_TOOLS: ReadonlySet<string> = new Set(["watch_video","listen_audio",+ "measure_level","detect_silence","detect_beats","refine_speech_cuts",