Open source CapCut alternative.
TypeScript
fix(site): measure_level survives silent sources and matches from the media level
David Le3f70d5b8bba7
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.tsindex 88a7f3e3..57a8b556 100644--- a/site/src/cut/components/AiPanel.tools.ts+++ b/site/src/cut/components/AiPanel.tools.ts@@ -68,7 +68,7 @@ export const AI_PANEL_TOOLS = [{name: "measure_level",description:- "Measure how loud clips actually play — the level of each clip's audible sound over its trimmed range, in dBFS, with its current volume applied — so volumes are set from numbers, never by ear. Returns per clip: sourceDb (the media), volume, levelDb (what plays), peakDb, and for every clip except the target, volumeToMatch: the volume that lands it at the target's level. \"Make the voiceover as loud as the clip\", \"match the levels\", \"balance the music under the speech\": call this with every clip involved and target_id = the one to match, then set_clip_volume / update_audio with volumeToMatch (for a bed under speech, a fraction of it). A shortfallDb means the volume ceiling (3) cannot reach the target; lower the target instead. A clip marked muted plays nothing until set_clip_muted / update_audio unmutes it; its level is what it would play unmuted.",+ "Measure how loud clips actually play — the level of each clip's audible sound over its trimmed range, in dBFS, with its current volume applied — so volumes are set from numbers, never by ear. Returns per clip: sourceDb (the media), volume, levelDb (what plays), loudestFrameDb (the loudest 20 ms frame as RMS — a sample peak sits higher, so it is no clipping margin), and for every clip except the target, volumeToMatch: the volume that lands it at the target's level; a clip whose source has no audio track comes back with noAudio instead of levels. \"Make the voiceover as loud as the clip\", \"match the levels\", \"balance the music under the speech\": call this with every clip involved and target_id = the one to match, then set_clip_volume / update_audio with volumeToMatch (for a bed under speech, a fraction of it). A shortfallDb means the volume ceiling (3) cannot reach the target; lower the target instead. A clip marked muted plays nothing until set_clip_muted / update_audio unmutes it; its level is what it would play unmuted.",inputSchema: obj({ids: { type: "array", items: { type: "string" }, description: "Clip ids to measure — video clips (their own sound) and soundtrack clips alike" },diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 61cf7462..b3451184 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -1114,24 +1114,43 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {if (ids.length === 0) throw new ToolError("ids is required: the clip ids to measure.");const targetId = input.target_id === undefined || input.target_id === null ? ids[0] : String(input.target_id);if (!ids.includes(targetId)) ids.push(targetId);- const rows = new Map<- string,- { id: string; name: string; kind: "video" | "soundtrack"; sourceDb: number; peakDb: number; audibleSeconds: number; volume: number; levelDb: number; muted?: boolean }- >();+ type Row = {+ id: string;+ name: string;+ kind: "video" | "soundtrack";+ sourceDb: number;+ loudestFrameDb: number;+ audibleSeconds: number;+ volume: number;+ levelDb: number;+ muted?: boolean;+ };+ const rows = new Map<string, Row | { id: string; name: string; kind: "video" | "soundtrack"; noAudio: string }>();await eachScanned(ids, async (id) => {const { asset, clip, from, to } = resolveWatchRange(s, { clip_id: id });- if (asset.type === "image") throw new ToolError(`"${asset.name}" is an image — it has no sound.`);- const level = await measureSourceLevel(asset.url, { from, ...(to !== undefined ? { to } : {}) }).catch((e) => {- throw new ToolError(e instanceof Error ? e.message : "Could not read the audio.");- });const soundtrack = s.audioClips.some((a) => a.id === clip!.id);+ const kind = soundtrack ? "soundtrack" : "video";+ if (asset.type === "image") {+ rows.set(id, { id, name: asset.name, kind, noAudio: "An image has no sound." });+ return;+ }+ // A source with no audio track is reported, and the rest of the+ // list still measures: "balance these three" must not fail on the+ // one silent generated clip among them.+ let level: Awaited<ReturnType<typeof measureSourceLevel>>;+ try {+ level = await measureSourceLevel(asset.url, { from, ...(to !== undefined ? { to } : {}) });+ } catch (e) {+ rows.set(id, { id, name: asset.name, kind, noAudio: e instanceof Error ? e.message : "Could not read the audio." });+ return;+ }const { volume = 1, muted = false } = clip as { volume?: number; muted?: boolean };rows.set(id, {id,name: asset.name,- kind: soundtrack ? "soundtrack" : "video",+ kind,sourceDb: level.rmsDb,- peakDb: level.peakDb,+ loudestFrameDb: level.loudestFrameDb,audibleSeconds: level.audibleSeconds,volume: round2(volume),// A muted clip plays nothing; its level is what it would play unmuted.@@ -1140,13 +1159,15 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {});});const target = rows.get(targetId)!;+ if ("noAudio" in target) throw new ToolError(`"${target.name}" (the target) has no sound to match: ${target.noAudio}`);// The volume that would land each clip at the target's level, from the- // clip's own level under its current volume. The volume range is 0..3, so- // a clip too quiet to reach the target says how far it gets.+ // media's own level — so a clip turned all the way down still has a+ // number to come back up to. The volume range is 0..3, so a clip too+ // quiet to reach the target says how far it gets.const clips = ids.map((id) => {const row = rows.get(id)!;- if (id === targetId) return row;- const wanted = row.volume * 10 ** ((target.levelDb - row.levelDb) / 20);+ if (id === targetId || "noAudio" in row) return row;+ const wanted = 10 ** ((target.levelDb - row.sourceDb) / 20);const volumeToMatch = round2(clamp(wanted, 0, 3));return {...row,diff --git a/site/src/cut/lib/audioScan.test.ts b/site/src/cut/lib/audioScan.test.tsindex 2309ecc3..c836ad58 100644--- a/site/src/cut/lib/audioScan.test.ts+++ b/site/src/cut/lib/audioScan.test.ts@@ -164,7 +164,7 @@ describe("scanLevel", () => {const half = 20 * Math.log10(0.5);const level = await scanLevel(chunks([0.5, 0, 0.5]), { from: 0 });expect(level.rmsDb).toBeCloseTo(half, 1);- expect(level.peakDb).toBeCloseTo(half, 1);+ expect(level.loudestFrameDb).toBeCloseTo(half, 1);expect(level.audibleSeconds).toBeCloseTo(2, 1);});diff --git a/site/src/cut/lib/audioScan.ts b/site/src/cut/lib/audioScan.tsindex 7e3b9541..bbd9484b 100644--- a/site/src/cut/lib/audioScan.ts+++ b/site/src/cut/lib/audioScan.ts@@ -158,8 +158,8 @@ const LEVEL_FLOOR_DB = -45;export interface LevelScan {/** Power-mean level of the audible frames — what the ear averages. */rmsDb: number;- /** The loudest 20ms frame. */- peakDb: number;+ /** The loudest 20ms frame, as RMS — a sample peak sits higher. */+ loudestFrameDb: number;/** Seconds of the span that read as audible. */audibleSeconds: number;}@@ -188,7 +188,7 @@ export function levelFrom(env: Envelope): LevelScan {const mean = audible > 0 ? power / audible : heard > 0 ? allPower / heard : 0;return {rmsDb: round2(Math.max(QUIET, 10 * Math.log10(mean || 1e-10))),- peakDb: round2(peak),+ loudestFrameDb: round2(peak),audibleSeconds: round2(audible * env.hop),};}