Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(studio): even out levels lifts the right part of a sped-up or slowed clip (#4636)
Miguel Ángelea494b467237
diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsxindex dc8c8a66d7..43f95e36e2 100644--- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx+++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx@@ -80,6 +80,8 @@ function byTextButton(host: HTMLElement, text: string): HTMLButtonElement | undereturn Array.from(host.querySelectorAll("button")).find((b) => b.textContent?.includes(text));}+const SAMPLE_RATE = 48000;+function mount(dataAttributes: Record<string, string>, alone = false, voices = 2) {// Every write is quiet: persisted without the preview reload that would// restart every playing track, but with a selection resync so the panel sees@@ -459,29 +461,8 @@ describe("AudioFxGroup dynamic carve", () => {afterEach(() => vi.unstubAllGlobals());- /**- * Hover-auditioning the leveller has to measure before there is anything to- * hear, and measuring a long voiceover takes seconds — by which time the- * pointer has usually moved on. Applying then would put levelling on a track- * nobody asked to level, through a channel that does not persist: audible,- * absent from the document, and gone on the next reload.- */- it("levels the part of the file the clip plays, not the file from its start", async () => {- // A lane's `t` is seconds from the start of the CLIP, but the decode is the- // whole file — so a trimmed clip got an envelope offset by exactly the- // in-point, and every correction landed early.- //- // The file is loud 0-2s, quiet 2-5s, loud again 5-8s, and the clip trims the- // first 2s. Measured from the clip's own zero, t=0.5 sits in the quiet- // passage and wants a real lift; measured from the file's zero it sits in- // the loud head and wants none. That gap is the bug.- const sampleRate = 48000;- const data = new Float32Array(sampleRate * 8);- for (let i = 0; i < data.length; i++) {- const t = i / sampleRate;- const amp = t < 2 ? 0.5 : t < 5 ? 0.05 : 0.5;- data[i] = amp * Math.sin(2 * Math.PI * 300 * t);- }+ /** Runs Even Out Levels on a decode of `data` and returns the fx lane's nearest point to a clip time. */+ async function levelledLane(data: Float32Array, dataAttributes: Record<string, string>) {vi.stubGlobal("fetch",vi.fn(async () => ({ arrayBuffer: async () => new ArrayBuffer(8) })),@@ -489,16 +470,10 @@ describe("AudioFxGroup dynamic carve", () => {vi.stubGlobal("OfflineAudioContext",class {- decodeAudioData = async () => ({ sampleRate, getChannelData: () => data });+ decodeAudioData = async () => ({ sampleRate: SAMPLE_RATE, getChannelData: () => data });},);-- const { host, onSetAttributeQuiet } = mount({- "fx-chain": CHAIN,- "playback-start": "2",- "media-start": "0",- duration: "6",- });+ const { host, onSetAttributeQuiet } = mount({ "fx-chain": CHAIN, ...dataAttributes });document.getElementById("bed")?.setAttribute("src", "bed.wav");act(() => byTextButton(host, "Audio FX")?.click());act(() => byTextButton(host, "+ effect")?.click());@@ -514,12 +489,75 @@ describe("AudioFxGroup dynamic carve", () => {JSON.parse(String(write[1])).lanes as { target: string; points: { t: number; v: number }[] }[]).find((l) => l.target.startsWith("fx."));if (!lane) throw new Error("no fx lane");- const near = (t: number) =>+ return (t: number) =>lane.points.reduce((best, p) => (Math.abs(p.t - t) < Math.abs(best.t - t) ? p : best));- // The quiet passage, from the clip's zero, gets its lift.+ }++ /** A 300 Hz tone, 16 s long: quiet between `quietFrom` and `quietTo`, loud elsewhere. */+ function toneWithQuietPassage(quietFrom: number, quietTo: number): Float32Array {+ const data = new Float32Array(SAMPLE_RATE * 16);+ for (let i = 0; i < data.length; i++) {+ const t = i / SAMPLE_RATE;+ data[i] = (t >= quietFrom && t < quietTo ? 0.05 : 0.5) * Math.sin(2 * Math.PI * 300 * t);+ }+ return data;+ }++ it("levels the part of the file the clip plays, not the file from its start", async () => {+ // A lane's `t` is seconds from the start of the CLIP, but the decode is the+ // whole file. The file is quiet 2-5 s and the clip trims the first 2 s, so+ // from the clip's zero t=0.5 wants a real lift; from the file's zero it wants none.+ const near = await levelledLane(toneWithQuietPassage(2, 5), {+ "playback-start": "2",+ "media-start": "0",+ duration: "6",+ });expect(near(0.5).v).toBeGreaterThan(4);});+ it.each<{+ label: string;+ attrs: Record<string, string>;+ quiet: [number, number];+ quietAt: number;+ loudAt: number;+ }>([+ // 2x: 6 s of clip play source 0-12 s; source 2-6 s is quiet, so clip 1-3 s wants lift.+ {+ label: "2x",+ attrs: { "playback-rate": "2", duration: "6" },+ quiet: [2, 6],+ quietAt: 2,+ loudAt: 5,+ },+ // 0.5x: 8 s of clip play source 0-4 s; source 1-2 s is quiet, so clip 2-4 s wants lift and clip 1 s none.+ {+ label: "0.5x",+ attrs: { "playback-rate": "0.5", duration: "8" },+ quiet: [1, 2],+ quietAt: 3.5,+ loudAt: 1,+ },+ // A rate lane holding 2x: playback reads the lane, not the missing data-playback-rate.+ {+ label: "a 2x rate lane",+ attrs: {+ automation:+ '{"version":1,"lanes":[{"target":"rate","points":[{"t":0,"v":2},{"t":6,"v":2}]}]}',+ duration: "6",+ },+ quiet: [2, 6],+ quietAt: 2,+ loudAt: 5,+ },+ ])("levels on the clip's clock at $label", async (c) => {+ const near = await levelledLane(toneWithQuietPassage(...c.quiet), c.attrs);+ expect({ quiet: near(c.quietAt).v > 4, loud: near(c.loudAt).v < 2 }).toEqual({+ quiet: true,+ loud: true,+ });+ });+it("removes every lane a preset owned, not just the last node's", () => {// Each write is computed from the same render-time snapshot and replaces the// whole attribute, so removing lanes one node at a time kept only the finaldiff --git a/packages/studio/src/components/editor/useFxLevelling.ts b/packages/studio/src/components/editor/useFxLevelling.tsindex 13bcf502ae..441e7b9cb7 100644--- a/packages/studio/src/components/editor/useFxLevelling.ts+++ b/packages/studio/src/components/editor/useFxLevelling.ts@@ -14,7 +14,14 @@ import {type HfAudioFxChain,} from "@hyperframes/core/audio-fx";import { levellingResult, removeLevelling } from "@hyperframes/core/audio-leveller";-import type { HfAutomation } from "@hyperframes/core/audio-automation";+import type { HfAutomation, HfAutomationPoint } from "@hyperframes/core/audio-automation";+import {+ resolveRateSpec,+ sourceTimeAt,+ timeAtSourceTime,+ type RateSpec,+} from "@hyperframes/core/speed-ramp";+import { readPlaybackRate } from "@hyperframes/parsers/media-duration";import {automationAttrValue,HF_AUDIO_AUTOMATION_ATTR,@@ -85,33 +92,50 @@ export function useFxLevelling(* in-point. Slicing here is what puts the two clocks back on the same* zero.*/- const clipWindow = (audio: { samples: Float32Array; sampleRate: number }) => {+ const clipWindow = (audio: { samples: Float32Array; sampleRate: number }, rate: RateSpec) => {const { mediaStart } = readClipInPoint(element.dataAttributes);const duration = positiveFinite(Number(element.dataAttributes?.["duration"] ?? Number.NaN));const from = mediaStart? Math.min(audio.samples.length, Math.floor(mediaStart * audio.sampleRate)): 0;const to = duration- ? Math.min(audio.samples.length, from + Math.ceil(duration * audio.sampleRate))+ ? Math.min(+ audio.samples.length,+ from + Math.ceil(sourceTimeAt(rate, duration) * audio.sampleRate),+ ): audio.samples.length;return from === 0 && to === audio.samples.length? audio.samples: audio.samples.subarray(from, to);};+ /** The levelled chain and its lane, with the lane moved from the source's clock onto the clip's. */+ const measureLevelling = (audio: { samples: Float32Array; sampleRate: number }) => {+ const attr = (name: string) => element.dataAttributes?.[name.slice(5)];+ const rate = resolveRateSpec(attr("data-automation"), readPlaybackRate(attr));+ const result = levellingResult(chain, clipWindow(audio, rate), audio.sampleRate);+ if (!result) return null;+ const lane = result.automation.lanes[0];+ const toClipTime = (point: HfAutomationPoint) => ({+ ...point,+ t: Number(timeAtSourceTime(rate, point.t).toFixed(3)),+ });+ return { chain: result.chain, lane: lane && { ...lane, points: lane.points.map(toClipTime) } };+ };+const runLeveller = async (): Promise<void> => {setAnalysing(true);try {const audio = await decodeTrack();if (!audio) return;- const result = levellingResult(chain, clipWindow(audio), audio.sampleRate);+ const result = measureLevelling(audio);if (!result) return;trackLeveller("run");await onSetAttributeQuiet(HF_AUDIO_FX_ATTR, serializeAudioFxChain(result.chain));// Merged by target, never written wholesale: the script describes its own// lane only, and replacing the attribute would take the carve's lanes and// the volume lane with it.- const lane = result.automation.lanes[0];+ const lane = result.lane;if (lane) {void onSetAttributeQuiet(HF_AUDIO_AUTOMATION_ATTR,@@ -161,10 +185,10 @@ export function useFxLevelling(const audio = await decodeTrack();// Gone, or superseded by a later hover. Either way this result is stale.if (!audio || run !== auditionRun.current) return;- const result = levellingResult(chain, clipWindow(audio), audio.sampleRate);+ const result = measureLevelling(audio);if (!result || run !== auditionRun.current) return;void onSetAttributeLive(HF_AUDIO_FX_ATTR, serializeAudioFxChain(result.chain));- const lane = result.automation.lanes[0];+ const lane = result.lane;if (lane) {void onSetAttributeLive(HF_AUDIO_AUTOMATION_ATTR,