Skip to content

Write HTML. Render video. Built for agents.

fix(studio): even out levels lifts the right part of a sped-up or slowed clip (#4636)

Miguel Ángelea494b467237
diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx
index dc8c8a66d7..43f95e36e2 100644
--- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx
+++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx
@@ -80,6 +80,8 @@ function byTextButton(host: HTMLElement, text: string): HTMLButtonElement | unde
return Array.from(host.querySelectorAll("button")).find((b) => b.textContent?.includes(text));
}
+const SAMPLE_RATE = 48000;
+
function mount(dataAttributes: Record<string, string>, alone = false, voices = 2) {
// Every write is quiet: persisted without the preview reload that would
// restart every playing track, but with a selection resync so the panel sees
@@ -459,29 +461,8 @@ describe("AudioFxGroup dynamic carve", () => {
afterEach(() => vi.unstubAllGlobals());
- /**
- * Hover-auditioning the leveller has to measure before there is anything to
- * hear, and measuring a long voiceover takes seconds — by which time the
- * pointer has usually moved on. Applying then would put levelling on a track
- * nobody asked to level, through a channel that does not persist: audible,
- * absent from the document, and gone on the next reload.
- */
- it("levels the part of the file the clip plays, not the file from its start", async () => {
- // A lane's `t` is seconds from the start of the CLIP, but the decode is the
- // whole file — so a trimmed clip got an envelope offset by exactly the
- // in-point, and every correction landed early.
- //
- // The file is loud 0-2s, quiet 2-5s, loud again 5-8s, and the clip trims the
- // first 2s. Measured from the clip's own zero, t=0.5 sits in the quiet
- // passage and wants a real lift; measured from the file's zero it sits in
- // the loud head and wants none. That gap is the bug.
- const sampleRate = 48000;
- const data = new Float32Array(sampleRate * 8);
- for (let i = 0; i < data.length; i++) {
- const t = i / sampleRate;
- const amp = t < 2 ? 0.5 : t < 5 ? 0.05 : 0.5;
- data[i] = amp * Math.sin(2 * Math.PI * 300 * t);
- }
+ /** Runs Even Out Levels on a decode of `data` and returns the fx lane's nearest point to a clip time. */
+ async function levelledLane(data: Float32Array, dataAttributes: Record<string, string>) {
vi.stubGlobal(
"fetch",
vi.fn(async () => ({ arrayBuffer: async () => new ArrayBuffer(8) })),
@@ -489,16 +470,10 @@ describe("AudioFxGroup dynamic carve", () => {
vi.stubGlobal(
"OfflineAudioContext",
class {
- decodeAudioData = async () => ({ sampleRate, getChannelData: () => data });
+ decodeAudioData = async () => ({ sampleRate: SAMPLE_RATE, getChannelData: () => data });
},
);
-
- const { host, onSetAttributeQuiet } = mount({
- "fx-chain": CHAIN,
- "playback-start": "2",
- "media-start": "0",
- duration: "6",
- });
+ const { host, onSetAttributeQuiet } = mount({ "fx-chain": CHAIN, ...dataAttributes });
document.getElementById("bed")?.setAttribute("src", "bed.wav");
act(() => byTextButton(host, "Audio FX")?.click());
act(() => byTextButton(host, "+ effect")?.click());
@@ -514,12 +489,75 @@ describe("AudioFxGroup dynamic carve", () => {
JSON.parse(String(write[1])).lanes as { target: string; points: { t: number; v: number }[] }[]
).find((l) => l.target.startsWith("fx."));
if (!lane) throw new Error("no fx lane");
- const near = (t: number) =>
+ return (t: number) =>
lane.points.reduce((best, p) => (Math.abs(p.t - t) < Math.abs(best.t - t) ? p : best));
- // The quiet passage, from the clip's zero, gets its lift.
+ }
+
+ /** A 300 Hz tone, 16 s long: quiet between `quietFrom` and `quietTo`, loud elsewhere. */
+ function toneWithQuietPassage(quietFrom: number, quietTo: number): Float32Array {
+ const data = new Float32Array(SAMPLE_RATE * 16);
+ for (let i = 0; i < data.length; i++) {
+ const t = i / SAMPLE_RATE;
+ data[i] = (t >= quietFrom && t < quietTo ? 0.05 : 0.5) * Math.sin(2 * Math.PI * 300 * t);
+ }
+ return data;
+ }
+
+ it("levels the part of the file the clip plays, not the file from its start", async () => {
+ // A lane's `t` is seconds from the start of the CLIP, but the decode is the
+ // whole file. The file is quiet 2-5 s and the clip trims the first 2 s, so
+ // from the clip's zero t=0.5 wants a real lift; from the file's zero it wants none.
+ const near = await levelledLane(toneWithQuietPassage(2, 5), {
+ "playback-start": "2",
+ "media-start": "0",
+ duration: "6",
+ });
expect(near(0.5).v).toBeGreaterThan(4);
});
+ it.each<{
+ label: string;
+ attrs: Record<string, string>;
+ quiet: [number, number];
+ quietAt: number;
+ loudAt: number;
+ }>([
+ // 2x: 6 s of clip play source 0-12 s; source 2-6 s is quiet, so clip 1-3 s wants lift.
+ {
+ label: "2x",
+ attrs: { "playback-rate": "2", duration: "6" },
+ quiet: [2, 6],
+ quietAt: 2,
+ loudAt: 5,
+ },
+ // 0.5x: 8 s of clip play source 0-4 s; source 1-2 s is quiet, so clip 2-4 s wants lift and clip 1 s none.
+ {
+ label: "0.5x",
+ attrs: { "playback-rate": "0.5", duration: "8" },
+ quiet: [1, 2],
+ quietAt: 3.5,
+ loudAt: 1,
+ },
+ // A rate lane holding 2x: playback reads the lane, not the missing data-playback-rate.
+ {
+ label: "a 2x rate lane",
+ attrs: {
+ automation:
+ '{"version":1,"lanes":[{"target":"rate","points":[{"t":0,"v":2},{"t":6,"v":2}]}]}',
+ duration: "6",
+ },
+ quiet: [2, 6],
+ quietAt: 2,
+ loudAt: 5,
+ },
+ ])("levels on the clip's clock at $label", async (c) => {
+ const near = await levelledLane(toneWithQuietPassage(...c.quiet), c.attrs);
+ expect({ quiet: near(c.quietAt).v > 4, loud: near(c.loudAt).v < 2 }).toEqual({
+ quiet: true,
+ loud: true,
+ });
+ });
+
it("removes every lane a preset owned, not just the last node's", () => {
// Each write is computed from the same render-time snapshot and replaces the
// whole attribute, so removing lanes one node at a time kept only the final
diff --git a/packages/studio/src/components/editor/useFxLevelling.ts b/packages/studio/src/components/editor/useFxLevelling.ts
index 13bcf502ae..441e7b9cb7 100644
--- a/packages/studio/src/components/editor/useFxLevelling.ts
+++ b/packages/studio/src/components/editor/useFxLevelling.ts
@@ -14,7 +14,14 @@ import {
type HfAudioFxChain,
} from "@hyperframes/core/audio-fx";
import { levellingResult, removeLevelling } from "@hyperframes/core/audio-leveller";
-import type { HfAutomation } from "@hyperframes/core/audio-automation";
+import type { HfAutomation, HfAutomationPoint } from "@hyperframes/core/audio-automation";
+import {
+ resolveRateSpec,
+ sourceTimeAt,
+ timeAtSourceTime,
+ type RateSpec,
+} from "@hyperframes/core/speed-ramp";
+import { readPlaybackRate } from "@hyperframes/parsers/media-duration";
import {
automationAttrValue,
HF_AUDIO_AUTOMATION_ATTR,
@@ -85,33 +92,50 @@ export function useFxLevelling(
* in-point. Slicing here is what puts the two clocks back on the same
* zero.
*/
- const clipWindow = (audio: { samples: Float32Array; sampleRate: number }) => {
+ const clipWindow = (audio: { samples: Float32Array; sampleRate: number }, rate: RateSpec) => {
const { mediaStart } = readClipInPoint(element.dataAttributes);
const duration = positiveFinite(Number(element.dataAttributes?.["duration"] ?? Number.NaN));
const from = mediaStart
? Math.min(audio.samples.length, Math.floor(mediaStart * audio.sampleRate))
: 0;
const to = duration
- ? Math.min(audio.samples.length, from + Math.ceil(duration * audio.sampleRate))
+ ? Math.min(
+ audio.samples.length,
+ from + Math.ceil(sourceTimeAt(rate, duration) * audio.sampleRate),
+ )
: audio.samples.length;
return from === 0 && to === audio.samples.length
? audio.samples
: audio.samples.subarray(from, to);
};
+ /** The levelled chain and its lane, with the lane moved from the source's clock onto the clip's. */
+ const measureLevelling = (audio: { samples: Float32Array; sampleRate: number }) => {
+ const attr = (name: string) => element.dataAttributes?.[name.slice(5)];
+ const rate = resolveRateSpec(attr("data-automation"), readPlaybackRate(attr));
+ const result = levellingResult(chain, clipWindow(audio, rate), audio.sampleRate);
+ if (!result) return null;
+ const lane = result.automation.lanes[0];
+ const toClipTime = (point: HfAutomationPoint) => ({
+ ...point,
+ t: Number(timeAtSourceTime(rate, point.t).toFixed(3)),
+ });
+ return { chain: result.chain, lane: lane && { ...lane, points: lane.points.map(toClipTime) } };
+ };
+
const runLeveller = async (): Promise<void> => {
setAnalysing(true);
try {
const audio = await decodeTrack();
if (!audio) return;
- const result = levellingResult(chain, clipWindow(audio), audio.sampleRate);
+ const result = measureLevelling(audio);
if (!result) return;
trackLeveller("run");
await onSetAttributeQuiet(HF_AUDIO_FX_ATTR, serializeAudioFxChain(result.chain));
// Merged by target, never written wholesale: the script describes its own
// lane only, and replacing the attribute would take the carve's lanes and
// the volume lane with it.
- const lane = result.automation.lanes[0];
+ const lane = result.lane;
if (lane) {
void onSetAttributeQuiet(
HF_AUDIO_AUTOMATION_ATTR,
@@ -161,10 +185,10 @@ export function useFxLevelling(
const audio = await decodeTrack();
// Gone, or superseded by a later hover. Either way this result is stale.
if (!audio || run !== auditionRun.current) return;
- const result = levellingResult(chain, clipWindow(audio), audio.sampleRate);
+ const result = measureLevelling(audio);
if (!result || run !== auditionRun.current) return;
void onSetAttributeLive(HF_AUDIO_FX_ATTR, serializeAudioFxChain(result.chain));
- const lane = result.automation.lanes[0];
+ const lane = result.lane;
if (lane) {
void onSetAttributeLive(
HF_AUDIO_AUTOMATION_ATTR,