Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(studio): carve and duck follow each clip's trim and speed (#4661)
Miguel Ángel7fae35f36efb
diff --git a/packages/studio/src/components/editor/clipAudioClock.test.ts b/packages/studio/src/components/editor/clipAudioClock.test.tsnew file mode 100644index 0000000000..c401db7465--- /dev/null+++ b/packages/studio/src/components/editor/clipAudioClock.test.ts@@ -0,0 +1,61 @@+import { describe, expect, it } from "vitest";+import { clipAudioOnItsClock, readClipClock } from "./clipAudioClock";++const RATE = 48000;+const sine = (seconds: number, hz: number) =>+ Float32Array.from({ length: RATE * seconds }, (_, i) => Math.sin((2 * Math.PI * hz * i) / RATE));+const zeroCrossingsPerSecond = (x: Float32Array) => {+ let n = 0;+ for (let i = 1; i < x.length; i += 1) if (x[i - 1]! < 0 !== x[i]! < 0) n += 1;+ return n / (x.length / RATE);+};++describe("readClipClock", () => {+ it("reads the in-point, the rate lane over the constant, and a positive duration", () => {+ const attrs: Record<string, string> = {+ "data-playback-start": "1.5",+ "data-media-start": "9",+ "data-playback-rate": "3",+ "data-automation":+ '{"version":1,"lanes":[{"target":"rate","points":[{"t":0,"v":2},{"t":4,"v":2}]}]}',+ "data-duration": "4",+ };+ const clock = readClipClock((name) => attrs[name]);+ expect({ ...clock, rate: typeof clock.rate }).toEqual({+ inPoint: 1.5,+ rate: "object",+ duration: 4,+ });+ expect(readClipClock((name) => (name === "data-duration" ? "0" : null))).toEqual({+ inPoint: 0,+ rate: 1,+ duration: null,+ });+ });+});++describe("clipAudioOnItsClock", () => {+ it("is the exact slice the clip plays at rate 1", () => {+ const samples = sine(4, 440);+ const out = clipAudioOnItsClock(samples, RATE, { inPoint: 1, rate: 1, duration: 2 });+ expect(Array.from(out)).toEqual(Array.from(samples.subarray(RATE, 3 * RATE)));+ });++ it("lasts the clip's duration at 2x, consuming twice as much source", () => {+ // Each sample holds its own source time, so the output reads back where it was taken from.+ const samples = Float32Array.from({ length: RATE * 6 }, (_, i) => i / RATE);+ const out = clipAudioOnItsClock(samples, RATE, { inPoint: 1, rate: 2, duration: 2 });+ expect({ length: out.length, atOneSecond: Number(out[RATE]!.toFixed(1)) }).toEqual({+ length: 2 * RATE,+ atOneSecond: 3,+ });+ });++ it("keeps the pitch at 2x and 0.5x", () => {+ const samples = sine(4, 440);+ for (const rate of [2, 0.5]) {+ const out = clipAudioOnItsClock(samples, RATE, { inPoint: 0, rate, duration: 1.5 });+ expect(zeroCrossingsPerSecond(out) / 880).toBeCloseTo(1, 1);+ }+ });+});diff --git a/packages/studio/src/components/editor/clipAudioClock.ts b/packages/studio/src/components/editor/clipAudioClock.tsnew file mode 100644index 0000000000..385ad2fcc9--- /dev/null+++ b/packages/studio/src/components/editor/clipAudioClock.ts@@ -0,0 +1,78 @@+import {+ resolveRateSpec,+ sourceTimeAt,+ timeAtSourceTime,+ type RateSpec,+} from "@hyperframes/core/speed-ramp";+import {+ readDataDurationSeconds,+ readMediaOffsetSeconds,+ readPlaybackRate,+ type AttrReader,+} from "@hyperframes/parsers/media-duration";++/** Where a clip's audio starts in its file, how fast it plays, and for how long (null: to the end). */+export interface ClipClock {+ inPoint: number;+ rate: RateSpec;+ duration: number | null;+}++export function readClipClock(getAttr: AttrReader): ClipClock {+ return {+ inPoint: readMediaOffsetSeconds(getAttr),+ rate: resolveRateSpec(getAttr("data-automation"), readPlaybackRate(getAttr)),+ duration: readDataDurationSeconds(getAttr),+ };+}++/** The source samples a clip plays, still on the file's clock. */+export function clipSourceWindow(+ samples: Float32Array,+ sampleRate: number,+ clock: ClipClock,+): Float32Array {+ const from = Math.min(samples.length, Math.floor(clock.inPoint * sampleRate));+ const to = clock.duration+ ? Math.min(+ samples.length,+ from + Math.ceil(sourceTimeAt(clock.rate, clock.duration) * sampleRate),+ )+ : samples.length;+ return samples.subarray(from, to);+}++const GRAIN = 4096;+const HOP = GRAIN / 2;+const HANN = Float32Array.from(+ { length: GRAIN },+ (_, k) => 0.5 - 0.5 * Math.cos((2 * Math.PI * k) / GRAIN),+);++/**+ * The audio heard over clip time [0, duration]. Off rate 1, Hann grains overlap-added at the source+ * time each lands on: pitch kept, like export's atempo, close enough for band and level analysis.+ */+export function clipAudioOnItsClock(+ samples: Float32Array,+ sampleRate: number,+ clock: ClipClock,+): Float32Array {+ const source = clipSourceWindow(samples, sampleRate, clock);+ if (clock.rate === 1) return source;+ const out = new Float32Array(+ Math.round(timeAtSourceTime(clock.rate, source.length / sampleRate) * sampleRate),+ );+ // Grains are not phase-aligned, so off rate 1 the level reads ~1.3 dB low (up to ~3 dB on+ // tones); a WSOLA stretch would keep it.+ for (let at = -HOP; at < out.length; at += HOP) {+ const centre = Math.round(sourceTimeAt(clock.rate, (at + HOP) / sampleRate) * sampleRate);+ for (let k = 0; k < GRAIN; k += 1) {+ const o = at + k;+ const s = centre - HOP + k;+ if (o < 0 || o >= out.length || s < 0 || s >= source.length) continue;+ out[o] = (out[o] ?? 0) + (HANN[k] ?? 0) * (source[s] ?? 0);+ }+ }+ return out;+}diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsxindex 43f95e36e2..7991bfcd4f 100644--- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx+++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.test.tsx@@ -989,6 +989,141 @@ describe("AudioFxGroup dynamic carve", () => {expect(onSetAttributeLive.mock.calls.every((c) => c[0] === "data-fx-carve")).toBe(true);expect(onSetAttributeQuiet.mock.calls.some((c) => c[0] === "data-fx-chain")).toBe(false);});++ describe("on each clip's own clock", () => {+ type Point = { t: number; v: number };+ /** `hz` at `amp` over [from, to) of a `seconds`-long file, silence elsewhere. */+ const tone = (seconds: number, [from, to]: [number, number], hz = 1000, amp = 0.7) =>+ Float32Array.from({ length: SAMPLE_RATE * seconds }, (_, i) => {+ const t = i / SAMPLE_RATE;+ return t >= from && t < to ? amp * Math.sin(2 * Math.PI * hz * t) : 0;+ });++ /** Each file decodes to its own samples, found by the URL that was fetched. */+ function stubFiles(files: Record<string, Float32Array>): void {+ vi.stubGlobal(+ "fetch",+ vi.fn(async (url: string) => ({ arrayBuffer: async () => url })),+ );+ vi.stubGlobal(+ "OfflineAudioContext",+ class {+ decodeAudioData = async (url: string) => {+ const data = Object.entries(files).find(([name]) => url.endsWith(name))![1];+ return { sampleRate: SAMPLE_RATE, getChannelData: () => data };+ };+ },+ );+ }++ const valueAt = (points: Point[], t: number): number => {+ const i = points.findIndex((p) => p.t >= t);+ if (i <= 0) return points.at(i)!.v;+ const [a, b] = [points[i - 1]!, points[i]!];+ return a.v + ((b.v - a.v) * (t - a.t)) / (b.t - a.t);+ };++ /** Carves the bed against `vo` and reads back the lane driving the first node of `type`. */+ async function carveLane(+ type: string,+ voice: Record<string, string>,+ bed: Record<string, string> = {},+ strength = 0,+ ) {+ const { host, onSetAttributeQuiet } = mount({+ "fx-chain": carvedChain,+ "fx-carve": settings(true, { strength, sources: [] }),+ start: "0",+ ...bed,+ });+ const vo = document.getElementById("vo")!;+ vo.setAttribute("src", "voice.wav");+ for (const [name, value] of Object.entries(voice)) vo.setAttribute(`data-${name}`, value);+ document.getElementById("bed")!.setAttribute("src", "bed.wav");+ await act(async () => {+ pickSource(host, "vo");+ });+ const calls = onSetAttributeQuiet.mock.calls;+ const nodes = parseWrite(writeTo(calls, "data-fx-chain")!).nodes as {+ id: string;+ type: string;+ params: { gain: number; frequency: number };+ }[];+ const node = nodes.find((n) => n.type === type)!;+ const lanes = parseWrite(writeTo(calls, "data-automation")!).lanes as {+ target: string;+ points: Point[];+ }[];+ const points = lanes.find((l) => l.target === `fx.${node.id}.gain`)!.points;+ return {+ depth: node.params.gain,+ frequency: node.params.frequency,+ points,+ at: (t: number) => valueAt(points, t),+ };+ }++ /** Quiet 1 kHz speech at 0-2 s, loud 4 kHz speech at 4-6 s, in an 8 s file. */+ const lateSpeech = () => {+ const after = tone(8, [4, 6], 4000, 0.9);+ return tone(8, [0, 2], 1000, 0.3).map((v, i) => v + after[i]!);+ };++ it("ignores speech after the bed has stopped playing", async () => {+ // The bed file runs 8 s, but the clip plays only its first 3 s.+ stubFiles({ "voice.wav": lateSpeech(), "bed.wav": tone(8, [0, 8], 200) });+ const lane = await carveLane("peaking", { start: "0" }, { duration: "3" });+ expect(lane.frequency).toBe(1000);+ });++ it("ignores speech after a bed with no duration runs out, when it ducks", async () => {+ // No data-duration: the bed plays its 3 s file. Strength 0.05 ducks and keeps one band.+ stubFiles({ "voice.wav": lateSpeech(), "bed.wav": tone(3, [0, 3], 200) });+ const lane = await carveLane("peaking", { start: "0" }, {}, 0.05);+ expect(lane.frequency).toBe(1000);+ });++ it.each([+ ["no duration", {}],+ ["a duration longer than its file", { duration: "6" }],+ ])("ignores speech after a 3 s bed file with %s, without ducking", async (_, bed) => {+ stubFiles({ "voice.wav": lateSpeech(), "bed.wav": tone(3, [0, 3], 200) });+ const lane = await carveLane("peaking", { start: "0" }, bed);+ expect(lane.frequency).toBe(1000);+ });++ it("cuts where a trimmed voice speaks, not where its file does", async () => {+ // File speech 2-3.5 s; the clip plays 2-4 s of it, so the bed hears it at 0-1.5 s.+ stubFiles({ "voice.wav": tone(6, [2, 3.5]), "bed.wav": tone(8, [0, 8], 200) });+ const lane = await carveLane("peaking", { start: "0", "media-start": "2", duration: "2" });+ expect({ speaking: lane.at(0.8) <= lane.depth * 0.5, after: lane.at(3) }).toEqual({+ speaking: true,+ after: 0,+ });+ });++ it("cuts over half the source length for a voice at 2x", async () => {+ // File speech 1-3 s at 2x: the bed hears it at 0.5-1.5 s.+ stubFiles({ "voice.wav": tone(4, [1, 3]), "bed.wav": tone(8, [0, 8], 200) });+ const lane = await carveLane("peaking", { start: "0", "playback-rate": "2", duration: "2" });+ expect({+ speaking: lane.at(0.8) <= lane.depth * 0.5 && lane.at(1.3) <= lane.depth * 0.5,+ after: lane.at(2.5),+ }).toEqual({ speaking: true, after: 0 });+ });++ it("ducks against the part of the bed that plays", async () => {+ // The bed file is silent for 4 s and loud after; the clip starts 4 s in, so it is loud under the voice.+ stubFiles({ "voice.wav": tone(4, [1, 3]), "bed.wav": tone(8, [4, 8], 200, 0.9) });+ const lane = await carveLane(+ "gain",+ { start: "0" },+ { "media-start": "4", duration: "4" },+ 1,+ );+ expect(Math.min(...lane.points.map((p) => p.v))).toBeLessThan(0);+ });+ });});describe("AudioFxGroup successive edits", () => {diff --git a/packages/studio/src/components/editor/useFxCarve.ts b/packages/studio/src/components/editor/useFxCarve.tsindex d4e4479877..27e9ae7b1b 100644--- a/packages/studio/src/components/editor/useFxCarve.ts+++ b/packages/studio/src/components/editor/useFxCarve.ts@@ -33,7 +33,8 @@ import { automationAttrValue, HF_AUDIO_AUTOMATION_ATTR } from "./propertyPanelAuimport { trackCarveChanged } from "./audioFxTelemetry.js";import type { DomEditSelection } from "./domEditingTypes";import { usePlayerStore } from "../../player";-import { carveLanes, measureCarve, mintCarveNodes } from "./useFxCarveNodes.js";+import { carveLanes, measureCarve, mintCarveNodes, type CarveClip } from "./useFxCarveNodes.js";+import { readClipClock } from "./clipAudioClock.js";import { spanOf } from "./propertyPanelAudioFxGroupUtils.js";import type { AudioTrackOption } from "./propertyPanelFxCarveModule.js";@@ -93,11 +94,8 @@ function carveNeedsReanalysis(* what lets the src and the start be non-null by construction downstream* instead of by assertion.*/-function resolveCarveVoices(- doc: Document,- sources: readonly string[],-): { src: string; start: string | null }[] {- const voices: { src: string; start: string | null }[] = [];+function resolveCarveVoices(doc: Document, sources: readonly string[]): CarveClip[] {+ const voices: CarveClip[] = [];for (const id of sources) {const el = doc.getElementById(id);// By tag name, not `instanceof HTMLAudioElement`: these elements belong to@@ -106,7 +104,8 @@ function resolveCarveVoices(if (el?.tagName !== "AUDIO") continue;const src = el.getAttribute("src");if (!src) continue;- voices.push({ src, start: el.getAttribute("data-start") });+ const clock = readClipClock((name) => el.getAttribute(name));+ voices.push({ src, start: el.getAttribute("data-start"), clock });}return voices;}@@ -363,14 +362,11 @@ export function useFxCarve(if (voices.length === 0) return;setAnalysing(true);try {- const bedSrc = element.element?.getAttribute("src");- const measured = await measureCarve(- doc,- voices,- active.strength,- element.dataAttributes?.["start"],- bedSrc,- );+ const measured = await measureCarve(doc, voices, active.strength, {+ src: element.element?.getAttribute("src"),+ start: element.dataAttributes?.["start"],+ clock: readClipClock((name) => element.dataAttributes?.[name.slice(5)]),+ });if (!measured) return;await persistMeasuredCarve(measured);} catch {diff --git a/packages/studio/src/components/editor/useFxCarveNodes.ts b/packages/studio/src/components/editor/useFxCarveNodes.tsindex d518aae768..f74e9c532a 100644--- a/packages/studio/src/components/editor/useFxCarveNodes.ts+++ b/packages/studio/src/components/editor/useFxCarveNodes.ts@@ -23,6 +23,7 @@ import {} from "@hyperframes/core/audio-carve";import { fxAutomationTarget, type HfAutomationLane } from "@hyperframes/core/audio-automation";import { clipStart } from "./propertyPanelAudioFxGroupUtils.js";+import { clipAudioOnItsClock, type ClipClock } from "./clipAudioClock.js";/** Decode rate every carve measurement shares — see `measureCarve`. */const DECODE_SAMPLE_RATE = 48000;@@ -60,6 +61,13 @@ export function mintCarveNodes(};}+/** A clip to decode, where it starts on the timeline, and how it plays its file. */+export interface CarveClip {+ src: string;+ start: string | null | undefined;+ clock: ClipClock;+}+/*** Decode every voice, mix them onto the bed's own clock, and measure the* bands (and, if the profile calls for it, the ducking envelope) from that@@ -68,10 +76,9 @@ export function mintCarveNodes(*/export async function measureCarve(doc: Document,- voices: { src: string; start: string | null }[],+ voices: CarveClip[],strength: number,- bedStartAttr: string | null | undefined,- bedSrc: string | null | undefined,+ bed: Omit<CarveClip, "src"> & { src: string | null | undefined },): Promise<{bands: ReturnType<typeof analyseCarveBands>;carved: HfAudioFxChain;@@ -86,11 +93,15 @@ export async function measureCarve((window as unknown as { webkitOfflineAudioContext?: typeof OfflineAudioContext }).webkitOfflineAudioContext;if (!Ctor) return null;- const decode = async (relative: string): Promise<AudioBuffer> => {+ // Each clip as it plays: trimmed to its in-point and duration, and at its own speed.+ const decode = async (relative: string, clock: ClipClock): Promise<Float32Array> => {const res = await fetch(new URL(relative, doc.baseURI).href);- return new Ctor(1, 1, DECODE_SAMPLE_RATE).decodeAudioData(await res.arrayBuffer());+ const buffer = await new Ctor(1, 1, DECODE_SAMPLE_RATE).decodeAudioData(+ await res.arrayBuffer(),+ );+ return clipAudioOnItsClock(buffer.getChannelData(0), DECODE_SAMPLE_RATE, clock);};- const bedStart = clipStart(bedStartAttr);+ const bedStart = clipStart(bed.start);// Every voice, summed onto the bed's own clock. One question — where and// when is speech masking this bed — with one answer, even when the answer// comes from three people talking at different times. Doing this before the@@ -98,23 +109,24 @@ export async function measureCarve(// the chain is fixed, so there is no per-voice filter to switch between.const decoded = await Promise.all(voices.map(async (voice) => ({- samples: (await decode(voice.src)).getChannelData(0),+ samples: await decode(voice.src, voice.clock),offsetSeconds: clipStart(voice.start) - bedStart,})),);- const voiceMix = mixCarveSources(decoded, DECODE_SAMPLE_RATE);- if (voiceMix.length === 0) return null;+ const mixed = mixCarveSources(decoded, DECODE_SAMPLE_RATE);+ if (mixed.length === 0) return null;// Strength is what the author set; these are the numbers it means.const profile = carveProfile(strength);- // The bed as well as the voice, when the carve is asked to match levels:- // "how far over the voice is this bed" cannot be answered by listening to- // one of them.- const bedBuffer = profile.duckDb > 0 && bedSrc ? await decode(bedSrc).catch(() => null) : null;+ // The bed as it plays: its length bounds the speech worth carving, and the duck+ // compares against it ("how far over the voice is this bed" needs both).+ const bedSamples = bed.src ? await decode(bed.src, bed.clock).catch(() => null) : null;+ // Speech after the bed stops has no bed to carve.+ const voiceMix = bedSamples ? mixed.subarray(0, bedSamples.length) : mixed;const bands = analyseCarveBands(voiceMix, DECODE_SAMPLE_RATE, profile);// The level half of the carve, measured against the speech it has to sit// under. No offset to apply: the mix is already on the bed's clock.- const duck = bedBuffer- ? analyseCarveDuck(voiceMix, bedBuffer.getChannelData(0), DECODE_SAMPLE_RATE, profile, 0)+ const duck = bedSamples+ ? analyseCarveDuck(voiceMix, bedSamples, DECODE_SAMPLE_RATE, profile, 0): [];return { bands, carved: carveBandsToChain(bands), duck, voiceMix };}diff --git a/packages/studio/src/components/editor/useFxLevelling.ts b/packages/studio/src/components/editor/useFxLevelling.tsindex 441e7b9cb7..d075188455 100644--- a/packages/studio/src/components/editor/useFxLevelling.ts+++ b/packages/studio/src/components/editor/useFxLevelling.ts@@ -15,13 +15,7 @@ import {} from "@hyperframes/core/audio-fx";import { levellingResult, removeLevelling } from "@hyperframes/core/audio-leveller";import type { HfAutomation, HfAutomationPoint } from "@hyperframes/core/audio-automation";-import {- resolveRateSpec,- sourceTimeAt,- timeAtSourceTime,- type RateSpec,-} from "@hyperframes/core/speed-ramp";-import { readPlaybackRate } from "@hyperframes/parsers/media-duration";+import { timeAtSourceTime } from "@hyperframes/core/speed-ramp";import {automationAttrValue,HF_AUDIO_AUTOMATION_ATTR,@@ -30,7 +24,7 @@ import {} from "./propertyPanelAutomation";import { trackLeveller } from "./audioFxTelemetry.js";import type { DomEditSelection } from "./domEditingTypes";-import { readClipInPoint } from "./propertyPanelHelpers";+import { clipSourceWindow, readClipClock } from "./clipAudioClock.js";import { useAuditionTransport } from "./useAuditionTransport.js";/**@@ -39,11 +33,6 @@ import { useAuditionTransport } from "./useAuditionTransport.js";*/const DECODE_SAMPLE_RATE = 48000;-/** A parsed attribute, kept only when it is a usable positive number. */-function positiveFinite(n: number): number | null {- return Number.isFinite(n) && n > 0 ? n : null;-}-export function useFxLevelling(element: DomEditSelection,chain: HfAudioFxChain,@@ -83,42 +72,17 @@ export function useFxLevelling(return next;};- /**- * The part of the decoded file this clip actually plays.- *- * A lane's `t` is seconds from the start of the CLIP, but the decode is the- * whole file from its first sample — so measuring a trimmed clip produced an- * envelope offset by the trim, and every correction landed early by exactly the- * in-point. Slicing here is what puts the two clocks back on the same- * zero.- */- const clipWindow = (audio: { samples: Float32Array; sampleRate: number }, rate: RateSpec) => {- const { mediaStart } = readClipInPoint(element.dataAttributes);- const duration = positiveFinite(Number(element.dataAttributes?.["duration"] ?? Number.NaN));- const from = mediaStart- ? Math.min(audio.samples.length, Math.floor(mediaStart * audio.sampleRate))- : 0;- const to = duration- ? Math.min(- audio.samples.length,- from + Math.ceil(sourceTimeAt(rate, duration) * audio.sampleRate),- )- : audio.samples.length;- return from === 0 && to === audio.samples.length- ? audio.samples- : audio.samples.subarray(from, to);- };-/** The levelled chain and its lane, with the lane moved from the source's clock onto the clip's. */const measureLevelling = (audio: { samples: Float32Array; sampleRate: number }) => {- const attr = (name: string) => element.dataAttributes?.[name.slice(5)];- const rate = resolveRateSpec(attr("data-automation"), readPlaybackRate(attr));- const result = levellingResult(chain, clipWindow(audio, rate), audio.sampleRate);+ // Measured on the part of the file the clip plays, so lane times map from that window's zero.+ const clock = readClipClock((name) => element.dataAttributes?.[name.slice(5)]);+ const played = clipSourceWindow(audio.samples, audio.sampleRate, clock);+ const result = levellingResult(chain, played, audio.sampleRate);if (!result) return null;const lane = result.automation.lanes[0];const toClipTime = (point: HfAutomationPoint) => ({...point,- t: Number(timeAtSourceTime(rate, point.t).toFixed(3)),+ t: Number(timeAtSourceTime(clock.rate, point.t).toFixed(3)),});return { chain: result.chain, lane: lane && { ...lane, points: lane.points.map(toClipTime) } };};