Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(core): a clip's volume lane stays smooth in preview while the page is busy (#6057)
Miguel Ángelcfff4493dfc6
diff --git a/packages/core/src/runtime/audioAutomationVolume.ts b/packages/core/src/runtime/audioAutomationVolume.tsindex 3a4ea306d1..1708941252 100644--- a/packages/core/src/runtime/audioAutomationVolume.ts+++ b/packages/core/src/runtime/audioAutomationVolume.ts@@ -1,11 +1,9 @@/*** The volume lane's gain for an element at a moment in its clip.*- * Volume is not scheduled onto the gain AudioParam the way FX parameters are.- * The runtime rewrites that gain every tick from `data-volume` and the GSAP- * seeked value, which would erase a scheduled envelope within a frame — so the- * lane feeds the same per-tick path the probed `volumeKeyframes` already use,- * and the render still bakes it into the samples.+ * Read per tick for a clip that sounds through `el.volume`. A clip the Web Audio+ * transport routes plays its lane scheduled on a gain of its own+ * (`attachClipStages`), out of reach of the per-tick gain write.*/import {diff --git a/packages/core/src/runtime/audioFx.ts b/packages/core/src/runtime/audioFx.tsindex 2a4376ecca..7fc374384d 100644--- a/packages/core/src/runtime/audioFx.ts+++ b/packages/core/src/runtime/audioFx.ts@@ -15,12 +15,15 @@ import {HF_AUDIO_AUTOMATION_ATTR,parseAutomation,resolveAutomation,+ VOLUME_RANGE,type HfAutomation,} from "../audioAutomation.js";import {cancelParamLane,clearParamLane,scheduleChainAutomation,+ scheduleParamLane,+ volumeLane,type AutomationTiming,} from "../audio/audioFxAutomation.js";import type { FxParamTarget } from "../audio/audioFxGraph.js";@@ -123,6 +126,18 @@ export interface FxAttributes {automation: string | null;}+/** The element's volume lane on `param`, as a target to cancel later; none when it has no lane. */+function scheduleVolume(+ param: AudioParam,+ automation: HfAutomation,+ at: AutomationTiming,+): FxParamTarget[] {+ const lane = volumeLane(automation);+ if (!lane) return [];+ scheduleParamLane([{ param }], lane, VOLUME_RANGE.scale, at);+ return [{ param }];+}+export function attachElementFxChain(ctx: BaseAudioContext,el: { getAttribute?(name: string): string | null },@@ -130,6 +145,7 @@ export function attachElementFxChain(destination: AudioNode,timing?: AutomationTiming,attrs: FxAttributes = { automation: HF_AUDIO_AUTOMATION_ATTR },+ volume?: AudioParam,): ElementFxHandle | null {const { chain } = readChain(el);@@ -205,16 +221,17 @@ export function attachElementFxChain(};const scheduleFor = (next: HfAudioFxChain, at: AutomationTiming | null): void => {- automated =- at && handle- ? scheduleChainAutomation(- readAutomation(el, next, attrs.automation),- next,- handle.nodes,- at,- handle.presets,- )- : [];+ if (!at) {+ automated = [];+ return;+ }+ const automation = readAutomation(el, next, attrs.automation);+ automated = [+ ...(handle+ ? scheduleChainAutomation(automation, next, handle.nodes, at, handle.presets)+ : []),+ ...(volume ? scheduleVolume(volume, automation, at) : []),+ ];};// The reference frame every later reschedule measures from. Mutable because adiff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.tsindex 52b5968df6..f47882ba06 100644--- a/packages/core/src/runtime/media.test.ts+++ b/packages/core/src/runtime/media.test.ts@@ -464,6 +464,22 @@ describe("syncRuntimeMedia", () => {expect(onElementVolume).toHaveBeenLastCalledWith(clip.el, 0.4, 0.8);});+ it("leaves a routed clip's lane to the transport, sending only what multiplies it", () => {+ const clip = createMockClip({ start: 0, end: 10, volume: 0.55 });+ clip.el.setAttribute("data-automation", DUCK);+ const onElementVolume = vi.fn();+ syncRuntimeMedia({+ clips: [clip],+ timeSeconds: 5,+ playing: true,+ playbackRate: 1,+ onElementVolume,+ isWebAudioRouted: () => true,+ });++ expect(onElementVolume).toHaveBeenLastCalledWith(clip.el, 1, 1);+ });+it("ramps between points across ticks", () => {const [a, b, c] = volumesAt([2, 2.5, 3], DUCK);expect(a).toBeCloseTo(0.8, 5);diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.tsindex b91d3288dc..72d195cd43 100644--- a/packages/core/src/runtime/media.ts+++ b/packages/core/src/runtime/media.ts@@ -457,7 +457,8 @@ export function syncRuntimeMedia(params: {// there is one time base, and this is it.const laneGain = elementVolumeLaneGain(el, params.timeSeconds - origin);if (laneGain !== null) {- authorVolume = clampAudioGain(laneGain);+ const transportOwnsLane = params.isWebAudioRouted?.(el);+ authorVolume = transportOwnsLane ? 1 : clampAudioGain(laneGain);} else if (clip.volumeKeyframes && clip.volumeKeyframes.length > 0) {// Keyframes probed from the GSAP timeline — same source as the renderer.// Use the interpolated envelope value directly; no need to track GSAP changes.diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.tsindex 79217cebe7..3f0b1adaff 100644--- a/packages/core/src/runtime/webAudioTransport.test.ts+++ b/packages/core/src/runtime/webAudioTransport.test.ts@@ -890,15 +890,158 @@ describe("WebAudioTransport", () => {document.body.innerHTML = "";});+ it.each([+ [+ "decoded",+ (t: WebAudioTransport, el: HTMLMediaElement, gen: number) =>+ t.schedulePlayback(el, mockBuffer, 0, 0, () => 0, 1, gen),+ ],+ [+ "media-element",+ (t: WebAudioTransport, el: HTMLMediaElement, gen: number) =>+ t.scheduleMediaElementPlayback(el, 0, 0, 0, 1, gen),+ ],+ ])(+ "keeps a %s clip's volume lane off the gain each tick rewrites",+ async (_route, schedule) => {+ const { transport, mock, gen } = setupGroupTransport();+ const el = groupedAudioEl("voice");+ const points = [+ { t: 0, v: 0 },+ { t: 3, v: 1 },+ { t: 6, v: 0 },+ ];+ el.setAttribute(+ "data-automation",+ JSON.stringify({ version: 1, lanes: [{ target: "volume", points }] }),+ );++ expect(await schedule(transport, el, gen)).not.toBeNull();+ type Ramps = { linearRampToValueAtTime: ReturnType<typeof vi.fn> };+ const laneGains = mock.gainNodes.filter(+ (node) => (node.gain as unknown as Ramps).linearRampToValueAtTime.mock.calls.length > 0,+ );+ expect(laneGains).toHaveLength(1);+ const laneValue = laneGains[0]!.gain.value;++ transport.setElementVolume(el, 0.5);++ expect(laneGains[0]!.gain.value).toBe(laneValue);+ expect(mock.gainNodes.filter((node) => node.gain.value === 0.5)).toHaveLength(1);+ },+ );++ describe("volume lanes follow the transport mid-play", () => {+ const lane = (points: { t: number; v: number }[]) =>+ JSON.stringify({ version: 1, lanes: [{ target: "volume", points }] });+ const FADE = lane([+ { t: 0, v: 0 },+ { t: 3, v: 1 },+ { t: 6, v: 0 },+ ]);+ type Param = Record<+ "linearRampToValueAtTime" | "cancelScheduledValues",+ ReturnType<typeof vi.fn>+ >;+ const param = (node: { gain: unknown }) => node.gain as Param;+ const rampsAfter = (node: { gain: unknown }, from: number) =>+ param(node).linearRampToValueAtTime.mock.calls.slice(from);+ /** Let the MutationObserver's microtask run. */+ const settle = () => new Promise((r) => setTimeout(r, 0));++ // An unbounded media-element clip: the runtime does not stopAll and reschedule it on a rate change.+ async function playBed(automation?: string) {+ const setup = setupGroupTransport();+ const el = groupedAudioEl("bed");+ if (automation) el.setAttribute("data-automation", automation);+ await setup.transport.scheduleMediaElementPlayback(el, 0, 0, 0, 1, setup.gen);+ expect(setup.transport.hasBoundedActiveSources()).toBe(false);+ return { ...setup, el, laneGain: setup.mock.gainNodes[1]! };+ }++ it("re-aims a clip's lane at a new rate", async () => {+ const { transport, laneGain } = await playBed(FADE);+ expect(rampsAfter(laneGain, 0)).toContainEqual([1, 103]);+ const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;++ transport.setRate(2);++ expect(rampsAfter(laneGain, before)).toContainEqual([1, 101.5]);+ });++ it("re-books a clip's lane when it is edited", async () => {+ const { el, laneGain } = await playBed(FADE);+ const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;++ el.setAttribute(+ "data-automation",+ lane([+ { t: 0, v: 0 },+ { t: 2, v: 0.5 },+ ]),+ );+ await settle();++ expect(rampsAfter(laneGain, before)).toContainEqual([0.5, 102]);+ });++ it("clears a clip's lane when it is removed", async () => {+ const { el, laneGain } = await playBed(FADE);+ const ramps = param(laneGain).linearRampToValueAtTime.mock.calls.length;+ const cancels = param(laneGain).cancelScheduledValues.mock.calls.length;++ el.removeAttribute("data-automation");+ await settle();++ expect(param(laneGain).cancelScheduledValues.mock.calls.slice(cancels)).toContainEqual([0]);+ expect(rampsAfter(laneGain, ramps)).toEqual([]);+ });++ it("books a lane added to a playing clip", async () => {+ const { el, laneGain } = await playBed();+ expect(laneGain).toBeDefined();++ el.setAttribute("data-automation", FADE);+ await settle();++ expect(rampsAfter(laneGain, 0)).toContainEqual([1, 103]);+ });++ it("re-aims a group fader's lane at a new rate and on an edit", async () => {+ document.body.innerHTML = `<hf-audio-group id="vo" data-volume="0.5"></hf-audio-group>`;+ document.getElementById("vo")!.setAttribute("data-automation", FADE);+ const { transport, mock, gen } = setupGroupTransport();+ await scheduleGrouped(transport, gen, "a", "vo");+ const fader = mock.gainNodes[4]!;+ expect(rampsAfter(fader, 0)).toContainEqual([1, 103]);++ let before = param(fader).linearRampToValueAtTime.mock.calls.length;+ transport.setRate(2);+ expect(rampsAfter(fader, before)).toContainEqual([1, 101.5]);++ before = param(fader).linearRampToValueAtTime.mock.calls.length;+ document.getElementById("vo")!.setAttribute(+ "data-automation",+ lane([+ { t: 0, v: 0 },+ { t: 4, v: 0.5 },+ ]),+ );+ await settle();+ expect(rampsAfter(fader, before)).toContainEqual([0.5, 102]);+ });+ });+it("routes an ungrouped clip straight to master through its own gain", async () => {const { transport, mock, gen } = setupGroupTransport();await scheduleGrouped(transport, gen, "lone");- // One gain per clip now that solo is gone — it goes straight to master.- expect(mock.gainNodes).toHaveLength(1);- const [clipGain] = mock.gainNodes;+ // The clip's gain goes straight to master; its volume-lane gain feeds it.+ expect(mock.gainNodes).toHaveLength(2);+ const [clipGain, laneGain] = mock.gainNodes;expect(clipGain!.connect).toHaveBeenCalledWith(mock.masterGain);+ expect(laneGain!.connect).toHaveBeenCalledWith(clipGain);});// The media-element transport is the PRIMARY path for audio — the runtime@@ -953,8 +1096,9 @@ describe("WebAudioTransport", () => {await transport.scheduleMediaElementPlayback(el, 0, 0, 0, 1, gen, 1);- expect(mock.gainNodes).toHaveLength(1);+ expect(mock.gainNodes).toHaveLength(2);expect(mock.gainNodes[0]!.connect).toHaveBeenCalledWith(mock.masterGain);+ expect(mock.gainNodes[1]!.connect).toHaveBeenCalledWith(mock.gainNodes[0]);});it("two members of the same group land on ONE shared group gain, not master directly", async () => {@@ -965,14 +1109,14 @@ describe("WebAudioTransport", () => {// Creation order for a: a-gain(0), groupInput(1), groupOutput(2),// muteGain(3), fader(4) — the group bus is built lazily inside a's- // schedule call. Then b: b-gain(5).- expect(mock.gainNodes.length).toBeGreaterThanOrEqual(6);+ // schedule call — then a's lane gain(5). Then b: b-gain(6), b's lane gain(7).+ expect(mock.gainNodes.length).toBeGreaterThanOrEqual(8);const aGain = mock.gainNodes[0]!;const groupInput = firstGroupInput(mock);const groupOutput = mock.gainNodes[2]!;const muteGain = mock.gainNodes[3]!;const fader = mock.gainNodes[4]!;- const bGain = mock.gainNodes[5]!;+ const bGain = mock.gainNodes[6]!;// Both members feed the shared bus — neither connects straight to master.expect(aGain.connect).toHaveBeenCalledWith(groupInput);@@ -996,11 +1140,11 @@ describe("WebAudioTransport", () => {const { transport, mock, gen } = setupGroupTransport();await scheduleGrouped(transport, gen, "a", "vo");- const gainCountAfterFirst = mock.gainNodes.length; // a-gain + group input/output/mute/fader+ const gainCountAfterFirst = mock.gainNodes.length; // a's gains + group input/output/mute/faderawait scheduleGrouped(transport, gen, "b", "vo");- // Only b's own gain is new — no second group bus minted.- expect(mock.gainNodes.length).toBe(gainCountAfterFirst + 1);+ // Only b's own gain and lane gain are new — no second group bus minted.+ expect(mock.gainNodes.length).toBe(gainCountAfterFirst + 2);});it("a group id with no matching <hf-audio-group> element still gets a flat bus", async () => {@@ -1193,8 +1337,8 @@ describe("WebAudioTransport", () => {const nodesAfterFirst = setup.mock.gainNodes.length;await scheduleGrouped(setup.transport, setup.gen, "b");- // The second member adds its own element gain and nothing else.- expect(setup.mock.gainNodes.length).toBe(nodesAfterFirst + 1);+ // The second member adds its own element gain and lane gain, and nothing else.+ expect(setup.mock.gainNodes.length).toBe(nodesAfterFirst + 2);expect(setup.mock.masterGain.disconnect).toHaveBeenCalledTimes(1);});});diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.tsindex 8d90eb4e0d..c8395b8a36 100644--- a/packages/core/src/runtime/webAudioTransport.ts+++ b/packages/core/src/runtime/webAudioTransport.ts@@ -65,9 +65,9 @@ function logFallbackHandoff(el: HTMLMediaElement, priorMuted: boolean): void {/*** Start a buffer source, bounding it to the clip's authored window* (`data-duration`) so a trimmed clip stops at its edge instead of running the- * buffer to the source file's natural end. `clipSourceLen` is the clip span in- * buffer seconds; the third `start()` arg is the portion to play from the- * offset. An infinite `clipDuration` plays unbounded (legacy behavior).+ * buffer to the source file's natural end. `remaining`, the third `start()` arg,+ * is the rest of the clip from the offset in buffer seconds. An infinite+ * `clipDuration` plays unbounded (legacy behavior).** Returns false when the playhead is already past the clip end (nothing to* play); the caller should discard the source.@@ -112,9 +112,26 @@ function scheduleVolumeLane(scheduleParamLane([{ param: gainNode.gain }], lane, VOLUME_RANGE.scale, timing);}+// FX, then the volume lane on its own gain ahead of `gainNode` (rewritten every tick); the FX+// handle re-aims the lane on rate changes and edits.+function attachClipStages(+ ctx: AudioContext,+ el: HTMLMediaElement,+ source: AudioNode,+ gainNode: GainNode,+ timing: AutomationTiming,+): { fx: ElementFxHandle | null; laneNode: GainNode } {+ const laneNode = ctx.createGain();+ laneNode.connect(gainNode);+ const fx = attachElementFxChain(ctx, el, source, laneNode, timing, undefined, laneNode.gain);+ return { fx, laneNode };+}+type ScheduledSourceBase = {el: HTMLMediaElement;gainNode: GainNode;+ /** The clip's volume lane, between its FX and `gainNode`; unity when it has none. */+ laneNode: GainNode;/** FX chain spliced between source and gain, when the element carries one. */fx?: ElementFxHandle | null;compositionStart: number;@@ -414,7 +431,6 @@ export class WebAudioTransport {const scheduledAt = this._ctx.currentTime;const elapsed = compositionTime - compositionStart;const timing: AutomationTiming = { scheduledAt, elapsed, rate: safeRate };- const fx = attachElementFxChain(this._ctx, el, sourceNode, gainNode, timing);// The group bus, not master, for a member — same as the decoded-buffer// path. This transport is the PRIMARY one for audio (the decode path is// its fallback), so routing it at master would have left every grouped@@ -423,7 +439,7 @@ export class WebAudioTransport {gainNode.connect(this.resolveDestination(el, scheduledAt, compositionTime, safeRate) ?? this._masterGain,);- scheduleVolumeLane(el, gainNode, timing);+ const { fx, laneNode } = attachClipStages(this._ctx, el, sourceNode, gainNode, timing);this._rate = safeRate;this._rateAnchorCtx = scheduledAt;@@ -435,6 +451,7 @@ export class WebAudioTransport {sourceNode,sourceKind: "media-element",gainNode,+ laneNode,compositionStart,mediaStart: _mediaStart,scheduledAt,@@ -525,8 +542,9 @@ export class WebAudioTransport {input,fader,timing,+ undefined,+ groupEl ? fader.gain : undefined,);- if (groupEl) scheduleVolumeLane(groupEl, fader, timing);this._groups.set(groupId, {input,@@ -537,19 +555,16 @@ export class WebAudioTransport {fx,generation: this._playGeneration,reanchor: (at: AutomationTiming) => {- // Cleared BEFORE the value write, and unconditionally. `scheduleVolumeLane`- // clears as part of scheduling, but returns early when the group no- // longer has a lane — and a scheduled envelope outranks a `.value`- // write, so deleting a group's automation mid-session otherwise left- // the previous pass's ramps still owning the param (for a fade-out,- // silence) for the rest of the session.+ // Cleared BEFORE the value write, and unconditionally: a scheduled envelope+ // outranks a `.value` write, so a deleted lane's ramps would keep the param.clearParamLane([{ param: fader.gain }]);// Re-resolved, not the element captured at build time — see `resolveEl`.const live = resolveEl();fader.gain.value = clampGroupVolume(readAudioGroupVolume(live));muteGain.gain.value = live?.hasAttribute("data-hidden") ? 0 : 1;fx?.reanchor(at);- if (live) scheduleVolumeLane(live, fader, at);+ // The handle books the fader from the element the bus was built with.+ if (live && live !== groupEl) scheduleVolumeLane(live, fader, at);},dispose: () => {try {@@ -688,6 +703,7 @@ export class WebAudioTransport {try {sourceNode.disconnect();scheduled.fx?.dispose();+ scheduled.laneNode.disconnect();scheduled.gainNode.disconnect();} catch {// Already torn down.@@ -741,12 +757,10 @@ export class WebAudioTransport {// so effects see the raw signal and volume automation rides on their// output — the same order the offline render uses. Preview and render run// the identical graph builders, so what is heard here is what is written.- const fx = attachElementFxChain(this._ctx, el, sourceNode, gainNode, timing);gainNode.connect(this.resolveDestination(el, scheduledAt, compositionTime, safeRate) ?? this._masterGain,);-- scheduleVolumeLane(el, gainNode, timing);+ const { fx, laneNode } = attachClipStages(this._ctx, el, sourceNode, gainNode, timing);this._rate = safeRate;this._rateAnchorCtx = scheduledAt;@@ -766,6 +780,7 @@ export class WebAudioTransport {// Playhead already past the clip end — discard the nodes we built.sourceNode.disconnect();fx?.dispose();+ laneNode.disconnect();gainNode.disconnect();return null;}@@ -780,6 +795,7 @@ export class WebAudioTransport {sourceNode,sourceKind: "buffer",gainNode,+ laneNode,compositionStart,mediaStart,scheduledAt,@@ -865,6 +881,7 @@ export class WebAudioTransport {if (isBufferSource(source)) source.sourceNode.stop();source.sourceNode.disconnect();source.fx?.dispose();+ source.laneNode.disconnect();source.gainNode.disconnect();} catch {// already stoppeddiff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.jsonindex fd9d1d0f98..23cb53d3c2 100644--- a/scripts/package-size-budgets.json+++ b/scripts/package-size-budgets.json@@ -1,7 +1,7 @@{"@hyperframes/aws-lambda": { "packed": 70874, "unpacked": 258166 },"@hyperframes/cli": { "packed": 7902966, "unpacked": 23722874 },- "@hyperframes/core": { "packed": 1636442, "unpacked": 3935667 },+ "@hyperframes/core": { "packed": 1636442, "unpacked": 3936240 },"@hyperframes/engine": { "packed": 428347, "unpacked": 1436293 },"@hyperframes/gcp-cloud-run": { "packed": 49412, "unpacked": 192672 },"@hyperframes/inspector": { "packed": 808905, "unpacked": 3645432 },