Skip to content

Write HTML. Render video. Built for agents.

fix(core): a clip's volume lane stays smooth in preview while the page is busy (#6057)

Miguel Ángelcfff4493dfc6
diff --git a/packages/core/src/runtime/audioAutomationVolume.ts b/packages/core/src/runtime/audioAutomationVolume.ts
index 3a4ea306d1..1708941252 100644
--- a/packages/core/src/runtime/audioAutomationVolume.ts
+++ b/packages/core/src/runtime/audioAutomationVolume.ts
@@ -1,11 +1,9 @@
/**
* The volume lane's gain for an element at a moment in its clip.
*
- * Volume is not scheduled onto the gain AudioParam the way FX parameters are.
- * The runtime rewrites that gain every tick from `data-volume` and the GSAP
- * seeked value, which would erase a scheduled envelope within a frame — so the
- * lane feeds the same per-tick path the probed `volumeKeyframes` already use,
- * and the render still bakes it into the samples.
+ * Read per tick for a clip that sounds through `el.volume`. A clip the Web Audio
+ * transport routes plays its lane scheduled on a gain of its own
+ * (`attachClipStages`), out of reach of the per-tick gain write.
*/
import {
diff --git a/packages/core/src/runtime/audioFx.ts b/packages/core/src/runtime/audioFx.ts
index 2a4376ecca..7fc374384d 100644
--- a/packages/core/src/runtime/audioFx.ts
+++ b/packages/core/src/runtime/audioFx.ts
@@ -15,12 +15,15 @@ import {
HF_AUDIO_AUTOMATION_ATTR,
parseAutomation,
resolveAutomation,
+ VOLUME_RANGE,
type HfAutomation,
} from "../audioAutomation.js";
import {
cancelParamLane,
clearParamLane,
scheduleChainAutomation,
+ scheduleParamLane,
+ volumeLane,
type AutomationTiming,
} from "../audio/audioFxAutomation.js";
import type { FxParamTarget } from "../audio/audioFxGraph.js";
@@ -123,6 +126,18 @@ export interface FxAttributes {
automation: string | null;
}
+/** The element's volume lane on `param`, as a target to cancel later; none when it has no lane. */
+function scheduleVolume(
+ param: AudioParam,
+ automation: HfAutomation,
+ at: AutomationTiming,
+): FxParamTarget[] {
+ const lane = volumeLane(automation);
+ if (!lane) return [];
+ scheduleParamLane([{ param }], lane, VOLUME_RANGE.scale, at);
+ return [{ param }];
+}
+
export function attachElementFxChain(
ctx: BaseAudioContext,
el: { getAttribute?(name: string): string | null },
@@ -130,6 +145,7 @@ export function attachElementFxChain(
destination: AudioNode,
timing?: AutomationTiming,
attrs: FxAttributes = { automation: HF_AUDIO_AUTOMATION_ATTR },
+ volume?: AudioParam,
): ElementFxHandle | null {
const { chain } = readChain(el);
@@ -205,16 +221,17 @@ export function attachElementFxChain(
};
const scheduleFor = (next: HfAudioFxChain, at: AutomationTiming | null): void => {
- automated =
- at && handle
- ? scheduleChainAutomation(
- readAutomation(el, next, attrs.automation),
- next,
- handle.nodes,
- at,
- handle.presets,
- )
- : [];
+ if (!at) {
+ automated = [];
+ return;
+ }
+ const automation = readAutomation(el, next, attrs.automation);
+ automated = [
+ ...(handle
+ ? scheduleChainAutomation(automation, next, handle.nodes, at, handle.presets)
+ : []),
+ ...(volume ? scheduleVolume(volume, automation, at) : []),
+ ];
};
// The reference frame every later reschedule measures from. Mutable because a
diff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.ts
index 52b5968df6..f47882ba06 100644
--- a/packages/core/src/runtime/media.test.ts
+++ b/packages/core/src/runtime/media.test.ts
@@ -464,6 +464,22 @@ describe("syncRuntimeMedia", () => {
expect(onElementVolume).toHaveBeenLastCalledWith(clip.el, 0.4, 0.8);
});
+ it("leaves a routed clip's lane to the transport, sending only what multiplies it", () => {
+ const clip = createMockClip({ start: 0, end: 10, volume: 0.55 });
+ clip.el.setAttribute("data-automation", DUCK);
+ const onElementVolume = vi.fn();
+ syncRuntimeMedia({
+ clips: [clip],
+ timeSeconds: 5,
+ playing: true,
+ playbackRate: 1,
+ onElementVolume,
+ isWebAudioRouted: () => true,
+ });
+
+ expect(onElementVolume).toHaveBeenLastCalledWith(clip.el, 1, 1);
+ });
+
it("ramps between points across ticks", () => {
const [a, b, c] = volumesAt([2, 2.5, 3], DUCK);
expect(a).toBeCloseTo(0.8, 5);
diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts
index b91d3288dc..72d195cd43 100644
--- a/packages/core/src/runtime/media.ts
+++ b/packages/core/src/runtime/media.ts
@@ -457,7 +457,8 @@ export function syncRuntimeMedia(params: {
// there is one time base, and this is it.
const laneGain = elementVolumeLaneGain(el, params.timeSeconds - origin);
if (laneGain !== null) {
- authorVolume = clampAudioGain(laneGain);
+ const transportOwnsLane = params.isWebAudioRouted?.(el);
+ authorVolume = transportOwnsLane ? 1 : clampAudioGain(laneGain);
} else if (clip.volumeKeyframes && clip.volumeKeyframes.length > 0) {
// Keyframes probed from the GSAP timeline — same source as the renderer.
// Use the interpolated envelope value directly; no need to track GSAP changes.
diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts
index 79217cebe7..3f0b1adaff 100644
--- a/packages/core/src/runtime/webAudioTransport.test.ts
+++ b/packages/core/src/runtime/webAudioTransport.test.ts
@@ -890,15 +890,158 @@ describe("WebAudioTransport", () => {
document.body.innerHTML = "";
});
+ it.each([
+ [
+ "decoded",
+ (t: WebAudioTransport, el: HTMLMediaElement, gen: number) =>
+ t.schedulePlayback(el, mockBuffer, 0, 0, () => 0, 1, gen),
+ ],
+ [
+ "media-element",
+ (t: WebAudioTransport, el: HTMLMediaElement, gen: number) =>
+ t.scheduleMediaElementPlayback(el, 0, 0, 0, 1, gen),
+ ],
+ ])(
+ "keeps a %s clip's volume lane off the gain each tick rewrites",
+ async (_route, schedule) => {
+ const { transport, mock, gen } = setupGroupTransport();
+ const el = groupedAudioEl("voice");
+ const points = [
+ { t: 0, v: 0 },
+ { t: 3, v: 1 },
+ { t: 6, v: 0 },
+ ];
+ el.setAttribute(
+ "data-automation",
+ JSON.stringify({ version: 1, lanes: [{ target: "volume", points }] }),
+ );
+
+ expect(await schedule(transport, el, gen)).not.toBeNull();
+ type Ramps = { linearRampToValueAtTime: ReturnType<typeof vi.fn> };
+ const laneGains = mock.gainNodes.filter(
+ (node) => (node.gain as unknown as Ramps).linearRampToValueAtTime.mock.calls.length > 0,
+ );
+ expect(laneGains).toHaveLength(1);
+ const laneValue = laneGains[0]!.gain.value;
+
+ transport.setElementVolume(el, 0.5);
+
+ expect(laneGains[0]!.gain.value).toBe(laneValue);
+ expect(mock.gainNodes.filter((node) => node.gain.value === 0.5)).toHaveLength(1);
+ },
+ );
+
+ describe("volume lanes follow the transport mid-play", () => {
+ const lane = (points: { t: number; v: number }[]) =>
+ JSON.stringify({ version: 1, lanes: [{ target: "volume", points }] });
+ const FADE = lane([
+ { t: 0, v: 0 },
+ { t: 3, v: 1 },
+ { t: 6, v: 0 },
+ ]);
+ type Param = Record<
+ "linearRampToValueAtTime" | "cancelScheduledValues",
+ ReturnType<typeof vi.fn>
+ >;
+ const param = (node: { gain: unknown }) => node.gain as Param;
+ const rampsAfter = (node: { gain: unknown }, from: number) =>
+ param(node).linearRampToValueAtTime.mock.calls.slice(from);
+ /** Let the MutationObserver's microtask run. */
+ const settle = () => new Promise((r) => setTimeout(r, 0));
+
+ // An unbounded media-element clip: the runtime does not stopAll and reschedule it on a rate change.
+ async function playBed(automation?: string) {
+ const setup = setupGroupTransport();
+ const el = groupedAudioEl("bed");
+ if (automation) el.setAttribute("data-automation", automation);
+ await setup.transport.scheduleMediaElementPlayback(el, 0, 0, 0, 1, setup.gen);
+ expect(setup.transport.hasBoundedActiveSources()).toBe(false);
+ return { ...setup, el, laneGain: setup.mock.gainNodes[1]! };
+ }
+
+ it("re-aims a clip's lane at a new rate", async () => {
+ const { transport, laneGain } = await playBed(FADE);
+ expect(rampsAfter(laneGain, 0)).toContainEqual([1, 103]);
+ const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
+
+ transport.setRate(2);
+
+ expect(rampsAfter(laneGain, before)).toContainEqual([1, 101.5]);
+ });
+
+ it("re-books a clip's lane when it is edited", async () => {
+ const { el, laneGain } = await playBed(FADE);
+ const before = param(laneGain).linearRampToValueAtTime.mock.calls.length;
+
+ el.setAttribute(
+ "data-automation",
+ lane([
+ { t: 0, v: 0 },
+ { t: 2, v: 0.5 },
+ ]),
+ );
+ await settle();
+
+ expect(rampsAfter(laneGain, before)).toContainEqual([0.5, 102]);
+ });
+
+ it("clears a clip's lane when it is removed", async () => {
+ const { el, laneGain } = await playBed(FADE);
+ const ramps = param(laneGain).linearRampToValueAtTime.mock.calls.length;
+ const cancels = param(laneGain).cancelScheduledValues.mock.calls.length;
+
+ el.removeAttribute("data-automation");
+ await settle();
+
+ expect(param(laneGain).cancelScheduledValues.mock.calls.slice(cancels)).toContainEqual([0]);
+ expect(rampsAfter(laneGain, ramps)).toEqual([]);
+ });
+
+ it("books a lane added to a playing clip", async () => {
+ const { el, laneGain } = await playBed();
+ expect(laneGain).toBeDefined();
+
+ el.setAttribute("data-automation", FADE);
+ await settle();
+
+ expect(rampsAfter(laneGain, 0)).toContainEqual([1, 103]);
+ });
+
+ it("re-aims a group fader's lane at a new rate and on an edit", async () => {
+ document.body.innerHTML = `<hf-audio-group id="vo" data-volume="0.5"></hf-audio-group>`;
+ document.getElementById("vo")!.setAttribute("data-automation", FADE);
+ const { transport, mock, gen } = setupGroupTransport();
+ await scheduleGrouped(transport, gen, "a", "vo");
+ const fader = mock.gainNodes[4]!;
+ expect(rampsAfter(fader, 0)).toContainEqual([1, 103]);
+
+ let before = param(fader).linearRampToValueAtTime.mock.calls.length;
+ transport.setRate(2);
+ expect(rampsAfter(fader, before)).toContainEqual([1, 101.5]);
+
+ before = param(fader).linearRampToValueAtTime.mock.calls.length;
+ document.getElementById("vo")!.setAttribute(
+ "data-automation",
+ lane([
+ { t: 0, v: 0 },
+ { t: 4, v: 0.5 },
+ ]),
+ );
+ await settle();
+ expect(rampsAfter(fader, before)).toContainEqual([0.5, 102]);
+ });
+ });
+
it("routes an ungrouped clip straight to master through its own gain", async () => {
const { transport, mock, gen } = setupGroupTransport();
await scheduleGrouped(transport, gen, "lone");
- // One gain per clip now that solo is gone — it goes straight to master.
- expect(mock.gainNodes).toHaveLength(1);
- const [clipGain] = mock.gainNodes;
+ // The clip's gain goes straight to master; its volume-lane gain feeds it.
+ expect(mock.gainNodes).toHaveLength(2);
+ const [clipGain, laneGain] = mock.gainNodes;
expect(clipGain!.connect).toHaveBeenCalledWith(mock.masterGain);
+ expect(laneGain!.connect).toHaveBeenCalledWith(clipGain);
});
// The media-element transport is the PRIMARY path for audio — the runtime
@@ -953,8 +1096,9 @@ describe("WebAudioTransport", () => {
await transport.scheduleMediaElementPlayback(el, 0, 0, 0, 1, gen, 1);
- expect(mock.gainNodes).toHaveLength(1);
+ expect(mock.gainNodes).toHaveLength(2);
expect(mock.gainNodes[0]!.connect).toHaveBeenCalledWith(mock.masterGain);
+ expect(mock.gainNodes[1]!.connect).toHaveBeenCalledWith(mock.gainNodes[0]);
});
it("two members of the same group land on ONE shared group gain, not master directly", async () => {
@@ -965,14 +1109,14 @@ describe("WebAudioTransport", () => {
// Creation order for a: a-gain(0), groupInput(1), groupOutput(2),
// muteGain(3), fader(4) — the group bus is built lazily inside a's
- // schedule call. Then b: b-gain(5).
- expect(mock.gainNodes.length).toBeGreaterThanOrEqual(6);
+ // schedule call — then a's lane gain(5). Then b: b-gain(6), b's lane gain(7).
+ expect(mock.gainNodes.length).toBeGreaterThanOrEqual(8);
const aGain = mock.gainNodes[0]!;
const groupInput = firstGroupInput(mock);
const groupOutput = mock.gainNodes[2]!;
const muteGain = mock.gainNodes[3]!;
const fader = mock.gainNodes[4]!;
- const bGain = mock.gainNodes[5]!;
+ const bGain = mock.gainNodes[6]!;
// Both members feed the shared bus — neither connects straight to master.
expect(aGain.connect).toHaveBeenCalledWith(groupInput);
@@ -996,11 +1140,11 @@ describe("WebAudioTransport", () => {
const { transport, mock, gen } = setupGroupTransport();
await scheduleGrouped(transport, gen, "a", "vo");
- const gainCountAfterFirst = mock.gainNodes.length; // a-gain + group input/output/mute/fader
+ const gainCountAfterFirst = mock.gainNodes.length; // a's gains + group input/output/mute/fader
await scheduleGrouped(transport, gen, "b", "vo");
- // Only b's own gain is new — no second group bus minted.
- expect(mock.gainNodes.length).toBe(gainCountAfterFirst + 1);
+ // Only b's own gain and lane gain are new — no second group bus minted.
+ expect(mock.gainNodes.length).toBe(gainCountAfterFirst + 2);
});
it("a group id with no matching <hf-audio-group> element still gets a flat bus", async () => {
@@ -1193,8 +1337,8 @@ describe("WebAudioTransport", () => {
const nodesAfterFirst = setup.mock.gainNodes.length;
await scheduleGrouped(setup.transport, setup.gen, "b");
- // The second member adds its own element gain and nothing else.
- expect(setup.mock.gainNodes.length).toBe(nodesAfterFirst + 1);
+ // The second member adds its own element gain and lane gain, and nothing else.
+ expect(setup.mock.gainNodes.length).toBe(nodesAfterFirst + 2);
expect(setup.mock.masterGain.disconnect).toHaveBeenCalledTimes(1);
});
});
diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts
index 8d90eb4e0d..c8395b8a36 100644
--- a/packages/core/src/runtime/webAudioTransport.ts
+++ b/packages/core/src/runtime/webAudioTransport.ts
@@ -65,9 +65,9 @@ function logFallbackHandoff(el: HTMLMediaElement, priorMuted: boolean): void {
/**
* Start a buffer source, bounding it to the clip's authored window
* (`data-duration`) so a trimmed clip stops at its edge instead of running the
- * buffer to the source file's natural end. `clipSourceLen` is the clip span in
- * buffer seconds; the third `start()` arg is the portion to play from the
- * offset. An infinite `clipDuration` plays unbounded (legacy behavior).
+ * buffer to the source file's natural end. `remaining`, the third `start()` arg,
+ * is the rest of the clip from the offset in buffer seconds. An infinite
+ * `clipDuration` plays unbounded (legacy behavior).
*
* Returns false when the playhead is already past the clip end (nothing to
* play); the caller should discard the source.
@@ -112,9 +112,26 @@ function scheduleVolumeLane(
scheduleParamLane([{ param: gainNode.gain }], lane, VOLUME_RANGE.scale, timing);
}
+// FX, then the volume lane on its own gain ahead of `gainNode` (rewritten every tick); the FX
+// handle re-aims the lane on rate changes and edits.
+function attachClipStages(
+ ctx: AudioContext,
+ el: HTMLMediaElement,
+ source: AudioNode,
+ gainNode: GainNode,
+ timing: AutomationTiming,
+): { fx: ElementFxHandle | null; laneNode: GainNode } {
+ const laneNode = ctx.createGain();
+ laneNode.connect(gainNode);
+ const fx = attachElementFxChain(ctx, el, source, laneNode, timing, undefined, laneNode.gain);
+ return { fx, laneNode };
+}
+
type ScheduledSourceBase = {
el: HTMLMediaElement;
gainNode: GainNode;
+ /** The clip's volume lane, between its FX and `gainNode`; unity when it has none. */
+ laneNode: GainNode;
/** FX chain spliced between source and gain, when the element carries one. */
fx?: ElementFxHandle | null;
compositionStart: number;
@@ -414,7 +431,6 @@ export class WebAudioTransport {
const scheduledAt = this._ctx.currentTime;
const elapsed = compositionTime - compositionStart;
const timing: AutomationTiming = { scheduledAt, elapsed, rate: safeRate };
- const fx = attachElementFxChain(this._ctx, el, sourceNode, gainNode, timing);
// The group bus, not master, for a member — same as the decoded-buffer
// path. This transport is the PRIMARY one for audio (the decode path is
// its fallback), so routing it at master would have left every grouped
@@ -423,7 +439,7 @@ export class WebAudioTransport {
gainNode.connect(
this.resolveDestination(el, scheduledAt, compositionTime, safeRate) ?? this._masterGain,
);
- scheduleVolumeLane(el, gainNode, timing);
+ const { fx, laneNode } = attachClipStages(this._ctx, el, sourceNode, gainNode, timing);
this._rate = safeRate;
this._rateAnchorCtx = scheduledAt;
@@ -435,6 +451,7 @@ export class WebAudioTransport {
sourceNode,
sourceKind: "media-element",
gainNode,
+ laneNode,
compositionStart,
mediaStart: _mediaStart,
scheduledAt,
@@ -525,8 +542,9 @@ export class WebAudioTransport {
input,
fader,
timing,
+ undefined,
+ groupEl ? fader.gain : undefined,
);
- if (groupEl) scheduleVolumeLane(groupEl, fader, timing);
this._groups.set(groupId, {
input,
@@ -537,19 +555,16 @@ export class WebAudioTransport {
fx,
generation: this._playGeneration,
reanchor: (at: AutomationTiming) => {
- // Cleared BEFORE the value write, and unconditionally. `scheduleVolumeLane`
- // clears as part of scheduling, but returns early when the group no
- // longer has a lane — and a scheduled envelope outranks a `.value`
- // write, so deleting a group's automation mid-session otherwise left
- // the previous pass's ramps still owning the param (for a fade-out,
- // silence) for the rest of the session.
+ // Cleared BEFORE the value write, and unconditionally: a scheduled envelope
+ // outranks a `.value` write, so a deleted lane's ramps would keep the param.
clearParamLane([{ param: fader.gain }]);
// Re-resolved, not the element captured at build time — see `resolveEl`.
const live = resolveEl();
fader.gain.value = clampGroupVolume(readAudioGroupVolume(live));
muteGain.gain.value = live?.hasAttribute("data-hidden") ? 0 : 1;
fx?.reanchor(at);
- if (live) scheduleVolumeLane(live, fader, at);
+ // The handle books the fader from the element the bus was built with.
+ if (live && live !== groupEl) scheduleVolumeLane(live, fader, at);
},
dispose: () => {
try {
@@ -688,6 +703,7 @@ export class WebAudioTransport {
try {
sourceNode.disconnect();
scheduled.fx?.dispose();
+ scheduled.laneNode.disconnect();
scheduled.gainNode.disconnect();
} catch {
// Already torn down.
@@ -741,12 +757,10 @@ export class WebAudioTransport {
// so effects see the raw signal and volume automation rides on their
// output — the same order the offline render uses. Preview and render run
// the identical graph builders, so what is heard here is what is written.
- const fx = attachElementFxChain(this._ctx, el, sourceNode, gainNode, timing);
gainNode.connect(
this.resolveDestination(el, scheduledAt, compositionTime, safeRate) ?? this._masterGain,
);
-
- scheduleVolumeLane(el, gainNode, timing);
+ const { fx, laneNode } = attachClipStages(this._ctx, el, sourceNode, gainNode, timing);
this._rate = safeRate;
this._rateAnchorCtx = scheduledAt;
@@ -766,6 +780,7 @@ export class WebAudioTransport {
// Playhead already past the clip end — discard the nodes we built.
sourceNode.disconnect();
fx?.dispose();
+ laneNode.disconnect();
gainNode.disconnect();
return null;
}
@@ -780,6 +795,7 @@ export class WebAudioTransport {
sourceNode,
sourceKind: "buffer",
gainNode,
+ laneNode,
compositionStart,
mediaStart,
scheduledAt,
@@ -865,6 +881,7 @@ export class WebAudioTransport {
if (isBufferSource(source)) source.sourceNode.stop();
source.sourceNode.disconnect();
source.fx?.dispose();
+ source.laneNode.disconnect();
source.gainNode.disconnect();
} catch {
// already stopped
diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json
index fd9d1d0f98..23cb53d3c2 100644
--- a/scripts/package-size-budgets.json
+++ b/scripts/package-size-budgets.json
@@ -1,7 +1,7 @@
{
"@hyperframes/aws-lambda": { "packed": 70874, "unpacked": 258166 },
"@hyperframes/cli": { "packed": 7902966, "unpacked": 23722874 },
- "@hyperframes/core": { "packed": 1636442, "unpacked": 3935667 },
+ "@hyperframes/core": { "packed": 1636442, "unpacked": 3936240 },
"@hyperframes/engine": { "packed": 428347, "unpacked": 1436293 },
"@hyperframes/gcp-cloud-run": { "packed": 49412, "unpacked": 192672 },
"@hyperframes/inspector": { "packed": 808905, "unpacked": 3645432 },