Skip to content

Write HTML. Render video. Built for agents.

fix(core): a video's sound plays at its boosted volume in the preview (#4746)

Miguel Ángel2fc5a37fceba
diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts
index b333cbb4a5..43c1beac68 100644
--- a/packages/core/src/runtime/init.test.ts
+++ b/packages/core/src/runtime/init.test.ts
@@ -4945,7 +4945,11 @@ describe("initSandboxRuntimeModular", () => {
await Promise.resolve();
}
- function mountAudio(src: string, attrs: Record<string, string> = {}) {
+ function mountAudio(
+ src: string,
+ attrs: Record<string, string> = {},
+ tag: "audio" | "video" = "audio",
+ ) {
const root = document.createElement("div");
root.setAttribute("data-composition-id", "main");
root.setAttribute("data-root", "true");
@@ -4955,7 +4959,7 @@ describe("initSandboxRuntimeModular", () => {
root.setAttribute("data-height", "1080");
document.body.appendChild(root);
- const audio = document.createElement("audio");
+ const audio = document.createElement(tag);
audio.setAttribute("data-start", "0");
audio.setAttribute("data-duration", "10");
audio.setAttribute("src", src);
@@ -5042,7 +5046,7 @@ describe("initSandboxRuntimeModular", () => {
expect(String(line?.[0])).toContain("fx-chain");
});
- it("says nothing about a cross-origin <video>, which never routes through Web Audio", () => {
+ it("says nothing about a cross-origin <video> at unity, which never routes through Web Audio", () => {
const root = document.createElement("div");
root.setAttribute("data-composition-id", "main");
root.setAttribute("data-root", "true");
@@ -5128,6 +5132,82 @@ describe("initSandboxRuntimeModular", () => {
});
});
+ // `el.volume` stops at 1, so a video's authored boost can only be carried by a Web Audio gain.
+ describe("a video's sound above unity", () => {
+ const ctx = useMockAudioContext();
+
+ it("plays a video at data-volume 1.5 through a Web Audio gain of 1.5", async () => {
+ const video = mountAudio("/assets/broll.mp4", { "data-volume": "1.5" }, "video");
+ const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback");
+
+ await startPlayback();
+ const scheduled = await captureSpy.mock.results[0]?.value;
+
+ expect(captureSpy.mock.calls[0]?.[0]).toBe(video);
+ expect(ctx.mediaElementSources).toBe(1);
+ expect(scheduled?.gainNode.gain.value).toBe(1.5);
+ expect(video.volume).toBe(1);
+ expect(video.muted).toBe(false);
+ });
+
+ it("leaves a video at unity on its native output", async () => {
+ mountAudio("/assets/broll.mp4", { "data-volume": "1" }, "video");
+ const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback");
+
+ await startPlayback();
+
+ expect(captureSpy).not.toHaveBeenCalled();
+ expect(ctx.mediaElementSources).toBe(0);
+ });
+
+ it("silences a captured video hidden after its volume drops to 1", async () => {
+ const raf = createManualRaf();
+ vi.spyOn(performance, "now").mockImplementation(() => raf.now());
+ window.requestAnimationFrame =
+ raf.requestAnimationFrame as typeof window.requestAnimationFrame;
+ window.cancelAnimationFrame = raf.cancelAnimationFrame as typeof window.cancelAnimationFrame;
+ const video = mountAudio("/assets/broll.mp4", { "data-volume": "1.5" }, "video");
+ Object.defineProperty(video, "paused", { value: false, configurable: true });
+ Object.defineProperty(video, "readyState", { value: 4, configurable: true });
+ const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback");
+ const step = async () => {
+ for (let frame = 0; frame < 5; frame++) {
+ ctx.time += 1 / 60;
+ raf.step(1000 / 60);
+ await Promise.resolve();
+ }
+ };
+
+ await startPlayback();
+ await captureSpy.mock.results[0]?.value;
+ const transport = captureSpy.mock.contexts[0] as WebAudioTransport;
+ await step();
+ video.setAttribute("data-volume", "1");
+ await step();
+ video.setAttribute("data-hidden", "");
+ await step();
+
+ expect(transport.routesElement(video)).toBe(false);
+ expect(video.volume).toBe(0);
+ });
+
+ it("keeps a cross-origin boosted video native and never decodes it", async () => {
+ const video = mountAudio(
+ "https://cdn.example.com/broll.mp4",
+ { "data-volume": "1.5" },
+ "video",
+ );
+ vi.spyOn(console, "info").mockImplementation(() => {});
+ const decodeSpy = vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement");
+
+ await startPlayback();
+
+ expect(ctx.mediaElementSources).toBe(0);
+ expect(decodeSpy).not.toHaveBeenCalled();
+ expect(video.muted).toBe(false);
+ });
+ });
+
describe("a voiceover routed through Web Audio", () => {
const ctx = useMockAudioContext();
diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts
index 774fb9e579..21a4dbeb07 100644
--- a/packages/core/src/runtime/init.ts
+++ b/packages/core/src/runtime/init.ts
@@ -329,6 +329,13 @@ function pageAnimationsForOnePass(): () => Animation[] {
return () => (list ??= document.getAnimations());
}
+// A `<video>` joins only for a gain `el.volume` cannot express: capture is a one-way door.
+const joinsWebAudio = (el: Element): el is HTMLMediaElement =>
+ isAudioElement(el) || (isVideoElement(el) && Number.parseFloat(el.dataset.volume ?? "") > 1);
+const WEB_AUDIO_MEDIA = "audio[data-start], video[data-start]";
+const webAudioMediaIn = (root: ParentNode): HTMLMediaElement[] =>
+ Array.from(root.querySelectorAll(WEB_AUDIO_MEDIA)).filter(joinsWebAudio);
+
export function initSandboxRuntimeModular(): void {
const state = createRuntimeState();
authoredMediaObserver?.disconnect();
@@ -2379,12 +2386,10 @@ export function initSandboxRuntimeModular(): void {
}
};
- // Only `<audio>` reaches `createMediaElementSource` (see
- // `scheduleWebAudioForActiveClips`, which queries `audio[data-start]`), so a
- // cross-origin `<video>` is not affected and must not be reported as if it
- // were.
+ // Only media `joinsWebAudio` admits when a play is scheduled reaches
+ // `createMediaElementSource`, so no other cross-origin media may be reported as if it did.
const reportWebAudioRoute = (mediaEl: HTMLMediaElement) => {
- if (!isAudioElement(mediaEl)) return;
+ if (!joinsWebAudio(mediaEl)) return;
// Before resource selection settles, the verdict is built from `<source>`
// children the browser might still pass over — good enough for the
// schedule path's conservative withhold, not good enough to put in front
@@ -2571,8 +2576,12 @@ export function initSandboxRuntimeModular(): void {
// started a second buffer source for every in-window clip on top of the ones
// still sounding: the whole mix audibly doubled, slightly out of phase.
let hiddenAudioDirty = false;
+ const affectsAudio = (el: Element): boolean =>
+ isMediaElement(el) &&
+ el.hasAttribute("data-start") &&
+ (joinsWebAudio(el) || webAudio.routesElement(el));
const nodeAffectsAudio = (node: HTMLElement): boolean =>
- node.matches("audio[data-start]") || node.querySelector("audio[data-start]") !== null;
+ affectsAudio(node) || Array.from(node.querySelectorAll(WEB_AUDIO_MEDIA)).some(affectsAudio);
// An `<hf-audio-group>` carries no `data-start`, so it is never among
// `visibilityNodes` above — group mute needs its own small diff pass.
@@ -2796,8 +2805,8 @@ export function initSandboxRuntimeModular(): void {
const groupNeedsCapture = syncAudioGroupMute(currentTime);
if ((hiddenAudioDirty || groupNeedsCapture) && clock.isPlaying()) {
webAudio.stopAll();
- for (const el of document.querySelectorAll("audio[data-start]")) {
- if (isMediaElement(el) && isSilencedByHidden(el)) el.volume = 0;
+ for (const el of webAudioMediaIn(document)) {
+ if (isSilencedByHidden(el)) el.volume = 0;
}
scheduleWebAudioForActiveClips();
}
@@ -4765,9 +4774,7 @@ export function initSandboxRuntimeModular(): void {
const scheduleWebAudioForActiveClips = () => {
if (state.nativeMediaSyncDisabled || state.webAudioMediaDisabled) return;
const gen = webAudio.startGeneration();
- const audioEls = document.querySelectorAll("audio[data-start]");
- for (const rawEl of audioEls) {
- if (!isMediaElement(rawEl) || !rawEl.isConnected) continue;
+ for (const rawEl of webAudioMediaIn(document)) {
if (isSilencedByHidden(rawEl)) continue;
const compStart = resolveAbsoluteMediaStartSeconds(rawEl);
if (!Number.isFinite(compStart)) continue;
@@ -4814,7 +4821,9 @@ export function initSandboxRuntimeModular(): void {
: Promise.resolve(null);
void capture.then((scheduled) => {
const replacedByNewerPass = gen !== webAudio.currentGeneration();
- if (scheduled || !clock.isPlaying() || replacedByNewerPass) return;
+ // A video's picture must keep playing from the element, so it has no decode fallback.
+ if (scheduled || !isAudioElement(rawEl) || !clock.isPlaying() || replacedByNewerPass)
+ return;
const effectiveRate = state.playbackRate * readElementPlaybackRate(rawEl);
// Deliberately the FX/automation pair and NOT
// `nativeUnexpressibleProcessing()`, which this route's diagnostic uses.