Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(core): a video's sound plays at its boosted volume in the preview (#4746)
Miguel Ángel2fc5a37fceba
diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.tsindex b333cbb4a5..43c1beac68 100644--- a/packages/core/src/runtime/init.test.ts+++ b/packages/core/src/runtime/init.test.ts@@ -4945,7 +4945,11 @@ describe("initSandboxRuntimeModular", () => {await Promise.resolve();}- function mountAudio(src: string, attrs: Record<string, string> = {}) {+ function mountAudio(+ src: string,+ attrs: Record<string, string> = {},+ tag: "audio" | "video" = "audio",+ ) {const root = document.createElement("div");root.setAttribute("data-composition-id", "main");root.setAttribute("data-root", "true");@@ -4955,7 +4959,7 @@ describe("initSandboxRuntimeModular", () => {root.setAttribute("data-height", "1080");document.body.appendChild(root);- const audio = document.createElement("audio");+ const audio = document.createElement(tag);audio.setAttribute("data-start", "0");audio.setAttribute("data-duration", "10");audio.setAttribute("src", src);@@ -5042,7 +5046,7 @@ describe("initSandboxRuntimeModular", () => {expect(String(line?.[0])).toContain("fx-chain");});- it("says nothing about a cross-origin <video>, which never routes through Web Audio", () => {+ it("says nothing about a cross-origin <video> at unity, which never routes through Web Audio", () => {const root = document.createElement("div");root.setAttribute("data-composition-id", "main");root.setAttribute("data-root", "true");@@ -5128,6 +5132,82 @@ describe("initSandboxRuntimeModular", () => {});});+ // `el.volume` stops at 1, so a video's authored boost can only be carried by a Web Audio gain.+ describe("a video's sound above unity", () => {+ const ctx = useMockAudioContext();++ it("plays a video at data-volume 1.5 through a Web Audio gain of 1.5", async () => {+ const video = mountAudio("/assets/broll.mp4", { "data-volume": "1.5" }, "video");+ const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback");++ await startPlayback();+ const scheduled = await captureSpy.mock.results[0]?.value;++ expect(captureSpy.mock.calls[0]?.[0]).toBe(video);+ expect(ctx.mediaElementSources).toBe(1);+ expect(scheduled?.gainNode.gain.value).toBe(1.5);+ expect(video.volume).toBe(1);+ expect(video.muted).toBe(false);+ });++ it("leaves a video at unity on its native output", async () => {+ mountAudio("/assets/broll.mp4", { "data-volume": "1" }, "video");+ const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback");++ await startPlayback();++ expect(captureSpy).not.toHaveBeenCalled();+ expect(ctx.mediaElementSources).toBe(0);+ });++ it("silences a captured video hidden after its volume drops to 1", async () => {+ const raf = createManualRaf();+ vi.spyOn(performance, "now").mockImplementation(() => raf.now());+ window.requestAnimationFrame =+ raf.requestAnimationFrame as typeof window.requestAnimationFrame;+ window.cancelAnimationFrame = raf.cancelAnimationFrame as typeof window.cancelAnimationFrame;+ const video = mountAudio("/assets/broll.mp4", { "data-volume": "1.5" }, "video");+ Object.defineProperty(video, "paused", { value: false, configurable: true });+ Object.defineProperty(video, "readyState", { value: 4, configurable: true });+ const captureSpy = vi.spyOn(WebAudioTransport.prototype, "scheduleMediaElementPlayback");+ const step = async () => {+ for (let frame = 0; frame < 5; frame++) {+ ctx.time += 1 / 60;+ raf.step(1000 / 60);+ await Promise.resolve();+ }+ };++ await startPlayback();+ await captureSpy.mock.results[0]?.value;+ const transport = captureSpy.mock.contexts[0] as WebAudioTransport;+ await step();+ video.setAttribute("data-volume", "1");+ await step();+ video.setAttribute("data-hidden", "");+ await step();++ expect(transport.routesElement(video)).toBe(false);+ expect(video.volume).toBe(0);+ });++ it("keeps a cross-origin boosted video native and never decodes it", async () => {+ const video = mountAudio(+ "https://cdn.example.com/broll.mp4",+ { "data-volume": "1.5" },+ "video",+ );+ vi.spyOn(console, "info").mockImplementation(() => {});+ const decodeSpy = vi.spyOn(WebAudioTransport.prototype, "decodeAudioElement");++ await startPlayback();++ expect(ctx.mediaElementSources).toBe(0);+ expect(decodeSpy).not.toHaveBeenCalled();+ expect(video.muted).toBe(false);+ });+ });+describe("a voiceover routed through Web Audio", () => {const ctx = useMockAudioContext();diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.tsindex 774fb9e579..21a4dbeb07 100644--- a/packages/core/src/runtime/init.ts+++ b/packages/core/src/runtime/init.ts@@ -329,6 +329,13 @@ function pageAnimationsForOnePass(): () => Animation[] {return () => (list ??= document.getAnimations());}+// A `<video>` joins only for a gain `el.volume` cannot express: capture is a one-way door.+const joinsWebAudio = (el: Element): el is HTMLMediaElement =>+ isAudioElement(el) || (isVideoElement(el) && Number.parseFloat(el.dataset.volume ?? "") > 1);+const WEB_AUDIO_MEDIA = "audio[data-start], video[data-start]";+const webAudioMediaIn = (root: ParentNode): HTMLMediaElement[] =>+ Array.from(root.querySelectorAll(WEB_AUDIO_MEDIA)).filter(joinsWebAudio);+export function initSandboxRuntimeModular(): void {const state = createRuntimeState();authoredMediaObserver?.disconnect();@@ -2379,12 +2386,10 @@ export function initSandboxRuntimeModular(): void {}};- // Only `<audio>` reaches `createMediaElementSource` (see- // `scheduleWebAudioForActiveClips`, which queries `audio[data-start]`), so a- // cross-origin `<video>` is not affected and must not be reported as if it- // were.+ // Only media `joinsWebAudio` admits when a play is scheduled reaches+ // `createMediaElementSource`, so no other cross-origin media may be reported as if it did.const reportWebAudioRoute = (mediaEl: HTMLMediaElement) => {- if (!isAudioElement(mediaEl)) return;+ if (!joinsWebAudio(mediaEl)) return;// Before resource selection settles, the verdict is built from `<source>`// children the browser might still pass over — good enough for the// schedule path's conservative withhold, not good enough to put in front@@ -2571,8 +2576,12 @@ export function initSandboxRuntimeModular(): void {// started a second buffer source for every in-window clip on top of the ones// still sounding: the whole mix audibly doubled, slightly out of phase.let hiddenAudioDirty = false;+ const affectsAudio = (el: Element): boolean =>+ isMediaElement(el) &&+ el.hasAttribute("data-start") &&+ (joinsWebAudio(el) || webAudio.routesElement(el));const nodeAffectsAudio = (node: HTMLElement): boolean =>- node.matches("audio[data-start]") || node.querySelector("audio[data-start]") !== null;+ affectsAudio(node) || Array.from(node.querySelectorAll(WEB_AUDIO_MEDIA)).some(affectsAudio);// An `<hf-audio-group>` carries no `data-start`, so it is never among// `visibilityNodes` above — group mute needs its own small diff pass.@@ -2796,8 +2805,8 @@ export function initSandboxRuntimeModular(): void {const groupNeedsCapture = syncAudioGroupMute(currentTime);if ((hiddenAudioDirty || groupNeedsCapture) && clock.isPlaying()) {webAudio.stopAll();- for (const el of document.querySelectorAll("audio[data-start]")) {- if (isMediaElement(el) && isSilencedByHidden(el)) el.volume = 0;+ for (const el of webAudioMediaIn(document)) {+ if (isSilencedByHidden(el)) el.volume = 0;}scheduleWebAudioForActiveClips();}@@ -4765,9 +4774,7 @@ export function initSandboxRuntimeModular(): void {const scheduleWebAudioForActiveClips = () => {if (state.nativeMediaSyncDisabled || state.webAudioMediaDisabled) return;const gen = webAudio.startGeneration();- const audioEls = document.querySelectorAll("audio[data-start]");- for (const rawEl of audioEls) {- if (!isMediaElement(rawEl) || !rawEl.isConnected) continue;+ for (const rawEl of webAudioMediaIn(document)) {if (isSilencedByHidden(rawEl)) continue;const compStart = resolveAbsoluteMediaStartSeconds(rawEl);if (!Number.isFinite(compStart)) continue;@@ -4814,7 +4821,9 @@ export function initSandboxRuntimeModular(): void {: Promise.resolve(null);void capture.then((scheduled) => {const replacedByNewerPass = gen !== webAudio.currentGeneration();- if (scheduled || !clock.isPlaying() || replacedByNewerPass) return;+ // A video's picture must keep playing from the element, so it has no decode fallback.+ if (scheduled || !isAudioElement(rawEl) || !clock.isPlaying() || replacedByNewerPass)+ return;const effectiveRate = state.playbackRate * readElementPlaybackRate(rawEl);// Deliberately the FX/automation pair and NOT// `nativeUnexpressibleProcessing()`, which this route's diagnostic uses.