Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(audio): master bus follow-ups: a scene renders with its own chain; long mixes render in windows (#5474)
Jerrai30359f803826
diff --git a/packages/core/src/audio/audioFxGraph.ts b/packages/core/src/audio/audioFxGraph.tsindex 3d0a03b08f..fcee382345 100644--- a/packages/core/src/audio/audioFxGraph.ts+++ b/packages/core/src/audio/audioFxGraph.ts@@ -253,8 +253,10 @@ function onePoleBuilder(kind: "highpass" | "lowpass"): Builder {}function workletBuilder(processor: string, resettable = false): Builder {- return (ctx, p) => {- const node = new AudioWorkletNode(ctx, processor, { processorOptions: { ...p } });+ return (ctx, p, elapsed) => {+ const node = new AudioWorkletNode(ctx, processor, {+ processorOptions: { ...p, __hfElapsed: elapsed },+ });return {input: node,output: node,diff --git a/packages/core/src/audio/audioFxTail.test.ts b/packages/core/src/audio/audioFxTail.test.tsindex 562c18bd35..2a363feed4 100644--- a/packages/core/src/audio/audioFxTail.test.ts+++ b/packages/core/src/audio/audioFxTail.test.ts@@ -1,5 +1,11 @@import { describe, expect, it } from "vitest";-import { chainLatencySamples, chainTailSeconds, MAX_FX_TAIL_SECONDS } from "./audioFxTail.js";+import {+ chainLatencySamples,+ chainTailSeconds,+ delayTailSeconds,+ MAX_FX_TAIL_SECONDS,+ reverbRoomSeconds,+} from "./audioFxTail.js";import { truePeakLatencySamples } from "./audioFxTruePeak.js";import { synthesizeReverbImpulse } from "./audioFxGraph.js";import type { HfAudioFxChain } from "../audioFx.js";@@ -130,3 +136,15 @@ describe("chainLatencySamples", () => {);});});++describe("tail helpers shared with the engine's window warm-up", () => {+ it("counts a delay's repeats to -60 dB, uncapped", () => {+ expect(delayTailSeconds(5000, 0.95)).toBeCloseTo(135 * 5, 5);+ expect(delayTailSeconds(400, 0.5)).toBeCloseTo(10 * 0.4, 5);+ });++ it("is the reverb impulse's length", () => {+ expect(reverbRoomSeconds(0.5)).toBeCloseTo(1.9, 5);+ expect(reverbRoomSeconds(4)).toBeCloseTo(3.2, 5);+ });+});diff --git a/packages/core/src/audio/audioFxTail.ts b/packages/core/src/audio/audioFxTail.tsindex 963664e481..6fbe901093 100644--- a/packages/core/src/audio/audioFxTail.ts+++ b/packages/core/src/audio/audioFxTail.ts@@ -56,7 +56,7 @@ function knobMax(node: HfAudioFxNode, key: string, automation?: HfAutomation): n* Repeats until a feedback loop falls under the floor, times the gap between* them. `feedback` is capped below 1 by the registry, so this terminates.*/-function delayTail(time: number, feedback: number): number {+export function delayTailSeconds(time: number, feedback: number): number {const gap = Math.min(5, time / 1000);if (gap <= 0) return 0;const fb = Math.max(0, Math.min(0.999, feedback));@@ -66,15 +66,19 @@ function delayTail(time: number, feedback: number): number {// Exactly the generated impulse's length — see synthesizeReverbImpulse, which// is the same expression. A convolution is as long as its impulse.+export function reverbRoomSeconds(size: number): number {+ return 0.6 + Math.max(0, Math.min(1, size)) * 2.6;+}+function reverbTail(node: HfAudioFxNode, automation?: HfAutomation): number {return knobMax(node, "wet", automation) > 0- ? 0.6 + Math.max(0, Math.min(1, knobMax(node, "size", automation))) * 2.6+ ? reverbRoomSeconds(knobMax(node, "size", automation)): 0;}function delayNodeTail(node: HfAudioFxNode, automation?: HfAutomation): number {return knobMax(node, "mix", automation) > 0- ? delayTail(knobMax(node, "time", automation), knobMax(node, "feedback", automation))+ ? delayTailSeconds(knobMax(node, "time", automation), knobMax(node, "feedback", automation)): 0;}diff --git a/packages/core/src/audio/audioFxWorklets.ts b/packages/core/src/audio/audioFxWorklets.tsindex 9f9354401c..fba4d3de12 100644--- a/packages/core/src/audio/audioFxWorklets.ts+++ b/packages/core/src/audio/audioFxWorklets.ts@@ -192,6 +192,7 @@ class HfBitcrush extends AudioWorkletProcessor {this.p = o.processorOptions || {};this.holds = [];this.held = [];+ this.start = Math.round((this.p.__hfElapsed ?? 0) * sampleRate);this.port.onmessage = (e) => {if (e.data && e.data.__hfDispose) { this.dead = true; return; }this.p = { ...this.p, ...e.data };@@ -208,7 +209,8 @@ class HfBitcrush extends AudioWorkletProcessor {for (let ch = 0; ch < i.length; ch++) {const inp = i[ch], out = o[ch];if (this.held[ch] === undefined) this.held[ch] = 0;- if (this.holds[ch] === undefined) this.holds[ch] = 0;+ // The hold counter runs from the file's first sample, so a node built part-way in starts where it would be.+ if (this.holds[ch] === undefined) this.holds[ch] = this.start % step;for (let n = 0; n < inp.length; n++) {// Per channel: advancing one shared counter only on the last channel// left every earlier channel either unheld or frozen for a whole@@ -255,7 +257,11 @@ class HfPitchshift extends AudioWorkletProcessor {this.grain = Math.round(sampleRate * 0.1);this.buf = [];this.write = 0;- this.phase = 0;+ // The taps sweep at a constant rate from the start of the file, so a node built+ // part-way in joins the sweep where it would already be.+ const ratio0 = Math.pow(2, Math.max(-12, Math.min(12, this.p.semitones ?? 0)) / 12);+ const turns = (this.p.__hfElapsed ?? 0) * sampleRate * ((1 - ratio0) / this.grain);+ this.phase = turns - Math.floor(turns);// Samples written so far, capped at one grain. The taps read up to a grain// behind the write head, so until this fills they would read the ring's// zeros, so the head of every clip came out attenuated or silent.diff --git a/packages/core/src/compiler/standaloneEntry.test.ts b/packages/core/src/compiler/standaloneEntry.test.tsindex 448f427c40..2939f5157c 100644--- a/packages/core/src/compiler/standaloneEntry.test.ts+++ b/packages/core/src/compiler/standaloneEntry.test.ts@@ -1,3 +1,4 @@+import { parseHTML } from "linkedom";import { describe, expect, it } from "vitest";import { extractStandaloneEntryFromIndex } from "./standaloneEntry";@@ -112,4 +113,77 @@ describe("extractStandaloneEntryFromIndex", () => {expect(extracted).toContain('data-duration="2"');expect(extracted).not.toContain('data-duration="12"');});+ describe("master bus chain", () => {+ const hostIndexHtml = `<!DOCTYPE html>+<html>+<body>+ <div id="scene1" data-composition-id="scene1" data-composition-src="compositions/scene1.html" data-width="640" data-height="360" data-fx-chain="{"version":1,"nodes":[{"type":"truepeak","id":"index"}]}"></div>+</body>+</html>`;+ const indexHtml = `<!DOCTYPE html>+<html>+<body>+ <div data-composition-id="master" data-width="640" data-height="360" data-fx-chain="{"version":1,"nodes":[{"type":"truepeak","id":"index"}]}">+ <div id="scene1" data-composition-id="scene1" data-composition-src="compositions/scene1.html" data-start="0"></div>+ </div>+</body>+</html>`;+ const chainOf = (html: string): string | null => {+ const { document } = parseHTML(html);+ const roots = Array.from(document.querySelectorAll("[data-composition-id]")) as Element[];+ const top = roots.find((el) => !el.parentElement?.closest("[data-composition-id]"));+ return top?.getAttribute("data-fx-chain") ?? null;+ };++ it("carries the scene's own chain, not the index's", () => {+ const chain = '{"version":1,"nodes":[{"type":"truepeak","id":"scene"}]}';+ const sceneHtml = `<template id="t"><div data-composition-id="scene1" data-width="640" data-height="360" data-fx-chain='${chain}'></div></template>`;++ const extracted = extractStandaloneEntryFromIndex(+ indexHtml,+ "compositions/scene1.html",+ sceneHtml,+ );++ expect(chainOf(extracted ?? "")).toBe(chain);+ });++ it("drops the index's chain when the scene declares none", () => {+ const sceneHtml = `<template id="t"><div data-composition-id="scene1" data-width="640" data-height="360"></div></template>`;++ const extracted = extractStandaloneEntryFromIndex(+ indexHtml,+ "compositions/scene1.html",+ sceneHtml,+ );++ expect(chainOf(extracted ?? "")).toBeNull();+ });++ it("carries the scene's chain when the index's top-level root is the mount itself", () => {+ const chain = '{"version":1,"nodes":[{"type":"truepeak","id":"scene"}]}';+ const sceneHtml = `<template id="t"><div data-composition-id="scene1" data-width="640" data-height="360" data-fx-chain='${chain}'></div></template>`;++ const extracted = extractStandaloneEntryFromIndex(+ hostIndexHtml,+ "compositions/scene1.html",+ sceneHtml,+ );++ expect(chainOf(extracted ?? "")).toBe(chain);+ });++ it("drops the index's chain from a top-level mount when the scene declares none", () => {+ const sceneHtml = `<template id="t"><div data-composition-id="scene1" data-width="640" data-height="360"></div></template>`;++ const extracted = extractStandaloneEntryFromIndex(+ hostIndexHtml,+ "compositions/scene1.html",+ sceneHtml,+ );++ expect(extracted).not.toBeNull();+ expect(chainOf(extracted ?? "")).toBeNull();+ });+ });});diff --git a/packages/core/src/compiler/standaloneEntry.ts b/packages/core/src/compiler/standaloneEntry.tsindex b03435ac79..89b2b4c4dc 100644--- a/packages/core/src/compiler/standaloneEntry.ts+++ b/packages/core/src/compiler/standaloneEntry.ts@@ -1,4 +1,5 @@import { parseHTML } from "linkedom";+import { HF_AUDIO_FX_ATTR } from "../audioFx";export function isTemplateEntry(html: string): boolean {return html.trimStart().startsWith("<template");@@ -9,24 +10,36 @@ function normalizeCompositionSrcPath(srcPath: string): string {}// linkedom has no inert <template> content, so the template's innerHTML is re-parsed.-function readSceneRootDuration(entryHtml: string | undefined): string | null {+function readSceneRoot(entryHtml: string | undefined): Element | null {if (!entryHtml) return null;const { document } = parseHTML(entryHtml);const template = document.querySelector("template");const scope = template ? parseHTML(template.innerHTML).document : document;- const root = scope.querySelector("[data-composition-id]") as Element | null;- return root?.getAttribute("data-duration") ?? null;+ return scope.querySelector("[data-composition-id]") as Element | null;+}++// The scene's own chain replaces the index's, which belongs to the whole project.+function setSceneFxChain(el: Element, sceneFxChain: string | null): void {+ if (sceneFxChain) {+ el.setAttribute(HF_AUDIO_FX_ATTR, sceneFxChain);+ } else {+ el.removeAttribute(HF_AUDIO_FX_ATTR);+ }}function createStandaloneEntryRenderClone(root: Element,host: Element,sceneDuration: string | null,+ sceneFxChain: string | null,): Element {const hostClone = host.cloneNode(true) as Element;hostClone.setAttribute("data-start", "0");- if (root === host) return hostClone;+ if (root === host) {+ setSceneFxChain(hostClone, sceneFxChain);+ return hostClone;+ }const rootClone = root.cloneNode(false) as Element;// The wrapper takes the scene's length, not the whole project's; with none declared@@ -36,6 +49,7 @@ function createStandaloneEntryRenderClone(} else {rootClone.removeAttribute("data-duration");}+ setSceneFxChain(rootClone, sceneFxChain);rootClone.appendChild(hostClone);return rootClone;}@@ -78,9 +92,12 @@ export function extractStandaloneEntryFromIndex(if (!root) return null;// The scene file owns its duration; the mount's window is the fallback.- const sceneDuration = readSceneRootDuration(entryHtml) ?? host.getAttribute("data-duration");+ const sceneRoot = readSceneRoot(entryHtml);+ const sceneDuration =+ sceneRoot?.getAttribute("data-duration") ?? host.getAttribute("data-duration");+ const sceneFxChain = sceneRoot?.getAttribute(HF_AUDIO_FX_ATTR) || null;- const renderClone = createStandaloneEntryRenderClone(root, host, sceneDuration);+ const renderClone = createStandaloneEntryRenderClone(root, host, sceneDuration, sceneFxChain);replaceBodyWithRenderClone(body, renderClone);return document.toString();diff --git a/packages/core/stubs/audio-fx-runtime-entry.ts b/packages/core/stubs/audio-fx-runtime-entry.tsindex 786122a325..4bd56f9d5a 100644--- a/packages/core/stubs/audio-fx-runtime-entry.ts+++ b/packages/core/stubs/audio-fx-runtime-entry.ts@@ -7,7 +7,7 @@* ends share a single implementation rather than two that have to be kept in* agreement.*- * Exposes `window.__HF_AUDIO_FX.render(pcm, sampleRate, chain, automation)`,+ * Exposes `window.__HF_AUDIO_FX.render(pcm, sampleRate, chain, automation, elapsed)`,* which returns the processed samples.*/@@ -29,6 +29,7 @@ declare global {sampleRate: number,chainJson: string,automationJson?: string,+ elapsed?: number,): Promise<Float32Array[]>;};}@@ -45,6 +46,9 @@ declare global {* comes from the settings (`chainTailSeconds`), capped, and the returned planes* are correspondingly longer than what came in — the mixer decides how much of* that it lets through past the clip's end.+ *+ * `elapsed` is where in the file `planes` begins, for a window cut from a longer+ * input: modulated effects follow the file's clock, not the window's.*//** The clip's audio as an AudioBuffer, a plane per channel. */@@ -77,6 +81,7 @@ async function render(sampleRate: number,chainJson: string,automationJson?: string,+ elapsed = 0,): Promise<Float32Array[]> {const chain: HfAudioFxChain = parseAudioFxChain(chainJson);const channels = Math.max(1, planes.length);@@ -98,9 +103,11 @@ async function render(if (chainNeedsWorklets(chain)) await ensureAudioFxWorklets(ctx);const source = ctx.createBufferSource();- source.buffer = toBuffer(ctx, planes, channels, frames, sampleRate);+ // Padded with silence by the limiter's lookahead: once a source ends its worklets stop being+ // fed, so without it the last `latency` samples of the clip never leave the lookahead line.+ source.buffer = toBuffer(ctx, planes, channels, frames + latency, sampleRate);- const fx = buildFxChain(ctx, chain);+ const fx = buildFxChain(ctx, chain, elapsed);// The input WAV is the clip's own audio from its first sample, so clip-local// time is offline time — the envelope needs no offset here. Same scheduler asdiff --git a/packages/engine/src/services/audioFxRender.test.ts b/packages/engine/src/services/audioFxRender.test.tsindex 6d2c944f97..eec49fb363 100644--- a/packages/engine/src/services/audioFxRender.test.ts+++ b/packages/engine/src/services/audioFxRender.test.ts@@ -1,11 +1,28 @@import { execFileSync, spawnSync } from "node:child_process";-import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import {+ closeSync,+ existsSync,+ mkdtempSync,+ openSync,+ readFileSync,+ rmSync,+ truncateSync,+ writeFileSync,+} from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";-import { afterEach, beforeEach, describe, expect, it } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { applyVolumeEnvelopeToWav } from "./audioVolumeEnvelope.js";import { defaultAudioFxParams, type HfAudioFxChain } from "@hyperframes/core/audio-fx";-import { applyAudioFxChain, AudioFxRenderError, readWav, writeWav } from "./audioFxRender.js";+import {+ applyAudioFxChain,+ AudioFxRenderError,+ planAudioFxSegments,+ readWav,+ readWavHeader,+ resolveDataBytes,+ writeWav,+} from "./audioFxRender.js";import {MIXED_AUDIO_FILENAME,parseAudioElements,@@ -15,6 +32,29 @@ import {import { resolveHeadlessShellPath } from "./browserManager.js";import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";+const wholeFileReads = vi.hoisted(() => ({ paths: [] as string[] }));+const openedFds = vi.hoisted(() => ({+ log: [] as { path: string; fd: number; closed: boolean }[],+}));+vi.mock("node:fs", async (importOriginal) => {+ const actual = await importOriginal<typeof import("node:fs")>();+ const readFileSync = ((path: Parameters<typeof actual.readFileSync>[0], ...rest: unknown[]) => {+ wholeFileReads.paths.push(String(path));+ return (actual.readFileSync as (...a: unknown[]) => unknown)(path, ...rest);+ }) as typeof actual.readFileSync;+ const openSync = ((path: Parameters<typeof actual.openSync>[0], ...rest: unknown[]) => {+ const fd = (actual.openSync as (...a: unknown[]) => number)(path, ...rest);+ openedFds.log.push({ path: String(path), fd, closed: false });+ return fd;+ }) as typeof actual.openSync;+ const closeSync = ((fd: number) => {+ const open = openedFds.log.findLast((entry) => entry.fd === fd && !entry.closed);+ if (open) open.closed = true;+ return actual.closeSync(fd);+ }) as typeof actual.closeSync;+ return { ...actual, readFileSync, openSync, closeSync };+});+/** Same probe the ffmpeg-dependent suites use: ask the binary, don't assume it. */const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;@@ -54,7 +94,11 @@ const SR = 48000;const chainOf = (...types: string[]): HfAudioFxChain => ({version: 1,- nodes: types.map((t) => ({ type: t, enabled: true, params: defaultAudioFxParams(t) })),+ nodes: types.map((t) => ({+ type: t,+ enabled: true,+ params: defaultAudioFxParams(t),+ })),});let dir: string;@@ -232,7 +276,13 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {input,{version: 1,- nodes: [{ type: "peaking", enabled: true, params: { frequency: 440, gain: -30, q: 1 } }],+ nodes: [+ {+ type: "peaking",+ enabled: true,+ params: { frequency: 440, gain: -30, q: 1 },+ },+ ],},outPath,{ trackId: "t" },@@ -258,7 +308,11 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {{type: "compressor",enabled: true,- params: { ...defaultAudioFxParams("compressor"), threshold: -40, ratio: 20 },+ params: {+ ...defaultAudioFxParams("compressor"),+ threshold: -40,+ ratio: 20,+ },},],},@@ -280,7 +334,11 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {{type: "pitchshift",enabled: true,- params: { ...defaultAudioFxParams("pitchshift"), semitones: 12, mix: 1 },+ params: {+ ...defaultAudioFxParams("pitchshift"),+ semitones: 12,+ mix: 1,+ },},],},@@ -344,7 +402,14 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {input,{version: 1,- nodes: [{ type: "lowpass", id: "n1", enabled: true, params: { frequency: 300, q: 0.707 } }],+ nodes: [+ {+ type: "lowpass",+ id: "n1",+ enabled: true,+ params: { frequency: 300, q: 0.707 },+ },+ ],},outPath,{@@ -398,7 +463,13 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {input,{version: 1,- nodes: [{ type: "peaking", enabled: true, params: { frequency: 440, gain: 6, q: 1 } }],+ nodes: [+ {+ type: "peaking",+ enabled: true,+ params: { frequency: 440, gain: 6, q: 1 },+ },+ ],},outPath,{@@ -470,7 +541,13 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {// input and any difference is the transfer's doing.{version: 1,- nodes: [{ type: "peaking", enabled: true, params: { frequency: 1000, gain: 0, q: 1 } }],+ nodes: [+ {+ type: "peaking",+ enabled: true,+ params: { frequency: 1000, gain: 0, q: 1 },+ },+ ],},outPath,{ trackId: "t" },@@ -552,7 +629,11 @@ function maxVolumeDb(path: string): number {const chainJson = (...nodes: Record<string, unknown>[]): string =>JSON.stringify({version: 1,- nodes: nodes.map((node, i) => ({ id: `n${i + 1}`, enabled: true, ...node })),+ nodes: nodes.map((node, i) => ({+ id: `n${i + 1}`,+ enabled: true,+ ...node,+ })),});describe("master bus: reading it", () => {@@ -730,3 +811,363 @@ describe.skipIf(!HAS_BROWSER || !HAS_FFMPEG)("master bus: rendering it", () => {expect(await mixedPeakDb("ordered", [element("g", group)], masterFxChain)).toBeCloseTo(-14, 0);}, 180_000);});++describe("windowed rendering of a long input", () => {+ const limits = {+ aboveBytes: 0,+ segmentSeconds: 1,+ prerollSeconds: 1,+ maxPrerollSeconds: 1,+ postrollSeconds: 0.1,+ };++ it("tiles the input exactly, each window carrying its pre- and post-roll", () => {+ const segments = planAudioFxSegments(Math.round(3.5 * SR), SR, limits);+ expect(segments.map((s) => [s.keepFrom, s.keepTo])).toEqual([+ [0, SR],+ [SR, 2 * SR],+ [2 * SR, 3 * SR],+ [3 * SR, Math.round(3.5 * SR)],+ ]);+ expect(segments.map((s) => s.last)).toEqual([false, false, false, true]);+ expect(segments[0]).toMatchObject({ readFrom: 0, readTo: SR + SR / 10 });+ expect(segments[2]).toMatchObject({+ readFrom: SR,+ readTo: 3 * SR + SR / 10,+ });+ expect(segments[3]?.readTo).toBe(Math.round(3.5 * SR));+ });++ it("starts every window on a render quantum, so block-level state lines up with a whole render", () => {+ const odd = { ...limits, prerollSeconds: 0.337 };+ const segments = planAudioFxSegments(Math.round(3.5 * SR), SR, odd);+ for (const seg of segments) {+ expect(seg.readFrom % 128).toBe(0);+ expect(seg.keepFrom - seg.readFrom).toBeGreaterThanOrEqual(+ Math.min(seg.keepFrom, Math.round(0.337 * SR)),+ );+ }+ });++ it("reads the head of a file Node cannot read whole", () => {+ // 2.5 GiB of samples: past the 2 GiB ceiling readFileSync enforces, sparse on disk.+ const path = join(dir, "huge.wav");+ writeWav(path, new Float32Array(0), SR, 2, true);+ const fd = openSync(path, "r+");+ truncateSync(path, 2.5 * 1024 ** 3);+ closeSync(fd);+ expect(() => readWav(path)).toThrow(/greater than 2 GiB/);+ const header = readWavHeader(path);+ expect(header).toMatchObject({+ channels: 2,+ sampleRate: SR,+ float: true,+ dataOffset: 44,+ });+ });++ it("reads a data size of 0xFFFFFFFF, or 0, as running to the end of the file", () => {+ const FIVE_GIB = 5 * 1024 ** 3;+ expect(resolveDataBytes(0xffffffff, FIVE_GIB)).toBe(FIVE_GIB);+ expect(resolveDataBytes(0, FIVE_GIB)).toBe(FIVE_GIB);+ expect(resolveDataBytes(1000, FIVE_GIB)).toBe(1000);+ expect(resolveDataBytes(2000, 1500)).toBe(1500);++ // The same header as ffmpeg leaves on a stream, on a file small enough to write.+ const path = join(dir, "stream.wav");+ writeWav(path, new Float32Array(100), SR, 2, true);+ const bytes = readFileSync(path);+ bytes.writeUInt32LE(0xffffffff, 40);+ writeFileSync(path, bytes);+ expect(readWavHeader(path)).toMatchObject({+ dataOffset: 44,+ dataBytes: 400,+ });+ });++ it("names the cause when a file past 2 GiB cannot be streamed", async () => {+ const path = join(dir, "huge-bad-header.wav");+ writeFileSync(path, "not a wav");+ truncateSync(path, 2.5 * 1024 ** 3);+ await expect(+ applyAudioFxChain(path, chainOf("gain"), join(dir, "out.wav"), {+ trackId: "t",+ }),+ ).rejects.toThrow(/WAV header of an input over 2 GiB could not be read/);+ });++ it("reads a data size of 0 or 0xFFFFFFFF as the rest of the file in both readers", () => {+ for (const declared of [0, 0xffffffff]) {+ const path = join(dir, `declared-${declared}.wav`);+ writeWav(path, new Float32Array(200), SR, 2, true);+ const bytes = readFileSync(path);+ bytes.writeUInt32LE(declared, 40);+ writeFileSync(path, bytes);+ expect(readWav(path).samples.length).toBe(200);+ expect(readWavHeader(path)?.dataBytes).toBe(800);+ }+ });++ it.skipIf(!HAS_BROWSER)("warns when a chain remembers longer than the pre-roll cap", async () => {+ const input = join(dir, "warn.wav");+ writeWav(input, new Float32Array(SR), SR, 1, true);+ const delay = (feedback: number): HfAudioFxChain => ({+ version: 1,+ nodes: [{ type: "delay", id: "d", enabled: true, params: { time: 400, feedback, mix: 0.5 } }],+ });+ const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);+ try {+ const streaming = { ...limits, prerollSeconds: 0.5, maxPrerollSeconds: 2 };+ await applyAudioFxChain(input, delay(0.95), join(dir, "w1.wav"), {+ trackId: "t1",+ streaming,+ });+ expect(warn).toHaveBeenCalledTimes(1);+ expect(String(warn.mock.calls[0]?.[0])).toContain("track t1");+ await applyAudioFxChain(input, delay(0.1), join(dir, "w2.wav"), { trackId: "t2", streaming });+ expect(warn).toHaveBeenCalledTimes(1);+ } finally {+ warn.mockRestore();+ }+ });++ it.skipIf(!HAS_BROWSER)("closes the input when the output cannot be opened", async () => {+ const input = join(dir, "in.wav");+ writeWav(input, new Float32Array(SR * 2), SR, 1, true);+ openedFds.log.length = 0;+ await expect(+ applyAudioFxChain(input, chainOf("gain"), join(dir, "no-such-dir", "out.wav"), {+ trackId: "t",+ streaming: limits,+ }),+ ).rejects.toThrow();+ const onInput = openedFds.log.filter((entry) => entry.path === input);+ expect(onInput.length).toBeGreaterThan(0);+ expect(onInput.every((entry) => entry.closed)).toBe(true);+ });++ it("names the unsupported case when automation meets a file past 2 GiB", async () => {+ const path = join(dir, "huge-auto.wav");+ writeWav(path, new Float32Array(0), SR, 2, true);+ truncateSync(path, 2.5 * 1024 ** 3);+ await expect(+ applyAudioFxChain(path, chainOf("gain"), join(dir, "out.wav"), {+ trackId: "t",+ automation: { version: 1, lanes: [] },+ }),+ ).rejects.toThrow(/parameter automation is not supported on inputs over 2 GiB/);+ });++ const hot = (seconds: number): Float32Array => {+ const frames = Math.floor(SR * seconds);+ const s = new Float32Array(frames * 2);+ for (let i = 0; i < frames; i++) {+ const swell = 0.5 + 0.9 * Math.abs(Math.sin((2 * Math.PI * i) / (SR * 1.7)));+ s[i * 2] = swell * Math.sin((2 * Math.PI * 300 * i) / SR);+ s[i * 2 + 1] = swell * Math.sin((2 * Math.PI * 450 * i) / SR);+ }+ return s;+ };+ const limiter: HfAudioFxChain = {+ version: 1,+ nodes: [+ {+ type: "truepeak",+ id: "m1",+ enabled: true,+ params: { ceiling: -6, lookahead: 3, release: 80 },+ },+ ],+ };++ it.skipIf(!HAS_BROWSER)(+ "renders a long mix window by window, never reading the file whole, and matches the whole render",+ async () => {+ const input = join(dir, "long.wav");+ writeWav(input, hot(4.4), SR, 2, true);+ const whole = join(dir, "whole.wav");+ const windowed = join(dir, "windowed.wav");+ await applyAudioFxChain(input, limiter, whole, { trackId: "whole" });++ wholeFileReads.paths.length = 0;+ const result = await applyAudioFxChain(input, limiter, windowed, {+ trackId: "windowed",+ streaming: limits,+ });++ expect(result.path).toBe(windowed);+ expect(wholeFileReads.paths).not.toContain(input);+ const a = readWav(whole);+ const b = readWav(windowed);+ expect(b).toMatchObject({ channels: 2, sampleRate: SR, float: true });+ expect(b.samples.length).toBe(a.samples.length);+ let worst = 0;+ // The final 10 ms is left out: a lookahead limiter's last frames come back silent or not+ // depending on the input's length, whole or windowed alike.+ for (let i = 0; i < a.samples.length - SR / 50; i++) {+ worst = Math.max(worst, Math.abs((a.samples[i] ?? 0) - (b.samples[i] ?? 0)));+ }+ expect(worst).toBeLessThan(0.01);+ },+ 180_000,+ );++ /** A tone with a loud burst ending just ahead of each 5 s seam. */+ const bursts = (seconds: number): Float32Array => {+ const frames = Math.floor(SR * seconds);+ const s = new Float32Array(frames * 2);+ for (let i = 0; i < frames; i++) {+ const t = i / SR;+ const loud = t % 5 >= 3.4 && t % 5 < 3.9;+ const amp = loud ? 0.9 : 0.12;+ s[i * 2] = amp * Math.sin((2 * Math.PI * 330 * i) / SR);+ s[i * 2 + 1] = amp * Math.sin((2 * Math.PI * 520 * i) / SR);+ }+ return s;+ };+ const memoryChain = (): HfAudioFxChain => ({+ version: 1,+ nodes: [+ // A hold of 5 does not divide the windows' 128-aligned starts, so an unseeded counter drifts.+ { type: "bitcrush", id: "b", enabled: true, params: { bits: 5, samples: 5, mix: 0.7 } },+ {+ type: "chorus",+ id: "c",+ enabled: true,+ params: { ...defaultAudioFxParams("chorus"), speed: 0.25 },+ },+ {+ type: "pitchshift",+ id: "p",+ enabled: true,+ params: { semitones: 3, mix: 1 },+ },+ {+ type: "compressor",+ id: "k",+ enabled: true,+ params: {+ ...defaultAudioFxParams("compressor"),+ threshold: -30,+ ratio: 10,+ makeup: 20,+ release: 700,+ },+ },+ {+ type: "delay",+ id: "d",+ enabled: true,+ params: { time: 400, feedback: 0.5, mix: 0.5 },+ },+ {+ type: "reverb",+ id: "r",+ enabled: true,+ params: { size: 0.5, wet: 0.3 },+ },+ ],+ });+ const memoryLimits = {+ ...limits,+ segmentSeconds: 5,+ prerollSeconds: 0.5,+ maxPrerollSeconds: 12,+ };++ const renderBoth = async (+ streaming: typeof memoryLimits,+ ): Promise<{ whole: Float32Array; windowed: Float32Array }> => {+ const input = join(dir, "mem.wav");+ writeWav(input, bursts(22), SR, 2, true);+ await applyAudioFxChain(input, memoryChain(), join(dir, "mem-whole.wav"), {+ trackId: "w",+ });+ await applyAudioFxChain(input, memoryChain(), join(dir, "mem-windowed.wav"), {+ trackId: "s",+ streaming,+ });+ return {+ whole: readWav(join(dir, "mem-whole.wav")).samples,+ windowed: readWav(join(dir, "mem-windowed.wav")).samples,+ };+ };+ const worstDifference = (a: Float32Array, b: Float32Array): number => {+ let worst = 0;+ for (let i = 0; i < a.length; i++) worst = Math.max(worst, Math.abs((a[i] ?? 0) - (b[i] ?? 0)));+ return worst;+ };++ it.skipIf(!HAS_BROWSER)(+ "carries bitcrush, modulation, delay, reverb and compressor state across window seams",+ async () => {+ const { whole, windowed } = await renderBoth(memoryLimits);+ // The tail of the last window is kept: dropping it would shorten the file.+ expect(windowed.length).toBe(whole.length);+ expect(whole.length).toBeGreaterThan(22 * SR * 2);+ expect(worstDifference(whole, windowed)).toBeLessThan(0.001);+ },+ 240_000,+ );++ it.skipIf(!HAS_BROWSER)(+ "cannot match the whole render without the pre-roll the chain asks for",+ async () => {+ const { whole, windowed } = await renderBoth({+ ...memoryLimits,+ maxPrerollSeconds: 0.5,+ });+ expect(worstDifference(whole, windowed)).toBeGreaterThan(0.05);+ },+ 240_000,+ );++ it.skipIf(!HAS_BROWSER)(+ "a lookahead limiter keeps the last of its input and the file's length",+ async () => {+ const input = join(dir, "tp.wav");+ const frames = Math.round(0.5 * SR);+ const steady = new Float32Array(frames).fill(0.3);+ writeWav(input, steady, SR, 1, true);+ const out = join(dir, "tp-out.wav");+ await applyAudioFxChain(input, chainOf("truepeak"), out, { trackId: "tp" });+ const samples = readWav(out).samples;+ expect(samples.length).toBe(frames);+ // The lookahead is 7 ms by default; the final 5 ms must still carry the signal.+ expect(sliceRms(samples, 0.495, 0.5)).toBeGreaterThan(0.25);+ },+ 120_000,+ );++ it.skipIf(!HAS_BROWSER)(+ "bakes a volume envelope on the file's own clock across window boundaries",+ async () => {+ const input = join(dir, "env.wav");+ writeWav(input, new Float32Array(SR * 3 * 2).fill(0.25), SR, 2, true);+ const out = join(dir, "env-out.wav");+ const gain: HfAudioFxChain = {+ version: 1,+ nodes: [{ type: "gain", id: "g", enabled: true, params: { gain: 0 } }],+ };+ const result = await applyAudioFxChain(input, gain, out, {+ trackId: "env",+ streaming: limits,+ envelope: {+ keyframes: [+ { time: 0, volume: 1 },+ { time: 1.5, volume: 1 },+ { time: 1.5001, volume: 0.5 },+ { time: 3, volume: 0.5 },+ ],+ trackStart: 0,+ baseVolume: 1,+ },+ });+ expect(result.envelopeBaked).toBe(true);+ const samples = readWav(out).samples;+ expect(samples[Math.floor(1.2 * SR) * 2]).toBeCloseTo(0.25, 2);+ expect(samples[Math.floor(2.2 * SR) * 2]).toBeCloseTo(0.125, 2);+ },+ 180_000,+ );+});diff --git a/packages/engine/src/services/audioFxRender.ts b/packages/engine/src/services/audioFxRender.tsindex ad1dc0edc2..597498af6e 100644--- a/packages/engine/src/services/audioFxRender.ts+++ b/packages/engine/src/services/audioFxRender.ts@@ -12,7 +12,18 @@* same way, so preview would quietly stop predicting the render.*/-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import {+ closeSync,+ existsSync,+ mkdtempSync,+ openSync,+ readFileSync,+ readSync,+ rmSync,+ statSync,+ writeFileSync,+ writeSync,+} from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";import { pathToFileURL } from "node:url";@@ -22,6 +33,7 @@ import { serializeAutomation, type HfAutomation } from "@hyperframes/core/audio-import { acquireBrowser } from "./browserManager.js";import { createEnvelopeWalker } from "./audioVolumeEnvelope.js";import { riffChunks, wavFormatTag } from "./wavChunks.js";+import { windowPrerollSeconds } from "./audioFxWarmup.js";import type { AudioVolumeKeyframe } from "./audioMixer.types.js";export class AudioFxRenderError extends Error {@@ -52,9 +64,13 @@ function readWavChunks(buf: Buffer): {sampleRate: number;bits: number;data?: Buffer;+ dataBody?: number;+ dataSize?: number;} {const head = { format: 1, channels: 1, sampleRate: 48000, bits: 16 };let data: Buffer | undefined;+ let dataBody: number | undefined;+ let dataSize: number | undefined;for (const { id, body, size } of riffChunks(buf)) {if (id === "fmt ") {const format = wavFormatTag(buf, body, size);@@ -64,7 +80,9 @@ function readWavChunks(buf: Buffer): {head.sampleRate = buf.readUInt32LE(body + 4);head.bits = buf.readUInt16LE(body + 14);} else if (id === "data") {- data = buf.subarray(body, Math.min(buf.length, body + size));+ data = buf.subarray(body, body + resolveDataBytes(size, buf.length - body));+ dataBody = body;+ dataSize = size;// The payload is the rest of the file for anything the mixer writes, and// reading past it buys nothing: `fmt ` precedes `data` in every WAV these// steps produce, and the alternative is walking a several-hundred-megabyte@@ -72,7 +90,55 @@ function readWavChunks(buf: Buffer): {break;}}- return { ...head, data };+ return { ...head, data, dataBody, dataSize };+}++/** Where a WAV's samples sit, read from its head alone. */+export interface WavHeader {+ format: number;+ channels: number;+ sampleRate: number;+ bits: number;+ float: boolean;+ dataOffset: number;+ dataBytes: number;+}++/** The most `readFileSync` returns in one call. */+const MAX_WHOLE_FILE_BYTES = 2 ** 31 - 1;++const HEADER_PROBE_BYTES = 64 * 1024;++/** A declared size of 0 or 0xFFFFFFFF (live-written, or past 4 GiB), or one past the end, means "to the end"; both WAV readers use it. */+export function resolveDataBytes(declared: number, availableBytes: number): number {+ return declared === 0 || declared === 0xffffffff+ ? availableBytes+ : Math.min(declared, availableBytes);+}++/** The header of `path`, or null when the head holds no readable `fmt `/`data` pair. */+export function readWavHeader(path: string): WavHeader | null {+ const fd = openSync(path, "r");+ try {+ const head = Buffer.alloc(HEADER_PROBE_BYTES);+ const read = readSync(fd, head, 0, head.length, 0);+ const buf = head.subarray(0, read);+ if (buf.length < 44 || buf.toString("ascii", 0, 4) !== "RIFF") return null;+ const { format, channels, sampleRate, bits, dataBody, dataSize } = readWavChunks(buf);+ if (dataBody === undefined || dataSize === undefined) return null;+ const dataBytes = resolveDataBytes(dataSize, statSync(path).size - dataBody);+ return {+ format,+ channels,+ sampleRate,+ bits,+ float: format === 3 && bits === 32,+ dataOffset: dataBody,+ dataBytes,+ };+ } finally {+ closeSync(fd);+ }}export function readWav(path: string): WavData {@@ -111,6 +177,41 @@ function decodeSamples(data: Buffer, format: number, bits: number, path: string)throw new AudioFxRenderError(`Unsupported WAV format ${format}/${bits}-bit: ${path}`);}+function wavHeader(bytes: number, sampleRate: number, channels: number, float: boolean): Buffer {+ const bytesPerSample = float ? 4 : 2;+ const buf = Buffer.alloc(44);+ buf.write("RIFF", 0, "ascii");+ buf.writeUInt32LE(Math.min(0xffffffff, 36 + bytes), 4);+ buf.write("WAVE", 8, "ascii");+ buf.write("fmt ", 12, "ascii");+ buf.writeUInt32LE(16, 16);+ buf.writeUInt16LE(float ? 3 : 1, 20); // IEEE_FLOAT / PCM+ buf.writeUInt16LE(channels, 22);+ buf.writeUInt32LE(sampleRate, 24);+ buf.writeUInt32LE(sampleRate * channels * bytesPerSample, 28);+ buf.writeUInt16LE(channels * bytesPerSample, 32);+ buf.writeUInt16LE(float ? 32 : 16, 34);+ buf.write("data", 36, "ascii");+ buf.writeUInt32LE(Math.min(0xffffffff, bytes), 40);+ return buf;+}++function encodeSamples(samples: Float32Array, float: boolean): Buffer {+ const n = samples.length;+ const buf = Buffer.alloc(n * (float ? 4 : 2));+ for (let i = 0; i < n; i++) {+ if (float) {+ buf.writeFloatLE(samples[i] ?? 0, i * 4);+ continue;+ }+ // Clamp before scaling: a limiter set to 0 dB or a resonant filter can push+ // past full scale, and wrapping would turn that into a click.+ const v = Math.max(-1, Math.min(1, samples[i] ?? 0));+ buf.writeInt16LE(Math.round(v * 32767), i * 2);+ }+ return buf;+}+/*** Write interleaved samples, preserving the channel count.*@@ -135,33 +236,8 @@ export function writeWav(channels = 1,float = false,): void {- const n = samples.length;- const bytesPerSample = float ? 4 : 2;- const bytes = n * bytesPerSample;- const buf = Buffer.alloc(44 + bytes);- buf.write("RIFF", 0, "ascii");- buf.writeUInt32LE(36 + bytes, 4);- buf.write("WAVE", 8, "ascii");- buf.write("fmt ", 12, "ascii");- buf.writeUInt32LE(16, 16);- buf.writeUInt16LE(float ? 3 : 1, 20); // IEEE_FLOAT / PCM- buf.writeUInt16LE(channels, 22);- buf.writeUInt32LE(sampleRate, 24);- buf.writeUInt32LE(sampleRate * channels * bytesPerSample, 28);- buf.writeUInt16LE(channels * bytesPerSample, 32);- buf.writeUInt16LE(float ? 32 : 16, 34);- buf.write("data", 36, "ascii");- buf.writeUInt32LE(bytes, 40);- for (let i = 0; i < n; i++) {- if (float) {- buf.writeFloatLE(samples[i] ?? 0, 44 + i * 4);- continue;- }- // Clamp before scaling: a limiter set to 0 dB or a resonant filter can push- // past full scale, and wrapping would turn that into a click.- const v = Math.max(-1, Math.min(1, samples[i] ?? 0));- buf.writeInt16LE(Math.round(v * 32767), 44 + i * 2);- }+ const body = encodeSamples(samples, float);+ const buf = Buffer.concat([wavHeader(body.length, sampleRate, channels, float), body]);// lgtm[js/insecure-temporary-file] — `path` is always inside a directory the// caller made with `mkdtempSync`, never a name assembled directly under// `tmpdir()`. Both routes here are covered: the browser host page writes into@@ -226,7 +302,13 @@ interface AudioFxPageIo {interface AudioFxWindow {__HF_AUDIO_FX?: {- render(p: Float32Array[], r: number, c: string, a?: string): Promise<Float32Array[]>;+ render(+ p: Float32Array[],+ r: number,+ c: string,+ a?: string,+ elapsed?: number,+ ): Promise<Float32Array[]>;};__HF_FX_IO?: AudioFxPageIo;}@@ -242,14 +324,179 @@ function applyEnvelopeToPlanes(planes: readonly Float32Array[],sampleRate: number,gainAt: (time: number) => number,+ startFrame = 0,): void {const frames = planes[0]?.length ?? 0;for (let frame = 0; frame < frames; frame += 1) {- const gain = gainAt(frame / sampleRate);+ const gain = gainAt((startFrame + frame) / sampleRate);for (const plane of planes) plane[frame] = (plane[frame] ?? 0) * gain;}}+/** Inputs over `aboveBytes` render in windows: Node reads no file past 2 GiB whole, and the page would hold it twice. */+export interface AudioFxStreamingLimits {+ aboveBytes: number;+ segmentSeconds: number;+ /** Audio rendered ahead of each window and discarded; the least, when the chain needs more. */+ prerollSeconds: number;+ /** The most pre-roll a chain's own memory can ask for. */+ maxPrerollSeconds: number;+ /** Audio rendered after each window and discarded, for look-ahead. */+ postrollSeconds: number;+}++const DEFAULT_STREAMING_LIMITS: AudioFxStreamingLimits = {+ aboveBytes: 512 * 1024 * 1024,+ segmentSeconds: 300,+ prerollSeconds: 10,+ maxPrerollSeconds: 300,+ postrollSeconds: 1,+};++export interface AudioFxSegment {+ /** The frames this window contributes to the output. */+ keepFrom: number;+ keepTo: number;+ /** The frames rendered to produce them. */+ readFrom: number;+ readTo: number;+ last: boolean;+}++/** Frames the audio graph processes at a time; a window starting on one sees the same blocks a whole render does. */+const RENDER_QUANTUM = 128;++/** Tile `totalFrames` into windows that each carry their pre/post-roll. */+export function planAudioFxSegments(+ totalFrames: number,+ sampleRate: number,+ limits: AudioFxStreamingLimits,+): AudioFxSegment[] {+ const step = Math.max(1, Math.round(limits.segmentSeconds * sampleRate));+ const pre = Math.round(limits.prerollSeconds * sampleRate);+ const post = Math.round(limits.postrollSeconds * sampleRate);+ const segments: AudioFxSegment[] = [];+ for (let keepFrom = 0; keepFrom < totalFrames; keepFrom += step) {+ const keepTo = Math.min(totalFrames, keepFrom + step);+ segments.push({+ keepFrom,+ keepTo,+ readFrom: Math.max(0, Math.floor((keepFrom - pre) / RENDER_QUANTUM) * RENDER_QUANTUM),+ readTo: Math.min(totalFrames, keepTo + post),+ last: keepTo === totalFrames,+ });+ }+ return segments;+}++function readFrames(fd: number, header: WavHeader, from: number, to: number): Float32Array {+ const frameBytes = (header.channels * header.bits) / 8;+ const data = Buffer.alloc((to - from) * frameBytes);+ for (let filled = 0; filled < data.length; ) {+ const n = readSync(+ fd,+ data,+ filled,+ data.length - filled,+ header.dataOffset + from * frameBytes + filled,+ );+ if (n === 0) break;+ filled += n;+ }+ return decodeSamples(data, header.format, header.bits, "streamed input");+}++/** A chain remembers longer than the floor: size the pre-roll to its slowest state, up to the cap. */+function chainPrerollSeconds(+ chain: HfAudioFxChain,+ limits: AudioFxStreamingLimits,+ trackId: string,+): number {+ const { seconds, capped } = windowPrerollSeconds(+ chain,+ limits.prerollSeconds,+ limits.maxPrerollSeconds,+ );+ if (capped) {+ console.warn(+ `Audio FX for track ${trackId}: the chain remembers longer than the ${limits.maxPrerollSeconds}s pre-roll cap, so its oldest repeats are cut at window seams`,+ );+ }+ return seconds;+}++async function renderInSegments(+ openPage: () => Promise<FxPage>,+ inputWav: string,+ header: WavHeader,+ chain: HfAudioFxChain,+ outputWav: string,+ options: {+ trackId: string;+ signal?: AbortSignal;+ envelope?: AudioFxEnvelope;+ },+ limits: AudioFxStreamingLimits,+): Promise<boolean> {+ const { sampleRate, channels, float } = header;+ const totalFrames = Math.floor(header.dataBytes / ((channels * header.bits) / 8));+ const gainAt = envelopeWalkerFor(options.envelope);+ const prerollSeconds = chainPrerollSeconds(chain, limits, options.trackId);+ const inFd = openSync(inputWav, "r");+ let outFd: number | undefined;+ try {+ // Same directory rule as writeWav: the caller's mkdtemp work dir.+ outFd = openSync(outputWav, "w");+ writeSync(outFd, wavHeader(0, sampleRate, channels, float));+ let written = 0;+ for (const seg of planAudioFxSegments(totalFrames, sampleRate, {+ ...limits,+ prerollSeconds,+ })) {+ if (options.signal?.aborted) {+ throw new AudioFxRenderError(`Audio FX cancelled for track ${options.trackId}`);+ }+ const planes = deinterleave(readFrames(inFd, header, seg.readFrom, seg.readTo), channels);+ // A fresh page per window, so the browser never holds more than one window of audio.+ const page = await openPage();+ let outPlanes: Float32Array[];+ try {+ await sendPlanesToPage(page, planes);+ outPlanes = await readPlanesFromPage(+ page,+ await renderPlanesInPage(page, sampleRate, chain, "", seg.readFrom / sampleRate),+ );+ } finally {+ await page.close().catch(() => undefined);+ }+ const rendered = outPlanes[0]?.length ?? 0;+ if (rendered === 0) {+ throw new AudioFxRenderError(`Audio FX produced no samples for track ${options.trackId}`);+ }+ // The last window keeps the chain's tail (reverb, delay), as a whole-file render does.+ const skip = seg.keepFrom - seg.readFrom;+ const kept = outPlanes.map((plane) =>+ plane.subarray(skip, seg.last ? rendered : seg.keepTo - seg.readFrom),+ );+ if (gainAt) applyEnvelopeToPlanes(kept, sampleRate, gainAt, seg.keepFrom);+ const bytes = encodeSamples(interleave(kept), float);+ writeFileSync(outFd, bytes);+ written += bytes.length;+ }+ writeSync(outFd, wavHeader(written, sampleRate, channels, float), 0, 44, 0);+ } finally {+ closeSync(inFd);+ if (outFd !== undefined) closeSync(outFd);+ }+ return gainAt !== null;+}++type AudioFxEnvelope = {+ keyframes: AudioVolumeKeyframe[];+ trackStart: number;+ baseVolume: number;+};+/*** Run a chain over `inputWav`, writing `outputWav`. Resolves to the path to use* downstream — `outputWav` when the chain did something, `inputWav` untouched@@ -275,7 +522,8 @@ export async function applyAudioFxChain(trackId: string;signal?: AbortSignal;automation?: HfAutomation;- envelope?: { keyframes: AudioVolumeKeyframe[]; trackStart: number; baseVolume: number };+ envelope?: AudioFxEnvelope;+ streaming?: Partial<AudioFxStreamingLimits>;},): Promise<{ path: string; envelopeBaked: boolean }> {if (enabledAudioFxNodes(chain).length === 0) return { path: inputWav, envelopeBaked: false };@@ -283,8 +531,34 @@ export async function applyAudioFxChain(throw new AudioFxRenderError(`Audio FX input is missing: ${inputWav}`);}- const { samples, sampleRate, channels, float } = readWav(inputWav);- const planes = deinterleave(samples, channels);+ // Automation is clocked from the file's own start, so a windowed render leaves it to the whole path.+ const limits = { ...DEFAULT_STREAMING_LIMITS, ...options.streaming };+ const header = options.automation ? null : readWavHeader(inputWav);+ const streamed = header !== null && header.dataBytes > limits.aboveBytes;++ if (streamed) {+ return withFxBrowser(options, async (openPage) => ({+ path: outputWav,+ envelopeBaked: await renderInSegments(+ openPage,+ inputWav,+ header,+ chain,+ outputWav,+ options,+ limits,+ ),+ }));+ }++ if (statSync(inputWav).size > MAX_WHOLE_FILE_BYTES) {+ const why = options.automation+ ? "parameter automation is not supported on inputs over 2 GiB"+ : "the WAV header of an input over 2 GiB could not be read";+ throw new AudioFxRenderError(`Audio FX for track ${options.trackId}: ${why} (${inputWav})`);+ }+ const input = readWav(inputWav);+ const planes = deinterleave(input.samples, input.channels);// An empty track has nothing to process — and an OfflineAudioContext of zero// length throws, which is fatal for the WHOLE render rather than this track:// the error travels past the mixer's per-track failure collector. ffmpeg@@ -294,36 +568,13 @@ export async function applyAudioFxChain(// never costs a browser.if ((planes[0]?.length ?? 0) === 0) return { path: inputWav, envelopeBaked: false };- // Both resources are taken INSIDE the try that releases them. The lease used- // to be acquired above it, with the mkdtemp between — so a failure there- // (a full disk, a read-only tmpdir) leaked a pooled browser, and a pool with- // no leases left hangs every later render rather than failing one.- const hostDir = mkdtempSync(join(tmpdir(), "hf-fx-host-"));- let lease: Awaited<ReturnType<typeof acquireBrowser>> | null = null;- try {- // Audio processing needs no GPU or special capture mode; a plain sandboxed- // browser is enough, and the lease pool reuses one across tracks.- // Checked before the lease, not after: an already-cancelled track has no- // reason to take a browser out of the pool just to hand it straight back.- if (options.signal?.aborted) {- throw new AudioFxRenderError(`Audio FX cancelled for track ${options.trackId}`);- }- lease = await acquireBrowser(["--no-sandbox", "--autoplay-policy=no-user-gesture-required"]);- const page = await lease.browser.newPage();+ return withFxBrowser(options, async (openPage) => {+ const page = await openPage();try {- // AudioWorklet is only exposed in a secure context, and about:blank is- // not one — the module would fail with an opaque error. A file:// page- // qualifies and needs no listening socket.- const hostPage = join(hostDir, "audio-fx.html");- writeFileSync(hostPage, "<!doctype html><meta charset=utf-8><title>audio fx</title>");- await page.goto(pathToFileURL(hostPage).href, { waitUntil: "domcontentloaded" });- await page.addScriptTag({ content: getAudioFxRuntimeScript() });-await sendPlanesToPage(page, planes);-const outLengths = await renderPlanesInPage(page,- sampleRate,+ input.sampleRate,chain,options.automation ? serializeAutomation(options.automation) : "",);@@ -335,14 +586,35 @@ export async function applyAudioFxChain(// own paths still own this track's gain, so say so rather than claiming// a bake that never happened.const gainAt = envelopeWalkerFor(options.envelope);- if (gainAt) applyEnvelopeToPlanes(outPlanes, sampleRate, gainAt);+ if (gainAt) applyEnvelopeToPlanes(outPlanes, input.sampleRate, gainAt);// Same format in as out: a float input is a group sub-mix whose headroom// must survive to its fader (see writeWav).- writeWav(outputWav, interleave(outPlanes), sampleRate, outPlanes.length, float);+ writeWav(outputWav, interleave(outPlanes), input.sampleRate, outPlanes.length, input.float);return { path: outputWav, envelopeBaked: gainAt !== null };} finally {await page.close().catch(() => undefined);}+ });+}++/** Run `body` with a leased browser and a way to open pages on the runtime; both are released in the same try that takes them. */+async function withFxBrowser<T>(+ options: { trackId: string; signal?: AbortSignal },+ body: (openPage: () => Promise<FxPage>) => Promise<T>,+): Promise<T> {+ const hostDir = mkdtempSync(join(tmpdir(), "hf-fx-host-"));+ let lease: Awaited<ReturnType<typeof acquireBrowser>> | null = null;+ try {+ // Audio processing needs no GPU or special capture mode; a plain sandboxed+ // browser is enough, and the lease pool reuses one across tracks.+ // Checked before the lease, not after: an already-cancelled track has no+ // reason to take a browser out of the pool just to hand it straight back.+ if (options.signal?.aborted) {+ throw new AudioFxRenderError(`Audio FX cancelled for track ${options.trackId}`);+ }+ lease = await acquireBrowser(["--no-sandbox", "--autoplay-policy=no-user-gesture-required"]);+ const { browser } = lease;+ return await body(() => openFxPage(browser, hostDir));} catch (err) {if (err instanceof AudioFxRenderError) throw err;throw new AudioFxRenderError(@@ -357,6 +629,28 @@ export async function applyAudioFxChain(/** The page handle `acquireBrowser().browser.newPage()` returns. */type FxPage = Awaited<ReturnType<Awaited<ReturnType<typeof acquireBrowser>>["browser"]["newPage"]>>;+async function openFxPage(+ browser: Awaited<ReturnType<typeof acquireBrowser>>["browser"],+ hostDir: string,+): Promise<FxPage> {+ const page = await browser.newPage();+ try {+ // AudioWorklet is only exposed in a secure context, and about:blank is+ // not one — the module would fail with an opaque error. A file:// page+ // qualifies and needs no listening socket.+ const hostPage = join(hostDir, "audio-fx.html");+ writeFileSync(hostPage, "<!doctype html><meta charset=utf-8><title>audio fx</title>");+ await page.goto(pathToFileURL(hostPage).href, {+ waitUntil: "domcontentloaded",+ });+ await page.addScriptTag({ content: getAudioFxRuntimeScript() });+ return page;+ } catch (err) {+ await page.close().catch(() => undefined);+ throw err;+ }+}+/*** Hand the input planes to the page a chunk at a time.*@@ -397,9 +691,10 @@ async function renderPlanesInPage(sampleRate: number,chain: HfAudioFxChain,automationJson: string,+ elapsed = 0,): Promise<number[]> {return (await page.evaluate(- async ([rate, chainJson, automation]: [number, string, string]) => {+ async ([rate, chainJson, automation, at]: [number, string, string, number]) => {const w = window as unknown as AudioFxWindow;const io = w.__HF_FX_IO;if (!w.__HF_AUDIO_FX || !io) throw new Error("audio FX runtime failed to load");@@ -415,10 +710,15 @@ async function renderPlanesInPage(// Dropped before the render allocates its own buffers, so the page does// not hold two copies of the track at once.io.in = [];- io.out = await w.__HF_AUDIO_FX.render(inPlanes, rate, chainJson, automation || undefined);+ io.out = await w.__HF_AUDIO_FX.render(inPlanes, rate, chainJson, automation || undefined, at);return io.out.map((plane) => plane.length);},- [sampleRate, JSON.stringify(chain), automationJson] as [number, string, string],+ [sampleRate, JSON.stringify(chain), automationJson, elapsed] as [+ number,+ string,+ string,+ number,+ ],)) as number[];}@@ -470,7 +770,11 @@ async function readPlanesFromPage(page: FxPage, outLengths: number[]): Promise<F/** The envelope walker for a track that has keyframes, or null when it has none. */function envelopeWalkerFor(envelope:- | { keyframes: AudioVolumeKeyframe[]; trackStart: number; baseVolume: number }+ | {+ keyframes: AudioVolumeKeyframe[];+ trackStart: number;+ baseVolume: number;+ }| undefined,): ReturnType<typeof createEnvelopeWalker> | null {if (!envelope) return null;diff --git a/packages/engine/src/services/audioFxWarmup.test.ts b/packages/engine/src/services/audioFxWarmup.test.tsnew file mode 100644index 0000000000..d8843ce58a--- /dev/null+++ b/packages/engine/src/services/audioFxWarmup.test.ts@@ -0,0 +1,59 @@+import { describe, expect, it } from "vitest";+import type { HfAudioFxChain } from "@hyperframes/core/audio-fx";+import { chainWarmupSeconds, windowPrerollSeconds } from "./audioFxWarmup.js";++const chainOf = (...nodes: HfAudioFxChain["nodes"]): HfAudioFxChain => ({ version: 1, nodes });++describe("chainWarmupSeconds", () => {+ it("is the time a delay's repeats take to fall to -60 dB", () => {+ const delay = {+ type: "delay",+ enabled: true,+ params: { time: 5000, feedback: 0.95, mix: 0.5 },+ };+ expect(chainWarmupSeconds(chainOf(delay))).toBeCloseTo(135 * 5, 5);+ });++ it("is seven release times for a compressor", () => {+ const comp = { type: "compressor", enabled: true, params: { release: 9000 } };+ expect(chainWarmupSeconds(chainOf(comp))).toBeCloseTo(9 * Math.log(1000), 5);+ });++ it("adds a reverb's room and sums a serial chain", () => {+ const reverb = { type: "reverb", enabled: true, params: { size: 0.5, wet: 0.3 } };+ const delay = { type: "delay", enabled: true, params: { time: 300, feedback: 0.4, mix: 0.4 } };+ expect(chainWarmupSeconds(chainOf(reverb))).toBeCloseTo(0.6 + 1.3, 5);+ expect(chainWarmupSeconds(chainOf(delay, reverb))).toBeCloseTo(0.3 * 8 + 1.9, 5);+ });++ it("ignores nodes that are off or mixed out", () => {+ const off = { type: "delay", enabled: false, params: { time: 5000, feedback: 0.9, mix: 0.5 } };+ const dry = { type: "reverb", enabled: true, params: { size: 1, wet: 0 } };+ const mixedOut = {+ type: "delay",+ enabled: true,+ params: { time: 5000, feedback: 0.9, mix: 0 },+ };+ expect(chainWarmupSeconds(chainOf(off, dry, mixedOut))).toBe(0);+ });+});++describe("windowPrerollSeconds", () => {+ const delay = (time: number, feedback: number): HfAudioFxChain =>+ chainOf({ type: "delay", enabled: true, params: { time, feedback, mix: 0.5 } });++ it("never goes below the floor", () => {+ expect(windowPrerollSeconds(delay(100, 0.1), 10, 300)).toEqual({ seconds: 10, capped: false });+ });++ it("follows the chain between the floor and the cap", () => {+ expect(windowPrerollSeconds(delay(400, 0.5), 1, 300)).toEqual({ seconds: 4, capped: false });+ });++ it("stops at the cap and says so", () => {+ expect(windowPrerollSeconds(delay(5000, 0.95), 10, 300)).toEqual({+ seconds: 300,+ capped: true,+ });+ });+});diff --git a/packages/engine/src/services/audioFxWarmup.ts b/packages/engine/src/services/audioFxWarmup.tsnew file mode 100644index 0000000000..b21a21695e--- /dev/null+++ b/packages/engine/src/services/audioFxWarmup.ts@@ -0,0 +1,58 @@+import { delayTailSeconds, reverbRoomSeconds } from "@hyperframes/core/audio-fx-tail";+import {+ enabledAudioFxNodes,+ normalizeAudioFxParams,+ type HfAudioFxChain,+ type HfAudioFxNode,+} from "@hyperframes/core/audio-fx";++/** Where state stops mattering: -60 dB, i.e. e^-6.9 of what it started as. */+const DECAY_TO_FLOOR = Math.log(1000);++const param = (node: HfAudioFxNode, key: string): number =>+ Number(normalizeAudioFxParams(node.type, node.params)[key] ?? 0);++const reverbMemory = (node: HfAudioFxNode): number =>+ param(node, "wet") > 0 ? reverbRoomSeconds(param(node, "size")) : 0;++const delayMemory = (node: HfAudioFxNode): number =>+ param(node, "mix") > 0 ? delayTailSeconds(param(node, "time"), param(node, "feedback")) : 0;++const releaseMemory = (node: HfAudioFxNode): number =>+ (DECAY_TO_FLOOR * param(node, "release")) / 1000;++/** Seconds of input a node still remembers, to -60 dB. */+const MEMORY: Record<string, (node: HfAudioFxNode) => number> = {+ reverb: reverbMemory,+ delay: delayMemory,+ compressor: releaseMemory,+ limiter: releaseMemory,+ truepeak: releaseMemory,+ gate: releaseMemory,+};++/**+ * How far ahead of a window a chain must be fed for its state to match a whole render.+ *+ * Summed, since the chain is serial: a delay ahead of a reverb hands the reverb every repeat.+ * Biquads, chorus and pitch shift settle well inside the caller's floor.+ */+export function chainWarmupSeconds(chain: HfAudioFxChain): number {+ return enabledAudioFxNodes(chain).reduce(+ (sum, node) => sum + (MEMORY[node.type]?.(node) ?? 0),+ 0,+ );+}++/** The pre-roll a chain gets: its own memory, between `floorSeconds` and `capSeconds`. */+export function windowPrerollSeconds(+ chain: HfAudioFxChain,+ floorSeconds: number,+ capSeconds: number,+): { seconds: number; capped: boolean } {+ const needed = chainWarmupSeconds(chain);+ return {+ seconds: Math.min(capSeconds, Math.max(floorSeconds, needed)),+ capped: needed > capSeconds,+ };+}