Skip to content

Write HTML. Render video. Built for agents.

fix(audio): master bus follow-ups: a scene renders with its own chain; long mixes render in windows (#5474)

Jerrai30359f803826
diff --git a/packages/core/src/audio/audioFxGraph.ts b/packages/core/src/audio/audioFxGraph.ts
index 3d0a03b08f..fcee382345 100644
--- a/packages/core/src/audio/audioFxGraph.ts
+++ b/packages/core/src/audio/audioFxGraph.ts
@@ -253,8 +253,10 @@ function onePoleBuilder(kind: "highpass" | "lowpass"): Builder {
}
function workletBuilder(processor: string, resettable = false): Builder {
- return (ctx, p) => {
- const node = new AudioWorkletNode(ctx, processor, { processorOptions: { ...p } });
+ return (ctx, p, elapsed) => {
+ const node = new AudioWorkletNode(ctx, processor, {
+ processorOptions: { ...p, __hfElapsed: elapsed },
+ });
return {
input: node,
output: node,
diff --git a/packages/core/src/audio/audioFxTail.test.ts b/packages/core/src/audio/audioFxTail.test.ts
index 562c18bd35..2a363feed4 100644
--- a/packages/core/src/audio/audioFxTail.test.ts
+++ b/packages/core/src/audio/audioFxTail.test.ts
@@ -1,5 +1,11 @@
import { describe, expect, it } from "vitest";
-import { chainLatencySamples, chainTailSeconds, MAX_FX_TAIL_SECONDS } from "./audioFxTail.js";
+import {
+ chainLatencySamples,
+ chainTailSeconds,
+ delayTailSeconds,
+ MAX_FX_TAIL_SECONDS,
+ reverbRoomSeconds,
+} from "./audioFxTail.js";
import { truePeakLatencySamples } from "./audioFxTruePeak.js";
import { synthesizeReverbImpulse } from "./audioFxGraph.js";
import type { HfAudioFxChain } from "../audioFx.js";
@@ -130,3 +136,15 @@ describe("chainLatencySamples", () => {
);
});
});
+
+describe("tail helpers shared with the engine's window warm-up", () => {
+ it("counts a delay's repeats to -60 dB, uncapped", () => {
+ expect(delayTailSeconds(5000, 0.95)).toBeCloseTo(135 * 5, 5);
+ expect(delayTailSeconds(400, 0.5)).toBeCloseTo(10 * 0.4, 5);
+ });
+
+ it("is the reverb impulse's length", () => {
+ expect(reverbRoomSeconds(0.5)).toBeCloseTo(1.9, 5);
+ expect(reverbRoomSeconds(4)).toBeCloseTo(3.2, 5);
+ });
+});
diff --git a/packages/core/src/audio/audioFxTail.ts b/packages/core/src/audio/audioFxTail.ts
index 963664e481..6fbe901093 100644
--- a/packages/core/src/audio/audioFxTail.ts
+++ b/packages/core/src/audio/audioFxTail.ts
@@ -56,7 +56,7 @@ function knobMax(node: HfAudioFxNode, key: string, automation?: HfAutomation): n
* Repeats until a feedback loop falls under the floor, times the gap between
* them. `feedback` is capped below 1 by the registry, so this terminates.
*/
-function delayTail(time: number, feedback: number): number {
+export function delayTailSeconds(time: number, feedback: number): number {
const gap = Math.min(5, time / 1000);
if (gap <= 0) return 0;
const fb = Math.max(0, Math.min(0.999, feedback));
@@ -66,15 +66,19 @@ function delayTail(time: number, feedback: number): number {
// Exactly the generated impulse's length — see synthesizeReverbImpulse, which
// is the same expression. A convolution is as long as its impulse.
+export function reverbRoomSeconds(size: number): number {
+ return 0.6 + Math.max(0, Math.min(1, size)) * 2.6;
+}
+
function reverbTail(node: HfAudioFxNode, automation?: HfAutomation): number {
return knobMax(node, "wet", automation) > 0
- ? 0.6 + Math.max(0, Math.min(1, knobMax(node, "size", automation))) * 2.6
+ ? reverbRoomSeconds(knobMax(node, "size", automation))
: 0;
}
function delayNodeTail(node: HfAudioFxNode, automation?: HfAutomation): number {
return knobMax(node, "mix", automation) > 0
- ? delayTail(knobMax(node, "time", automation), knobMax(node, "feedback", automation))
+ ? delayTailSeconds(knobMax(node, "time", automation), knobMax(node, "feedback", automation))
: 0;
}
diff --git a/packages/core/src/audio/audioFxWorklets.ts b/packages/core/src/audio/audioFxWorklets.ts
index 9f9354401c..fba4d3de12 100644
--- a/packages/core/src/audio/audioFxWorklets.ts
+++ b/packages/core/src/audio/audioFxWorklets.ts
@@ -192,6 +192,7 @@ class HfBitcrush extends AudioWorkletProcessor {
this.p = o.processorOptions || {};
this.holds = [];
this.held = [];
+ this.start = Math.round((this.p.__hfElapsed ?? 0) * sampleRate);
this.port.onmessage = (e) => {
if (e.data && e.data.__hfDispose) { this.dead = true; return; }
this.p = { ...this.p, ...e.data };
@@ -208,7 +209,8 @@ class HfBitcrush extends AudioWorkletProcessor {
for (let ch = 0; ch < i.length; ch++) {
const inp = i[ch], out = o[ch];
if (this.held[ch] === undefined) this.held[ch] = 0;
- if (this.holds[ch] === undefined) this.holds[ch] = 0;
+ // The hold counter runs from the file's first sample, so a node built part-way in starts where it would be.
+ if (this.holds[ch] === undefined) this.holds[ch] = this.start % step;
for (let n = 0; n < inp.length; n++) {
// Per channel: advancing one shared counter only on the last channel
// left every earlier channel either unheld or frozen for a whole
@@ -255,7 +257,11 @@ class HfPitchshift extends AudioWorkletProcessor {
this.grain = Math.round(sampleRate * 0.1);
this.buf = [];
this.write = 0;
- this.phase = 0;
+ // The taps sweep at a constant rate from the start of the file, so a node built
+ // part-way in joins the sweep where it would already be.
+ const ratio0 = Math.pow(2, Math.max(-12, Math.min(12, this.p.semitones ?? 0)) / 12);
+ const turns = (this.p.__hfElapsed ?? 0) * sampleRate * ((1 - ratio0) / this.grain);
+ this.phase = turns - Math.floor(turns);
// Samples written so far, capped at one grain. The taps read up to a grain
// behind the write head, so until this fills they would read the ring's
// zeros, so the head of every clip came out attenuated or silent.
diff --git a/packages/core/src/compiler/standaloneEntry.test.ts b/packages/core/src/compiler/standaloneEntry.test.ts
index 448f427c40..2939f5157c 100644
--- a/packages/core/src/compiler/standaloneEntry.test.ts
+++ b/packages/core/src/compiler/standaloneEntry.test.ts
@@ -1,3 +1,4 @@
+import { parseHTML } from "linkedom";
import { describe, expect, it } from "vitest";
import { extractStandaloneEntryFromIndex } from "./standaloneEntry";
@@ -112,4 +113,77 @@ describe("extractStandaloneEntryFromIndex", () => {
expect(extracted).toContain('data-duration="2"');
expect(extracted).not.toContain('data-duration="12"');
});
+ describe("master bus chain", () => {
+ const hostIndexHtml = `<!DOCTYPE html>
+<html>
+<body>
+ <div id="scene1" data-composition-id="scene1" data-composition-src="compositions/scene1.html" data-width="640" data-height="360" data-fx-chain="{&quot;version&quot;:1,&quot;nodes&quot;:[{&quot;type&quot;:&quot;truepeak&quot;,&quot;id&quot;:&quot;index&quot;}]}"></div>
+</body>
+</html>`;
+ const indexHtml = `<!DOCTYPE html>
+<html>
+<body>
+ <div data-composition-id="master" data-width="640" data-height="360" data-fx-chain="{&quot;version&quot;:1,&quot;nodes&quot;:[{&quot;type&quot;:&quot;truepeak&quot;,&quot;id&quot;:&quot;index&quot;}]}">
+ <div id="scene1" data-composition-id="scene1" data-composition-src="compositions/scene1.html" data-start="0"></div>
+ </div>
+</body>
+</html>`;
+ const chainOf = (html: string): string | null => {
+ const { document } = parseHTML(html);
+ const roots = Array.from(document.querySelectorAll("[data-composition-id]")) as Element[];
+ const top = roots.find((el) => !el.parentElement?.closest("[data-composition-id]"));
+ return top?.getAttribute("data-fx-chain") ?? null;
+ };
+
+ it("carries the scene's own chain, not the index's", () => {
+ const chain = '{"version":1,"nodes":[{"type":"truepeak","id":"scene"}]}';
+ const sceneHtml = `<template id="t"><div data-composition-id="scene1" data-width="640" data-height="360" data-fx-chain='${chain}'></div></template>`;
+
+ const extracted = extractStandaloneEntryFromIndex(
+ indexHtml,
+ "compositions/scene1.html",
+ sceneHtml,
+ );
+
+ expect(chainOf(extracted ?? "")).toBe(chain);
+ });
+
+ it("drops the index's chain when the scene declares none", () => {
+ const sceneHtml = `<template id="t"><div data-composition-id="scene1" data-width="640" data-height="360"></div></template>`;
+
+ const extracted = extractStandaloneEntryFromIndex(
+ indexHtml,
+ "compositions/scene1.html",
+ sceneHtml,
+ );
+
+ expect(chainOf(extracted ?? "")).toBeNull();
+ });
+
+ it("carries the scene's chain when the index's top-level root is the mount itself", () => {
+ const chain = '{"version":1,"nodes":[{"type":"truepeak","id":"scene"}]}';
+ const sceneHtml = `<template id="t"><div data-composition-id="scene1" data-width="640" data-height="360" data-fx-chain='${chain}'></div></template>`;
+
+ const extracted = extractStandaloneEntryFromIndex(
+ hostIndexHtml,
+ "compositions/scene1.html",
+ sceneHtml,
+ );
+
+ expect(chainOf(extracted ?? "")).toBe(chain);
+ });
+
+ it("drops the index's chain from a top-level mount when the scene declares none", () => {
+ const sceneHtml = `<template id="t"><div data-composition-id="scene1" data-width="640" data-height="360"></div></template>`;
+
+ const extracted = extractStandaloneEntryFromIndex(
+ hostIndexHtml,
+ "compositions/scene1.html",
+ sceneHtml,
+ );
+
+ expect(extracted).not.toBeNull();
+ expect(chainOf(extracted ?? "")).toBeNull();
+ });
+ });
});
diff --git a/packages/core/src/compiler/standaloneEntry.ts b/packages/core/src/compiler/standaloneEntry.ts
index b03435ac79..89b2b4c4dc 100644
--- a/packages/core/src/compiler/standaloneEntry.ts
+++ b/packages/core/src/compiler/standaloneEntry.ts
@@ -1,4 +1,5 @@
import { parseHTML } from "linkedom";
+import { HF_AUDIO_FX_ATTR } from "../audioFx";
export function isTemplateEntry(html: string): boolean {
return html.trimStart().startsWith("<template");
@@ -9,24 +10,36 @@ function normalizeCompositionSrcPath(srcPath: string): string {
}
// linkedom has no inert <template> content, so the template's innerHTML is re-parsed.
-function readSceneRootDuration(entryHtml: string | undefined): string | null {
+function readSceneRoot(entryHtml: string | undefined): Element | null {
if (!entryHtml) return null;
const { document } = parseHTML(entryHtml);
const template = document.querySelector("template");
const scope = template ? parseHTML(template.innerHTML).document : document;
- const root = scope.querySelector("[data-composition-id]") as Element | null;
- return root?.getAttribute("data-duration") ?? null;
+ return scope.querySelector("[data-composition-id]") as Element | null;
+}
+
+// The scene's own chain replaces the index's, which belongs to the whole project.
+function setSceneFxChain(el: Element, sceneFxChain: string | null): void {
+ if (sceneFxChain) {
+ el.setAttribute(HF_AUDIO_FX_ATTR, sceneFxChain);
+ } else {
+ el.removeAttribute(HF_AUDIO_FX_ATTR);
+ }
}
function createStandaloneEntryRenderClone(
root: Element,
host: Element,
sceneDuration: string | null,
+ sceneFxChain: string | null,
): Element {
const hostClone = host.cloneNode(true) as Element;
hostClone.setAttribute("data-start", "0");
- if (root === host) return hostClone;
+ if (root === host) {
+ setSceneFxChain(hostClone, sceneFxChain);
+ return hostClone;
+ }
const rootClone = root.cloneNode(false) as Element;
// The wrapper takes the scene's length, not the whole project's; with none declared
@@ -36,6 +49,7 @@ function createStandaloneEntryRenderClone(
} else {
rootClone.removeAttribute("data-duration");
}
+ setSceneFxChain(rootClone, sceneFxChain);
rootClone.appendChild(hostClone);
return rootClone;
}
@@ -78,9 +92,12 @@ export function extractStandaloneEntryFromIndex(
if (!root) return null;
// The scene file owns its duration; the mount's window is the fallback.
- const sceneDuration = readSceneRootDuration(entryHtml) ?? host.getAttribute("data-duration");
+ const sceneRoot = readSceneRoot(entryHtml);
+ const sceneDuration =
+ sceneRoot?.getAttribute("data-duration") ?? host.getAttribute("data-duration");
+ const sceneFxChain = sceneRoot?.getAttribute(HF_AUDIO_FX_ATTR) || null;
- const renderClone = createStandaloneEntryRenderClone(root, host, sceneDuration);
+ const renderClone = createStandaloneEntryRenderClone(root, host, sceneDuration, sceneFxChain);
replaceBodyWithRenderClone(body, renderClone);
return document.toString();
diff --git a/packages/core/stubs/audio-fx-runtime-entry.ts b/packages/core/stubs/audio-fx-runtime-entry.ts
index 786122a325..4bd56f9d5a 100644
--- a/packages/core/stubs/audio-fx-runtime-entry.ts
+++ b/packages/core/stubs/audio-fx-runtime-entry.ts
@@ -7,7 +7,7 @@
* ends share a single implementation rather than two that have to be kept in
* agreement.
*
- * Exposes `window.__HF_AUDIO_FX.render(pcm, sampleRate, chain, automation)`,
+ * Exposes `window.__HF_AUDIO_FX.render(pcm, sampleRate, chain, automation, elapsed)`,
* which returns the processed samples.
*/
@@ -29,6 +29,7 @@ declare global {
sampleRate: number,
chainJson: string,
automationJson?: string,
+ elapsed?: number,
): Promise<Float32Array[]>;
};
}
@@ -45,6 +46,9 @@ declare global {
* comes from the settings (`chainTailSeconds`), capped, and the returned planes
* are correspondingly longer than what came in — the mixer decides how much of
* that it lets through past the clip's end.
+ *
+ * `elapsed` is where in the file `planes` begins, for a window cut from a longer
+ * input: modulated effects follow the file's clock, not the window's.
*/
/** The clip's audio as an AudioBuffer, a plane per channel. */
@@ -77,6 +81,7 @@ async function render(
sampleRate: number,
chainJson: string,
automationJson?: string,
+ elapsed = 0,
): Promise<Float32Array[]> {
const chain: HfAudioFxChain = parseAudioFxChain(chainJson);
const channels = Math.max(1, planes.length);
@@ -98,9 +103,11 @@ async function render(
if (chainNeedsWorklets(chain)) await ensureAudioFxWorklets(ctx);
const source = ctx.createBufferSource();
- source.buffer = toBuffer(ctx, planes, channels, frames, sampleRate);
+ // Padded with silence by the limiter's lookahead: once a source ends its worklets stop being
+ // fed, so without it the last `latency` samples of the clip never leave the lookahead line.
+ source.buffer = toBuffer(ctx, planes, channels, frames + latency, sampleRate);
- const fx = buildFxChain(ctx, chain);
+ const fx = buildFxChain(ctx, chain, elapsed);
// The input WAV is the clip's own audio from its first sample, so clip-local
// time is offline time — the envelope needs no offset here. Same scheduler as
diff --git a/packages/engine/src/services/audioFxRender.test.ts b/packages/engine/src/services/audioFxRender.test.ts
index 6d2c944f97..eec49fb363 100644
--- a/packages/engine/src/services/audioFxRender.test.ts
+++ b/packages/engine/src/services/audioFxRender.test.ts
@@ -1,11 +1,28 @@
import { execFileSync, spawnSync } from "node:child_process";
-import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import {
+ closeSync,
+ existsSync,
+ mkdtempSync,
+ openSync,
+ readFileSync,
+ rmSync,
+ truncateSync,
+ writeFileSync,
+} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { applyVolumeEnvelopeToWav } from "./audioVolumeEnvelope.js";
import { defaultAudioFxParams, type HfAudioFxChain } from "@hyperframes/core/audio-fx";
-import { applyAudioFxChain, AudioFxRenderError, readWav, writeWav } from "./audioFxRender.js";
+import {
+ applyAudioFxChain,
+ AudioFxRenderError,
+ planAudioFxSegments,
+ readWav,
+ readWavHeader,
+ resolveDataBytes,
+ writeWav,
+} from "./audioFxRender.js";
import {
MIXED_AUDIO_FILENAME,
parseAudioElements,
@@ -15,6 +32,29 @@ import {
import { resolveHeadlessShellPath } from "./browserManager.js";
import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
+const wholeFileReads = vi.hoisted(() => ({ paths: [] as string[] }));
+const openedFds = vi.hoisted(() => ({
+ log: [] as { path: string; fd: number; closed: boolean }[],
+}));
+vi.mock("node:fs", async (importOriginal) => {
+ const actual = await importOriginal<typeof import("node:fs")>();
+ const readFileSync = ((path: Parameters<typeof actual.readFileSync>[0], ...rest: unknown[]) => {
+ wholeFileReads.paths.push(String(path));
+ return (actual.readFileSync as (...a: unknown[]) => unknown)(path, ...rest);
+ }) as typeof actual.readFileSync;
+ const openSync = ((path: Parameters<typeof actual.openSync>[0], ...rest: unknown[]) => {
+ const fd = (actual.openSync as (...a: unknown[]) => number)(path, ...rest);
+ openedFds.log.push({ path: String(path), fd, closed: false });
+ return fd;
+ }) as typeof actual.openSync;
+ const closeSync = ((fd: number) => {
+ const open = openedFds.log.findLast((entry) => entry.fd === fd && !entry.closed);
+ if (open) open.closed = true;
+ return actual.closeSync(fd);
+ }) as typeof actual.closeSync;
+ return { ...actual, readFileSync, openSync, closeSync };
+});
+
/** Same probe the ffmpeg-dependent suites use: ask the binary, don't assume it. */
const HAS_FFMPEG = spawnSync(getFfmpegBinary(), ["-version"], { encoding: "utf-8" }).status === 0;
@@ -54,7 +94,11 @@ const SR = 48000;
const chainOf = (...types: string[]): HfAudioFxChain => ({
version: 1,
- nodes: types.map((t) => ({ type: t, enabled: true, params: defaultAudioFxParams(t) })),
+ nodes: types.map((t) => ({
+ type: t,
+ enabled: true,
+ params: defaultAudioFxParams(t),
+ })),
});
let dir: string;
@@ -232,7 +276,13 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {
input,
{
version: 1,
- nodes: [{ type: "peaking", enabled: true, params: { frequency: 440, gain: -30, q: 1 } }],
+ nodes: [
+ {
+ type: "peaking",
+ enabled: true,
+ params: { frequency: 440, gain: -30, q: 1 },
+ },
+ ],
},
outPath,
{ trackId: "t" },
@@ -258,7 +308,11 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {
{
type: "compressor",
enabled: true,
- params: { ...defaultAudioFxParams("compressor"), threshold: -40, ratio: 20 },
+ params: {
+ ...defaultAudioFxParams("compressor"),
+ threshold: -40,
+ ratio: 20,
+ },
},
],
},
@@ -280,7 +334,11 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {
{
type: "pitchshift",
enabled: true,
- params: { ...defaultAudioFxParams("pitchshift"), semitones: 12, mix: 1 },
+ params: {
+ ...defaultAudioFxParams("pitchshift"),
+ semitones: 12,
+ mix: 1,
+ },
},
],
},
@@ -344,7 +402,14 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {
input,
{
version: 1,
- nodes: [{ type: "lowpass", id: "n1", enabled: true, params: { frequency: 300, q: 0.707 } }],
+ nodes: [
+ {
+ type: "lowpass",
+ id: "n1",
+ enabled: true,
+ params: { frequency: 300, q: 0.707 },
+ },
+ ],
},
outPath,
{
@@ -398,7 +463,13 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {
input,
{
version: 1,
- nodes: [{ type: "peaking", enabled: true, params: { frequency: 440, gain: 6, q: 1 } }],
+ nodes: [
+ {
+ type: "peaking",
+ enabled: true,
+ params: { frequency: 440, gain: 6, q: 1 },
+ },
+ ],
},
outPath,
{
@@ -470,7 +541,13 @@ describe.skipIf(!HAS_BROWSER)("browser render", () => {
// input and any difference is the transfer's doing.
{
version: 1,
- nodes: [{ type: "peaking", enabled: true, params: { frequency: 1000, gain: 0, q: 1 } }],
+ nodes: [
+ {
+ type: "peaking",
+ enabled: true,
+ params: { frequency: 1000, gain: 0, q: 1 },
+ },
+ ],
},
outPath,
{ trackId: "t" },
@@ -552,7 +629,11 @@ function maxVolumeDb(path: string): number {
const chainJson = (...nodes: Record<string, unknown>[]): string =>
JSON.stringify({
version: 1,
- nodes: nodes.map((node, i) => ({ id: `n${i + 1}`, enabled: true, ...node })),
+ nodes: nodes.map((node, i) => ({
+ id: `n${i + 1}`,
+ enabled: true,
+ ...node,
+ })),
});
describe("master bus: reading it", () => {
@@ -730,3 +811,363 @@ describe.skipIf(!HAS_BROWSER || !HAS_FFMPEG)("master bus: rendering it", () => {
expect(await mixedPeakDb("ordered", [element("g", group)], masterFxChain)).toBeCloseTo(-14, 0);
}, 180_000);
});
+
+describe("windowed rendering of a long input", () => {
+ const limits = {
+ aboveBytes: 0,
+ segmentSeconds: 1,
+ prerollSeconds: 1,
+ maxPrerollSeconds: 1,
+ postrollSeconds: 0.1,
+ };
+
+ it("tiles the input exactly, each window carrying its pre- and post-roll", () => {
+ const segments = planAudioFxSegments(Math.round(3.5 * SR), SR, limits);
+ expect(segments.map((s) => [s.keepFrom, s.keepTo])).toEqual([
+ [0, SR],
+ [SR, 2 * SR],
+ [2 * SR, 3 * SR],
+ [3 * SR, Math.round(3.5 * SR)],
+ ]);
+ expect(segments.map((s) => s.last)).toEqual([false, false, false, true]);
+ expect(segments[0]).toMatchObject({ readFrom: 0, readTo: SR + SR / 10 });
+ expect(segments[2]).toMatchObject({
+ readFrom: SR,
+ readTo: 3 * SR + SR / 10,
+ });
+ expect(segments[3]?.readTo).toBe(Math.round(3.5 * SR));
+ });
+
+ it("starts every window on a render quantum, so block-level state lines up with a whole render", () => {
+ const odd = { ...limits, prerollSeconds: 0.337 };
+ const segments = planAudioFxSegments(Math.round(3.5 * SR), SR, odd);
+ for (const seg of segments) {
+ expect(seg.readFrom % 128).toBe(0);
+ expect(seg.keepFrom - seg.readFrom).toBeGreaterThanOrEqual(
+ Math.min(seg.keepFrom, Math.round(0.337 * SR)),
+ );
+ }
+ });
+
+ it("reads the head of a file Node cannot read whole", () => {
+ // 2.5 GiB of samples: past the 2 GiB ceiling readFileSync enforces, sparse on disk.
+ const path = join(dir, "huge.wav");
+ writeWav(path, new Float32Array(0), SR, 2, true);
+ const fd = openSync(path, "r+");
+ truncateSync(path, 2.5 * 1024 ** 3);
+ closeSync(fd);
+ expect(() => readWav(path)).toThrow(/greater than 2 GiB/);
+ const header = readWavHeader(path);
+ expect(header).toMatchObject({
+ channels: 2,
+ sampleRate: SR,
+ float: true,
+ dataOffset: 44,
+ });
+ });
+
+ it("reads a data size of 0xFFFFFFFF, or 0, as running to the end of the file", () => {
+ const FIVE_GIB = 5 * 1024 ** 3;
+ expect(resolveDataBytes(0xffffffff, FIVE_GIB)).toBe(FIVE_GIB);
+ expect(resolveDataBytes(0, FIVE_GIB)).toBe(FIVE_GIB);
+ expect(resolveDataBytes(1000, FIVE_GIB)).toBe(1000);
+ expect(resolveDataBytes(2000, 1500)).toBe(1500);
+
+ // The same header as ffmpeg leaves on a stream, on a file small enough to write.
+ const path = join(dir, "stream.wav");
+ writeWav(path, new Float32Array(100), SR, 2, true);
+ const bytes = readFileSync(path);
+ bytes.writeUInt32LE(0xffffffff, 40);
+ writeFileSync(path, bytes);
+ expect(readWavHeader(path)).toMatchObject({
+ dataOffset: 44,
+ dataBytes: 400,
+ });
+ });
+
+ it("names the cause when a file past 2 GiB cannot be streamed", async () => {
+ const path = join(dir, "huge-bad-header.wav");
+ writeFileSync(path, "not a wav");
+ truncateSync(path, 2.5 * 1024 ** 3);
+ await expect(
+ applyAudioFxChain(path, chainOf("gain"), join(dir, "out.wav"), {
+ trackId: "t",
+ }),
+ ).rejects.toThrow(/WAV header of an input over 2 GiB could not be read/);
+ });
+
+ it("reads a data size of 0 or 0xFFFFFFFF as the rest of the file in both readers", () => {
+ for (const declared of [0, 0xffffffff]) {
+ const path = join(dir, `declared-${declared}.wav`);
+ writeWav(path, new Float32Array(200), SR, 2, true);
+ const bytes = readFileSync(path);
+ bytes.writeUInt32LE(declared, 40);
+ writeFileSync(path, bytes);
+ expect(readWav(path).samples.length).toBe(200);
+ expect(readWavHeader(path)?.dataBytes).toBe(800);
+ }
+ });
+
+ it.skipIf(!HAS_BROWSER)("warns when a chain remembers longer than the pre-roll cap", async () => {
+ const input = join(dir, "warn.wav");
+ writeWav(input, new Float32Array(SR), SR, 1, true);
+ const delay = (feedback: number): HfAudioFxChain => ({
+ version: 1,
+ nodes: [{ type: "delay", id: "d", enabled: true, params: { time: 400, feedback, mix: 0.5 } }],
+ });
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+ try {
+ const streaming = { ...limits, prerollSeconds: 0.5, maxPrerollSeconds: 2 };
+ await applyAudioFxChain(input, delay(0.95), join(dir, "w1.wav"), {
+ trackId: "t1",
+ streaming,
+ });
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(String(warn.mock.calls[0]?.[0])).toContain("track t1");
+ await applyAudioFxChain(input, delay(0.1), join(dir, "w2.wav"), { trackId: "t2", streaming });
+ expect(warn).toHaveBeenCalledTimes(1);
+ } finally {
+ warn.mockRestore();
+ }
+ });
+
+ it.skipIf(!HAS_BROWSER)("closes the input when the output cannot be opened", async () => {
+ const input = join(dir, "in.wav");
+ writeWav(input, new Float32Array(SR * 2), SR, 1, true);
+ openedFds.log.length = 0;
+ await expect(
+ applyAudioFxChain(input, chainOf("gain"), join(dir, "no-such-dir", "out.wav"), {
+ trackId: "t",
+ streaming: limits,
+ }),
+ ).rejects.toThrow();
+ const onInput = openedFds.log.filter((entry) => entry.path === input);
+ expect(onInput.length).toBeGreaterThan(0);
+ expect(onInput.every((entry) => entry.closed)).toBe(true);
+ });
+
+ it("names the unsupported case when automation meets a file past 2 GiB", async () => {
+ const path = join(dir, "huge-auto.wav");
+ writeWav(path, new Float32Array(0), SR, 2, true);
+ truncateSync(path, 2.5 * 1024 ** 3);
+ await expect(
+ applyAudioFxChain(path, chainOf("gain"), join(dir, "out.wav"), {
+ trackId: "t",
+ automation: { version: 1, lanes: [] },
+ }),
+ ).rejects.toThrow(/parameter automation is not supported on inputs over 2 GiB/);
+ });
+
+ const hot = (seconds: number): Float32Array => {
+ const frames = Math.floor(SR * seconds);
+ const s = new Float32Array(frames * 2);
+ for (let i = 0; i < frames; i++) {
+ const swell = 0.5 + 0.9 * Math.abs(Math.sin((2 * Math.PI * i) / (SR * 1.7)));
+ s[i * 2] = swell * Math.sin((2 * Math.PI * 300 * i) / SR);
+ s[i * 2 + 1] = swell * Math.sin((2 * Math.PI * 450 * i) / SR);
+ }
+ return s;
+ };
+ const limiter: HfAudioFxChain = {
+ version: 1,
+ nodes: [
+ {
+ type: "truepeak",
+ id: "m1",
+ enabled: true,
+ params: { ceiling: -6, lookahead: 3, release: 80 },
+ },
+ ],
+ };
+
+ it.skipIf(!HAS_BROWSER)(
+ "renders a long mix window by window, never reading the file whole, and matches the whole render",
+ async () => {
+ const input = join(dir, "long.wav");
+ writeWav(input, hot(4.4), SR, 2, true);
+ const whole = join(dir, "whole.wav");
+ const windowed = join(dir, "windowed.wav");
+ await applyAudioFxChain(input, limiter, whole, { trackId: "whole" });
+
+ wholeFileReads.paths.length = 0;
+ const result = await applyAudioFxChain(input, limiter, windowed, {
+ trackId: "windowed",
+ streaming: limits,
+ });
+
+ expect(result.path).toBe(windowed);
+ expect(wholeFileReads.paths).not.toContain(input);
+ const a = readWav(whole);
+ const b = readWav(windowed);
+ expect(b).toMatchObject({ channels: 2, sampleRate: SR, float: true });
+ expect(b.samples.length).toBe(a.samples.length);
+ let worst = 0;
+ // The final 10 ms is left out: a lookahead limiter's last frames come back silent or not
+ // depending on the input's length, whole or windowed alike.
+ for (let i = 0; i < a.samples.length - SR / 50; i++) {
+ worst = Math.max(worst, Math.abs((a.samples[i] ?? 0) - (b.samples[i] ?? 0)));
+ }
+ expect(worst).toBeLessThan(0.01);
+ },
+ 180_000,
+ );
+
+ /** A tone with a loud burst ending just ahead of each 5 s seam. */
+ const bursts = (seconds: number): Float32Array => {
+ const frames = Math.floor(SR * seconds);
+ const s = new Float32Array(frames * 2);
+ for (let i = 0; i < frames; i++) {
+ const t = i / SR;
+ const loud = t % 5 >= 3.4 && t % 5 < 3.9;
+ const amp = loud ? 0.9 : 0.12;
+ s[i * 2] = amp * Math.sin((2 * Math.PI * 330 * i) / SR);
+ s[i * 2 + 1] = amp * Math.sin((2 * Math.PI * 520 * i) / SR);
+ }
+ return s;
+ };
+ const memoryChain = (): HfAudioFxChain => ({
+ version: 1,
+ nodes: [
+ // A hold of 5 does not divide the windows' 128-aligned starts, so an unseeded counter drifts.
+ { type: "bitcrush", id: "b", enabled: true, params: { bits: 5, samples: 5, mix: 0.7 } },
+ {
+ type: "chorus",
+ id: "c",
+ enabled: true,
+ params: { ...defaultAudioFxParams("chorus"), speed: 0.25 },
+ },
+ {
+ type: "pitchshift",
+ id: "p",
+ enabled: true,
+ params: { semitones: 3, mix: 1 },
+ },
+ {
+ type: "compressor",
+ id: "k",
+ enabled: true,
+ params: {
+ ...defaultAudioFxParams("compressor"),
+ threshold: -30,
+ ratio: 10,
+ makeup: 20,
+ release: 700,
+ },
+ },
+ {
+ type: "delay",
+ id: "d",
+ enabled: true,
+ params: { time: 400, feedback: 0.5, mix: 0.5 },
+ },
+ {
+ type: "reverb",
+ id: "r",
+ enabled: true,
+ params: { size: 0.5, wet: 0.3 },
+ },
+ ],
+ });
+ const memoryLimits = {
+ ...limits,
+ segmentSeconds: 5,
+ prerollSeconds: 0.5,
+ maxPrerollSeconds: 12,
+ };
+
+ const renderBoth = async (
+ streaming: typeof memoryLimits,
+ ): Promise<{ whole: Float32Array; windowed: Float32Array }> => {
+ const input = join(dir, "mem.wav");
+ writeWav(input, bursts(22), SR, 2, true);
+ await applyAudioFxChain(input, memoryChain(), join(dir, "mem-whole.wav"), {
+ trackId: "w",
+ });
+ await applyAudioFxChain(input, memoryChain(), join(dir, "mem-windowed.wav"), {
+ trackId: "s",
+ streaming,
+ });
+ return {
+ whole: readWav(join(dir, "mem-whole.wav")).samples,
+ windowed: readWav(join(dir, "mem-windowed.wav")).samples,
+ };
+ };
+ const worstDifference = (a: Float32Array, b: Float32Array): number => {
+ let worst = 0;
+ for (let i = 0; i < a.length; i++) worst = Math.max(worst, Math.abs((a[i] ?? 0) - (b[i] ?? 0)));
+ return worst;
+ };
+
+ it.skipIf(!HAS_BROWSER)(
+ "carries bitcrush, modulation, delay, reverb and compressor state across window seams",
+ async () => {
+ const { whole, windowed } = await renderBoth(memoryLimits);
+ // The tail of the last window is kept: dropping it would shorten the file.
+ expect(windowed.length).toBe(whole.length);
+ expect(whole.length).toBeGreaterThan(22 * SR * 2);
+ expect(worstDifference(whole, windowed)).toBeLessThan(0.001);
+ },
+ 240_000,
+ );
+
+ it.skipIf(!HAS_BROWSER)(
+ "cannot match the whole render without the pre-roll the chain asks for",
+ async () => {
+ const { whole, windowed } = await renderBoth({
+ ...memoryLimits,
+ maxPrerollSeconds: 0.5,
+ });
+ expect(worstDifference(whole, windowed)).toBeGreaterThan(0.05);
+ },
+ 240_000,
+ );
+
+ it.skipIf(!HAS_BROWSER)(
+ "a lookahead limiter keeps the last of its input and the file's length",
+ async () => {
+ const input = join(dir, "tp.wav");
+ const frames = Math.round(0.5 * SR);
+ const steady = new Float32Array(frames).fill(0.3);
+ writeWav(input, steady, SR, 1, true);
+ const out = join(dir, "tp-out.wav");
+ await applyAudioFxChain(input, chainOf("truepeak"), out, { trackId: "tp" });
+ const samples = readWav(out).samples;
+ expect(samples.length).toBe(frames);
+ // The lookahead is 7 ms by default; the final 5 ms must still carry the signal.
+ expect(sliceRms(samples, 0.495, 0.5)).toBeGreaterThan(0.25);
+ },
+ 120_000,
+ );
+
+ it.skipIf(!HAS_BROWSER)(
+ "bakes a volume envelope on the file's own clock across window boundaries",
+ async () => {
+ const input = join(dir, "env.wav");
+ writeWav(input, new Float32Array(SR * 3 * 2).fill(0.25), SR, 2, true);
+ const out = join(dir, "env-out.wav");
+ const gain: HfAudioFxChain = {
+ version: 1,
+ nodes: [{ type: "gain", id: "g", enabled: true, params: { gain: 0 } }],
+ };
+ const result = await applyAudioFxChain(input, gain, out, {
+ trackId: "env",
+ streaming: limits,
+ envelope: {
+ keyframes: [
+ { time: 0, volume: 1 },
+ { time: 1.5, volume: 1 },
+ { time: 1.5001, volume: 0.5 },
+ { time: 3, volume: 0.5 },
+ ],
+ trackStart: 0,
+ baseVolume: 1,
+ },
+ });
+ expect(result.envelopeBaked).toBe(true);
+ const samples = readWav(out).samples;
+ expect(samples[Math.floor(1.2 * SR) * 2]).toBeCloseTo(0.25, 2);
+ expect(samples[Math.floor(2.2 * SR) * 2]).toBeCloseTo(0.125, 2);
+ },
+ 180_000,
+ );
+});
diff --git a/packages/engine/src/services/audioFxRender.ts b/packages/engine/src/services/audioFxRender.ts
index ad1dc0edc2..597498af6e 100644
--- a/packages/engine/src/services/audioFxRender.ts
+++ b/packages/engine/src/services/audioFxRender.ts
@@ -12,7 +12,18 @@
* same way, so preview would quietly stop predicting the render.
*/
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import {
+ closeSync,
+ existsSync,
+ mkdtempSync,
+ openSync,
+ readFileSync,
+ readSync,
+ rmSync,
+ statSync,
+ writeFileSync,
+ writeSync,
+} from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { pathToFileURL } from "node:url";
@@ -22,6 +33,7 @@ import { serializeAutomation, type HfAutomation } from "@hyperframes/core/audio-
import { acquireBrowser } from "./browserManager.js";
import { createEnvelopeWalker } from "./audioVolumeEnvelope.js";
import { riffChunks, wavFormatTag } from "./wavChunks.js";
+import { windowPrerollSeconds } from "./audioFxWarmup.js";
import type { AudioVolumeKeyframe } from "./audioMixer.types.js";
export class AudioFxRenderError extends Error {
@@ -52,9 +64,13 @@ function readWavChunks(buf: Buffer): {
sampleRate: number;
bits: number;
data?: Buffer;
+ dataBody?: number;
+ dataSize?: number;
} {
const head = { format: 1, channels: 1, sampleRate: 48000, bits: 16 };
let data: Buffer | undefined;
+ let dataBody: number | undefined;
+ let dataSize: number | undefined;
for (const { id, body, size } of riffChunks(buf)) {
if (id === "fmt ") {
const format = wavFormatTag(buf, body, size);
@@ -64,7 +80,9 @@ function readWavChunks(buf: Buffer): {
head.sampleRate = buf.readUInt32LE(body + 4);
head.bits = buf.readUInt16LE(body + 14);
} else if (id === "data") {
- data = buf.subarray(body, Math.min(buf.length, body + size));
+ data = buf.subarray(body, body + resolveDataBytes(size, buf.length - body));
+ dataBody = body;
+ dataSize = size;
// The payload is the rest of the file for anything the mixer writes, and
// reading past it buys nothing: `fmt ` precedes `data` in every WAV these
// steps produce, and the alternative is walking a several-hundred-megabyte
@@ -72,7 +90,55 @@ function readWavChunks(buf: Buffer): {
break;
}
}
- return { ...head, data };
+ return { ...head, data, dataBody, dataSize };
+}
+
+/** Where a WAV's samples sit, read from its head alone. */
+export interface WavHeader {
+ format: number;
+ channels: number;
+ sampleRate: number;
+ bits: number;
+ float: boolean;
+ dataOffset: number;
+ dataBytes: number;
+}
+
+/** The most `readFileSync` returns in one call. */
+const MAX_WHOLE_FILE_BYTES = 2 ** 31 - 1;
+
+const HEADER_PROBE_BYTES = 64 * 1024;
+
+/** A declared size of 0 or 0xFFFFFFFF (live-written, or past 4 GiB), or one past the end, means "to the end"; both WAV readers use it. */
+export function resolveDataBytes(declared: number, availableBytes: number): number {
+ return declared === 0 || declared === 0xffffffff
+ ? availableBytes
+ : Math.min(declared, availableBytes);
+}
+
+/** The header of `path`, or null when the head holds no readable `fmt `/`data` pair. */
+export function readWavHeader(path: string): WavHeader | null {
+ const fd = openSync(path, "r");
+ try {
+ const head = Buffer.alloc(HEADER_PROBE_BYTES);
+ const read = readSync(fd, head, 0, head.length, 0);
+ const buf = head.subarray(0, read);
+ if (buf.length < 44 || buf.toString("ascii", 0, 4) !== "RIFF") return null;
+ const { format, channels, sampleRate, bits, dataBody, dataSize } = readWavChunks(buf);
+ if (dataBody === undefined || dataSize === undefined) return null;
+ const dataBytes = resolveDataBytes(dataSize, statSync(path).size - dataBody);
+ return {
+ format,
+ channels,
+ sampleRate,
+ bits,
+ float: format === 3 && bits === 32,
+ dataOffset: dataBody,
+ dataBytes,
+ };
+ } finally {
+ closeSync(fd);
+ }
}
export function readWav(path: string): WavData {
@@ -111,6 +177,41 @@ function decodeSamples(data: Buffer, format: number, bits: number, path: string)
throw new AudioFxRenderError(`Unsupported WAV format ${format}/${bits}-bit: ${path}`);
}
+function wavHeader(bytes: number, sampleRate: number, channels: number, float: boolean): Buffer {
+ const bytesPerSample = float ? 4 : 2;
+ const buf = Buffer.alloc(44);
+ buf.write("RIFF", 0, "ascii");
+ buf.writeUInt32LE(Math.min(0xffffffff, 36 + bytes), 4);
+ buf.write("WAVE", 8, "ascii");
+ buf.write("fmt ", 12, "ascii");
+ buf.writeUInt32LE(16, 16);
+ buf.writeUInt16LE(float ? 3 : 1, 20); // IEEE_FLOAT / PCM
+ buf.writeUInt16LE(channels, 22);
+ buf.writeUInt32LE(sampleRate, 24);
+ buf.writeUInt32LE(sampleRate * channels * bytesPerSample, 28);
+ buf.writeUInt16LE(channels * bytesPerSample, 32);
+ buf.writeUInt16LE(float ? 32 : 16, 34);
+ buf.write("data", 36, "ascii");
+ buf.writeUInt32LE(Math.min(0xffffffff, bytes), 40);
+ return buf;
+}
+
+function encodeSamples(samples: Float32Array, float: boolean): Buffer {
+ const n = samples.length;
+ const buf = Buffer.alloc(n * (float ? 4 : 2));
+ for (let i = 0; i < n; i++) {
+ if (float) {
+ buf.writeFloatLE(samples[i] ?? 0, i * 4);
+ continue;
+ }
+ // Clamp before scaling: a limiter set to 0 dB or a resonant filter can push
+ // past full scale, and wrapping would turn that into a click.
+ const v = Math.max(-1, Math.min(1, samples[i] ?? 0));
+ buf.writeInt16LE(Math.round(v * 32767), i * 2);
+ }
+ return buf;
+}
+
/**
* Write interleaved samples, preserving the channel count.
*
@@ -135,33 +236,8 @@ export function writeWav(
channels = 1,
float = false,
): void {
- const n = samples.length;
- const bytesPerSample = float ? 4 : 2;
- const bytes = n * bytesPerSample;
- const buf = Buffer.alloc(44 + bytes);
- buf.write("RIFF", 0, "ascii");
- buf.writeUInt32LE(36 + bytes, 4);
- buf.write("WAVE", 8, "ascii");
- buf.write("fmt ", 12, "ascii");
- buf.writeUInt32LE(16, 16);
- buf.writeUInt16LE(float ? 3 : 1, 20); // IEEE_FLOAT / PCM
- buf.writeUInt16LE(channels, 22);
- buf.writeUInt32LE(sampleRate, 24);
- buf.writeUInt32LE(sampleRate * channels * bytesPerSample, 28);
- buf.writeUInt16LE(channels * bytesPerSample, 32);
- buf.writeUInt16LE(float ? 32 : 16, 34);
- buf.write("data", 36, "ascii");
- buf.writeUInt32LE(bytes, 40);
- for (let i = 0; i < n; i++) {
- if (float) {
- buf.writeFloatLE(samples[i] ?? 0, 44 + i * 4);
- continue;
- }
- // Clamp before scaling: a limiter set to 0 dB or a resonant filter can push
- // past full scale, and wrapping would turn that into a click.
- const v = Math.max(-1, Math.min(1, samples[i] ?? 0));
- buf.writeInt16LE(Math.round(v * 32767), 44 + i * 2);
- }
+ const body = encodeSamples(samples, float);
+ const buf = Buffer.concat([wavHeader(body.length, sampleRate, channels, float), body]);
// lgtm[js/insecure-temporary-file] — `path` is always inside a directory the
// caller made with `mkdtempSync`, never a name assembled directly under
// `tmpdir()`. Both routes here are covered: the browser host page writes into
@@ -226,7 +302,13 @@ interface AudioFxPageIo {
interface AudioFxWindow {
__HF_AUDIO_FX?: {
- render(p: Float32Array[], r: number, c: string, a?: string): Promise<Float32Array[]>;
+ render(
+ p: Float32Array[],
+ r: number,
+ c: string,
+ a?: string,
+ elapsed?: number,
+ ): Promise<Float32Array[]>;
};
__HF_FX_IO?: AudioFxPageIo;
}
@@ -242,14 +324,179 @@ function applyEnvelopeToPlanes(
planes: readonly Float32Array[],
sampleRate: number,
gainAt: (time: number) => number,
+ startFrame = 0,
): void {
const frames = planes[0]?.length ?? 0;
for (let frame = 0; frame < frames; frame += 1) {
- const gain = gainAt(frame / sampleRate);
+ const gain = gainAt((startFrame + frame) / sampleRate);
for (const plane of planes) plane[frame] = (plane[frame] ?? 0) * gain;
}
}
+/** Inputs over `aboveBytes` render in windows: Node reads no file past 2 GiB whole, and the page would hold it twice. */
+export interface AudioFxStreamingLimits {
+ aboveBytes: number;
+ segmentSeconds: number;
+ /** Audio rendered ahead of each window and discarded; the least, when the chain needs more. */
+ prerollSeconds: number;
+ /** The most pre-roll a chain's own memory can ask for. */
+ maxPrerollSeconds: number;
+ /** Audio rendered after each window and discarded, for look-ahead. */
+ postrollSeconds: number;
+}
+
+const DEFAULT_STREAMING_LIMITS: AudioFxStreamingLimits = {
+ aboveBytes: 512 * 1024 * 1024,
+ segmentSeconds: 300,
+ prerollSeconds: 10,
+ maxPrerollSeconds: 300,
+ postrollSeconds: 1,
+};
+
+export interface AudioFxSegment {
+ /** The frames this window contributes to the output. */
+ keepFrom: number;
+ keepTo: number;
+ /** The frames rendered to produce them. */
+ readFrom: number;
+ readTo: number;
+ last: boolean;
+}
+
+/** Frames the audio graph processes at a time; a window starting on one sees the same blocks a whole render does. */
+const RENDER_QUANTUM = 128;
+
+/** Tile `totalFrames` into windows that each carry their pre/post-roll. */
+export function planAudioFxSegments(
+ totalFrames: number,
+ sampleRate: number,
+ limits: AudioFxStreamingLimits,
+): AudioFxSegment[] {
+ const step = Math.max(1, Math.round(limits.segmentSeconds * sampleRate));
+ const pre = Math.round(limits.prerollSeconds * sampleRate);
+ const post = Math.round(limits.postrollSeconds * sampleRate);
+ const segments: AudioFxSegment[] = [];
+ for (let keepFrom = 0; keepFrom < totalFrames; keepFrom += step) {
+ const keepTo = Math.min(totalFrames, keepFrom + step);
+ segments.push({
+ keepFrom,
+ keepTo,
+ readFrom: Math.max(0, Math.floor((keepFrom - pre) / RENDER_QUANTUM) * RENDER_QUANTUM),
+ readTo: Math.min(totalFrames, keepTo + post),
+ last: keepTo === totalFrames,
+ });
+ }
+ return segments;
+}
+
+function readFrames(fd: number, header: WavHeader, from: number, to: number): Float32Array {
+ const frameBytes = (header.channels * header.bits) / 8;
+ const data = Buffer.alloc((to - from) * frameBytes);
+ for (let filled = 0; filled < data.length; ) {
+ const n = readSync(
+ fd,
+ data,
+ filled,
+ data.length - filled,
+ header.dataOffset + from * frameBytes + filled,
+ );
+ if (n === 0) break;
+ filled += n;
+ }
+ return decodeSamples(data, header.format, header.bits, "streamed input");
+}
+
+/** A chain remembers longer than the floor: size the pre-roll to its slowest state, up to the cap. */
+function chainPrerollSeconds(
+ chain: HfAudioFxChain,
+ limits: AudioFxStreamingLimits,
+ trackId: string,
+): number {
+ const { seconds, capped } = windowPrerollSeconds(
+ chain,
+ limits.prerollSeconds,
+ limits.maxPrerollSeconds,
+ );
+ if (capped) {
+ console.warn(
+ `Audio FX for track ${trackId}: the chain remembers longer than the ${limits.maxPrerollSeconds}s pre-roll cap, so its oldest repeats are cut at window seams`,
+ );
+ }
+ return seconds;
+}
+
+async function renderInSegments(
+ openPage: () => Promise<FxPage>,
+ inputWav: string,
+ header: WavHeader,
+ chain: HfAudioFxChain,
+ outputWav: string,
+ options: {
+ trackId: string;
+ signal?: AbortSignal;
+ envelope?: AudioFxEnvelope;
+ },
+ limits: AudioFxStreamingLimits,
+): Promise<boolean> {
+ const { sampleRate, channels, float } = header;
+ const totalFrames = Math.floor(header.dataBytes / ((channels * header.bits) / 8));
+ const gainAt = envelopeWalkerFor(options.envelope);
+ const prerollSeconds = chainPrerollSeconds(chain, limits, options.trackId);
+ const inFd = openSync(inputWav, "r");
+ let outFd: number | undefined;
+ try {
+ // Same directory rule as writeWav: the caller's mkdtemp work dir.
+ outFd = openSync(outputWav, "w");
+ writeSync(outFd, wavHeader(0, sampleRate, channels, float));
+ let written = 0;
+ for (const seg of planAudioFxSegments(totalFrames, sampleRate, {
+ ...limits,
+ prerollSeconds,
+ })) {
+ if (options.signal?.aborted) {
+ throw new AudioFxRenderError(`Audio FX cancelled for track ${options.trackId}`);
+ }
+ const planes = deinterleave(readFrames(inFd, header, seg.readFrom, seg.readTo), channels);
+ // A fresh page per window, so the browser never holds more than one window of audio.
+ const page = await openPage();
+ let outPlanes: Float32Array[];
+ try {
+ await sendPlanesToPage(page, planes);
+ outPlanes = await readPlanesFromPage(
+ page,
+ await renderPlanesInPage(page, sampleRate, chain, "", seg.readFrom / sampleRate),
+ );
+ } finally {
+ await page.close().catch(() => undefined);
+ }
+ const rendered = outPlanes[0]?.length ?? 0;
+ if (rendered === 0) {
+ throw new AudioFxRenderError(`Audio FX produced no samples for track ${options.trackId}`);
+ }
+ // The last window keeps the chain's tail (reverb, delay), as a whole-file render does.
+ const skip = seg.keepFrom - seg.readFrom;
+ const kept = outPlanes.map((plane) =>
+ plane.subarray(skip, seg.last ? rendered : seg.keepTo - seg.readFrom),
+ );
+ if (gainAt) applyEnvelopeToPlanes(kept, sampleRate, gainAt, seg.keepFrom);
+ const bytes = encodeSamples(interleave(kept), float);
+ writeFileSync(outFd, bytes);
+ written += bytes.length;
+ }
+ writeSync(outFd, wavHeader(written, sampleRate, channels, float), 0, 44, 0);
+ } finally {
+ closeSync(inFd);
+ if (outFd !== undefined) closeSync(outFd);
+ }
+ return gainAt !== null;
+}
+
+type AudioFxEnvelope = {
+ keyframes: AudioVolumeKeyframe[];
+ trackStart: number;
+ baseVolume: number;
+};
+
/**
* Run a chain over `inputWav`, writing `outputWav`. Resolves to the path to use
* downstream — `outputWav` when the chain did something, `inputWav` untouched
@@ -275,7 +522,8 @@ export async function applyAudioFxChain(
trackId: string;
signal?: AbortSignal;
automation?: HfAutomation;
- envelope?: { keyframes: AudioVolumeKeyframe[]; trackStart: number; baseVolume: number };
+ envelope?: AudioFxEnvelope;
+ streaming?: Partial<AudioFxStreamingLimits>;
},
): Promise<{ path: string; envelopeBaked: boolean }> {
if (enabledAudioFxNodes(chain).length === 0) return { path: inputWav, envelopeBaked: false };
@@ -283,8 +531,34 @@ export async function applyAudioFxChain(
throw new AudioFxRenderError(`Audio FX input is missing: ${inputWav}`);
}
- const { samples, sampleRate, channels, float } = readWav(inputWav);
- const planes = deinterleave(samples, channels);
+ // Automation is clocked from the file's own start, so a windowed render leaves it to the whole path.
+ const limits = { ...DEFAULT_STREAMING_LIMITS, ...options.streaming };
+ const header = options.automation ? null : readWavHeader(inputWav);
+ const streamed = header !== null && header.dataBytes > limits.aboveBytes;
+
+ if (streamed) {
+ return withFxBrowser(options, async (openPage) => ({
+ path: outputWav,
+ envelopeBaked: await renderInSegments(
+ openPage,
+ inputWav,
+ header,
+ chain,
+ outputWav,
+ options,
+ limits,
+ ),
+ }));
+ }
+
+ if (statSync(inputWav).size > MAX_WHOLE_FILE_BYTES) {
+ const why = options.automation
+ ? "parameter automation is not supported on inputs over 2 GiB"
+ : "the WAV header of an input over 2 GiB could not be read";
+ throw new AudioFxRenderError(`Audio FX for track ${options.trackId}: ${why} (${inputWav})`);
+ }
+ const input = readWav(inputWav);
+ const planes = deinterleave(input.samples, input.channels);
// An empty track has nothing to process — and an OfflineAudioContext of zero
// length throws, which is fatal for the WHOLE render rather than this track:
// the error travels past the mixer's per-track failure collector. ffmpeg
@@ -294,36 +568,13 @@ export async function applyAudioFxChain(
// never costs a browser.
if ((planes[0]?.length ?? 0) === 0) return { path: inputWav, envelopeBaked: false };
- // Both resources are taken INSIDE the try that releases them. The lease used
- // to be acquired above it, with the mkdtemp between — so a failure there
- // (a full disk, a read-only tmpdir) leaked a pooled browser, and a pool with
- // no leases left hangs every later render rather than failing one.
- const hostDir = mkdtempSync(join(tmpdir(), "hf-fx-host-"));
- let lease: Awaited<ReturnType<typeof acquireBrowser>> | null = null;
- try {
- // Audio processing needs no GPU or special capture mode; a plain sandboxed
- // browser is enough, and the lease pool reuses one across tracks.
- // Checked before the lease, not after: an already-cancelled track has no
- // reason to take a browser out of the pool just to hand it straight back.
- if (options.signal?.aborted) {
- throw new AudioFxRenderError(`Audio FX cancelled for track ${options.trackId}`);
- }
- lease = await acquireBrowser(["--no-sandbox", "--autoplay-policy=no-user-gesture-required"]);
- const page = await lease.browser.newPage();
+ return withFxBrowser(options, async (openPage) => {
+ const page = await openPage();
try {
- // AudioWorklet is only exposed in a secure context, and about:blank is
- // not one — the module would fail with an opaque error. A file:// page
- // qualifies and needs no listening socket.
- const hostPage = join(hostDir, "audio-fx.html");
- writeFileSync(hostPage, "<!doctype html><meta charset=utf-8><title>audio fx</title>");
- await page.goto(pathToFileURL(hostPage).href, { waitUntil: "domcontentloaded" });
- await page.addScriptTag({ content: getAudioFxRuntimeScript() });
-
await sendPlanesToPage(page, planes);
-
const outLengths = await renderPlanesInPage(
page,
- sampleRate,
+ input.sampleRate,
chain,
options.automation ? serializeAutomation(options.automation) : "",
);
@@ -335,14 +586,35 @@ export async function applyAudioFxChain(
// own paths still own this track's gain, so say so rather than claiming
// a bake that never happened.
const gainAt = envelopeWalkerFor(options.envelope);
- if (gainAt) applyEnvelopeToPlanes(outPlanes, sampleRate, gainAt);
+ if (gainAt) applyEnvelopeToPlanes(outPlanes, input.sampleRate, gainAt);
// Same format in as out: a float input is a group sub-mix whose headroom
// must survive to its fader (see writeWav).
- writeWav(outputWav, interleave(outPlanes), sampleRate, outPlanes.length, float);
+ writeWav(outputWav, interleave(outPlanes), input.sampleRate, outPlanes.length, input.float);
return { path: outputWav, envelopeBaked: gainAt !== null };
} finally {
await page.close().catch(() => undefined);
}
+ });
+}
+
+/** Run `body` with a leased browser and a way to open pages on the runtime; both are released in the same try that takes them. */
+async function withFxBrowser<T>(
+ options: { trackId: string; signal?: AbortSignal },
+ body: (openPage: () => Promise<FxPage>) => Promise<T>,
+): Promise<T> {
+ const hostDir = mkdtempSync(join(tmpdir(), "hf-fx-host-"));
+ let lease: Awaited<ReturnType<typeof acquireBrowser>> | null = null;
+ try {
+ // Audio processing needs no GPU or special capture mode; a plain sandboxed
+ // browser is enough, and the lease pool reuses one across tracks.
+ // Checked before the lease, not after: an already-cancelled track has no
+ // reason to take a browser out of the pool just to hand it straight back.
+ if (options.signal?.aborted) {
+ throw new AudioFxRenderError(`Audio FX cancelled for track ${options.trackId}`);
+ }
+ lease = await acquireBrowser(["--no-sandbox", "--autoplay-policy=no-user-gesture-required"]);
+ const { browser } = lease;
+ return await body(() => openFxPage(browser, hostDir));
} catch (err) {
if (err instanceof AudioFxRenderError) throw err;
throw new AudioFxRenderError(
@@ -357,6 +629,28 @@ export async function applyAudioFxChain(
/** The page handle `acquireBrowser().browser.newPage()` returns. */
type FxPage = Awaited<ReturnType<Awaited<ReturnType<typeof acquireBrowser>>["browser"]["newPage"]>>;
+async function openFxPage(
+ browser: Awaited<ReturnType<typeof acquireBrowser>>["browser"],
+ hostDir: string,
+): Promise<FxPage> {
+ const page = await browser.newPage();
+ try {
+ // AudioWorklet is only exposed in a secure context, and about:blank is
+ // not one — the module would fail with an opaque error. A file:// page
+ // qualifies and needs no listening socket.
+ const hostPage = join(hostDir, "audio-fx.html");
+ writeFileSync(hostPage, "<!doctype html><meta charset=utf-8><title>audio fx</title>");
+ await page.goto(pathToFileURL(hostPage).href, {
+ waitUntil: "domcontentloaded",
+ });
+ await page.addScriptTag({ content: getAudioFxRuntimeScript() });
+ return page;
+ } catch (err) {
+ await page.close().catch(() => undefined);
+ throw err;
+ }
+}
+
/**
* Hand the input planes to the page a chunk at a time.
*
@@ -397,9 +691,10 @@ async function renderPlanesInPage(
sampleRate: number,
chain: HfAudioFxChain,
automationJson: string,
+ elapsed = 0,
): Promise<number[]> {
return (await page.evaluate(
- async ([rate, chainJson, automation]: [number, string, string]) => {
+ async ([rate, chainJson, automation, at]: [number, string, string, number]) => {
const w = window as unknown as AudioFxWindow;
const io = w.__HF_FX_IO;
if (!w.__HF_AUDIO_FX || !io) throw new Error("audio FX runtime failed to load");
@@ -415,10 +710,15 @@ async function renderPlanesInPage(
// Dropped before the render allocates its own buffers, so the page does
// not hold two copies of the track at once.
io.in = [];
- io.out = await w.__HF_AUDIO_FX.render(inPlanes, rate, chainJson, automation || undefined);
+ io.out = await w.__HF_AUDIO_FX.render(inPlanes, rate, chainJson, automation || undefined, at);
return io.out.map((plane) => plane.length);
},
- [sampleRate, JSON.stringify(chain), automationJson] as [number, string, string],
+ [sampleRate, JSON.stringify(chain), automationJson, elapsed] as [
+ number,
+ string,
+ string,
+ number,
+ ],
)) as number[];
}
@@ -470,7 +770,11 @@ async function readPlanesFromPage(page: FxPage, outLengths: number[]): Promise<F
/** The envelope walker for a track that has keyframes, or null when it has none. */
function envelopeWalkerFor(
envelope:
- | { keyframes: AudioVolumeKeyframe[]; trackStart: number; baseVolume: number }
+ | {
+ keyframes: AudioVolumeKeyframe[];
+ trackStart: number;
+ baseVolume: number;
+ }
| undefined,
): ReturnType<typeof createEnvelopeWalker> | null {
if (!envelope) return null;
diff --git a/packages/engine/src/services/audioFxWarmup.test.ts b/packages/engine/src/services/audioFxWarmup.test.ts
new file mode 100644
index 0000000000..d8843ce58a
--- /dev/null
+++ b/packages/engine/src/services/audioFxWarmup.test.ts
@@ -0,0 +1,59 @@
+import { describe, expect, it } from "vitest";
+import type { HfAudioFxChain } from "@hyperframes/core/audio-fx";
+import { chainWarmupSeconds, windowPrerollSeconds } from "./audioFxWarmup.js";
+
+const chainOf = (...nodes: HfAudioFxChain["nodes"]): HfAudioFxChain => ({ version: 1, nodes });
+
+describe("chainWarmupSeconds", () => {
+ it("is the time a delay's repeats take to fall to -60 dB", () => {
+ const delay = {
+ type: "delay",
+ enabled: true,
+ params: { time: 5000, feedback: 0.95, mix: 0.5 },
+ };
+ expect(chainWarmupSeconds(chainOf(delay))).toBeCloseTo(135 * 5, 5);
+ });
+
+ it("is seven release times for a compressor", () => {
+ const comp = { type: "compressor", enabled: true, params: { release: 9000 } };
+ expect(chainWarmupSeconds(chainOf(comp))).toBeCloseTo(9 * Math.log(1000), 5);
+ });
+
+ it("adds a reverb's room and sums a serial chain", () => {
+ const reverb = { type: "reverb", enabled: true, params: { size: 0.5, wet: 0.3 } };
+ const delay = { type: "delay", enabled: true, params: { time: 300, feedback: 0.4, mix: 0.4 } };
+ expect(chainWarmupSeconds(chainOf(reverb))).toBeCloseTo(0.6 + 1.3, 5);
+ expect(chainWarmupSeconds(chainOf(delay, reverb))).toBeCloseTo(0.3 * 8 + 1.9, 5);
+ });
+
+ it("ignores nodes that are off or mixed out", () => {
+ const off = { type: "delay", enabled: false, params: { time: 5000, feedback: 0.9, mix: 0.5 } };
+ const dry = { type: "reverb", enabled: true, params: { size: 1, wet: 0 } };
+ const mixedOut = {
+ type: "delay",
+ enabled: true,
+ params: { time: 5000, feedback: 0.9, mix: 0 },
+ };
+ expect(chainWarmupSeconds(chainOf(off, dry, mixedOut))).toBe(0);
+ });
+});
+
+describe("windowPrerollSeconds", () => {
+ const delay = (time: number, feedback: number): HfAudioFxChain =>
+ chainOf({ type: "delay", enabled: true, params: { time, feedback, mix: 0.5 } });
+
+ it("never goes below the floor", () => {
+ expect(windowPrerollSeconds(delay(100, 0.1), 10, 300)).toEqual({ seconds: 10, capped: false });
+ });
+
+ it("follows the chain between the floor and the cap", () => {
+ expect(windowPrerollSeconds(delay(400, 0.5), 1, 300)).toEqual({ seconds: 4, capped: false });
+ });
+
+ it("stops at the cap and says so", () => {
+ expect(windowPrerollSeconds(delay(5000, 0.95), 10, 300)).toEqual({
+ seconds: 300,
+ capped: true,
+ });
+ });
+});
diff --git a/packages/engine/src/services/audioFxWarmup.ts b/packages/engine/src/services/audioFxWarmup.ts
new file mode 100644
index 0000000000..b21a21695e
--- /dev/null
+++ b/packages/engine/src/services/audioFxWarmup.ts
@@ -0,0 +1,58 @@
+import { delayTailSeconds, reverbRoomSeconds } from "@hyperframes/core/audio-fx-tail";
+import {
+ enabledAudioFxNodes,
+ normalizeAudioFxParams,
+ type HfAudioFxChain,
+ type HfAudioFxNode,
+} from "@hyperframes/core/audio-fx";
+
+/** Where state stops mattering: -60 dB, i.e. e^-6.9 of what it started as. */
+const DECAY_TO_FLOOR = Math.log(1000);
+
+const param = (node: HfAudioFxNode, key: string): number =>
+ Number(normalizeAudioFxParams(node.type, node.params)[key] ?? 0);
+
+const reverbMemory = (node: HfAudioFxNode): number =>
+ param(node, "wet") > 0 ? reverbRoomSeconds(param(node, "size")) : 0;
+
+const delayMemory = (node: HfAudioFxNode): number =>
+ param(node, "mix") > 0 ? delayTailSeconds(param(node, "time"), param(node, "feedback")) : 0;
+
+const releaseMemory = (node: HfAudioFxNode): number =>
+ (DECAY_TO_FLOOR * param(node, "release")) / 1000;
+
+/** Seconds of input a node still remembers, to -60 dB. */
+const MEMORY: Record<string, (node: HfAudioFxNode) => number> = {
+ reverb: reverbMemory,
+ delay: delayMemory,
+ compressor: releaseMemory,
+ limiter: releaseMemory,
+ truepeak: releaseMemory,
+ gate: releaseMemory,
+};
+
+/**
+ * How far ahead of a window a chain must be fed for its state to match a whole render.
+ *
+ * Summed, since the chain is serial: a delay ahead of a reverb hands the reverb every repeat.
+ * Biquads, chorus and pitch shift settle well inside the caller's floor.
+ */
+export function chainWarmupSeconds(chain: HfAudioFxChain): number {
+ return enabledAudioFxNodes(chain).reduce(
+ (sum, node) => sum + (MEMORY[node.type]?.(node) ?? 0),
+ 0,
+ );
+}
+
+/** The pre-roll a chain gets: its own memory, between `floorSeconds` and `capSeconds`. */
+export function windowPrerollSeconds(
+ chain: HfAudioFxChain,
+ floorSeconds: number,
+ capSeconds: number,
+): { seconds: number; capped: boolean } {
+ const needed = chainWarmupSeconds(chain);
+ return {
+ seconds: Math.min(capSeconds, Math.max(floorSeconds, needed)),
+ capped: needed > capSeconds,
+ };
+}