Skip to content

Write HTML. Render video. Built for agents.

fix(core): recover refused audio automation with ramps (#5927)

James Russoe44f456b7cbf
diff --git a/packages/core/src/audio/audioFxAutomation.clamping.test.ts b/packages/core/src/audio/audioFxAutomation.clamping.test.ts
new file mode 100644
index 0000000000..5f8b45ae96
--- /dev/null
+++ b/packages/core/src/audio/audioFxAutomation.clamping.test.ts
@@ -0,0 +1,92 @@
+// @vitest-environment happy-dom
+import { expect, it } from "vitest";
+import { scheduleParamLane } from "./audioFxAutomation.js";
+import type { HfAutomationLane } from "../audioAutomation.js";
+
+class ClampedParam implements AudioParam {
+ automationRate: AutomationRate = "a-rate";
+ defaultValue = 1;
+ minValue = -Infinity;
+ maxValue = Infinity;
+ value = 1;
+ ramps = 0;
+ curves: { time: number; duration: number }[] = [];
+
+ constructor(readonly currentTime: number) {}
+
+ private rejectOverlap(time: number): void {
+ if (this.curves.some((curve) => time >= curve.time && time < curve.time + curve.duration)) {
+ throw new DOMException("An event overlaps the active curve", "NotSupportedError");
+ }
+ }
+
+ setValueAtTime(value: number, time: number): AudioParam {
+ this.rejectOverlap(Math.max(time, this.currentTime));
+ this.value = value;
+ return this;
+ }
+
+ linearRampToValueAtTime(value: number, time: number): AudioParam {
+ this.setValueAtTime(value, time);
+ this.ramps++;
+ return this;
+ }
+
+ setValueCurveAtTime(
+ values: Float32Array<ArrayBuffer> | number[],
+ time: number,
+ duration: number,
+ ): AudioParam {
+ // WebAudio clamps the start to its current time without shortening the duration.
+ const start = Math.max(time, this.currentTime);
+ this.rejectOverlap(start);
+ this.curves.push({ time: start, duration });
+ this.value = values[values.length - 1] ?? this.value;
+ return this;
+ }
+
+ cancelScheduledValues(time: number): AudioParam {
+ const now = Math.max(time, this.currentTime);
+ this.curves = this.curves.filter((curve) => curve.time + curve.duration <= now);
+ return this;
+ }
+
+ cancelAndHoldAtTime(time: number): AudioParam {
+ return this.cancelScheduledValues(time);
+ }
+
+ exponentialRampToValueAtTime(value: number, time: number): AudioParam {
+ return this.linearRampToValueAtTime(value, time);
+ }
+
+ setTargetAtTime(value: number, time: number): AudioParam {
+ return this.setValueAtTime(value, time);
+ }
+}
+
+const lane: HfAutomationLane = {
+ target: "volume",
+ points: [
+ { t: 0, v: 0.1, curve: 0.7 },
+ { t: 0.004, v: 0.3, curve: 0.7 },
+ { t: 0.008, v: 0.5, curve: 0.7 },
+ { t: 0.012, v: 0.7 },
+ ],
+};
+
+it("finishes the lane when clamping a curve also blocks its fallback ramp", () => {
+ const param = new ClampedParam(0.032);
+ expect(() =>
+ scheduleParamLane([{ param }], lane, "linear", { scheduledAt: 0, elapsed: 0, rate: 1 }),
+ ).not.toThrow();
+ expect(param.ramps).toBe(3);
+ expect(param.curves).toHaveLength(0);
+ expect(param.value).toBeCloseTo(0.7);
+});
+
+it("retains each bend when the curves are scheduled ahead of the audio clock", () => {
+ const param = new ClampedParam(0.032);
+ scheduleParamLane([{ param }], lane, "linear", { scheduledAt: 1, elapsed: 0, rate: 1 });
+ expect(param.curves).toHaveLength(3);
+ expect(param.ramps).toBe(0);
+});
diff --git a/packages/core/src/audio/audioFxAutomation.ts b/packages/core/src/audio/audioFxAutomation.ts
index d818b961b6..2d0f253d58 100644
--- a/packages/core/src/audio/audioFxAutomation.ts
+++ b/packages/core/src/audio/audioFxAutomation.ts
@@ -136,21 +136,23 @@ function planOps(
return ops;
}
+/** Rebuild a refused plan with ramps so clamped curves cannot replay past audio. */
+function emitRamps(param: AudioParam, ops: readonly Op[]): void {
+ param.cancelScheduledValues(0);
+ for (const op of ops) {
+ if (op.kind === "set") param.setValueAtTime(op.value, op.time);
+ else if (op.kind === "ramp") param.linearRampToValueAtTime(op.value, op.time);
+ else {
+ param.setValueAtTime(op.values[0] ?? 0, op.time);
+ param.linearRampToValueAtTime(op.values[op.values.length - 1] ?? 0, op.time + op.duration);
+ }
+ }
+}
+
/**
- * Hand the plan to the AudioParam, degrading a curve the browser will not take.
- *
- * A value curve may not overlap another, and this has been seen refused in the
- * field while rescheduling mid-playback — two passes landing a render quantum
- * apart, which is what applying a carve while the transport runs does. Every
- * caller here cancels the parameter first, and measured against Chrome that is
- * enough to free even a curve already under way, so the mechanism behind those
- * reports is not understood and does not reproduce. This catch is the backstop
- * for it: unhandled, the exception reached the console and abandoned every
- * remaining segment of the envelope.
- *
- * A ramp to the same destination is always accepted, since it books no span. The
- * segment loses its bend until the next full reschedule, which is a far better
- * failure than the rest of the envelope never being scheduled at all.
+ * Hand the plan to the AudioParam. A clamped curve can overlap the next segment.
+ * On refusal, rebuild the whole lane as ramps: retrying later curves would clamp
+ * them forward too, replaying values that should already have elapsed.
*/
function emit(param: AudioParam, ops: readonly Op[]): void {
for (const op of ops) {
@@ -160,8 +162,8 @@ function emit(param: AudioParam, ops: readonly Op[]): void {
try {
param.setValueCurveAtTime(op.values, op.time, op.duration);
} catch {
- const end = op.values[op.values.length - 1] ?? 0;
- param.linearRampToValueAtTime(end, op.time + op.duration);
+ emitRamps(param, ops);
+ return;
}
}
}
diff --git a/packages/engine/src/services/audioFxAutomation.browser.test.ts b/packages/engine/src/services/audioFxAutomation.browser.test.ts
new file mode 100644
index 0000000000..65342d4aea
--- /dev/null
+++ b/packages/engine/src/services/audioFxAutomation.browser.test.ts
@@ -0,0 +1,84 @@
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { build } from "esbuild";
+import type { Browser } from "puppeteer-core";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { scheduleParamLane } from "../../../core/src/audio/audioFxAutomation.js";
+import { resolveHeadlessShellPath } from "./browserManager.js";
+import { closeTestChrome, launchTestChrome } from "./__fixtures__/loopback.js";
+
+declare global {
+ interface Window {
+ hfScheduleParamLane: typeof scheduleParamLane;
+ }
+}
+
+const chromePath = resolveHeadlessShellPath();
+const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;
+
+describe.skipIf(!hasBrowser)("clamped audio automation in WebAudio", () => {
+ let browser: Browser;
+ let scheduler: string;
+
+ beforeAll(async () => {
+ const entry = fileURLToPath(
+ new URL("../../../core/src/audio/audioFxAutomation.ts", import.meta.url),
+ );
+ const bundled = await build({
+ stdin: {
+ resolveDir: process.cwd(),
+ contents: `import { scheduleParamLane } from ${JSON.stringify(entry)}; globalThis.hfScheduleParamLane = scheduleParamLane;`,
+ },
+ bundle: true,
+ format: "iife",
+ write: false,
+ });
+ scheduler = bundled.outputFiles[0]!.text;
+ browser = await launchTestChrome(chromePath);
+ });
+ afterAll(() => closeTestChrome(browser));
+
+ it.each([0, 0.024])("renders the recovered envelope at scheduledAt=%s", async (scheduledAt) => {
+ const page = await browser.newPage();
+ try {
+ await page.addScriptTag({ content: scheduler });
+ const samples = await page.evaluate(async (at) => {
+ const ctx = new OfflineAudioContext(1, 8192, 48000);
+ const gain = ctx.createGain();
+ const source = ctx.createConstantSource();
+ source.connect(gain).connect(ctx.destination);
+ source.start();
+ const suspended = ctx.suspend(0.032);
+ const rendering = ctx.startRendering();
+ await suspended;
+ const firstFrame = Math.round(ctx.currentTime * ctx.sampleRate);
+ window.hfScheduleParamLane(
+ [{ param: gain.gain }],
+ {
+ target: "volume",
+ points: [
+ { t: 0, v: 0.1, curve: 0.7 },
+ { t: 0.004, v: 0.3, curve: 0.7 },
+ { t: 0.008, v: 0.5, curve: 0.7 },
+ { t: 0.012, v: 0.7 },
+ ],
+ },
+ "linear",
+ { scheduledAt: at, elapsed: 0, rate: 1 },
+ );
+ await ctx.resume();
+ const output = (await rendering).getChannelData(0);
+ return { firstFrame, values: Array.from(output.slice(firstFrame)) };
+ }, scheduledAt);
+ expect(samples.values.length).toBeGreaterThan(192);
+ const errors = samples.values.map((value, index) => {
+ const laneTime = (samples.firstFrame + index) / 48000 - scheduledAt;
+ const expected = Math.min(0.7, 0.5 + ((laneTime - 0.008) / 0.004) * 0.2);
+ return Math.abs(value - expected);
+ });
+ expect(Math.max(...errors)).toBeLessThan(0.000001);
+ } finally {
+ await page.close();
+ }
+ });
+});