Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(core): recover refused audio automation with ramps (#5927)
James Russoe44f456b7cbf
diff --git a/packages/core/src/audio/audioFxAutomation.clamping.test.ts b/packages/core/src/audio/audioFxAutomation.clamping.test.tsnew file mode 100644index 0000000000..5f8b45ae96--- /dev/null+++ b/packages/core/src/audio/audioFxAutomation.clamping.test.ts@@ -0,0 +1,92 @@+// @vitest-environment happy-dom+import { expect, it } from "vitest";+import { scheduleParamLane } from "./audioFxAutomation.js";+import type { HfAutomationLane } from "../audioAutomation.js";++class ClampedParam implements AudioParam {+ automationRate: AutomationRate = "a-rate";+ defaultValue = 1;+ minValue = -Infinity;+ maxValue = Infinity;+ value = 1;+ ramps = 0;+ curves: { time: number; duration: number }[] = [];++ constructor(readonly currentTime: number) {}++ private rejectOverlap(time: number): void {+ if (this.curves.some((curve) => time >= curve.time && time < curve.time + curve.duration)) {+ throw new DOMException("An event overlaps the active curve", "NotSupportedError");+ }+ }++ setValueAtTime(value: number, time: number): AudioParam {+ this.rejectOverlap(Math.max(time, this.currentTime));+ this.value = value;+ return this;+ }++ linearRampToValueAtTime(value: number, time: number): AudioParam {+ this.setValueAtTime(value, time);+ this.ramps++;+ return this;+ }++ setValueCurveAtTime(+ values: Float32Array<ArrayBuffer> | number[],+ time: number,+ duration: number,+ ): AudioParam {+ // WebAudio clamps the start to its current time without shortening the duration.+ const start = Math.max(time, this.currentTime);+ this.rejectOverlap(start);+ this.curves.push({ time: start, duration });+ this.value = values[values.length - 1] ?? this.value;+ return this;+ }++ cancelScheduledValues(time: number): AudioParam {+ const now = Math.max(time, this.currentTime);+ this.curves = this.curves.filter((curve) => curve.time + curve.duration <= now);+ return this;+ }++ cancelAndHoldAtTime(time: number): AudioParam {+ return this.cancelScheduledValues(time);+ }++ exponentialRampToValueAtTime(value: number, time: number): AudioParam {+ return this.linearRampToValueAtTime(value, time);+ }++ setTargetAtTime(value: number, time: number): AudioParam {+ return this.setValueAtTime(value, time);+ }+}++const lane: HfAutomationLane = {+ target: "volume",+ points: [+ { t: 0, v: 0.1, curve: 0.7 },+ { t: 0.004, v: 0.3, curve: 0.7 },+ { t: 0.008, v: 0.5, curve: 0.7 },+ { t: 0.012, v: 0.7 },+ ],+};++it("finishes the lane when clamping a curve also blocks its fallback ramp", () => {+ const param = new ClampedParam(0.032);+ expect(() =>+ scheduleParamLane([{ param }], lane, "linear", { scheduledAt: 0, elapsed: 0, rate: 1 }),+ ).not.toThrow();+ expect(param.ramps).toBe(3);+ expect(param.curves).toHaveLength(0);+ expect(param.value).toBeCloseTo(0.7);+});++it("retains each bend when the curves are scheduled ahead of the audio clock", () => {+ const param = new ClampedParam(0.032);+ scheduleParamLane([{ param }], lane, "linear", { scheduledAt: 1, elapsed: 0, rate: 1 });+ expect(param.curves).toHaveLength(3);+ expect(param.ramps).toBe(0);+});diff --git a/packages/core/src/audio/audioFxAutomation.ts b/packages/core/src/audio/audioFxAutomation.tsindex d818b961b6..2d0f253d58 100644--- a/packages/core/src/audio/audioFxAutomation.ts+++ b/packages/core/src/audio/audioFxAutomation.ts@@ -136,21 +136,23 @@ function planOps(return ops;}+/** Rebuild a refused plan with ramps so clamped curves cannot replay past audio. */+function emitRamps(param: AudioParam, ops: readonly Op[]): void {+ param.cancelScheduledValues(0);+ for (const op of ops) {+ if (op.kind === "set") param.setValueAtTime(op.value, op.time);+ else if (op.kind === "ramp") param.linearRampToValueAtTime(op.value, op.time);+ else {+ param.setValueAtTime(op.values[0] ?? 0, op.time);+ param.linearRampToValueAtTime(op.values[op.values.length - 1] ?? 0, op.time + op.duration);+ }+ }+}+/**- * Hand the plan to the AudioParam, degrading a curve the browser will not take.- *- * A value curve may not overlap another, and this has been seen refused in the- * field while rescheduling mid-playback — two passes landing a render quantum- * apart, which is what applying a carve while the transport runs does. Every- * caller here cancels the parameter first, and measured against Chrome that is- * enough to free even a curve already under way, so the mechanism behind those- * reports is not understood and does not reproduce. This catch is the backstop- * for it: unhandled, the exception reached the console and abandoned every- * remaining segment of the envelope.- *- * A ramp to the same destination is always accepted, since it books no span. The- * segment loses its bend until the next full reschedule, which is a far better- * failure than the rest of the envelope never being scheduled at all.+ * Hand the plan to the AudioParam. A clamped curve can overlap the next segment.+ * On refusal, rebuild the whole lane as ramps: retrying later curves would clamp+ * them forward too, replaying values that should already have elapsed.*/function emit(param: AudioParam, ops: readonly Op[]): void {for (const op of ops) {@@ -160,8 +162,8 @@ function emit(param: AudioParam, ops: readonly Op[]): void {try {param.setValueCurveAtTime(op.values, op.time, op.duration);} catch {- const end = op.values[op.values.length - 1] ?? 0;- param.linearRampToValueAtTime(end, op.time + op.duration);+ emitRamps(param, ops);+ return;}}}diff --git a/packages/engine/src/services/audioFxAutomation.browser.test.ts b/packages/engine/src/services/audioFxAutomation.browser.test.tsnew file mode 100644index 0000000000..65342d4aea--- /dev/null+++ b/packages/engine/src/services/audioFxAutomation.browser.test.ts@@ -0,0 +1,84 @@+import { spawnSync } from "node:child_process";+import { fileURLToPath } from "node:url";+import { build } from "esbuild";+import type { Browser } from "puppeteer-core";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import type { scheduleParamLane } from "../../../core/src/audio/audioFxAutomation.js";+import { resolveHeadlessShellPath } from "./browserManager.js";+import { closeTestChrome, launchTestChrome } from "./__fixtures__/loopback.js";++declare global {+ interface Window {+ hfScheduleParamLane: typeof scheduleParamLane;+ }+}++const chromePath = resolveHeadlessShellPath();+const hasBrowser = chromePath !== undefined && spawnSync(chromePath, ["--version"]).status === 0;++describe.skipIf(!hasBrowser)("clamped audio automation in WebAudio", () => {+ let browser: Browser;+ let scheduler: string;++ beforeAll(async () => {+ const entry = fileURLToPath(+ new URL("../../../core/src/audio/audioFxAutomation.ts", import.meta.url),+ );+ const bundled = await build({+ stdin: {+ resolveDir: process.cwd(),+ contents: `import { scheduleParamLane } from ${JSON.stringify(entry)}; globalThis.hfScheduleParamLane = scheduleParamLane;`,+ },+ bundle: true,+ format: "iife",+ write: false,+ });+ scheduler = bundled.outputFiles[0]!.text;+ browser = await launchTestChrome(chromePath);+ });+ afterAll(() => closeTestChrome(browser));++ it.each([0, 0.024])("renders the recovered envelope at scheduledAt=%s", async (scheduledAt) => {+ const page = await browser.newPage();+ try {+ await page.addScriptTag({ content: scheduler });+ const samples = await page.evaluate(async (at) => {+ const ctx = new OfflineAudioContext(1, 8192, 48000);+ const gain = ctx.createGain();+ const source = ctx.createConstantSource();+ source.connect(gain).connect(ctx.destination);+ source.start();+ const suspended = ctx.suspend(0.032);+ const rendering = ctx.startRendering();+ await suspended;+ const firstFrame = Math.round(ctx.currentTime * ctx.sampleRate);+ window.hfScheduleParamLane(+ [{ param: gain.gain }],+ {+ target: "volume",+ points: [+ { t: 0, v: 0.1, curve: 0.7 },+ { t: 0.004, v: 0.3, curve: 0.7 },+ { t: 0.008, v: 0.5, curve: 0.7 },+ { t: 0.012, v: 0.7 },+ ],+ },+ "linear",+ { scheduledAt: at, elapsed: 0, rate: 1 },+ );+ await ctx.resume();+ const output = (await rendering).getChannelData(0);+ return { firstFrame, values: Array.from(output.slice(firstFrame)) };+ }, scheduledAt);+ expect(samples.values.length).toBeGreaterThan(192);+ const errors = samples.values.map((value, index) => {+ const laneTime = (samples.firstFrame + index) / 48000 - scheduledAt;+ const expected = Math.min(0.7, 0.5 + ((laneTime - 0.008) / 0.004) * 0.2);+ return Math.abs(value - expected);+ });+ expect(Math.max(...errors)).toBeLessThan(0.000001);+ } finally {+ await page.close();+ }+ });+});