Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { initSandboxRuntimeModular } from "./init";
import type { RuntimeTimelineLike } from "./types";
import { WebAudioTransport } from "./webAudioTransport";
type GraphNode = {
kind: string;
outputs: Set<GraphNode>;
gain?: { value: number };
connect(dest: GraphNode): GraphNode;
disconnect(dest?: GraphNode): void;
};
function makeParam(value: number) {
return {
value,
cancelScheduledValues() {},
cancelAndHoldAtTime() {},
setValueAtTime() {},
linearRampToValueAtTime() {},
setTargetAtTime() {},
};
}
function makeNode(kind: string, extra: Partial<GraphNode> = {}): GraphNode {
const node: GraphNode = {
kind,
outputs: new Set(),
connect(dest) {
node.outputs.add(dest);
return dest;
},
disconnect(dest) {
if (dest) node.outputs.delete(dest);
else node.outputs.clear();
},
...extra,
};
return node;
}
const sources = new Map<HTMLMediaElement, GraphNode>();
let destination: GraphNode;
let pendingResume: Promise<void> | null = null;
class GraphAudioContext {
state = pendingResume ? "suspended" : "running";
currentTime = 0;
destination = (destination = makeNode("destination"));
resume() {
return pendingResume ?? Promise.resolve();
}
suspend() {
return Promise.resolve();
}
close() {
return Promise.resolve();
}
createGain() {
return makeNode("gain", { gain: makeParam(1) });
}
createMediaElementSource(el: HTMLMediaElement) {
const node = makeNode("media-element");
sources.set(el, node);
return node;
}
}
/** True when a signal entering `node` reaches the speakers through no silenced gain. */
function audible(node: GraphNode | undefined, seen = new Set<GraphNode>()): boolean {
if (!node || seen.has(node)) return false;
seen.add(node);
if (node === destination) return true;
if (node.gain && node.gain.value <= 0) return false;
for (const out of node.outputs) if (audible(out, seen)) return true;
return false;
}
function createTimeline(duration: number): RuntimeTimelineLike {
const s = { time: 0, paused: true };
return {
play: () => void (s.paused = false),
pause: () => void (s.paused = true),
seek: (t?: number) => (t === undefined ? s.time : (s.time = t)),
totalTime: (t?: number) => (t === undefined ? s.time : (s.time = t)),
time: () => s.time,
duration: () => duration,
add: () => {},
paused: (v?: boolean) => (typeof v === "boolean" ? (s.paused = v) : s.paused),
timeScale: () => {},
set: () => {},
getChildren: () => [],
};
}
function mount(bodyHtml: string): void {
document.body.innerHTML =
`<div data-composition-id="main" data-root="true" data-start="0" data-duration="10"` +
` data-width="1920" data-height="1080">${bodyHtml}</div>`;
for (const el of document.querySelectorAll("audio")) {
el.load = () => {};
el.play = vi.fn(() => Promise.resolve());
el.pause = vi.fn();
}
window.__timelines = { main: createTimeline(10) };
}
async function flush() {
for (let i = 0; i < 4; i++) await Promise.resolve();
}
let frameCallbacks: FrameRequestCallback[] = [];
let nowMs = 0;
function stepFrames(count: number) {
for (let i = 0; i < count; i++) {
nowMs += 1000 / 60;
const pending = frameCallbacks;
frameCallbacks = [];
for (const cb of pending) cb(nowMs);
}
}
const originalAudioContext = (globalThis as Record<string, unknown>).AudioContext;
const originalRaf = window.requestAnimationFrame;
const originalCaf = window.cancelAnimationFrame;
beforeEach(() => {
sources.clear();
nowMs = 0;
frameCallbacks = [];
(globalThis as Record<string, unknown>).AudioContext = GraphAudioContext;
(globalThis as typeof globalThis & { CSS?: { escape?: (v: string) => string } }).CSS ??= {};
globalThis.CSS.escape ??= (v: string) => v;
window.requestAnimationFrame = ((cb: FrameRequestCallback) => {
frameCallbacks.push(cb);
return frameCallbacks.length;
}) as typeof window.requestAnimationFrame;
window.cancelAnimationFrame = (() => {}) as typeof window.cancelAnimationFrame;
vi.spyOn(performance, "now").mockImplementation(() => nowMs);
});
afterEach(() => {
pendingResume = null;
window.__hfRuntimeTeardown?.();
document.body.innerHTML = "";
document.documentElement.removeAttribute("style");
document.body.removeAttribute("style");
window.__timelines = {} as Record<string, RuntimeTimelineLike>;
delete window.__player;
delete window.__hf;
vi.restoreAllMocks();
(globalThis as Record<string, unknown>).AudioContext = originalAudioContext;
window.requestAnimationFrame = originalRaf;
window.cancelAnimationFrame = originalCaf;
});
describe("an element Web Audio captured, outside a play", () => {
it("stays audible to a paused scrub after the first Play/Pause, through one path only", async () => {
mount(`<audio data-start="0" data-duration="10" src="/assets/music.mp3"></audio>`);
const music = document.querySelector("audio")!;
initSandboxRuntimeModular();
await flush();
window.__player?.play();
await flush();
const source = sources.get(music)!;
expect(audible(source)).toBe(true);
window.__player?.pause();
// What studio/src/player/lib/timelineIframeHelpers.ts applyScrub does on a paused drag.
window.__hf?.leasePausedMedia?.(music);
music.muted = false;
music.volume = 0.25;
void music.play();
expect(audible(source)).toBe(true);
window.__hf?.releasePausedMedia?.(music);
window.__player?.play();
await flush();
expect(audible(source)).toBe(true);
expect(source.outputs.has(destination)).toBe(false);
});
it("does not let a track hidden while playing sound for a frame on its idle route", async () => {
mount(`<audio data-start="0" data-duration="10" src="/assets/music.mp3"></audio>`);
const music = document.querySelector("audio")!;
initSandboxRuntimeModular();
await flush();
window.__player?.play();
await flush();
stepFrames(2);
music.setAttribute("data-hidden", "");
stepFrames(1);
expect(music.volume).toBe(0);
});
it("does not mute a processed track whose capture a newer pass replaced", async () => {
let release!: () => void;
pendingResume = new Promise<void>((r) => (release = r));
mount(
`<audio id="vo" data-start="0" data-duration="10" data-playback-rate="1.5" data-fx-chain="[]" src="/assets/vo.mp3"></audio>` +
`<audio id="sfx" data-start="0" data-duration="10" src="/assets/sfx.mp3"></audio>`,
);
const vo = document.getElementById("vo") as HTMLAudioElement;
initSandboxRuntimeModular();
await flush();
window.__player?.play();
await flush();
// A hide reschedules every track while the first captures still wait on resume().
document.getElementById("sfx")!.setAttribute("data-hidden", "");
stepFrames(1);
release();
pendingResume = null;
await flush();
expect(vo.muted).toBe(false);
expect(audible(sources.get(vo))).toBe(true);
});
});
/** On its idle route an element's own volume applies; inside a play only the graph's gains do. */
function heard(el: HTMLMediaElement): boolean {
const source = sources.get(el);
if (!source) return !el.muted && el.volume > 0;
if (source.outputs.has(destination)) return el.volume > 0;
return audible(source);
}
describe("an audio group muted or unmuted while playing", () => {
const groupHtml =
`<hf-audio-group id="music"></hf-audio-group>` +
`<audio id="bed" data-start="0" data-duration="10" data-audio-group="music" src="/assets/music.mp3"></audio>`;
it("silences its members at once", async () => {
mount(groupHtml);
const bed = document.getElementById("bed") as HTMLAudioElement;
initSandboxRuntimeModular();
await flush();
window.__player?.play();
await flush();
expect(heard(bed)).toBe(true);
document.getElementById("music")!.setAttribute("data-hidden", "");
stepFrames(1);
await flush();
expect(heard(bed)).toBe(false);
});
it("brings its members back through the group", async () => {
mount(groupHtml);
const bed = document.getElementById("bed") as HTMLAudioElement;
const group = document.getElementById("music")!;
initSandboxRuntimeModular();
await flush();
window.__player?.play();
await flush();
window.__player?.pause();
group.setAttribute("data-hidden", "");
stepFrames(1);
window.__player?.play();
await flush();
expect(heard(bed)).toBe(false);
group.removeAttribute("data-hidden");
stepFrames(1);
await flush();
expect(heard(bed)).toBe(true);
expect(sources.get(bed)!.outputs.has(destination)).toBe(false);
});
it("does not restart the rest of the mix to mute it", async () => {
mount(
groupHtml + `<audio id="vo" data-start="0" data-duration="10" src="/assets/vo.mp3"></audio>`,
);
initSandboxRuntimeModular();
await flush();
window.__player?.play();
await flush();
const stopAll = vi.spyOn(WebAudioTransport.prototype, "stopAll");
document.getElementById("music")!.setAttribute("data-hidden", "");
stepFrames(1);
await flush();
expect(heard(document.getElementById("bed") as HTMLAudioElement)).toBe(false);
expect(heard(document.getElementById("vo") as HTMLAudioElement)).toBe(true);
expect(stopAll).not.toHaveBeenCalled();
});
it("does not restart the mix to unmute it right after muting it", async () => {
mount(
groupHtml + `<audio id="vo" data-start="0" data-duration="10" src="/assets/vo.mp3"></audio>`,
);
const bed = document.getElementById("bed") as HTMLAudioElement;
const group = document.getElementById("music")!;
initSandboxRuntimeModular();
await flush();
window.__player?.play();
await flush();
group.setAttribute("data-hidden", "");
stepFrames(1);
const stopAll = vi.spyOn(WebAudioTransport.prototype, "stopAll");
group.removeAttribute("data-hidden");
stepFrames(1);
await flush();
expect(heard(bed)).toBe(true);
expect(stopAll).not.toHaveBeenCalled();
});
it("does not restart the mix to unmute a group whose members have all finished", async () => {
mount(
`<hf-audio-group id="music" data-hidden=""></hf-audio-group>` +
`<audio id="bed" data-start="0" data-duration="2" data-audio-group="music" src="/assets/music.mp3"></audio>` +
`<audio id="vo" data-start="0" data-duration="10" src="/assets/vo.mp3"></audio>`,
);
initSandboxRuntimeModular();
await flush();
window.__player?.seek(3);
window.__player?.play();
await flush();
const stopAll = vi.spyOn(WebAudioTransport.prototype, "stopAll");
document.getElementById("music")!.removeAttribute("data-hidden");
stepFrames(1);
await flush();
expect(stopAll).not.toHaveBeenCalled();
});
it("does not restart the mix to unmute a group whose unfinished member is muted itself", async () => {
mount(
`<hf-audio-group id="music" data-hidden=""></hf-audio-group>` +
`<audio id="bed" data-start="0" data-duration="10" data-audio-group="music" data-hidden="" src="/assets/music.mp3"></audio>` +
`<audio id="vo" data-start="0" data-duration="10" src="/assets/vo.mp3"></audio>`,
);
initSandboxRuntimeModular();
await flush();
window.__player?.play();
await flush();
const stopAll = vi.spyOn(WebAudioTransport.prototype, "stopAll");
document.getElementById("music")!.removeAttribute("data-hidden");
stepFrames(1);
await flush();
expect(stopAll).not.toHaveBeenCalled();
});
it("brings a member that starts after the unmute in through the group", async () => {
mount(
`<hf-audio-group id="music" data-hidden=""></hf-audio-group>` +
`<audio id="bed" data-start="5" data-duration="4" data-audio-group="music" src="/assets/music.mp3"></audio>`,
);
const bed = document.getElementById("bed") as HTMLAudioElement;
initSandboxRuntimeModular();
await flush();
window.__player?.play();
await flush();
expect(sources.get(bed)).toBeUndefined();
document.getElementById("music")!.removeAttribute("data-hidden");
stepFrames(1);
await flush();
expect(audible(sources.get(bed))).toBe(true);
expect(sources.get(bed)!.outputs.has(destination)).toBe(false);
});
});