Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
import type { RuntimeTimelineLike } from "./types";
import { clampAudioGain, withUnclampedVolume } from "../audioGain.js";
import { parseStrictFiniteTimingNumber } from "./playbackRate";
import { createRuntimeStartTimeResolver } from "./startResolver";
import { isMediaElement } from "./domRealm";
/**
* Shared volume-automation utilities used by both the renderer (offline PCM
* baking in audioVolumeEnvelope.ts) and the preview runtime (per-tick gain
* applied in syncRuntimeMedia).
*
* Keeping the two concerns in one place ensures preview and render derive the
* envelope from the same logic and the same probe samples.
*/
export interface VolumeKeyframe {
time: number;
volume: number;
}
/**
* Normalise raw keyframes to track-relative seconds: subtract `trackStart`,
* clamp to [0,1], sort, de-duplicate, and prepend a `baseVolume` anchor at
* t=0 when the first keyframe starts after the clip's begin.
*
* Returns an empty array when all keyframes are invalid — the caller should
* treat an empty envelope as "no automation, use static volume."
*/
export function normaliseEnvelope(
keyframes: VolumeKeyframe[],
trackStart: number,
baseVolume: number,
): VolumeKeyframe[] {
const points = keyframes
.filter((k) => Number.isFinite(k.time) && Number.isFinite(k.volume))
.map((k) => ({
time: Math.max(0, k.time - trackStart),
volume: clampAudioGain(k.volume),
}))
.sort((a, b) => a.time - b.time);
const deduped: VolumeKeyframe[] = [];
for (const point of points) {
const previous = deduped.at(-1);
if (previous && Math.abs(previous.time - point.time) < 1e-9) {
previous.volume = point.volume;
} else {
deduped.push(point);
}
}
if (deduped.length === 0) return deduped;
if (deduped[0]!.time > 0) {
deduped.unshift({ time: 0, volume: clampAudioGain(baseVolume) });
}
return deduped;
}
/**
* Linearly interpolate the gain at time `t` (track-relative seconds) from a
* normalised envelope produced by `normaliseEnvelope`. Returns 1 when the
* envelope is empty.
*/
export function interpolateVolumeGain(envelope: VolumeKeyframe[], t: number): number {
if (envelope.length === 0) return 1;
let segment = 0;
// The PCM baker intentionally inlines this lookup with a monotonic cursor
// because calling this preview-oriented helper per sample would be O(N×M).
// fallow-ignore-next-line code-duplication
while (segment < envelope.length - 2 && t >= envelope[segment + 1]!.time) {
segment += 1;
}
const a = envelope[segment]!;
const b = envelope[segment + 1] ?? a;
const span = b.time - a.time;
const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (t - a.time) / span));
return a.volume + (b.volume - a.volume) * progress;
}
function recordVolumeSample(
keyframes: VolumeKeyframe[],
previousSample: VolumeKeyframe | undefined,
sample: VolumeKeyframe,
isFinalSample: boolean,
): void {
const last = keyframes.at(-1);
if (!last || Math.abs(last.volume - sample.volume) > 0.0001) {
// Change-only compression must retain the preceding real sample so a
// flat run stays flat instead of being interpolated into the next value.
// During a continuous ramp, that sample is already the last keyframe.
if (last && previousSample && previousSample.time > last.time) {
keyframes.push(previousSample);
}
keyframes.push(sample);
} else if (isFinalSample && sample.time > last.time) {
keyframes.push(sample);
}
}
function parseVolumeNumber(value: string | undefined): number | undefined {
const parsed = Number.parseFloat(value ?? "");
return Number.isFinite(parsed) ? parsed : undefined;
}
interface ProbeWindow {
start: number;
end: number;
staticVolume: number;
}
function resolveVolumeProbeWindow(
el: HTMLAudioElement | HTMLVideoElement,
compositionDuration: number,
): ProbeWindow {
// Probe samples are stamped with ROOT-timeline seek times, and
// `normaliseEnvelope` rebases them by this start — so it has to be the same
// absolute start the transport plays the clip at. Reading `data-start`
// directly gave a composition-local value, which put a nested clip's whole
// envelope at the wrong origin.
const start = createRuntimeStartTimeResolver({
timelineRegistry: (window as Window & { __timelines?: Record<string, RuntimeTimelineLike> })
.__timelines,
includeAuthoredTimingAttrs: true,
}).resolveMediaStartForElement(el);
const endAttr = parseStrictFiniteTimingNumber(el.dataset.end) ?? undefined;
const durAttr = parseStrictFiniteTimingNumber(el.dataset.duration) ?? undefined;
let end = compositionDuration;
if (durAttr !== undefined && durAttr > 0) {
end = start + durAttr;
} else if (endAttr !== undefined && endAttr > start) {
end = endAttr;
}
const staticAttr = parseVolumeNumber(el.dataset.volume) ?? 1;
return { start, end, staticVolume: clampAudioGain(staticAttr) };
}
/**
* Probe a single media element's volume automation by seeking a GSAP timeline
* through the element's active window.
*
* Runs synchronously in the browser. The timeline is left at its current
* position after the probe (the next transport tick re-seeks it to `t`).
*
* Returns null when the element has no detectable automation (volume never
* changes from its initial `data-volume` value).
*/
export function probeElementVolumeKeyframes(
el: HTMLAudioElement | HTMLVideoElement,
seekTimeline: (t: number) => void,
compositionDuration: number,
sampleFps: number,
): VolumeKeyframe[] | null {
return probeKeyframesInWindow(
el,
seekTimeline,
compositionDuration,
sampleFps,
resolveVolumeProbeWindow(el, compositionDuration),
);
}
/** Sampling half of the probe, given an already-resolved window. Split out so
* `probeAndCacheElementVolume` resolves that window ONCE and reuses it for the
* envelope rebase, instead of deriving the same start twice per element. */
function probeKeyframesInWindow(
el: HTMLAudioElement | HTMLVideoElement,
seekTimeline: (t: number) => void,
compositionDuration: number,
sampleFps: number,
{ start, end, staticVolume }: ProbeWindow,
): VolumeKeyframe[] | null {
const step = 1 / Math.min(60, Math.max(1, sampleFps));
const sampleStart = Math.max(0, start);
const sampleEnd = Math.min(compositionDuration, end);
const keyframes: VolumeKeyframe[] = withUnclampedVolume(el, () => {
// Reset to data-volume so GSAP captures the correct FROM value. Above
// unity that only survives because the shadow accessor is installed.
el.volume = staticVolume;
const samples: VolumeKeyframe[] = [];
let previousSample: VolumeKeyframe | undefined;
for (let t = sampleStart; t <= sampleEnd + 1e-6; t = Math.min(sampleEnd, t + step)) {
seekTimeline(t);
const raw = Number(el.volume);
if (Number.isFinite(raw)) {
const sample = {
time: Number(t.toFixed(6)),
volume: Number(clampAudioGain(raw).toFixed(6)),
};
recordVolumeSample(samples, previousSample, sample, t === sampleEnd);
previousSample = sample;
}
if (t === sampleEnd) break;
}
return samples;
});
const hasAutomation = keyframes.some((kf) => Math.abs(kf.volume - staticVolume) > 0.0001);
return hasAutomation ? keyframes : null;
}
export type RuntimeTimelineRef = Partial<
Pick<RuntimeTimelineLike, "totalTime" | "seek" | "getChildren">
>;
export interface VolumeProbeOptions {
/**
* Render/probe pages must not sample the live visual timeline during runtime
* initialization. The producer discovers audio automation in its own
* isolated pass and bakes it before frame capture, so seeking here is both
* redundant and capable of materializing future zero-duration GSAP state.
*
* Preview callers omit this option and retain live automation discovery.
*/
allowLiveTimelineSeek?: boolean;
movers?: () => VolumeMovers;
}
function namesKey(vars: unknown, matches: (key: string) => boolean, depth = 0): boolean {
if (depth > 3 || vars === null || typeof vars !== "object") return false;
if (Array.isArray(vars)) return vars.some((item) => namesKey(item, matches, depth + 1));
if (Object.getPrototypeOf(vars) !== Object.prototype) return false;
return Object.entries(vars).some(
([key, value]) => matches(key) || namesKey(value, matches, depth + 1),
);
}
const isDomNode = (target: unknown): boolean =>
typeof (target as { nodeType?: unknown } | null)?.nodeType === "number";
function hasSetter(target: unknown, key: string): boolean {
for (let o = Object(target); o; o = Object.getPrototypeOf(o)) {
const descriptor = Object.getOwnPropertyDescriptor(o, key);
if (descriptor) return typeof descriptor.set === "function";
}
return false;
}
export interface VolumeMovers {
elements: Set<unknown>;
anySetter: boolean;
unknown: boolean;
}
export function volumeMoversOf(timeline: RuntimeTimelineRef): VolumeMovers {
const unknown: VolumeMovers = { elements: new Set(), anySetter: false, unknown: true };
if (typeof timeline.getChildren !== "function") return unknown;
try {
const movers: VolumeMovers = { elements: new Set(), anySetter: false, unknown: false };
for (const tween of timeline.getChildren(true, true, false)) {
const targets: unknown[] = tween.targets?.() ?? [];
for (const target of targets) {
if (isDomNode(target)) {
if (namesKey(tween.vars, (key) => key === "volume")) movers.elements.add(target);
} else if (!movers.anySetter && namesKey(tween.vars, (key) => hasSetter(target, key))) {
movers.anySetter = true;
}
}
}
return movers;
} catch {
return unknown;
}
}
const canMoveVolume = (movers: VolumeMovers, el: HTMLMediaElement): boolean =>
movers.unknown || movers.anySetter || movers.elements.has(el);
/**
* Probe a media element and, if volume automation is detected, store a
* NORMALISED envelope in `cache`. Safe to call with a null timeline — returns
* early.
*
* `probeElementVolumeKeyframes` stamps each keyframe with the timeline seek
* time it was sampled at. Everything downstream of this cache — like the
* renderer's PCM baker — indexes an envelope by track-relative seconds, so the
* rebase belongs here, at the one point that fills the cache, rather than at
* each read.
*/
export function probeAndCacheElementVolume(
mediaEl: HTMLMediaElement,
timeline: RuntimeTimelineRef | null | undefined,
compositionDuration: number,
cache: WeakMap<HTMLMediaElement, VolumeKeyframe[]>,
options: VolumeProbeOptions = {},
): void {
if (options.allowLiveTimelineSeek === false) return;
if (!timeline) return;
if (!isMediaElement(mediaEl)) return;
if (compositionDuration <= 0) return;
// Sampling is ~60 whole-timeline seeks per second of clip, paid again on every rebind.
if (!canMoveVolume(options.movers?.() ?? volumeMoversOf(timeline), mediaEl)) return;
const seekFn = (t: number) => {
try {
if (typeof timeline.totalTime === "function") {
timeline.totalTime(t, true);
} else if (typeof timeline.seek === "function") {
timeline.seek(t, true);
}
} catch {
// ignore seek failures during probe
}
};
// Sampling seeks the live timeline through the entire composition. Preserve
// its playhead so the probe cannot perturb the first rendered frame (or any
// user scrub in the preview).
const originalTime =
typeof timeline.totalTime === "function"
? Number(timeline.totalTime())
: typeof timeline.seek === "function"
? Number(timeline.seek())
: 0;
const probeWindow = resolveVolumeProbeWindow(mediaEl, compositionDuration);
const keyframes = probeKeyframesInWindow(mediaEl, seekFn, compositionDuration, 60, probeWindow);
if (Number.isFinite(originalTime)) seekFn(originalTime);
if (keyframes) {
const envelope = normaliseEnvelope(keyframes, probeWindow.start, probeWindow.staticVolume);
if (envelope.length > 0) cache.set(mediaEl, envelope);
}
}