Skip to content

Code-first video & image, rendered natively on WebGPU.

3.7 KB
Raw Blame
import { keyedGain } from "@framefields/compositions";
import type { LayerAnimation } from "@framefields/compositions/program";
import type { VirtualMediaData } from "@framefields/core";
import {
	type AudioProcessor,
	audioRegistry,
	resolveBoolean,
	resolveNumber,
} from "@framefields/node-sdk";
import { compileLayerTimeline } from "../shared/compiler.js";

interface CompositorOperation {
	op: "Compositor" | "CompositorLayer";
	id?: string;
	inputHandleId?: string;
	volume?: number;
	muted?: boolean;
	animation?: LayerAnimation;
}

export const compositorLayerAudioProcessor: AudioProcessor = async (
	channels: Float32Array[],
	sampleRate: number,
	virtualMedia: VirtualMediaData,
	ctx,
) => {
	const op = virtualMedia.operation as unknown as CompositorOperation;
	if (!op) return;

	const numChannels = channels.length;
	if (numChannels === 0) return;
	const numSamples = channels[0].length;
	if (numSamples === 0) return;

	// Master gain (L1): the audio extractor dispatches this processor for the
	// root Compositor op too (registered under the node type "Compositor").
	// The root's channels ARE the fully-mixed composition, so scale them
	// once by the program-level `volume` — the per-layer path below handles
	// each CompositorLayer op's own volume/mute animation.
	if (op.op === "Compositor") {
		const master = resolveNumber(op.volume, 1);
		if (master === 1) return;
		for (let c = 0; c < numChannels; c++) {
			const ch = channels[c];
			for (let i = 0; i < ch.length; i++) ch[i] *= master;
		}
		return;
	}

	const volume = resolveNumber(op.volume, 1);
	const muted = resolveBoolean(op.muted, false);
	const animation = op.animation;
	const fps = ctx?.fps ?? 24;

	const keyed = keyedGain(op);
	if (!keyed.volume && !keyed.muted) {
		// Nothing keyed: the mix already applied the resting volume and mute.
		return;
	}

	// Compile or retrieve the GSAP timeline for this layer
	const { tl, stub } = compileLayerTimeline(
		op.id || op.inputHandleId || "layer",
		animation,
		volume,
		muted,
		fps,
	);

	const elapsedMs = ctx?.elapsedMs ?? 0;
	const startTimeSec = elapsedMs / 1000;

	// Process in sub-chunks of 128 samples to be fast and smooth
	const subChunkSize = 128;

	// Seek to the starting time to establish baseline values
	tl.seek(startTimeSec);
	let lastVol = stub.volume;
	let lastMuted = stub.muted;

	// What the mix already applied: the resting values of the props this
	// layer does not key (see `keyedGain`). The keyed ones are applied here.
	const applied = (keyed.volume ? 1 : volume) * (!keyed.muted && muted ? 0 : 1);

	for (let i = 0; i < numSamples; i += subChunkSize) {
		const nextI = Math.min(numSamples, i + subChunkSize);
		const tNext = startTimeSec + nextI / sampleRate;
		tl.seek(tNext);
		const nextVol = stub.volume;
		const nextMuted = stub.muted;

		for (let j = i; j < nextI; j++) {
			const p = (j - i) / (nextI - i);
			// L3: clamp — keyframes are schema-bounded to 0–1, but a lerp
			// between in-range values can overshoot with custom eases; the
			// relativeGain compensation below must never amplify past the
			// channel's own scale.
			const currentVol = Math.min(
				1,
				Math.max(0, lastVol + (nextVol - lastVol) * p),
			);
			// For muted, if it changes mid-sub-chunk, toggle it at the halfway point
			const currentMuted = j - i < (nextI - i) / 2 ? lastMuted : nextMuted;

			const finalGain = currentMuted ? 0 : currentVol;
			const relativeGain = applied === 0 ? 0 : finalGain / applied;

			if (relativeGain !== 1) {
				for (let c = 0; c < numChannels; c++) {
					channels[c][j] *= relativeGain;
				}
			}
		}

		lastVol = nextVol;
		lastMuted = nextMuted;
	}
};

// Register immediately on load
audioRegistry.register("CompositorLayer", compositorLayerAudioProcessor);