Skip to content

Code-first video & image, rendered natively on WebGPU.

13.1 KB
Raw Blame
/// <reference types="@webgpu/types" />
import type {
	MediaOperation,
	NodeInputConnection,
	NodeOperation,
	VirtualMediaData,
} from "@framefields/core";
import type { RenderContextValue } from "@framefields/webgpu-renderers";

export type GPUCommandEncoder = globalThis.GPUCommandEncoder;
export type GPURenderPassEncoder = globalThis.GPURenderPassEncoder;
export type GPUTextureView = globalThis.GPUTextureView;
export type GPUTexture = globalThis.GPUTexture;

export type { MediaOperation, NodeInputConnection, NodeOperation };

export interface NodeRenderProps<
	TOperation extends MediaOperation = MediaOperation,
> {
	renderId: string;
	virtualMedia: VirtualMediaData<TOperation>;
	volume?: number;
	playbackRateOverride?: number;
	trimStartOverride?: number;
	trimEndOverride?: number;
	containerWidth: number;
	containerHeight: number;
	/**
	 * Target pixels per layout pixel, when a compositor runs an effect on a
	 * layer rasterized above layout size so it stays sharp scaled up; unset
	 * means 1. Only renderers that declare `maxPixelScale` get more than 1.
	 */
	pixelScale?: number;
	opacity?: number;
	frame?: number;
	compositionFrame?: number;
	/** Absolute source time for a remapped media leaf; effect clocks stay unchanged. */
	sourceTimeSec?: number;
	fps?: number;
	elapsedMs?: number;
	durationMs?: number;
	timestampSec?: number;
	renderingContext?: "visual" | "audio";
	inheritedSeekOffset?: number;
	inheritedClockOffset?: number;
	isHeadless?: boolean;
	forceWait?: boolean;
	isVideoMode?: boolean;
	isPlaying?: boolean;
	excludeTextures?: GPUTexture[];
	/**
	 * Render only to record the 3D scene's motion (shutter motion blur reads
	 * every draw one frame earlier); nothing drawn is shown.
	 */
	motionHistoryOnly?: boolean;
	/** Signal values that override the node's own (text animators read them). */
	signals?: Record<string, unknown>;
	renderChild?: (
		child: VirtualMediaData,
		overrides?: Partial<NodeRenderProps>,
	) => unknown;
}

/**
 * How far an effect draws past the pixels it is given, px, for one of its
 * operations; `null` when it isn't local (it reads or writes pixels anywhere,
 * like a mirror or a portal) or can't tell (a size driven by a signal).
 */
export type EffectReach = (operation: Record<string, unknown>) => number | null;

/**
 * How many target pixels per layout pixel (`props.pixelScale`) an effect can
 * run at and still look as it does at layout size, for one of its operations:
 * it reads its pixel-unit parameters (a blur radius, a shadow's distance) in
 * layout px and multiplies them by `pixelScale`, up to its own kernel limits.
 * `Infinity` when nothing in it is measured in pixels.
 */
export type EffectMaxPixelScale = (
	operation: Record<string, unknown>,
) => number;

/**
 * A node's GPU renderer. An effect renderer may carry `reach`: a compositor
 * then runs it over the layer's own region plus that margin instead of the
 * whole target. Without it the effect always gets the whole target. It may
 * also carry `maxPixelScale`: a compositor may then rasterize a layer it
 * styles above layout size (a layer scaled up stays sharp). Without it such a
 * layer keeps its layout size.
 */
export type WebGPUNodeRenderer<
	TOperation extends MediaOperation = MediaOperation,
> = WebGPUNodeRenderFn<TOperation> & {
	reach?: EffectReach;
	maxPixelScale?: EffectMaxPixelScale;
};

type WebGPUNodeRenderFn<TOperation extends MediaOperation> = (args: {
	ctx: RenderContextValue;
	encoder: GPUCommandEncoder;
	pass: GPURenderPassEncoder;
	targetView: GPUTextureView;
	targetTexture: GPUTexture;
	targetWidth: number;
	targetHeight: number;
	props: NodeRenderProps<TOperation>;
	drawChild: (
		child: VirtualMediaData,
		overrides?: Partial<NodeRenderProps>,
		targetViewOverride?: GPUTextureView,
		targetTextureOverride?: GPUTexture,
		targetWidthOverride?: number,
		targetHeightOverride?: number,
	) => Promise<void>;
}) => Promise<void> | void;

export interface AudioProcessorContext {
	device?: GPUDevice;
	frame?: number;
	compositionFrame?: number;
	fps?: number;
	elapsedMs?: number;
	durationMs?: number;
	renderId?: string;
}

export type AudioProcessor<TOperation extends MediaOperation = MediaOperation> =
	(
		channels: Float32Array[],
		sampleRate: number,
		virtualMedia: VirtualMediaData<TOperation>,
		ctx?: AudioProcessorContext,
	) => void | Promise<void>;

export {
	isSignal,
	resolveBoolean,
	resolveNumber,
	resolveParam,
	resolveString,
} from "@framefields/core";

/**
 * A numeric node parameter at `frame`: a number, a signal-like `get(ctx)` or
 * `{ value }`, a `(ctx) => number` function, or anything `Number()` accepts;
 * `fallback` when none of those yields a finite number.
 */
export function resolveFrameNumber(
	val: unknown,
	fallback: number,
	frame: number,
	fps: number,
): number {
	if (val === undefined || val === null) return fallback;
	if (typeof val === "number") return Number.isFinite(val) ? val : fallback;
	if (typeof val === "object" && val !== null) {
		if (
			"get" in (val as Record<string, unknown>) &&
			typeof (val as { get: unknown }).get === "function"
		) {
			try {
				const res = (val as { get: (ctx?: unknown) => unknown }).get({
					frame,
					fps,
					time: fps > 0 ? frame / fps : 0,
					duration: 0,
					durationMs: 0,
					progress: 0,
					deltaTime: fps > 0 ? 1 / fps : 0,
				});
				const n = Number(res);
				if (Number.isFinite(n)) return n;
			} catch {
				// fallback
			}
		}
		if ("value" in (val as Record<string, unknown>)) {
			const n = Number((val as { value: unknown }).value);
			if (Number.isFinite(n)) return n;
		}
	}
	if (typeof val === "function") {
		try {
			const n = Number(val({ frame, fps, time: fps > 0 ? frame / fps : 0 }));
			if (Number.isFinite(n)) return n;
		} catch {
			// fallback
		}
	}
	const n = Number(val);
	return Number.isFinite(n) ? n : fallback;
}

/**
 * A static node parameter at `frame`: calls `get(ctx)` / `(frame, fps)`
 * functions and reads `{ value }` wrappers, keeping a signal's GPU texture
 * binding when it has one. Anything else is returned as-is.
 */
export function unwrapFrameParam(
	raw: unknown,
	frame: number,
	fps: number,
): { value: unknown; gpuTextureView?: GPUTextureView } {
	if (raw === undefined || raw === null) return { value: raw };
	if (typeof raw === "function") {
		return { value: (raw as (f: number, fps: number) => unknown)(frame, fps) };
	}
	if (typeof raw !== "object") return { value: raw };
	const obj = raw as Record<string, unknown>;
	if (typeof obj.get === "function") {
		return {
			value: (obj.get as (ctx: { frame: number; fps: number }) => unknown)({
				frame,
				fps,
			}),
		};
	}
	if ("value" in obj) {
		const sig = raw as {
			value: unknown;
			gpuBinding?: { textureView?: GPUTextureView };
		};
		return { value: sig.value, gpuTextureView: sig.gpuBinding?.textureView };
	}
	return { value: raw };
}

/** The `inputs` map node operations carry: one entry per connected handle. */
export type NodeOperationInputs = Record<
	string,
	{
		connectionValid: boolean;
		outputItem: { type: string; data: unknown } | null;
	}
>;

export interface HandleParam<SD> {
	val: number;
	/** The value comes from a signal (`sd`) sampled on the GPU. */
	hasStaticSig: boolean;
	sd: SD | null;
	handleId: string | null | undefined;
}

/**
 * A numeric parameter fed through `handleId`: a connected Number (clamped), a
 * connected Signal (sampled on the GPU, `val` stays `fallbackVal`), or else
 * `fallbackVal` clamped to [clampMin, clampMax].
 */
export function resolveHandleParam<SD>(
	inputs: NodeOperationInputs | undefined,
	handleId: string | null | undefined,
	fallbackVal: number,
	clampMin: number,
	clampMax: number,
): HandleParam<SD> {
	const clamp = (v: number) => Math.max(clampMin, Math.min(clampMax, v));
	const input = handleId ? inputs?.[handleId] : undefined;
	if (input?.connectionValid) {
		const outputItem = input.outputItem;
		if (outputItem?.type === "Number") {
			const numVal =
				typeof outputItem.data === "number"
					? outputItem.data
					: ((outputItem.data as { value?: number })?.value ?? fallbackVal);
			return { val: clamp(numVal), hasStaticSig: false, sd: null, handleId };
		}
		if (outputItem?.type === "Signal" && outputItem.data) {
			return {
				val: fallbackVal,
				hasStaticSig: true,
				sd: outputItem.data as SD,
				handleId,
			};
		}
	}
	return { val: clamp(fallbackVal), hasStaticSig: false, sd: null, handleId };
}

/**
 * A numeric field `fieldName` of a node operation, fed through its
 * `${fieldName}HandleId` input: a connected Signal/Numeric (`sd`, sampled on
 * the GPU; `value` is its offset), a connected Number, or the operation's own
 * value — each clamped to [minVal, maxVal].
 */
export function resolveSignalField<SD>(
	op: { inputs?: NodeOperationInputs },
	fieldName: string,
	defaultValue: number,
	minVal = Number.NEGATIVE_INFINITY,
	maxVal = Number.POSITIVE_INFINITY,
): { hasSignal: boolean; sd: SD | null; value: number } {
	const clamp = (v: number) => Math.max(minVal, Math.min(maxVal, v));
	const fields = op as Record<string, unknown>;
	const handleId = fields[`${fieldName}HandleId`] as string | null | undefined;
	const signalInput = handleId ? op.inputs?.[handleId] : null;
	const outputType = signalInput?.connectionValid
		? signalInput.outputItem?.type
		: undefined;
	const hasSignal = outputType === "Signal" || outputType === "Numeric";
	const sd =
		hasSignal && signalInput?.outputItem?.data
			? (signalInput.outputItem.data as SD)
			: null;

	let value = defaultValue;
	if (!hasSignal) {
		const raw =
			outputType === "Number"
				? signalInput?.outputItem?.data
				: fields[fieldName];
		value = clamp(Number(raw ?? defaultValue));
	} else if (sd) {
		value = clamp(Number((sd as { offset?: unknown }).offset ?? defaultValue));
	}
	return { hasSignal, sd, value };
}

/**
 * Whether the handle whose id is stored at `op[handleIdKey]` has a valid
 * Signal or Number connection.
 */
export function isHandleConnected(
	op: Record<string, unknown>,
	handleIdKey: string,
): boolean {
	const handleId = op[handleIdKey];
	if (typeof handleId !== "string" || !handleId) return false;
	const input = (op.inputs as NodeOperationInputs | undefined)?.[handleId];
	return Boolean(
		input?.connectionValid &&
			input.outputItem &&
			(input.outputItem.type === "Signal" ||
				input.outputItem.type === "Number"),
	);
}

export interface NodeRendererPlugin {
	WebGPURenderer?: WebGPUNodeRenderer;
	audioProcessor?: AudioProcessor;
}

export function defineRenderer(
	plugin: NodeRendererPlugin,
): Readonly<NodeRendererPlugin> {
	return Object.freeze(plugin);
}

const GLOBAL_WEBGPU_REGISTRY_KEY = Symbol.for("framefields.webgpuRegistry");

export class WebGPURegistry {
	private renderers = new Map<string, WebGPUNodeRenderer>();

	register(op: string, renderer: WebGPUNodeRenderer): void {
		this.renderers.set(op, renderer);
	}

	get(op: string): WebGPUNodeRenderer | undefined {
		return this.renderers.get(op);
	}

	has(op: string): boolean {
		return this.renderers.has(op);
	}
}

export const webgpuRegistry: WebGPURegistry = (() => {
	const g = globalThis as typeof globalThis & {
		[GLOBAL_WEBGPU_REGISTRY_KEY]?: WebGPURegistry;
	};
	if (!g[GLOBAL_WEBGPU_REGISTRY_KEY]) {
		g[GLOBAL_WEBGPU_REGISTRY_KEY] = new WebGPURegistry();
	}
	return g[GLOBAL_WEBGPU_REGISTRY_KEY];
})();

export function registerWebGPURenderer(
	op: string,
	renderer: WebGPUNodeRenderer,
): void {
	webgpuRegistry.register(op, renderer);
}

/**
 * How far an operation's effect draws past its pixels, from its registered
 * renderer; `null` when unknown: no renderer, no `reach`, or no answer.
 */
export function effectReach(operation: Record<string, unknown>): number | null {
	const op = operation.op;
	if (typeof op !== "string") return null;
	return webgpuRegistry.get(op)?.reach?.(operation) ?? null;
}

/**
 * How far above layout resolution an operation's effect may run, from its
 * registered renderer; 1 when it doesn't say.
 */
export function effectMaxPixelScale(
	operation: Record<string, unknown>,
): number {
	const op = operation.op;
	if (typeof op !== "string") return 1;
	return webgpuRegistry.get(op)?.maxPixelScale?.(operation) ?? 1;
}

export type { WebGPUNodeRendererArgs } from "./gpu.js";

export interface TextureSourceOp {
	op: "TextureSource";
	sourceTexture: GPUTexture;
	opacity?: number;
	dataType?: "Image";
}

registerWebGPURenderer("TextureSource", (args) => {
	const op = args.props.virtualMedia.operation as unknown as TextureSourceOp;
	if (op?.sourceTexture) {
		args.ctx.renderer.drawTexture(
			args.pass,
			op.sourceTexture,
			{ x: 0, y: 0, width: args.targetWidth, height: args.targetHeight },
			{ opacity: Number(op.opacity ?? 1) },
		);
	}
});

class AudioProcessorRegistry {
	private processors = new Map<string, AudioProcessor>();

	register(opType: string, processor: AudioProcessor): void {
		this.processors.set(opType, processor);
	}

	get(opType: string): AudioProcessor | null {
		return this.processors.get(opType) || null;
	}
}

const GLOBAL_AUDIO_KEY = Symbol.for("framefields.audioRegistry");
export const audioRegistry: AudioProcessorRegistry = (() => {
	const g = globalThis as typeof globalThis & {
		[GLOBAL_AUDIO_KEY]?: AudioProcessorRegistry;
	};
	if (!g[GLOBAL_AUDIO_KEY]) {
		g[GLOBAL_AUDIO_KEY] = new AudioProcessorRegistry();
	}
	return g[GLOBAL_AUDIO_KEY];
})();

export * from "./homography.js";