Skip to content

Code-first video & image, rendered natively on WebGPU.

14.7 KB
Raw Blame
/// <reference lib="dom" />
/// <reference types="webgpu" />
/**
 * The browser side of the preview: runs the project's composition code and
 * renders its frames with WebGPU straight onto the page's canvas. Part of the
 * browser engine (bundle.ts); the project's bundle calls `startPlayer`.
 */

import { BUILTIN_NODE_RENDERERS } from "framefields-preview:renderers";
import {
	compositionStateStore,
	drawCompositionTree,
} from "@framefields/compositions";
import { updateClockSignals, type VirtualMediaData } from "@framefields/core";
import { preloadCompositionFonts } from "@framefields/renderers";
import {
	type OrtWebModule,
	setDefaultSessionProvider,
	WebGPUProvider,
} from "@framefields/vision/web";
import {
	BrowserSurfaceProvider,
	ensureDevice,
	getRenderer2D,
	lutStore,
	type RenderContextValue,
	SlugFontCache,
} from "@framefields/webgpu-renderers";
import {
	type AudioTarget,
	createAudioMixer,
	type PreviewMix,
} from "./audio-mixer.js";
import { Blitter } from "./blit.js";
import { type BuiltinRenderer, registerRenderers } from "./register.js";

export { startAudioMixer } from "./audio-mixer.js";
export type { BuiltinRenderer, PreviewMix };

export interface PlayerMeta {
	width: number;
	height: number;
	fps: number;
	frameCount: number;
	durationMs: number;
}

export interface PreviewPlayer {
	meta: PlayerMeta;
	canvas: HTMLCanvasElement;
	/**
	 * Draws a frame. Calls made while one is drawing collapse to the latest.
	 * `cache: false` leaves the drawn frame out of the step/scrub cache: for
	 * real-time playback, which draws every frame once and would otherwise pay
	 * a texture and a copy per frame.
	 */
	render(frame: number, options?: { cache?: boolean }): Promise<number>;
	/** Milliseconds the last frame took to draw. */
	lastRenderMs: number;
	/** True when the program runs vision models (worth analysing ahead). */
	hasVision: boolean;
	/**
	 * Draws a frame off-screen so per-frame work (vision analysis) is cached
	 * before it plays. Skipped (resolves false) while a frame is being drawn.
	 */
	warm(frame: number): Promise<boolean>;
	/**
	 * Mixes [startSec, endSec) of the soundtrack here, with the export's
	 * mixer: the same decoders (reading only the stretch it plays, so pinned
	 * assets stream from their bucket), audio effects on this page's GPU and
	 * signals, on `chain`'s effect state (see `createAudioMixer`). The page
	 * mixes the blocks around its playhead as it plays.
	 */
	mixAudio(
		range: { startSec: number; endSec: number },
		chain?: string,
	): Promise<PreviewMix>;
	/** A frame at full size as an image, e.g. to attach to a note. */
	snapshot(frame: number): Promise<ImageBitmap>;
	/**
	 * Stops drawing and frees the player's textures (its frame cache among
	 * them). The GPU device stays: players share one, as the text, shader and
	 * LUT caches they draw with are tied to it. `startPlayer` disposes the
	 * player it replaces.
	 */
	dispose(): Promise<void>;
}

type Renderable = AudioTarget & { fps?: number };

declare global {
	interface Window {
		framefieldsPlayer?: Promise<PreviewPlayer>;
	}
}

export interface PlayerSettings {
	/** Where cached vision models are (served by the preview server). */
	modelsDir: string;
	/** URL of an onnxruntime-web `dist/` folder, for vision nodes. */
	ortBase: string;
}

/**
 * Builds the module's composition and makes it the page's player
 * (`window.framefieldsPlayer`). Called again, e.g. with new props, it
 * replaces the player on the same canvas and GPU device.
 */
export function startPlayer(
	mod: Record<string, unknown>,
	exportName: string,
	settings: PlayerSettings,
): void {
	process.env.FRAMEFIELDS_MODELS_DIR = settings.modelsDir;
	// Vision nodes create their runners themselves; here they run on WebGPU,
	// with onnxruntime-web loaded on first use.
	setDefaultSessionProvider(
		() =>
			new WebGPUProvider({
				loader: async () => {
					const ort = await import(
						/* @vite-ignore */ `${settings.ortBase}ort.webgpu.bundle.min.mjs`
					);
					ort.env.wasm.wasmPaths = settings.ortBase;
					ort.env.logLevel = "error";
					// Threads need the wasm on this origin; a CDN build runs on one.
					if (!settings.ortBase.startsWith("/")) ort.env.wasm.numThreads = 1;
					return ort as OrtWebModule;
				},
			}),
	);
	const previous = window.framefieldsPlayer;
	window.framefieldsPlayer = (async () => {
		await previous?.then(
			(p) => p.dispose(),
			() => {},
		);
		return createPlayer(mod, exportName, BUILTIN_NODE_RENDERERS);
	})();
	// The page listens for this; a failure is shown there.
	window.dispatchEvent(new Event("framefields:player"));
}

async function createPlayer(
	mod: Record<string, unknown>,
	exportName: string,
	renderers: readonly BuiltinRenderer[],
): Promise<PreviewPlayer> {
	if (!("gpu" in navigator)) {
		throw new Error(
			"This browser has no WebGPU. Use a current Chrome, Edge or Safari.",
		);
	}
	await registerRenderers(renderers);

	const exported = mod[exportName];
	if (exported === undefined) {
		throw new Error(`The preview entry has no export named "${exportName}".`);
	}
	const target = (
		typeof exported === "function"
			? await (exported as () => unknown)()
			: exported
	) as Renderable;
	const vm = target.toVirtualMediaAsync
		? await target.toVirtualMediaAsync()
		: target.toVirtualMedia?.();
	if (!vm) throw new Error(`"${exportName}" is not a composition.`);

	const width = Math.round(vm.metadata?.width ?? 1920);
	const height = Math.round(vm.metadata?.height ?? 1080);
	const fps = target.fps ?? (vm.operation as { fps?: number })?.fps ?? 30;
	const durationMs = vm.metadata?.durationMs ?? 0;
	const meta: PlayerMeta = {
		width,
		height,
		fps,
		durationMs,
		frameCount: Math.max(1, Math.round((durationMs / 1000) * fps)),
	};
	const hasVision = containsOp(vm as VirtualMediaData, "Vision");
	// As the export decides it, so text entrances play the same.
	const isVideoMode = vm.operation?.dataType === "Video";

	// One device for every player on the page: glyph, shader and LUT caches
	// are global and keep the GPU objects of the device that made them.
	const device = await ensureDevice();
	const canvas =
		(document.getElementById("canvas") as HTMLCanvasElement | null) ??
		document.createElement("canvas");
	canvas.width = width % 2 ? width + 1 : width;
	canvas.height = height % 2 ? height + 1 : height;
	const surface = new BrowserSurfaceProvider(device, canvas);
	await preloadCompositionFonts(vm, device);

	const renderId = `preview-${Math.random().toString(36).slice(2)}`;
	let first = true;
	let drawing: Promise<void> | null = null;
	let wanted: number | null = null;
	let wantedCached = true;
	let waiters: Array<(f: number) => void> = [];
	let disposed = false;

	let offscreen: GPUTexture | undefined;

	// Frames drawn on screen are kept so stepping or scrubbing back to one is a
	// blit, not a re-render. Full resolution, bounded by a byte budget. Vision
	// compositions are excluded: their frames depend on the one before, so a
	// hit would skip that work.
	const renderCache = new Map<number, { tex: GPUTexture; used: number }>();
	const renderCacheMax = Math.max(
		1,
		Math.min(
			240,
			Math.floor(
				(128 * 1024 * 1024) / Math.max(1, canvas.width * canvas.height * 4),
			),
		),
	);
	let renderCacheClock = 0;

	function newRenderCacheTexture(): GPUTexture {
		return device.createTexture({
			size: [canvas.width, canvas.height],
			format: surface.colorFormat,
			usage:
				GPUTextureUsage.RENDER_ATTACHMENT |
				GPUTextureUsage.TEXTURE_BINDING |
				GPUTextureUsage.COPY_DST,
			label: "preview_frame_cache",
		});
	}

	function takeRenderCache(frame: number): GPUTexture | undefined {
		if (hasVision) return undefined;
		const hit = renderCache.get(frame);
		if (!hit) return undefined;
		hit.used = ++renderCacheClock;
		return hit.tex;
	}

	function storeRenderCache(frame: number, tex: GPUTexture): void {
		if (hasVision) {
			tex.destroy();
			return;
		}
		const existing = renderCache.get(frame);
		if (existing) {
			if (existing.tex !== tex) existing.tex.destroy();
			renderCache.delete(frame);
		}
		renderCache.set(frame, { tex, used: ++renderCacheClock });
		while (renderCache.size > renderCacheMax) {
			let oldestKey: number | undefined;
			let oldestUsed = Number.POSITIVE_INFINITY;
			for (const [key, entry] of renderCache) {
				if (entry.used < oldestUsed) {
					oldestUsed = entry.used;
					oldestKey = key;
				}
			}
			if (oldestKey === undefined) break;
			const evicted = renderCache.get(oldestKey);
			renderCache.delete(oldestKey);
			evicted?.tex.destroy();
		}
	}

	async function draw(
		frame: number,
		toScreen = true,
		cache = toScreen,
	): Promise<void> {
		const renderer = getRenderer2D(device, surface.colorFormat);
		const ctx: RenderContextValue = { device, renderer, surface };
		const w = surface.width;
		const h = surface.height;
		const props = {
			frame,
			compositionFrame: frame,
			fps,
			isHeadless: false,
			renderId,
			virtualMedia: vm as VirtualMediaData,
			containerWidth: w,
			containerHeight: h,
			isVideoMode,
			// A drawn frame can be cached, so it waits for its LUTs
			// and exact video frame as the export does, never a stand-in.
			forceWait: true,
		};
		updateClockSignals(frame, fps, durationMs || undefined);

		// Shader, LUT and glyph uploads finish on a pass whose output is dropped.
		if (first || lutStore.hasPending(device) || SlugFontCache.hasPending()) {
			first = false;
			const pre = device.createCommandEncoder();
			const tex = renderer.getTemporaryTexture(w, h);
			const view = tex.createView();
			renderer
				.beginFrame(pre, view, { r: 0, g: 0, b: 0, a: 0 }, w, h, "clear")
				.end();
			await drawCompositionTree(
				ctx,
				pre,
				view,
				tex,
				w,
				h,
				vm as VirtualMediaData,
				props,
			);
			device.queue.submit([pre.finish()]);
			await lutStore.awaitAllPending(device);
		}

		compositionStateStore.setState(renderId, frame, fps, true);
		const encoder = device.createCommandEncoder();
		if (!toScreen && !offscreen) {
			offscreen = device.createTexture({
				size: [w, h],
				format: surface.colorFormat,
				usage:
					GPUTextureUsage.RENDER_ATTACHMENT |
					GPUTextureUsage.TEXTURE_BINDING |
					GPUTextureUsage.COPY_SRC,
				label: "preview_warm_target",
			});
		}
		const tex =
			toScreen || !offscreen ? surface.getCurrentTexture() : offscreen;
		const view = tex.createView();
		renderer
			.beginFrame(encoder, view, { r: 0, g: 0, b: 0, a: 0 }, w, h, "clear")
			.end();
		await drawCompositionTree(
			ctx,
			encoder,
			view,
			tex,
			w,
			h,
			vm as VirtualMediaData,
			props,
		);
		let cacheTex: GPUTexture | undefined;
		if (cache) {
			cacheTex = newRenderCacheTexture();
			encoder.copyTextureToTexture({ texture: tex }, { texture: cacheTex }, [
				w,
				h,
			]);
		}
		device.queue.submit([encoder.finish()]);
		if (toScreen) surface.present();
		await device.queue.onSubmittedWorkDone();
		if (cacheTex) storeRenderCache(frame, cacheTex);
	}

	// One frame at a time. Requests made while a frame draws collapse to the
	// latest; warm-ups only run when nothing else is drawing.
	function pump(): void {
		if (disposed || drawing || wanted === null) return;
		// `drawing` is set before the loop runs: a frame from the cache is drawn
		// without awaiting, so the loop can finish (and clear it) synchronously.
		let finished!: () => void;
		drawing = new Promise<void>((resolve) => {
			finished = resolve;
		});
		void (async () => {
			while (wanted !== null) {
				const f = wanted;
				const cache = wantedCached;
				wanted = null;
				const started = performance.now();
				try {
					const cached = takeRenderCache(f);
					if (cached) {
						blitter.blit(cached, surface.getCurrentTexture());
						surface.present();
					} else {
						await draw(f, true, cache);
					}
				} catch (err) {
					console.error("[framefields] frame", f, err);
				}
				player.lastRenderMs = performance.now() - started;
				if (wanted === null) {
					const ready = waiters;
					waiters = [];
					for (const resolve of ready) resolve(f);
				}
			}
			drawing = null;
			finished();
		})();
	}

	const mixAudio = createAudioMixer(target, fps, device, `${renderId}-audio`);

	const blitter = new Blitter(device, surface.colorFormat);

	const player: PreviewPlayer = {
		meta,
		canvas,
		lastRenderMs: 0,
		hasVision,
		render(frame, options) {
			if (disposed) return Promise.resolve(frame);
			wanted = frame;
			wantedCached = options?.cache ?? true;
			const done = new Promise<number>((resolve) => waiters.push(resolve));
			pump();
			return done;
		},
		async warm(frame) {
			if (disposed || drawing || wanted !== null) return false;
			drawing = draw(frame, false).catch((err: unknown) =>
				console.warn("[framefields] warm-up of frame", frame, err),
			);
			await drawing;
			drawing = null;
			pump();
			return true;
		},
		mixAudio,
		async snapshot(frame) {
			if (disposed) throw new Error("the player was disposed");
			while (drawing) await drawing;
			const done = (async () => {
				await draw(frame, false);
				if (!offscreen) throw new Error("no frame to capture");
				return readTexture(device, offscreen);
			})();
			drawing = done.then(
				() => {},
				() => {},
			);
			try {
				return await done;
			} finally {
				drawing = null;
				pump();
			}
		},
		async dispose() {
			if (disposed) return;
			disposed = true;
			wanted = null;
			while (drawing) await drawing;
			for (const { tex } of renderCache.values()) tex.destroy();
			renderCache.clear();
			offscreen?.destroy();
			offscreen = undefined;
			surface.destroy();
		},
	};
	return player;
}

/** Reads a texture back as an image (RGBA or BGRA, as the canvas format is). */
async function readTexture(
	device: GPUDevice,
	texture: GPUTexture,
): Promise<ImageBitmap> {
	const { width, height } = texture;
	const bytesPerRow = Math.ceil((width * 4) / 256) * 256;
	const buffer = device.createBuffer({
		size: bytesPerRow * height,
		usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
	});
	const encoder = device.createCommandEncoder();
	encoder.copyTextureToBuffer(
		{ texture },
		{ buffer, bytesPerRow },
		{ width, height },
	);
	device.queue.submit([encoder.finish()]);
	await buffer.mapAsync(GPUMapMode.READ);
	const src = new Uint8Array(buffer.getMappedRange());
	const pixels = new Uint8ClampedArray(width * height * 4);
	const bgra = texture.format.startsWith("bgra");
	for (let y = 0; y < height; y++) {
		const row = src.subarray(y * bytesPerRow, y * bytesPerRow + width * 4);
		pixels.set(row, y * width * 4);
	}
	buffer.unmap();
	buffer.destroy();
	for (let i = 0; i < pixels.length; i += 4) {
		if (bgra) {
			const b = pixels[i];
			pixels[i] = pixels[i + 2];
			pixels[i + 2] = b;
		}
		pixels[i + 3] = 255;
	}
	return createImageBitmap(new ImageData(pixels, width, height));
}

function containsOp(node: VirtualMediaData, op: string): boolean {
	if ((node.operation as { op?: string } | undefined)?.op === op) return true;
	return (node.children ?? []).some((child) => containsOp(child, op));
}