Skip to content

Code-first video & image, rendered natively on WebGPU.

7.4 KB
Raw Blame
/**
 * What each 3D draw looked like one frame earlier, for shutter motion blur.
 * A scene records every mesh and plane (by a key stable across frames) and
 * its camera as it renders frame f; frame f + 1 reads them back. Instance
 * records and deformed vertices (morph targets, audio deformation) are
 * copied on the GPU, where they are made, into a pair of buffers by frame
 * parity so rendering a frame again still finds f - 1.
 * A frame whose predecessor is not recorded (a seek) renders f - 1 first
 * (`needsPrevious`), so every frame comes out the same in any order.
 */
import type {
	Camera3DPose,
	Mat4,
	MeshMotionPrev,
	MeshWind,
	SlugGlyphBatch,
} from "@framefields/webgpu-renderers";

export interface MeshMotionState {
	kind: "mesh";
	modelMatrix: Mat4;
	skinning?: Float32Array;
	/** Byte size of the instance records copied for this frame. */
	instanceBytes?: number;
	/** Byte size of the deformed vertices copied for this frame. */
	vertexBytes?: number;
	windTime?: number;
	crowdTime?: number;
}

export interface QuadMotionState {
	kind: "quad";
	modelMatrix: Mat4;
}

/** A text plane: its matrix and its glyph batches (records copied). */
export interface SlugMotionState {
	kind: "slug";
	modelMatrix: Mat4;
	batches: SlugGlyphBatch[];
}

type MotionState = MeshMotionState | QuadMotionState | SlugMotionState;

/** Two frames of one value: the latest and the one before it. */
class Slots<T> {
	frame = Number.NaN;
	value?: T;
	prevFrame = Number.NaN;
	prevValue?: T;

	write(frame: number, value: T): void {
		if (frame !== this.frame) {
			this.prevFrame = this.frame;
			this.prevValue = this.value;
		}
		this.frame = frame;
		this.value = value;
	}

	/** The value at `frame`, if it is one of the two held. */
	at(frame: number): T | undefined {
		if (frame === this.frame) return this.value;
		if (frame === this.prevFrame) return this.prevValue;
		return undefined;
	}
}

export class MotionHistory {
	private readonly states = new Map<string, Slots<MotionState>>();
	private readonly instances: GpuPairs;
	private readonly vertices: GpuPairs;
	private readonly camera = new Slots<Camera3DPose>();
	/** Frames recorded, latest last (two kept). */
	private readonly frames = new Slots<true>();

	constructor(device: GPUDevice) {
		this.instances = new GpuPairs(device, "Instances");
		this.vertices = new GpuPairs(device, "Vertices");
	}

	/** Whether rendering `frame` must render `frame - 1` first to know its motion. */
	needsPrevious(frame: number): boolean {
		return frame > 0 && this.frames.at(frame - 1) === undefined;
	}

	previousCamera(frame: number): Camera3DPose | undefined {
		return this.camera.at(frame - 1);
	}

	/** A mesh's state at `frame - 1`, shaped for the velocity pass. */
	previousMesh(
		key: string,
		frame: number,
		now: {
			skinning?: Float32Array;
			instanceBuffer?: GPUBuffer;
			/** Byte size of this frame's deformed vertices, when the GPU deforms the mesh. */
			vertexBytes?: number;
			wind?: MeshWind;
			crowd?: MeshMotionPrev["crowd"];
		},
	): MeshMotionPrev | undefined {
		const prev = this.states.get(key)?.at(frame - 1);
		if (prev?.kind !== "mesh") return undefined;
		const out: MeshMotionPrev = { modelMatrix: prev.modelMatrix };
		// A mesh that changed shape (other joints, other instance count) has
		// no comparable past: it moves by its matrix alone.
		if (now.skinning && prev.skinning?.length === now.skinning.length) {
			out.skinningMatrices = prev.skinning;
		}
		const copy = this.instances.at(key, frame - 1);
		if (
			now.instanceBuffer &&
			copy &&
			prev.instanceBytes === now.instanceBuffer.size
		) {
			out.instanceBuffer = copy;
		}
		const vertices = this.vertices.at(key, frame - 1);
		if (now.vertexBytes && vertices && prev.vertexBytes === now.vertexBytes) {
			out.vertexBuffer = vertices;
		}
		if (now.wind && prev.windTime !== undefined) {
			out.wind = { ...now.wind, time: prev.windTime };
		}
		if (now.crowd && prev.crowdTime !== undefined) {
			out.crowd = { ...now.crowd, time: prev.crowdTime };
		}
		return out;
	}

	previousQuad(key: string, frame: number): Mat4 | undefined {
		const prev = this.states.get(key)?.at(frame - 1);
		return prev?.kind === "quad" ? prev.modelMatrix : undefined;
	}

	/**
	 * Records a mesh at `frame`; its instance records and deformed vertices
	 * (when the GPU deforms it) are copied in `encoder`.
	 */
	recordMesh(
		encoder: GPUCommandEncoder,
		key: string,
		frame: number,
		state: Omit<MeshMotionState, "kind" | "instanceBytes" | "vertexBytes">,
		buffers: {
			instanceBuffer?: GPUBuffer;
			/** Deformed vertices and their byte size (the buffer may be pooled and larger). */
			vertices?: { buffer: GPUBuffer; bytes: number };
		},
	): void {
		const { instanceBuffer, vertices } = buffers;
		if (instanceBuffer) {
			this.instances.copy(
				encoder,
				key,
				frame,
				instanceBuffer,
				instanceBuffer.size,
				GPUBufferUsage.STORAGE,
			);
		}
		if (vertices) {
			this.vertices.copy(
				encoder,
				key,
				frame,
				vertices.buffer,
				vertices.bytes,
				GPUBufferUsage.VERTEX,
			);
		}
		this.slots(key).write(frame, {
			kind: "mesh",
			...state,
			...(state.skinning && { skinning: state.skinning.slice() }),
			instanceBytes: instanceBuffer?.size,
			vertexBytes: vertices?.bytes,
		});
	}

	previousSlug(
		key: string,
		frame: number,
	): Omit<SlugMotionState, "kind"> | undefined {
		const prev = this.states.get(key)?.at(frame - 1);
		return prev?.kind === "slug" ? prev : undefined;
	}

	/** Records a text plane at `frame`; glyph records are copied (batches are rebuilt each frame). */
	recordSlug(
		key: string,
		frame: number,
		modelMatrix: Mat4,
		batches: SlugGlyphBatch[],
	): void {
		this.slots(key).write(frame, {
			kind: "slug",
			modelMatrix,
			batches: batches.map((b) => ({
				...b,
				instances: b.instances.slice(0, b.instanceCount * 24),
			})),
		});
	}

	recordQuad(key: string, frame: number, modelMatrix: Mat4): void {
		this.slots(key).write(frame, { kind: "quad", modelMatrix });
	}

	recordFrame(frame: number, camera: Camera3DPose): void {
		this.camera.write(frame, camera);
		this.frames.write(frame, true);
	}

	private slots(key: string): Slots<MotionState> {
		let s = this.states.get(key);
		if (!s) {
			s = new Slots();
			this.states.set(key, s);
		}
		return s;
	}
}

/** GPU copies of a buffer per key, one per frame parity. */
class GpuPairs {
	private readonly pairs = new Map<string, [GPUBuffer?, GPUBuffer?]>();

	constructor(
		private readonly device: GPUDevice,
		private readonly label: string,
	) {}

	at(key: string, frame: number): GPUBuffer | undefined {
		return this.pairs.get(key)?.[frame & 1];
	}

	copy(
		encoder: GPUCommandEncoder,
		key: string,
		frame: number,
		source: GPUBuffer,
		bytes: number,
		usage: GPUBufferUsageFlags,
	): void {
		const pair = this.pairs.get(key) ?? [];
		const slot = frame & 1;
		let copy = pair[slot];
		if (!copy || copy.size !== bytes) {
			copy?.destroy();
			copy = this.device.createBuffer({
				label: `MotionHistory${this.label}:${key}`,
				size: bytes,
				usage: usage | GPUBufferUsage.COPY_DST,
			});
			pair[slot] = copy;
			this.pairs.set(key, pair);
		}
		encoder.copyBufferToBuffer(source, 0, copy, 0, bytes);
	}
}

const histories = new Map<string, MotionHistory>();

/** The motion history of one composition render (by render id). */
export function motionHistory(
	device: GPUDevice,
	renderId: string,
): MotionHistory {
	let h = histories.get(renderId);
	if (!h) {
		if (histories.size > 64) histories.clear();
		h = new MotionHistory(device);
		histories.set(renderId, h);
	}
	return h;
}