Skip to content

πŸŽ₯ Make videos programmatically with React

8.1 KB
Raw Blame
import React, {useCallback, useRef} from 'react';
import {
	AbsoluteFill,
	HtmlInCanvas,
	type HtmlInCanvasOnInit,
	type HtmlInCanvasOnPaint,
	useCurrentFrame,
	useVideoConfig,
} from 'remotion';

// Minimal WebGPU types β€” `@webgpu/types` is intentionally not a dependency,
// matching the convention in `packages/core/src/effects/gpu-device.ts`.
type Gpu = {
	requestAdapter(): Promise<GpuAdapter | null>;
	getPreferredCanvasFormat(): string;
};
type GpuAdapter = {requestDevice(): Promise<GpuDevice>};
type GpuTextureView = unknown;
type GpuTexture = {createView(): GpuTextureView; destroy(): void};
type GpuBuffer = {destroy(): void};
type GpuBindGroup = unknown;
type GpuPipeline = unknown;
type GpuSampler = unknown;
type GpuShaderModule = unknown;
type GpuDevice = {
	createShaderModule(d: {code: string}): GpuShaderModule;
	createRenderPipeline(d: unknown): GpuPipeline;
	createTexture(d: unknown): GpuTexture;
	createSampler(d?: unknown): GpuSampler;
	createBindGroup(d: unknown): GpuBindGroup;
	createBuffer(d: unknown): GpuBuffer;
	createCommandEncoder(): {
		beginRenderPass(d: unknown): {
			setPipeline(p: GpuPipeline): void;
			setBindGroup(i: number, b: GpuBindGroup): void;
			draw(n: number): void;
			end(): void;
		};
		finish(): unknown;
	};
	queue: {
		submit(c: unknown[]): void;
		writeBuffer(b: GpuBuffer, offset: number, data: BufferSource): void;
		drawElementImageToTexture?(
			source: {source: Element | ElementImage},
			destination: {
				texture: GpuTexture;
				size: {width: number; height: number};
			},
		): void;
		// Kept for Chrome builds predating drawElementImageToTexture.
		copyElementImageToTexture(
			source: Element | ElementImage,
			width: number,
			height: number,
			destination: {texture: GpuTexture},
		): void;
	};
};
type GpuCanvasContext = {
	configure(d: {
		device: GpuDevice;
		format: string;
		alphaMode: 'premultiplied' | 'opaque';
	}): void;
	getCurrentTexture(): GpuTexture;
};

const WGSL = /* wgsl */ `
struct VsOut {
  @builtin(position) pos: vec4f,
  @location(0) uv: vec2f,
};

@vertex
fn vs(@builtin(vertex_index) i: u32) -> VsOut {
  // Fullscreen triangle (slightly oversized β€” clipped to viewport).
  var p = array(vec2f(-1.0, -3.0), vec2f(-1.0, 1.0), vec2f(3.0, 1.0));
  var uv = array(vec2f(0.0, 2.0), vec2f(0.0, 0.0), vec2f(2.0, 0.0));
  var o: VsOut;
  o.pos = vec4f(p[i], 0.0, 1.0);
  o.uv = uv[i];
  return o;
}

struct U {
  time: f32,
  _pad: f32,
  resolution: vec2f,
};

@group(0) @binding(0) var samp: sampler;
@group(0) @binding(1) var tex: texture_2d<f32>;
@group(0) @binding(2) var<uniform> u: U;

@fragment
fn fs(in: VsOut) -> @location(0) vec4f {
  // Animate pixel cell size with a slow breathing motion.
  let cell = 6.0 + sin(u.time * 0.8) * 4.0;
  let snapped = floor(in.uv * u.resolution / cell) * cell / u.resolution;

  // Slight chromatic offset between channels β€” sampled from snapped centers.
  let off = vec2f(2.0, 0.0) / u.resolution;
  let r = textureSample(tex, samp, snapped + off).r;
  let g = textureSample(tex, samp, snapped).g;
  let b = textureSample(tex, samp, snapped - off).b;
  let a = textureSample(tex, samp, snapped).a;

  // Posterize to 5 levels per channel for a flatter, screenprint look.
  let levels = 5.0;
  let q = floor(vec3f(r, g, b) * levels) / (levels - 1.0);

  return vec4f(q, a);
}
`;

type GpuState = {
	device: GpuDevice;
	context: GpuCanvasContext;
	pipeline: GpuPipeline;
	sampler: GpuSampler;
	texture: GpuTexture;
	uniformBuffer: GpuBuffer;
	bindGroup: GpuBindGroup;
	width: number;
	height: number;
};

export const HtmlInCanvasComposeWebGPU: React.FC = () => {
	const frame = useCurrentFrame();
	const {width, height, fps} = useVideoConfig();
	const gpuRef = useRef<GpuState | null>(null);

	const time = frame / fps;

	const onInit: HtmlInCanvasOnInit = useCallback(async ({canvas}) => {
		if (typeof navigator === 'undefined' || !('gpu' in navigator)) {
			throw new Error('WebGPU is not available in this environment');
		}

		const gpu = (navigator as unknown as {gpu: Gpu}).gpu;
		const adapter = await gpu.requestAdapter();
		if (!adapter) {
			throw new Error('No WebGPU adapter available');
		}

		const device = await adapter.requestDevice();
		const context = (
			canvas as unknown as {
				getContext(id: 'webgpu'): GpuCanvasContext | null;
			}
		).getContext('webgpu');
		if (!context) {
			throw new Error('WebGPU context unavailable on OffscreenCanvas');
		}

		// Use the device's preferred swap-chain format (typically `bgra8unorm`)
		// to avoid an extra format-conversion copy on present.
		const presentationFormat = gpu.getPreferredCanvasFormat();

		context.configure({
			device,
			format: presentationFormat,
			alphaMode: 'premultiplied',
		});

		const module = device.createShaderModule({code: WGSL});

		const pipeline = device.createRenderPipeline({
			layout: 'auto',
			vertex: {module, entryPoint: 'vs'},
			fragment: {
				module,
				entryPoint: 'fs',
				targets: [{format: presentationFormat}],
			},
			primitive: {topology: 'triangle-list'},
		});

		const TextureUsage = (
			globalThis as unknown as {
				GPUTextureUsage: {
					COPY_DST: number;
					TEXTURE_BINDING: number;
					RENDER_ATTACHMENT: number;
				};
			}
		).GPUTextureUsage;
		const BufferUsage = (
			globalThis as unknown as {
				GPUBufferUsage: {UNIFORM: number; COPY_DST: number};
			}
		).GPUBufferUsage;

		const texture = device.createTexture({
			size: {width: canvas.width, height: canvas.height},
			format: 'rgba8unorm',
			usage:
				TextureUsage.COPY_DST |
				TextureUsage.TEXTURE_BINDING |
				TextureUsage.RENDER_ATTACHMENT,
		});

		const sampler = device.createSampler({
			magFilter: 'linear',
			minFilter: 'linear',
			addressModeU: 'clamp-to-edge',
			addressModeV: 'clamp-to-edge',
		});

		// 16 bytes: time (f32), pad (f32), resolution (vec2f).
		const uniformBuffer = device.createBuffer({
			size: 16,
			usage: BufferUsage.UNIFORM | BufferUsage.COPY_DST,
		});

		const bindGroup = device.createBindGroup({
			layout: (
				pipeline as unknown as {
					getBindGroupLayout(i: number): unknown;
				}
			).getBindGroupLayout(0),
			entries: [
				{binding: 0, resource: sampler},
				{binding: 1, resource: texture.createView()},
				{binding: 2, resource: {buffer: uniformBuffer}},
			],
		});

		gpuRef.current = {
			device,
			context,
			pipeline,
			sampler,
			texture,
			uniformBuffer,
			bindGroup,
			width: canvas.width,
			height: canvas.height,
		};

		return () => {
			texture.destroy();
			uniformBuffer.destroy();
			gpuRef.current = null;
		};
	}, []);

	const onPaint: HtmlInCanvasOnPaint = useCallback(
		({elementImage}) => {
			const gpu = gpuRef.current;
			if (!gpu) {
				return;
			}

			const {device, context, pipeline, texture, bindGroup, uniformBuffer} =
				gpu;

			if (device.queue.drawElementImageToTexture) {
				device.queue.drawElementImageToTexture(
					{source: elementImage},
					{texture, size: {width: gpu.width, height: gpu.height}},
				);
			} else {
				device.queue.copyElementImageToTexture(
					elementImage,
					gpu.width,
					gpu.height,
					{texture},
				);
			}

			const uniforms = new Float32Array([time, 0, gpu.width, gpu.height]);
			device.queue.writeBuffer(uniformBuffer, 0, uniforms);

			const encoder = device.createCommandEncoder();
			const view = context.getCurrentTexture().createView();
			const pass = encoder.beginRenderPass({
				colorAttachments: [
					{
						view,
						clearValue: {r: 0, g: 0, b: 0, a: 0},
						loadOp: 'clear',
						storeOp: 'store',
					},
				],
			});
			pass.setPipeline(pipeline);
			pass.setBindGroup(0, bindGroup);
			pass.draw(3);
			pass.end();
			device.queue.submit([encoder.finish()]);
		},
		[time],
	);

	return (
		<AbsoluteFill
			style={{
				justifyContent: 'center',
				alignItems: 'center',
			}}
		>
			<HtmlInCanvas
				width={width}
				height={height}
				onInit={onInit}
				onPaint={onPaint}
			>
				<AbsoluteFill
					style={{
						backgroundColor: 'white',
						color: 'black',
						justifyContent: 'center',
						alignItems: 'center',
						fontSize: 120,
						fontFamily: 'sans-serif',
						fontWeight: 'bold',
					}}
				>
					<h1>Hello, World!</h1>
				</AbsoluteFill>
			</HtmlInCanvas>
		</AbsoluteFill>
	);
};