Skip to content

πŸŽ₯ Make videos programmatically with React

35.3 KB
Raw Blame
import React, {
	createContext,
	forwardRef,
	useCallback,
	useContext,
	useLayoutEffect,
	useMemo,
	useRef,
} from 'react';
import type {SequenceControls} from './CompositionManager.js';
import {DefaultPremountContext} from './DefaultPremountContext.js';
import type {EffectsProp, EffectsOutputSize} from './effects/effect-types.js';
import {runEffectChain} from './effects/run-effect-chain.js';
import {useEffectChainState} from './effects/use-effect-chain-state.js';
import {
	useMemoizedEffectDefinitions,
	useMemoizedEffects,
} from './effects/use-memoized-effects.js';
import {addSequenceStackTraces} from './enable-sequence-stack-traces.js';
import {Freeze} from './freeze.js';
import {HtmlInCanvasFallback} from './HtmlInCanvasFallback.js';
import type {
	InteractiveBaseProps,
	InteractiveCropProps,
} from './Interactive.js';
import {
	backgroundSchema,
	baseSchema,
	premountSchema,
	borderRadiusSchema,
	borderSchema,
	cropSchema,
	transformSchema,
	type InteractivitySchema,
} from './interactivity-schema.js';
import {resolveSequenceDuration} from './resolve-sequence-duration.js';
import {Sequence} from './Sequence.js';
import type {AbsoluteFillLayout} from './Sequence.js';
import {useCropStyle} from './use-crop-style.js';
import {useCurrentFrame} from './use-current-frame.js';
import {useDelayRender} from './use-delay-render.js';
import {usePremounting} from './use-premounting.js';
import {useRemotionEnvironment} from './use-remotion-environment.js';
import {withInteractivitySchema} from './with-interactivity-schema.js';

// IDL: https://github.com/WICG/html-in-canvas#idl-changes
// WebGPU's `drawElementImageToTexture` is omitted β€” `GPUQueue` is not in
// lib.dom.d.ts and would require pulling in `@webgpu/types`.
declare global {
	interface DrawElementImageOptions {
		preserveElementGeometry?: boolean;
	}

	interface WebGLCopyElementImageConfig {
		sx?: GLfloat;
		sy?: GLfloat;
		swidth?: GLfloat;
		sheight?: GLfloat;
		width?: GLsizei;
		height?: GLsizei;
	}

	interface UpdateElementGeometryOptions {
		preserveHitTestOrder?: boolean;
		clip?: DOMRectInit;
		canvasTransform?: DOMMatrixInit;
	}

	interface ElementImage {
		readonly width: number;
		readonly height: number;
		close(): void;
	}

	interface CanvasRenderingContext2D {
		drawElementImage(
			element: Element | ElementImage,
			dx: number,
			dy: number,
			options?: DrawElementImageOptions,
		): void;
		drawElementImage(
			element: Element | ElementImage,
			dx: number,
			dy: number,
			dwidth: number,
			dheight: number,
			options?: DrawElementImageOptions,
		): void;
		drawElementImage(
			element: Element | ElementImage,
			sx: number,
			sy: number,
			swidth: number,
			sheight: number,
			dx: number,
			dy: number,
			options?: DrawElementImageOptions,
		): void;
		drawElementImage(
			element: Element | ElementImage,
			sx: number,
			sy: number,
			swidth: number,
			sheight: number,
			dx: number,
			dy: number,
			dwidth: number,
			dheight: number,
			options?: DrawElementImageOptions,
		): void;
	}

	interface OffscreenCanvasRenderingContext2D {
		drawElementImage(
			element: Element | ElementImage,
			dx: number,
			dy: number,
			options?: DrawElementImageOptions,
		): void;
		drawElementImage(
			element: Element | ElementImage,
			dx: number,
			dy: number,
			dwidth: number,
			dheight: number,
			options?: DrawElementImageOptions,
		): void;
		drawElementImage(
			element: Element | ElementImage,
			sx: number,
			sy: number,
			swidth: number,
			sheight: number,
			dx: number,
			dy: number,
			options?: DrawElementImageOptions,
		): void;
		drawElementImage(
			element: Element | ElementImage,
			sx: number,
			sy: number,
			swidth: number,
			sheight: number,
			dx: number,
			dy: number,
			dwidth: number,
			dheight: number,
			options?: DrawElementImageOptions,
		): void;
	}

	// Augmenting the base interface applies to both WebGL1 and WebGL2.
	interface WebGLRenderingContextBase {
		texElementSubImage2D(
			target: GLenum,
			level: GLint,
			xoffset: GLint,
			yoffset: GLint,
			element: Element | ElementImage,
			config?: WebGLCopyElementImageConfig,
		): void;

		// Older Chromium builds expose texElementImage2D during the migration.
		texElementImage2D(
			target: GLenum,
			level: GLint,
			internalformat: GLint,
			format: GLenum,
			type: GLenum,
			element: Element | ElementImage,
		): void;
		texElementImage2D(
			target: GLenum,
			level: GLint,
			internalformat: GLint,
			width: GLsizei,
			height: GLsizei,
			format: GLenum,
			type: GLenum,
			element: Element | ElementImage,
		): void;
		texElementImage2D(
			target: GLenum,
			level: GLint,
			internalformat: GLint,
			sx: GLfloat,
			sy: GLfloat,
			swidth: GLfloat,
			sheight: GLfloat,
			format: GLenum,
			type: GLenum,
			element: Element | ElementImage,
		): void;
		texElementImage2D(
			target: GLenum,
			level: GLint,
			internalformat: GLint,
			sx: GLfloat,
			sy: GLfloat,
			swidth: GLfloat,
			sheight: GLfloat,
			width: GLsizei,
			height: GLsizei,
			format: GLenum,
			type: GLenum,
			element: Element | ElementImage,
		): void;
	}

	interface HTMLCanvasElementEventMap {
		paint: Event;
	}

	interface HTMLCanvasElement {
		content: string;
		layoutSubtree?: boolean;
		onpaint: ((this: HTMLCanvasElement, ev: Event) => unknown) | null;
		requestPaint?(): void;
		captureElementImage(element: Element): ElementImage;
		updateElementGeometry(
			element: Element | ElementImage,
			options?: UpdateElementGeometryOptions,
		): void;
		clearElementGeometry(element: Element | ElementImage): void;
		getElementTransform(element: Element): DOMMatrix;
	}

	interface OffscreenCanvas {
		updateElementGeometry(
			element: Element | ElementImage,
			options?: UpdateElementGeometryOptions,
		): void;
		clearElementGeometry(element: Element | ElementImage): void;
	}
}

export type HtmlInCanvasOnPaintParams = {
	/**
	 * The paint target. During rendering, its pixels are copied to the layout
	 * `<canvas>` after painting; during preview, it controls that canvas directly.
	 */
	readonly canvas: OffscreenCanvas;
	readonly element: HTMLDivElement;
	readonly elementImage: ElementImage;
	readonly pixelDensity: number;
};

// `transferControlToOffscreen()` may only be called once per canvas β€” a second
// call throws an `InvalidStateError`. The layout effect that needs the
// `OffscreenCanvas` can run more than once for the same canvas element: its
// dependencies (e.g. the paint callback) can change identity without the
// canvas remounting, and React StrictMode intentionally double-invokes
// effects. Cache the transferred `OffscreenCanvas` per canvas element so
// re-runs reuse it instead of throwing.
const transferredOffscreenCanvases = new WeakMap<
	HTMLCanvasElement,
	OffscreenCanvas
>();

const getTransferredOffscreenCanvas = (
	canvas: HTMLCanvasElement,
): OffscreenCanvas => {
	const existing = transferredOffscreenCanvases.get(canvas);
	if (existing) {
		return existing;
	}

	const offscreen = canvas.transferControlToOffscreen();
	transferredOffscreenCanvases.set(canvas, offscreen);
	return offscreen;
};

// Memoize the support check across the session β€” neither the platform
// capability nor the chrome://flags toggle can change between calls.
// SSR results are not cached so the check runs again once `document` exists.
let cachedSupport: boolean | null = null;

export const isHtmlInCanvasSupported = (): boolean => {
	if (cachedSupport !== null) {
		return cachedSupport;
	}

	if (typeof document === 'undefined') {
		return false;
	}

	const canvas = document.createElement('canvas');
	const ctx = canvas.getContext('2d');
	cachedSupport =
		typeof ctx?.drawElementImage === 'function' &&
		typeof canvas.requestPaint === 'function' &&
		typeof canvas.captureElementImage === 'function' &&
		'transferControlToOffscreen' in HTMLCanvasElement.prototype;
	return cachedSupport;
};

const MINIMUM_CHROME_VERSION_FOR_NESTED_HTML_IN_CANVAS = 157;

export const isHtmlInCanvasNestingSupported = (): boolean => {
	if (!isHtmlInCanvasSupported() || typeof navigator === 'undefined') {
		return false;
	}

	const chromeVersion = navigator.userAgent.match(
		/(?:Chrome|Chromium)\/(\d+)/,
	)?.[1];
	return (
		chromeVersion !== undefined &&
		Number(chromeVersion) >= MINIMUM_CHROME_VERSION_FOR_NESTED_HTML_IN_CANVAS
	);
};

/** Generic fallback for consumers that cannot inspect the current browser. */
export const HTML_IN_CANVAS_UNSUPPORTED_MESSAGE =
	'HTML in Canvas requires Chrome 149 or newer with Canvas Draw Element enabled at chrome://flags/#canvas-draw-element.';

export const getHtmlInCanvasUnsupportedMessage = (): string => {
	if (typeof document === 'undefined') {
		return `HTML in Canvas is unavailable because there is no browser document. ${HTML_IN_CANVAS_UNSUPPORTED_MESSAGE}`;
	}

	const userAgent = typeof navigator === 'undefined' ? '' : navigator.userAgent;
	const chromiumVersion = userAgent.match(/(?:Chrome|Chromium)\/(\d+)/)?.[1];
	let browser = 'this browser';
	if (userAgent.includes('Edg/')) {
		browser = 'Microsoft Edge';
	} else if (userAgent.includes('Chromium/')) {
		browser = 'Chromium';
	} else if (chromiumVersion) {
		browser = 'Chrome';
	} else if (userAgent.includes('Firefox/')) {
		browser = 'Firefox';
	} else if (userAgent.includes('Safari/')) {
		browser = 'Safari';
	}

	if (chromiumVersion && Number(chromiumVersion) >= 149) {
		const flagUrl =
			browser === 'Microsoft Edge'
				? 'edge://flags/#canvas-draw-element'
				: 'chrome://flags/#canvas-draw-element';
		return `HTML in Canvas is unavailable. Enable Canvas Draw Element at ${flagUrl} and fully restart ${browser}.`;
	}

	if (chromiumVersion) {
		return `HTML in Canvas is not supported in ${browser} ${chromiumVersion}. Use a Chromium-based browser running version 149 or newer.`;
	}

	return `HTML in Canvas is not supported in ${browser}. Use Chrome 149 or newer.`;
};

export type HtmlInCanvasOnPaint = (
	params: HtmlInCanvasOnPaintParams,
) => void | Promise<void>;

export type HtmlInCanvasOnInitCleanup = () => void;

export type HtmlInCanvasOnInit = (
	params: HtmlInCanvasOnPaintParams,
) => HtmlInCanvasOnInitCleanup | Promise<HtmlInCanvasOnInitCleanup>;

export type HtmlInCanvasPixelDensity = number;

function assertHtmlInCanvasDimensions(width: unknown, height: unknown): void {
	if (typeof width !== 'number' || typeof height !== 'number') {
		throw new Error(
			`HtmlInCanvas: \`width\` and \`height\` must be numbers. Received width=${String(width)}, height=${String(height)}.`,
		);
	}

	if (!Number.isInteger(width) || width <= 0) {
		throw new Error(
			`HtmlInCanvas: \`width\` must be a positive integer. Received: ${String(width)}.`,
		);
	}

	if (!Number.isInteger(height) || height <= 0) {
		throw new Error(
			`HtmlInCanvas: \`height\` must be a positive integer. Received: ${String(height)}.`,
		);
	}
}

function resolveHtmlInCanvasPixelDensity(
	pixelDensity: HtmlInCanvasPixelDensity | undefined,
): number {
	if (pixelDensity === undefined) {
		return 1;
	}

	if (
		typeof pixelDensity !== 'number' ||
		!Number.isFinite(pixelDensity) ||
		pixelDensity <= 0
	) {
		throw new Error(
			`HtmlInCanvas: \`pixelDensity\` must be a positive finite number. Received: ${String(pixelDensity)}.`,
		);
	}

	return pixelDensity;
}

const isMissingPaintRecordError = (error: unknown): boolean => {
	return error instanceof DOMException && error.name === 'InvalidStateError';
};

const missingPaintRecordMessage =
	'HtmlInCanvas: Expected the element to be inside the viewport during rendering, but Chrome had no cached paint record for it.';

type HtmlInCanvasPaintTarget = HTMLCanvasElement | OffscreenCanvas;

const resizePaintTarget = ({
	target,
	width,
	height,
}: {
	target: HtmlInCanvasPaintTarget;
	width: number;
	height: number;
}) => {
	if (target.width !== width) {
		target.width = width;
	}

	if (target.height !== height) {
		target.height = height;
	}
};

const defaultOnPaint = ({
	canvas,
	elementImage,
}: Omit<HtmlInCanvasOnPaintParams, 'canvas'> & {
	readonly canvas: HtmlInCanvasPaintTarget;
}) => {
	const ctx = canvas.getContext('2d');
	if (!ctx) {
		throw new Error('Failed to acquire 2D context for <HtmlInCanvas> canvas');
	}

	ctx.reset();
	ctx.drawElementImage(elementImage, 0, 0);
};

/* eslint-disable react/require-default-props -- optional fields mirror `<Sequence>` / canvas hooks API */
export type HtmlInCanvasProps = Omit<InteractiveBaseProps, 'children'> &
	InteractiveCropProps &
	Omit<AbsoluteFillLayout, 'layout'> & {
		readonly durationInFrames?: number;
		readonly width: number;
		readonly height: number;
		readonly effects?: EffectsProp;
		readonly effectsOutputSize?: EffectsOutputSize;
		readonly children: React.ReactNode;
		readonly onPaint?: HtmlInCanvasOnPaint;
		readonly onInit?: HtmlInCanvasOnInit;
		readonly pixelDensity?: HtmlInCanvasPixelDensity;
		// captureElementImage() only accepts immediate children of the layout canvas.
		readonly _remotionInternalCanvasSiblings?: React.ReactNode;
		readonly _remotionInternalFallbackLabel?: string;
	};
/* eslint-enable react/require-default-props */

type HtmlInCanvasPaintState = {
	pendingChildren: number;
	generation: number;
	release: (() => void) | null;
	blockPaint: () => () => void;
};

const HtmlInCanvasAncestorContext =
	createContext<HtmlInCanvasPaintState | null>(null);

export const useHtmlInCanvasUnsupportedMessage = (): string | null => {
	const ancestorPaintState = useContext(HtmlInCanvasAncestorContext);
	if (!isHtmlInCanvasSupported()) {
		return getHtmlInCanvasUnsupportedMessage();
	}

	if (ancestorPaintState && !isHtmlInCanvasNestingSupported()) {
		return `Nested <HtmlInCanvas> components require Chrome ${MINIMUM_CHROME_VERSION_FOR_NESTED_HTML_IN_CANVAS} or newer with HTML-in-canvas enabled.`;
	}

	return null;
};

type HtmlInCanvasContentProps = {
	readonly width: number;
	readonly height: number;
	readonly effects: EffectsProp;
	readonly effectsOutputSize: EffectsOutputSize | null;
	readonly children: React.ReactNode;
	readonly canvasSiblings: React.ReactNode | null;
	readonly onPaint: HtmlInCanvasOnPaint | undefined;
	readonly onInit: HtmlInCanvasOnInit | undefined;
	readonly pixelDensity: HtmlInCanvasPixelDensity | undefined;
	readonly controls: SequenceControls | undefined;
	readonly style: React.CSSProperties | undefined;
};

const HtmlInCanvasContent = forwardRef<
	HTMLCanvasElement,
	HtmlInCanvasContentProps & {readonly renderFrame: number | null}
>(
	(
		{
			width,
			height,
			effects,
			effectsOutputSize,
			children,
			canvasSiblings,
			onPaint,
			onInit,
			pixelDensity,
			controls,
			style,
			renderFrame,
		},
		ref,
	) => {
		const ancestorPaintState = useContext(HtmlInCanvasAncestorContext);
		assertHtmlInCanvasDimensions(width, height);
		if (ancestorPaintState && !isHtmlInCanvasNestingSupported()) {
			throw new Error(
				`Nested <HtmlInCanvas> components require Chrome ${MINIMUM_CHROME_VERSION_FOR_NESTED_HTML_IN_CANVAS} or newer with HTML-in-canvas enabled.`,
			);
		}

		const resolvedPixelDensity = resolveHtmlInCanvasPixelDensity(pixelDensity);
		const canvasWidth = Math.ceil(width * resolvedPixelDensity);
		const canvasHeight = Math.ceil(height * resolvedPixelDensity);
		const {delayRender, continueRender, cancelRender} = useDelayRender();
		const {isClientSideRendering, isRendering} = useRemotionEnvironment();
		const canRetryMissingPaintRecord = !isRendering || isClientSideRendering;
		const usesDirectLayoutCanvas =
			onPaint === undefined && onInit === undefined;
		// Keep a nested drawable canvas separate from the bitmap captured by its
		// parent. Reusing it for both can crash Chromium during raster invalidation.
		const paintTargetMode = ancestorPaintState
			? 'copied'
			: usesDirectLayoutCanvas
				? 'direct'
				: isRendering
					? 'copied'
					: 'transferred';

		if (!isHtmlInCanvasSupported()) {
			cancelRender(new Error(getHtmlInCanvasUnsupportedMessage()));
		}

		const canvas2dRef = useRef<HTMLCanvasElement | null>(null);
		const outputCanvasRef = useRef<HTMLCanvasElement | null>(null);
		const paintTargetRef = useRef<HtmlInCanvasPaintTarget | null>(null);
		const divRef = useRef<HTMLDivElement | null>(null);
		const canvasSizeKey = `${width}x${height}@${resolvedPixelDensity}-${paintTargetMode}`;
		const paintState = useMemo<HtmlInCanvasPaintState>(() => {
			const state: HtmlInCanvasPaintState = {
				pendingChildren: 0,
				generation: 0,
				release: null,
				blockPaint: () => {
					state.pendingChildren++;
					state.generation++;
					if (state.release === null) {
						const handle = delayRender('waiting for nested HtmlInCanvas paint');
						const releaseAncestor = ancestorPaintState?.blockPaint();
						state.release = () => {
							continueRender(handle);
							releaseAncestor?.();
						};
					}

					let released = false;
					return () => {
						if (released) {
							return;
						}

						released = true;
						state.pendingChildren--;
						if (state.pendingChildren === 0) {
							// A child can finish asynchronously after our paint record was
							// taken. Keep capture blocked until we paint its finished pixels.
							canvas2dRef.current?.requestPaint?.();
						}
					};
				},
			};
			return state;
		}, [continueRender, delayRender, ancestorPaintState]);

		const setVisibleCanvasRef = useCallback(
			(node: HTMLCanvasElement | null) => {
				if (paintTargetMode === 'copied') {
					outputCanvasRef.current = node;
				} else {
					canvas2dRef.current = node;
				}

				if (typeof ref === 'function') {
					ref(node);
				} else if (ref) {
					(ref as React.RefObject<HTMLCanvasElement | null>).current = node;
				}
			},
			[ref, paintTargetMode],
		);

		const chainState = useEffectChainState(effectsOutputSize);

		const memoizedEffects = useMemoizedEffects({
			effects,
			overrideId: controls?.overrideId ?? null,
			videoConfigValues: controls?.videoConfigValues ?? null,
		});

		// Refs so the paint handler always reads fresh values.
		const effectsRef = useRef(memoizedEffects);
		effectsRef.current = memoizedEffects;
		const onPaintRef = useRef(onPaint);
		onPaintRef.current = onPaint;
		const onInitRef = useRef(onInit);
		onInitRef.current = onInit;
		const initializedRef = useRef(false);
		const onInitCleanupRef = useRef<HtmlInCanvasOnInitCleanup | null>(null);
		const unmountedRef = useRef(false);
		const effectScratchCanvasesRef = useRef<{
			source: HTMLCanvasElement;
			previousOutput: HTMLCanvasElement;
		} | null>(null);
		const paintingTargets = useRef(
			new WeakMap<HtmlInCanvasPaintTarget, {repaint: boolean}>(),
		);

		const onPaintCb = useCallback(async () => {
			if (isRendering && paintState.pendingChildren > 0) {
				return;
			}

			const childrenGeneration = paintState.generation;
			const element = divRef.current;

			if (!element) {
				throw new Error('Canvas or scene element not found');
			}

			const paintTarget = paintTargetRef.current;
			if (!paintTarget) {
				throw new Error(
					'HtmlInCanvas: paint target is not ready because the canvas is remounting',
				);
			}

			const inProgress = paintingTargets.current.get(paintTarget);
			if (isRendering && inProgress) {
				// In particular, do not run onPaint while an earlier event is still
				// awaiting onInit (e.g. a WebGPU device).
				inProgress.repaint = true;
				return;
			}

			const painting = {repaint: false};
			if (isRendering) {
				paintingTargets.current.set(paintTarget, painting);
			}

			resizePaintTarget({
				target: paintTarget,
				width: canvasWidth,
				height: canvasHeight,
			});

			const releaseAncestor = isRendering
				? ancestorPaintState?.blockPaint()
				: null;
			let canReleasePaint = false;
			try {
				const placeholderCanvas = canvas2dRef.current;
				if (!placeholderCanvas) {
					throw new Error('Canvas not found');
				}

				const outputCanvas = outputCanvasRef.current;
				const handle = delayRender('onPaint');
				if (!initializedRef.current) {
					const currentOnInit = onInitRef.current;
					if (!currentOnInit) {
						initializedRef.current = true;
					} else {
						// `onInit` may be async (e.g. WebGPU
						// `requestAdapter`/`requestDevice`). Do not reuse this capture for
						// `onPaint`: awaiting initialization can invalidate its paint context.
						let initImage: ElementImage;
						try {
							initImage = placeholderCanvas.captureElementImage(element);
						} catch (error) {
							if (
								isMissingPaintRecordError(error) &&
								canRetryMissingPaintRecord
							) {
								// The web renderer explicitly drives additional paint cycles, so a
								// transient missing record can be retried without failing the render.
								canReleasePaint = true;
								continueRender(handle);
								return;
							}

							if (isMissingPaintRecordError(error)) {
								throw new Error(missingPaintRecordMessage);
							}

							throw error;
						}

						initializedRef.current = true;
						try {
							if (paintTarget instanceof HTMLCanvasElement) {
								throw new Error(
									'HtmlInCanvas: onInit requires an OffscreenCanvas paint target',
								);
							}

							const cleanup = await currentOnInit({
								canvas: paintTarget,
								element,
								elementImage: initImage,
								pixelDensity: resolvedPixelDensity,
							});
							if (typeof cleanup !== 'function') {
								throw new Error(
									'HtmlInCanvas: when `onInit` is provided, it must return a cleanup function, or a Promise that resolves to one.',
								);
							}

							if (
								unmountedRef.current ||
								paintTargetRef.current !== paintTarget
							) {
								cleanup();
								continueRender(handle);
								return;
							}

							onInitCleanupRef.current = cleanup;
						} finally {
							initImage.close();
						}
					}
				}

				let elImage: ElementImage;
				try {
					elImage = placeholderCanvas.captureElementImage(element);
				} catch (error) {
					// `captureElementImage` throws `InvalidStateError` when the
					// element is outside the viewport (no cached paint record).
					// Skip this paint cycle β€” the canvas retains its last state.
					if (isMissingPaintRecordError(error) && canRetryMissingPaintRecord) {
						canReleasePaint = true;
						continueRender(handle);
						return;
					}

					if (isMissingPaintRecordError(error)) {
						throw new Error(missingPaintRecordMessage);
					}

					throw error;
				}

				let effectSourceCanvas: HTMLCanvasElement | null = null;
				try {
					const currentOnPaint = onPaintRef.current;
					if (currentOnPaint) {
						if (paintTarget instanceof HTMLCanvasElement) {
							throw new Error(
								'HtmlInCanvas: onPaint requires an OffscreenCanvas paint target',
							);
						}

						const paintResult = currentOnPaint({
							canvas: paintTarget,
							element,
							elementImage: elImage,
							pixelDensity: resolvedPixelDensity,
						});
						if (paintResult) {
							await paintResult;
						}
					} else if (
						paintTarget instanceof HTMLCanvasElement &&
						effectsRef.current.some(
							(effect) => !(effect.params as {disabled?: boolean}).disabled,
						)
					) {
						// The element image can only be drawn into the visible layout
						// canvas, but effects finish asynchronously (e.g. while the
						// WebGPU device initializes). Move the unprocessed pixels to a
						// scratch canvas and restore the previous output within this task,
						// so the browser never presents the HTML without its effects.
						effectScratchCanvasesRef.current ??= {
							source: document.createElement('canvas'),
							previousOutput: document.createElement('canvas'),
						};
						const {source: effectSource, previousOutput} =
							effectScratchCanvasesRef.current;
						resizePaintTarget({
							target: effectSource,
							width: canvasWidth,
							height: canvasHeight,
						});
						resizePaintTarget({
							target: previousOutput,
							width: canvasWidth,
							height: canvasHeight,
						});
						const effectSourceCtx = effectSource.getContext('2d');
						const previousOutputCtx = previousOutput.getContext('2d');
						const paintTargetCtx = paintTarget.getContext('2d');
						if (!effectSourceCtx || !previousOutputCtx || !paintTargetCtx) {
							throw new Error(
								'Failed to acquire 2D context for <HtmlInCanvas> canvas',
							);
						}

						previousOutputCtx.reset();
						previousOutputCtx.drawImage(paintTarget, 0, 0);
						defaultOnPaint({
							canvas: paintTarget,
							element,
							elementImage: elImage,
							pixelDensity: resolvedPixelDensity,
						});
						effectSourceCtx.reset();
						effectSourceCtx.drawImage(paintTarget, 0, 0);
						paintTargetCtx.reset();
						paintTargetCtx.drawImage(previousOutput, 0, 0);
						effectSourceCanvas = effectSource;
					} else {
						defaultOnPaint({
							canvas: paintTarget,
							element,
							elementImage: elImage,
							pixelDensity: resolvedPixelDensity,
						});
					}

					// `null` once unmounted, e.g. when an async `onPaint` resolves late.
					const state = chainState.get(canvasWidth, canvasHeight);
					if (state) {
						await runEffectChain({
							state,
							source: effectSourceCanvas ?? paintTarget,
							effects: effectsRef.current,
							output: paintTarget,
							width: canvasWidth,
							height: canvasHeight,
						});
					}
				} finally {
					elImage.close();
				}

				if (
					paintTargetMode === 'copied' &&
					paintTargetRef.current === paintTarget &&
					outputCanvas
				) {
					const ctx = outputCanvas.getContext('2d');
					if (!ctx) {
						throw new Error(
							'HtmlInCanvas: could not get layout canvas context',
						);
					}

					// A transferred OffscreenCanvas submits its pixels asynchronously.
					// Copy before releasing the render handle so capture sees this paint.
					// drawImage preserves the source for handlers that paint incrementally.
					ctx.reset();
					ctx.drawImage(paintTarget, 0, 0);
				}

				canReleasePaint = true;
				continueRender(handle);
			} catch (error) {
				cancelRender(error);
			} finally {
				if (
					canReleasePaint &&
					paintState.pendingChildren === 0 &&
					paintState.generation === childrenGeneration
				) {
					const {release} = paintState;
					paintState.release = null;
					release?.();
				}

				releaseAncestor?.();
				paintingTargets.current.delete(paintTarget);
				if (painting.repaint && paintTargetRef.current === paintTarget) {
					canvas2dRef.current?.requestPaint?.();
				}
			}
		}, [
			canvasHeight,
			canvasWidth,
			chainState,
			continueRender,
			cancelRender,
			delayRender,
			resolvedPixelDensity,
			canRetryMissingPaintRecord,
			paintTargetMode,
			isRendering,
			paintState,
			ancestorPaintState,
		]);

		useLayoutEffect(() => {
			return () => {
				paintState.release?.();
				paintState.release = null;
			};
		}, [paintState]);

		// Non-nested default handlers paint directly on the layout canvas. Nested
		// canvases and custom handlers retain a transferred target because an
		// ElementImage can only be drawn into its associated canvas. Nested canvases
		// and rendering copy it to a separate visible canvas without waiting for
		// the transferred canvas to present.
		useLayoutEffect(() => {
			const placeholder = canvas2dRef.current;
			if (!placeholder) {
				throw new Error('Canvas not found');
			}

			placeholder.setAttribute('content', 'drawable');
			placeholder.layoutSubtree = true;
			divRef.current?.setAttribute('drawable', '');

			const paintTarget =
				paintTargetMode === 'direct'
					? placeholder
					: getTransferredOffscreenCanvas(placeholder);

			paintTargetRef.current = paintTarget;
			resizePaintTarget({
				target: paintTarget,
				width: canvasWidth,
				height: canvasHeight,
			});

			initializedRef.current = false;
			unmountedRef.current = false;

			placeholder.addEventListener('paint', onPaintCb);

			return () => {
				placeholder.removeEventListener('paint', onPaintCb);
				paintTargetRef.current = null;
				initializedRef.current = false;
				unmountedRef.current = true;
				onInitCleanupRef.current?.();
				onInitCleanupRef.current = null;
			};
		}, [onPaintCb, cancelRender, canvasWidth, canvasHeight, paintTargetMode]);

		const onPaintChangedRef = useRef(false);
		useLayoutEffect(() => {
			if (!onPaintChangedRef.current) {
				onPaintChangedRef.current = true;
				return;
			}

			const canvas = canvas2dRef.current;
			if (!canvas) {
				return;
			}

			canvas.requestPaint?.();
		}, [onPaint, memoizedEffects]);

		useLayoutEffect(() => {
			const canvas = canvas2dRef.current;
			if (!canvas) {
				return;
			}

			// Block capture before the paint event begins, including updates that
			// keep the same canvas. onPaintCb holds its own handle through async work.
			const handle = delayRender('waiting for HtmlInCanvas paint');
			const releaseAncestor = isRendering
				? ancestorPaintState?.blockPaint()
				: null;
			const onPainted = () => {
				continueRender(handle);
				releaseAncestor?.();
			};

			canvas.addEventListener('paint', onPainted, {once: true});
			canvas.requestPaint?.();

			return () => {
				canvas.removeEventListener('paint', onPainted);
				onPainted();
			};
		}, [
			width,
			height,
			continueRender,
			delayRender,
			canvasSizeKey,
			renderFrame,
			isRendering,
			ancestorPaintState,
		]);

		const innerStyle = useMemo(() => {
			return {
				width,
				height,
			};
		}, [width, height]);

		const canvasStyle = useMemo(() => {
			return {
				width,
				height,
				...(style ?? {}),
			};
		}, [height, style, width]);

		return (
			<HtmlInCanvasAncestorContext.Provider value={paintState}>
				<canvas
					key={canvasSizeKey}
					ref={paintTargetMode === 'copied' ? canvas2dRef : setVisibleCanvasRef}
					width={canvasWidth}
					height={canvasHeight}
					style={
						paintTargetMode === 'copied'
							? {
									...canvasStyle,
									position: 'absolute',
									// Hide the source canvas while keeping its drawable children
									// visible to captureElementImage. opacity: 0 drops paint records.
									visibility: 'hidden',
									pointerEvents: 'none',
								}
							: canvasStyle
					}
				>
					<DefaultPremountContext.Provider value={0}>
						<div
							ref={divRef}
							style={
								paintTargetMode === 'copied'
									? {...innerStyle, visibility: 'visible'}
									: innerStyle
							}
						>
							{children}
						</div>
						{canvasSiblings}
					</DefaultPremountContext.Provider>
				</canvas>
				{paintTargetMode === 'copied' ? (
					<canvas
						key={`output-${canvasSizeKey}`}
						ref={setVisibleCanvasRef}
						width={canvasWidth}
						height={canvasHeight}
						style={canvasStyle}
					/>
				) : null}
			</HtmlInCanvasAncestorContext.Provider>
		);
	},
);

HtmlInCanvasContent.displayName = 'HtmlInCanvasContent';

// Subscribe to frame changes only while rendering. Preview keeps relying on
// the browser's paints and does not need a capture barrier on every frame.
const HtmlInCanvasContentForRendering = forwardRef<
	HTMLCanvasElement,
	HtmlInCanvasContentProps
>((props, ref) => {
	const frame = useCurrentFrame();
	return <HtmlInCanvasContent {...props} ref={ref} renderFrame={frame} />;
});

HtmlInCanvasContentForRendering.displayName = 'HtmlInCanvasContentForRendering';

const HtmlInCanvasInner = forwardRef<
	HTMLCanvasElement,
	HtmlInCanvasProps & {
		readonly controls: SequenceControls | undefined;
	}
>(
	(
		{
			width,
			height,
			effects = [],
			effectsOutputSize,
			children,
			onPaint,
			onInit,
			pixelDensity,
			_remotionInternalCanvasSiblings,
			_remotionInternalFallbackLabel = 'Effects are not visible',
			controls,
			style,
			cropLeft,
			cropRight,
			cropTop,
			cropBottom,
			durationInFrames,
			premountFor,
			postmountFor,
			styleWhilePremounted,
			styleWhilePostmounted,
			name,
			...sequenceProps
		},
		ref,
	) => {
		const {isRendering} = useRemotionEnvironment();
		const unsupportedMessage = useHtmlInCanvasUnsupportedMessage();
		assertHtmlInCanvasDimensions(width, height);
		resolveHtmlInCanvasPixelDensity(pixelDensity);
		const Content = isRendering
			? HtmlInCanvasContentForRendering
			: HtmlInCanvasContent;
		const memoizedEffectDefinitions = useMemoizedEffectDefinitions(effects);
		const actualRef = useRef<HTMLCanvasElement | null>(null);
		const setCanvasRef = useCallback(
			(node: HTMLCanvasElement | null) => {
				actualRef.current = node;
				if (typeof ref === 'function') {
					ref(node);
				} else if (ref) {
					(ref as React.RefObject<HTMLCanvasElement | null>).current = node;
				}
			},
			[ref],
		);

		const {
			effectivePremountFor,
			effectivePostmountFor,
			freezeFrame,
			isPremountingOrPostmounting,
			premountingActive,
			postmountingActive,
			premountingStyle,
		} = usePremounting({
			from: sequenceProps.from ?? 0,
			durationInFrames: resolveSequenceDuration({
				durationInFrames,
				playbackRate: sequenceProps.playbackRate,
				loop: sequenceProps.loop,
			}),
			premountFor: premountFor ?? null,
			postmountFor: postmountFor ?? null,
			style: style ?? null,
			styleWhilePremounted: styleWhilePremounted ?? null,
			styleWhilePostmounted: styleWhilePostmounted ?? null,
			hideWhilePremounted: 'opacity',
		});
		const croppedStyle = useCropStyle({
			cropLeft,
			cropRight,
			cropTop,
			cropBottom,
			style: premountingStyle,
			componentName: '<HtmlInCanvas />',
		});

		return (
			<Freeze
				frame={freezeFrame}
				active={isPremountingOrPostmounting}
				_remotionInternalIsPremounting={premountingActive}
			>
				<Sequence
					layout="none"
					durationInFrames={durationInFrames}
					name={name ?? '<HtmlInCanvas>'}
					_remotionInternalDocumentationLink="https://www.remotion.dev/docs/remotion/html-in-canvas"
					controls={controls}
					_remotionInternalEffects={memoizedEffectDefinitions}
					{...sequenceProps}
					_remotionInternalPremountDisplay={effectivePremountFor || null}
					_remotionInternalPostmountDisplay={effectivePostmountFor || null}
					_remotionInternalIsPremounting={premountingActive}
					_remotionInternalIsPostmounting={postmountingActive}
				>
					{!isRendering && unsupportedMessage !== null ? (
						<HtmlInCanvasFallback
							style={{width, height, ...croppedStyle}}
							label={_remotionInternalFallbackLabel}
						>
							<DefaultPremountContext.Provider value={0}>
								<div style={{width, height}}>{children}</div>
							</DefaultPremountContext.Provider>
						</HtmlInCanvasFallback>
					) : (
						<Content
							ref={setCanvasRef}
							renderFrame={null}
							width={width}
							height={height}
							effects={effects}
							effectsOutputSize={effectsOutputSize ?? null}
							onPaint={onPaint}
							onInit={onInit}
							pixelDensity={pixelDensity}
							canvasSiblings={_remotionInternalCanvasSiblings ?? null}
							controls={controls}
							style={croppedStyle ?? undefined}
						>
							{children}
						</Content>
					)}
				</Sequence>
			</Freeze>
		);
	},
);

HtmlInCanvasInner.displayName = 'HtmlInCanvas';

export const htmlInCanvasSchema = {
	...baseSchema,
	...premountSchema,
	pixelDensity: {
		type: 'number',
		min: 1,
		max: 3,
		step: 0.1,
		default: 1,
		description: 'Pixel density',
		hiddenFromList: false,
	},
	...transformSchema,
	...backgroundSchema,
	...borderSchema,
	...borderRadiusSchema,
	...cropSchema,
} as const satisfies InteractivitySchema;

const HtmlInCanvasWrapped = withInteractivitySchema({
	Component: HtmlInCanvasInner,
	componentName: '<HtmlInCanvas>',
	componentIdentity: 'dev.remotion.remotion.HtmlInCanvas',
	schema: htmlInCanvasSchema,
	supportsEffects: true,
});

export const HtmlInCanvas = Object.assign(HtmlInCanvasWrapped, {
	isSupported: isHtmlInCanvasSupported,
	isNestingSupported: isHtmlInCanvasNestingSupported,
}) as React.ForwardRefExoticComponent<
	HtmlInCanvasProps & React.RefAttributes<HTMLCanvasElement>
> & {
	readonly isSupported: typeof isHtmlInCanvasSupported;
	readonly isNestingSupported: typeof isHtmlInCanvasNestingSupported;
};

HtmlInCanvas.displayName = 'HtmlInCanvas';

addSequenceStackTraces(HtmlInCanvas);