π₯ Make videos programmatically with React
Adapt HTML-in-canvas to the drawable API
JonnyBurgerc8202eae50ed
diff --git a/packages/canvas-capture-extension/README.md b/packages/canvas-capture-extension/README.mdindex 4e082cda75..ef3b7f818d 100644--- a/packages/canvas-capture-extension/README.md+++ b/packages/canvas-capture-extension/README.md@@ -77,4 +77,4 @@ recordings. Stop the recording, then open it directly in[remotion.dev/new](https://remotion.dev/new), inspect it in[remotion.dev/convert](https://remotion.dev/convert), or save it to disk.-The page contents are temporarily placed inside a `layoutSubtree` canvas while recording and restored afterward. Websites that rely on direct-child CSS selectors may look different during capture. Chrome's own pages and the Chrome Web Store do not allow extension script injection.+The page contents are temporarily placed inside a `content="drawable"` canvas while recording and restored afterward. The extension also sets `layoutSubtree` for older Chromium versions. Websites that rely on direct-child CSS selectors may look different during capture. Chrome's own pages and the Chrome Web Store do not allow extension script injection.diff --git a/packages/canvas-capture-extension/src/capture.ts b/packages/canvas-capture-extension/src/capture.tsindex 1517bb7892..40da973a9b 100644--- a/packages/canvas-capture-extension/src/capture.ts+++ b/packages/canvas-capture-extension/src/capture.ts@@ -137,8 +137,10 @@ const wrapWholePage = (): WrappedPage => {const minimumSize = getWholePageSize();const canvas = document.createElement('canvas') as HtmlInCanvasElement;const content = document.createElement('div');+ canvas.setAttribute('content', 'drawable');canvas.layoutSubtree = true;canvas.setAttribute('layoutsubtree', '');+ content.setAttribute('drawable', '');canvas.style.display = 'block';canvas.style.position = 'relative';content.style.position = 'absolute';diff --git a/packages/codex-plugin/skills/remotion-best-practices/remotion-markup/html-in-canvas.md b/packages/codex-plugin/skills/remotion-best-practices/remotion-markup/html-in-canvas.mdindex d2638015e4..b78e9dd4ed 100644--- a/packages/codex-plugin/skills/remotion-best-practices/remotion-markup/html-in-canvas.md+++ b/packages/codex-plugin/skills/remotion-best-practices/remotion-markup/html-in-canvas.md@@ -47,7 +47,7 @@ export const MyComp = () => {## 2D effect with `onPaint`-`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM, and assign the returned transform to `element.style.transform` so DOM selection still aligns with the painted output.+`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM. Current Chromium synchronizes the element's geometry automatically; older versions return a transform that must be assigned to `element.style.transform`.```tsximport {@@ -73,7 +73,9 @@ export const Blur = () => {ctx.reset();ctx.filter = `blur(${blurPx}px)`;const transform = ctx.drawElementImage(elementImage, 0, 0);- element.style.transform = transform.toString();+ if (transform) {+ element.style.transform = transform.toString();+ }},[frame, fps],);@@ -98,28 +100,7 @@ export const Blur = () => {## WebGL effects-For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Inside `onPaint`, upload the captured DOM with `gl.texElementImage2D(...)` and draw.--```tsx-const onInit: HtmlInCanvasOnInit = useCallback(({ canvas }) => {- const gl = canvas.getContext("webgl2", { alpha: true, premultipliedAlpha: true });- if (!gl) {- throw new Error(- "WebGL2 unavailable. Try rendering with the --gl=angle option. See https://remotion.dev/docs/gl-options.",- );- }- gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);- // compile program, create texture, set up VAO...- return () => {- // delete program, texture, buffers...- };-}, []);--const onPaint: HtmlInCanvasOnPaint = useCallback(({ elementImage }) => {- gl.texElementImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, elementImage);- gl.drawArrays(gl.TRIANGLES, 0, 6);-}, []);-```+For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Allocate the texture with `gl.texImage2D(..., null)` when `elementImage` changes size. Inside `onPaint`, upload with `gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage)` and draw. Older Chromium versions only expose `gl.texElementImage2D()`; use it when the new method is absent.For a fully working minimal example, see https://github.com/remotion-dev/remotion/blob/main/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx.diff --git a/packages/codex-plugin/skills/remotion-markup/html-in-canvas.md b/packages/codex-plugin/skills/remotion-markup/html-in-canvas.mdindex d2638015e4..b78e9dd4ed 100644--- a/packages/codex-plugin/skills/remotion-markup/html-in-canvas.md+++ b/packages/codex-plugin/skills/remotion-markup/html-in-canvas.md@@ -47,7 +47,7 @@ export const MyComp = () => {## 2D effect with `onPaint`-`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM, and assign the returned transform to `element.style.transform` so DOM selection still aligns with the painted output.+`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM. Current Chromium synchronizes the element's geometry automatically; older versions return a transform that must be assigned to `element.style.transform`.```tsximport {@@ -73,7 +73,9 @@ export const Blur = () => {ctx.reset();ctx.filter = `blur(${blurPx}px)`;const transform = ctx.drawElementImage(elementImage, 0, 0);- element.style.transform = transform.toString();+ if (transform) {+ element.style.transform = transform.toString();+ }},[frame, fps],);@@ -98,28 +100,7 @@ export const Blur = () => {## WebGL effects-For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Inside `onPaint`, upload the captured DOM with `gl.texElementImage2D(...)` and draw.--```tsx-const onInit: HtmlInCanvasOnInit = useCallback(({ canvas }) => {- const gl = canvas.getContext("webgl2", { alpha: true, premultipliedAlpha: true });- if (!gl) {- throw new Error(- "WebGL2 unavailable. Try rendering with the --gl=angle option. See https://remotion.dev/docs/gl-options.",- );- }- gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);- // compile program, create texture, set up VAO...- return () => {- // delete program, texture, buffers...- };-}, []);--const onPaint: HtmlInCanvasOnPaint = useCallback(({ elementImage }) => {- gl.texElementImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, elementImage);- gl.drawArrays(gl.TRIANGLES, 0, 6);-}, []);-```+For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Allocate the texture with `gl.texImage2D(..., null)` when `elementImage` changes size. Inside `onPaint`, upload with `gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage)` and draw. Older Chromium versions only expose `gl.texElementImage2D()`; use it when the new method is absent.For a fully working minimal example, see https://github.com/remotion-dev/remotion/blob/main/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx.diff --git a/packages/core/src/HtmlInCanvas.tsx b/packages/core/src/HtmlInCanvas.tsxindex cd73ba4c82..2744bb55b2 100644--- a/packages/core/src/HtmlInCanvas.tsx+++ b/packages/core/src/HtmlInCanvas.tsx@@ -41,9 +41,28 @@ import {useRemotionEnvironment} from './use-remotion-environment.js';import {withInteractivitySchema} from './with-interactivity-schema.js';// IDL: https://github.com/WICG/html-in-canvas#idl-changes-// WebGPU's `copyElementImageToTexture` is omitted β `GPUQueue` is not in+// WebGPU's `drawElementImageToTexture` is omitted β `GPUQueue` is not in// lib.dom.d.ts and would require pulling in `@webgpu/types`.declare global {+ interface DrawElementImageOptions {+ preserveElementGeometry?: boolean;+ }++ interface WebGLCopyElementImageConfig {+ sx?: GLfloat;+ sy?: GLfloat;+ swidth?: GLfloat;+ sheight?: GLfloat;+ width?: GLsizei;+ height?: GLsizei;+ }++ interface UpdateElementGeometryOptions {+ preserveHitTestOrder?: boolean;+ clip?: DOMRectInit;+ canvasTransform?: DOMMatrixInit;+ }+interface ElementImage {readonly width: number;readonly height: number;@@ -55,14 +74,16 @@ declare global {element: Element | ElementImage,dx: number,dy: number,- ): DOMMatrix;+ options?: DrawElementImageOptions,+ ): DOMMatrix | void;drawElementImage(element: Element | ElementImage,dx: number,dy: number,dwidth: number,dheight: number,- ): DOMMatrix;+ options?: DrawElementImageOptions,+ ): DOMMatrix | void;drawElementImage(element: Element | ElementImage,sx: number,@@ -71,7 +92,8 @@ declare global {sheight: number,dx: number,dy: number,- ): DOMMatrix;+ options?: DrawElementImageOptions,+ ): DOMMatrix | void;drawElementImage(element: Element | ElementImage,sx: number,@@ -82,7 +104,8 @@ declare global {dy: number,dwidth: number,dheight: number,- ): DOMMatrix;+ options?: DrawElementImageOptions,+ ): DOMMatrix | void;}interface OffscreenCanvasRenderingContext2D {@@ -90,14 +113,16 @@ declare global {element: Element | ElementImage,dx: number,dy: number,- ): DOMMatrix;+ options?: DrawElementImageOptions,+ ): DOMMatrix | void;drawElementImage(element: Element | ElementImage,dx: number,dy: number,dwidth: number,dheight: number,- ): DOMMatrix;+ options?: DrawElementImageOptions,+ ): DOMMatrix | void;drawElementImage(element: Element | ElementImage,sx: number,@@ -106,7 +131,8 @@ declare global {sheight: number,dx: number,dy: number,- ): DOMMatrix;+ options?: DrawElementImageOptions,+ ): DOMMatrix | void;drawElementImage(element: Element | ElementImage,sx: number,@@ -117,11 +143,22 @@ declare global {dy: number,dwidth: number,dheight: number,- ): DOMMatrix;+ options?: DrawElementImageOptions,+ ): DOMMatrix | void;}// Augmenting the base interface applies to both WebGL1 and WebGL2.interface WebGLRenderingContextBase {+ texElementSubImage2D(+ target: GLenum,+ level: GLint,+ xoffset: GLint,+ yoffset: GLint,+ element: Element | ElementImage,+ config?: WebGLCopyElementImageConfig,+ ): void;++ // Older Chromium builds expose texElementImage2D during the migration.texElementImage2D(target: GLenum,level: GLint,@@ -173,15 +210,30 @@ declare global {}interface HTMLCanvasElement {+ content: string;layoutSubtree?: boolean;onpaint: ((this: HTMLCanvasElement, ev: Event) => unknown) | null;requestPaint?(): void;captureElementImage(element: Element): ElementImage;+ updateElementGeometry(+ element: Element | ElementImage,+ options?: UpdateElementGeometryOptions,+ ): void;+ clearElementGeometry(element: Element | ElementImage): void;+ getElementTransform(element: Element): DOMMatrix;getElementTransform(element: Element | ElementImage,drawTransform: DOMMatrix,): DOMMatrix;}++ interface OffscreenCanvas {+ updateElementGeometry(+ element: Element | ElementImage,+ options?: UpdateElementGeometryOptions,+ ): void;+ clearElementGeometry(element: Element | ElementImage): void;+ }}export type HtmlInCanvasOnPaintParams = {@@ -376,7 +428,10 @@ const defaultOnPaint = ({ctx.reset();const transform = ctx.drawElementImage(elementImage, 0, 0);- element.style.transform = transform.toString();+ if (transform) {+ // Older Chromium requires manual geometry synchronization.+ element.style.transform = transform.toString();+ }};/* eslint-disable react/require-default-props -- optional fields mirror `<Sequence>` / canvas hooks API */@@ -660,7 +715,9 @@ const HtmlInCanvasContent = forwardRef<throw new Error('Canvas not found');}+ placeholder.setAttribute('content', 'drawable');placeholder.layoutSubtree = true;+ divRef.current?.setAttribute('drawable', '');const paintTarget = usesDirectLayoutCanvas? placeholderdiff --git a/packages/docs/components/demos/HtmlInCanvasDocsDemo2DBlur.tsx b/packages/docs/components/demos/HtmlInCanvasDocsDemo2DBlur.tsxindex 26eed98f80..d3a3a5a1ea 100644--- a/packages/docs/components/demos/HtmlInCanvasDocsDemo2DBlur.tsx+++ b/packages/docs/components/demos/HtmlInCanvasDocsDemo2DBlur.tsx@@ -31,7 +31,9 @@ const HtmlInCanvasDocsDemo2DBlurInner: React.FC = () => {ctx.reset();ctx.filter = `blur(${blurPx}px)`;const transform = ctx.drawElementImage(elementImage, 0, 0);- element.style.transform = transform.toString();+ if (transform) {+ element.style.transform = transform.toString();+ }},[frame, fps],);diff --git a/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx b/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsxindex 2cad5c5d83..de1a684892 100644--- a/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx+++ b/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx@@ -20,6 +20,8 @@ type GlState = {uTex: WebGLUniformLocation | null;uTime: WebGLUniformLocation | null;texture: WebGLTexture;+ textureWidth: number;+ textureHeight: number;vao: WebGLVertexArrayObject;};@@ -118,7 +120,16 @@ const HtmlInCanvasDocsWebGLInner: React.FC = () => {gl.enableVertexAttribArray(locUv);gl.vertexAttribPointer(locUv, 2, gl.FLOAT, false, 16, 8);- gpuRef.current = {gl, program, uTex, uTime, texture, vao};+ gpuRef.current = {+ gl,+ program,+ uTex,+ uTime,+ texture,+ textureWidth: 0,+ textureHeight: 0,+ vao,+ };return () => {gl.deleteProgram(program);@@ -142,14 +153,39 @@ const HtmlInCanvasDocsWebGLInner: React.FC = () => {gl.activeTexture(gl.TEXTURE0);gl.bindTexture(gl.TEXTURE_2D, gpu.texture);- gl.texElementImage2D(- gl.TEXTURE_2D,- 0,- gl.RGBA,- gl.RGBA,- gl.UNSIGNED_BYTE,- elementImage,- );+ if (typeof gl.texElementSubImage2D === 'function') {+ const imageWidth = Math.max(1, Math.ceil(elementImage.width));+ const imageHeight = Math.max(1, Math.ceil(elementImage.height));+ if (+ gpu.textureWidth !== imageWidth ||+ gpu.textureHeight !== imageHeight+ ) {+ gl.texImage2D(+ gl.TEXTURE_2D,+ 0,+ gl.RGBA8,+ imageWidth,+ imageHeight,+ 0,+ gl.RGBA,+ gl.UNSIGNED_BYTE,+ null,+ );+ gpu.textureWidth = imageWidth;+ gpu.textureHeight = imageHeight;+ }++ gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage);+ } else {+ gl.texElementImage2D(+ gl.TEXTURE_2D,+ 0,+ gl.RGBA,+ gl.RGBA,+ gl.UNSIGNED_BYTE,+ elementImage,+ );+ }if (gpu.uTex) {gl.uniform1i(gpu.uTex, 0);diff --git a/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGPU.tsx b/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGPU.tsxindex 167ff68238..662a85eb12 100644--- a/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGPU.tsx+++ b/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGPU.tsx@@ -43,6 +43,14 @@ type GpuDevice = {queue: {submit(c: unknown[]): void;writeBuffer(b: GpuBuffer, offset: number, data: BufferSource): void;+ drawElementImageToTexture?(+ source: {source: Element | ElementImage},+ destination: {+ texture: GpuTexture;+ size: {width: number; height: number};+ },+ ): void;+ // Kept for Chrome builds predating drawElementImageToTexture.copyElementImageToTexture(source: Element | ElementImage,width: number,@@ -247,12 +255,19 @@ const HtmlInCanvasDocsWebGPUInner: React.FC = () => {const {device, context, pipeline, texture, bindGroup, uniformBuffer} =gpu;- device.queue.copyElementImageToTexture(- elementImage,- gpu.width,- gpu.height,- {texture},- );+ if (device.queue.drawElementImageToTexture) {+ device.queue.drawElementImageToTexture(+ {source: elementImage},+ {texture, size: {width: gpu.width, height: gpu.height}},+ );+ } else {+ device.queue.copyElementImageToTexture(+ elementImage,+ gpu.width,+ gpu.height,+ {texture},+ );+ }const uniforms = new Float32Array([time, 0, gpu.width, gpu.height]);device.queue.writeBuffer(uniformBuffer, 0, uniforms);diff --git a/packages/docs/docs/html-in-canvas-guide.mdx b/packages/docs/docs/html-in-canvas-guide.mdxindex 556678bbeb..34c2d5834d 100644--- a/packages/docs/docs/html-in-canvas-guide.mdx+++ b/packages/docs/docs/html-in-canvas-guide.mdx@@ -50,7 +50,9 @@ const onPaint: HtmlInCanvasOnPaint = ({canvas, element, elementImage}) => {ctx.reset();ctx.filter = 'blur(8px)';const transform = ctx.drawElementImage(elementImage, 0, 0);- element.style.transform = transform.toString();+ if (transform) {+ element.style.transform = transform.toString();+ }};export const MyComp: React.FC = () => {diff --git a/packages/docs/docs/remotion/html-in-canvas.mdx b/packages/docs/docs/remotion/html-in-canvas.mdxindex 574f6692a0..63f5e50b86 100644--- a/packages/docs/docs/remotion/html-in-canvas.mdx+++ b/packages/docs/docs/remotion/html-in-canvas.mdx@@ -92,7 +92,9 @@ const onPaint: HtmlInCanvasOnPaint = ({canvas, element, elementImage, pixelDensictx.reset();ctx.filter = `blur(${10 * pixelDensity}px)`;const transform = ctx.drawElementImage(elementImage, 0, 0);- element.style.transform = transform.toString();+ if (transform) {+ element.style.transform = transform.toString();+ }};```@@ -108,7 +110,7 @@ You should paint to this canvas.#### `element`The inner [`HTMLDivElement`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLDivElement) that wraps `children`.-You should apply the return value of [`drawElementImage`](https://github.com/WICG/html-in-canvas#2-drawelementimage-and-webglwebgpu-equivalents) to this element's `style.transform` property.+Current Chromium synchronizes its position and hit testing when you call `drawElementImage()`. Older versions return a matrix that you should apply to `element.style.transform`.#### `elementImage`@@ -150,7 +152,7 @@ From <AvailableFrom v="4.0.528" inline />, it also inherits [`premountFor`](/doc### `ref?`You can add a [React ref](https://react.dev/learn/manipulating-the-dom-with-refs) to `<HtmlInCanvas />`.-It is attached to the layout [`HTMLCanvasElement`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement) β the canvas that hosts the laid-out subtree (`layoutSubtree`).+It is attached to the [`HTMLCanvasElement`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement) that hosts the drawable subtree.If you use TypeScript, type the ref with `HTMLCanvasElement`:@@ -173,8 +175,8 @@ export const Example: React.FC = () => {### 2D-Call [`drawElementImage()`](https://github.com/WICG/html-in-canvas#2-drawelementimage-and-webglwebgpu-equivalents) on the 2D context to draw `elementImage` into the bitmap.-HTML-in-canvas recommends that you use the return value and assign it to `element.style.transform`, so that the original DOM element matches the transform, so that selection still works.+Call [`drawElementImage()`](https://github.com/WICG/html-in-canvas#idl-changes) on the 2D context to draw `elementImage` into the bitmap.+Current Chromium updates the element's position and hit testing automatically. Older versions return a transform that must be assigned to `element.style.transform` for selection and input to follow the drawing.For 2D, you usually do not need `onInit`.@@ -213,7 +215,9 @@ export const HtmlInCanvas2DBlur: React.FC = () => {ctx.reset();ctx.filter = `blur(${blurPx * pixelDensity}px)`;const transform = ctx.drawElementImage(elementImage, 0, 0);- element.style.transform = transform.toString();+ if (transform) {+ element.style.transform = transform.toString();+ }},[frame, fps],);@@ -242,7 +246,8 @@ export const HtmlInCanvas2DBlur: React.FC = () => {Do all setup in [`onInit`](#oninit) such as getting the WebGL context and compiling the shader.`onInit` must return a cleanup function that destroys the resources that were created in `onInit`.-Use [`texElementImage2D`](https://github.com/WICG/html-in-canvas#2-drawelementimage-and-webglwebgpu-equivalents) to turn the `elementImage` into a texture.+Use [`texElementSubImage2D`](https://github.com/WICG/html-in-canvas#idl-changes) to upload `elementImage` into a preallocated texture. Older Chromium versions expose `texElementImage2D` instead.+WebGL does not update the DOM element's hit testing geometry automatically. If the content needs to remain interactive, call `canvas.updateElementGeometry(element, {canvasTransform})` with a transform matching where your shader draws it.<Demo type="html-in-canvas-webgl" />@@ -268,6 +273,8 @@ type GlState = {uTex: WebGLUniformLocation | null;uTime: WebGLUniformLocation | null;texture: WebGLTexture;+ textureWidth: number;+ textureHeight: number;vao: WebGLVertexArrayObject;};@@ -366,7 +373,16 @@ export const HtmlInCanvasDocsMinimalWebGL: React.FC = () => {gl.enableVertexAttribArray(locUv);gl.vertexAttribPointer(locUv, 2, gl.FLOAT, false, 16, 8);- gpuRef.current = {gl, program, uTex, uTime, texture, vao};+ gpuRef.current = {+ gl,+ program,+ uTex,+ uTime,+ texture,+ textureWidth: 0,+ textureHeight: 0,+ vao,+ };return () => {gl.deleteProgram(program);@@ -390,14 +406,39 @@ export const HtmlInCanvasDocsMinimalWebGL: React.FC = () => {gl.activeTexture(gl.TEXTURE0);gl.bindTexture(gl.TEXTURE_2D, gpu.texture);- gl.texElementImage2D(- gl.TEXTURE_2D,- 0,- gl.RGBA,- gl.RGBA,- gl.UNSIGNED_BYTE,- elementImage,- );+ if (typeof gl.texElementSubImage2D === 'function') {+ const imageWidth = Math.max(1, Math.ceil(elementImage.width));+ const imageHeight = Math.max(1, Math.ceil(elementImage.height));+ if (+ gpu.textureWidth !== imageWidth ||+ gpu.textureHeight !== imageHeight+ ) {+ gl.texImage2D(+ gl.TEXTURE_2D,+ 0,+ gl.RGBA8,+ imageWidth,+ imageHeight,+ 0,+ gl.RGBA,+ gl.UNSIGNED_BYTE,+ null,+ );+ gpu.textureWidth = imageWidth;+ gpu.textureHeight = imageHeight;+ }++ gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage);+ } else {+ gl.texElementImage2D(+ gl.TEXTURE_2D,+ 0,+ gl.RGBA,+ gl.RGBA,+ gl.UNSIGNED_BYTE,+ elementImage,+ );+ }if (gpu.uTex) {gl.uniform1i(gpu.uTex, 0);@@ -439,7 +480,8 @@ export const HtmlInCanvasDocsMinimalWebGL: React.FC = () => {Use [`onInit`](#oninit) to get a WebGPU context, to request a GPU and to compile shaders.`onInit` must return a cleanup function that destroys the resources that were created in `onInit`.-In `onPaint`, draw the `elementImage` to the canvas using [`copyElementImageToTexture`](https://github.com/WICG/html-in-canvas#2-drawelementimage-and-webglwebgpu-equivalents).+In `onPaint`, upload `elementImage` using [`drawElementImageToTexture`](https://github.com/WICG/html-in-canvas#idl-changes). Older Chromium versions expose `copyElementImageToTexture` instead.+WebGPU also requires `canvas.updateElementGeometry(element, {canvasTransform})` when the drawn content needs hit testing.<Demo type="html-in-canvas-webgpu" />@@ -490,6 +532,13 @@ type GpuDevice = {queue: {submit(c: unknown[]): void;writeBuffer(b: GpuBuffer, offset: number, data: BufferSource): void;+ drawElementImageToTexture?(+ source: {source: Element | ElementImage},+ destination: {+ texture: GpuTexture;+ size: {width: number; height: number};+ },+ ): void;copyElementImageToTexture(source: Element | ElementImage,width: number,@@ -697,12 +746,19 @@ export const HtmlInCanvasComposeWebGPU: React.FC = () => {const {device, context, pipeline, texture, bindGroup, uniformBuffer} =gpu;- device.queue.copyElementImageToTexture(- elementImage,- gpu.width,- gpu.height,- {texture},- );+ if (device.queue.drawElementImageToTexture) {+ device.queue.drawElementImageToTexture(+ {source: elementImage},+ {texture, size: {width: gpu.width, height: gpu.height}},+ );+ } else {+ device.queue.copyElementImageToTexture(+ elementImage,+ gpu.width,+ gpu.height,+ {texture},+ );+ }const uniforms = new Float32Array([time, 0, gpu.width, gpu.height]);device.queue.writeBuffer(uniformBuffer, 0, uniforms);diff --git a/packages/example/src/HtmlInCanvas/changing-size.tsx b/packages/example/src/HtmlInCanvas/changing-size.tsxindex 94d5975095..eb19627b22 100644--- a/packages/example/src/HtmlInCanvas/changing-size.tsx+++ b/packages/example/src/HtmlInCanvas/changing-size.tsx@@ -38,7 +38,9 @@ export const HtmlInCanvasChangingSize: React.FC = () => {canvas.width,canvas.height,);- element.style.transform = transform.toString();+ if (transform) {+ element.style.transform = transform.toString();+ }const {data: pixels} = ctx.getImageData(0,0,diff --git a/packages/example/src/HtmlInCanvas/complex-text.tsx b/packages/example/src/HtmlInCanvas/complex-text.tsxindex 48f31ed2fd..875d2aaf5c 100644--- a/packages/example/src/HtmlInCanvas/complex-text.tsx+++ b/packages/example/src/HtmlInCanvas/complex-text.tsx@@ -28,7 +28,9 @@ export const HtmlInCanvasComplexText: React.FC = () => {ctx.rotate((15 * Math.PI) / 180);ctx.translate(160, -40);const transform = ctx.drawElementImage(elementImage, 0, 0);- element.style.transform = transform.toString();+ if (transform) {+ element.style.transform = transform.toString();+ }}}><divdiff --git a/packages/example/src/HtmlInCanvas/compose-webgl-crt.tsx b/packages/example/src/HtmlInCanvas/compose-webgl-crt.tsxindex 9b69f6725e..86f37484ff 100644--- a/packages/example/src/HtmlInCanvas/compose-webgl-crt.tsx+++ b/packages/example/src/HtmlInCanvas/compose-webgl-crt.tsx@@ -15,6 +15,8 @@ type GpuState = {uTime: WebGLUniformLocation | null;uResolution: WebGLUniformLocation | null;texture: WebGLTexture;+ textureWidth: number;+ textureHeight: number;vao: WebGLVertexArrayObject;buffer: WebGLBuffer;};@@ -166,6 +168,8 @@ export const HtmlInCanvasComposeWebGLCrt: React.FC = () => {uTime,uResolution,texture,+ textureWidth: 0,+ textureHeight: 0,vao,buffer,};@@ -192,14 +196,39 @@ export const HtmlInCanvasComposeWebGLCrt: React.FC = () => {gl.activeTexture(gl.TEXTURE0);gl.bindTexture(gl.TEXTURE_2D, gpu.texture);- gl.texElementImage2D(- gl.TEXTURE_2D,- 0,- gl.RGBA,- gl.RGBA,- gl.UNSIGNED_BYTE,- elementImage,- );+ if (typeof gl.texElementSubImage2D === 'function') {+ const imageWidth = Math.max(1, Math.ceil(elementImage.width));+ const imageHeight = Math.max(1, Math.ceil(elementImage.height));+ if (+ gpu.textureWidth !== imageWidth ||+ gpu.textureHeight !== imageHeight+ ) {+ gl.texImage2D(+ gl.TEXTURE_2D,+ 0,+ gl.RGBA8,+ imageWidth,+ imageHeight,+ 0,+ gl.RGBA,+ gl.UNSIGNED_BYTE,+ null,+ );+ gpu.textureWidth = imageWidth;+ gpu.textureHeight = imageHeight;+ }++ gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage);+ } else {+ gl.texElementImage2D(+ gl.TEXTURE_2D,+ 0,+ gl.RGBA,+ gl.RGBA,+ gl.UNSIGNED_BYTE,+ elementImage,+ );+ }if (gpu.uTex) {gl.uniform1i(gpu.uTex, 0);diff --git a/packages/example/src/HtmlInCanvas/compose-webgl.tsx b/packages/example/src/HtmlInCanvas/compose-webgl.tsxindex 47a8e5b1f0..8f604f360d 100644--- a/packages/example/src/HtmlInCanvas/compose-webgl.tsx+++ b/packages/example/src/HtmlInCanvas/compose-webgl.tsx@@ -16,6 +16,8 @@ type GpuState = {uMatrix: WebGLUniformLocation | null;uTex: WebGLUniformLocation | null;texture: WebGLTexture;+ textureWidth: number;+ textureHeight: number;vao: WebGLVertexArrayObject;buffer: WebGLBuffer;};@@ -119,7 +121,17 @@ export const HtmlInCanvasComposeWebGL: React.FC = () => {gl.enableVertexAttribArray(locUv);gl.vertexAttribPointer(locUv, 2, gl.FLOAT, false, 16, 8);- gpuRef.current = {gl, program, uMatrix, uTex, texture, vao, buffer};+ gpuRef.current = {+ gl,+ program,+ uMatrix,+ uTex,+ texture,+ textureWidth: 0,+ textureHeight: 0,+ vao,+ buffer,+ };return () => {gl.deleteProgram(program);@@ -147,14 +159,39 @@ export const HtmlInCanvasComposeWebGL: React.FC = () => {gl.activeTexture(gl.TEXTURE0);gl.bindTexture(gl.TEXTURE_2D, gpu.texture);- gl.texElementImage2D(- gl.TEXTURE_2D,- 0,- gl.RGBA,- gl.RGBA,- gl.UNSIGNED_BYTE,- elementImage,- );+ if (typeof gl.texElementSubImage2D === 'function') {+ const imageWidth = Math.max(1, Math.ceil(elementImage.width));+ const imageHeight = Math.max(1, Math.ceil(elementImage.height));+ if (+ gpu.textureWidth !== imageWidth ||+ gpu.textureHeight !== imageHeight+ ) {+ gl.texImage2D(+ gl.TEXTURE_2D,+ 0,+ gl.RGBA8,+ imageWidth,+ imageHeight,+ 0,+ gl.RGBA,+ gl.UNSIGNED_BYTE,+ null,+ );+ gpu.textureWidth = imageWidth;+ gpu.textureHeight = imageHeight;+ }++ gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage);+ } else {+ gl.texElementImage2D(+ gl.TEXTURE_2D,+ 0,+ gl.RGBA,+ gl.RGBA,+ gl.UNSIGNED_BYTE,+ elementImage,+ );+ }if (gpu.uTex) {gl.uniform1i(gpu.uTex, 0);diff --git a/packages/example/src/HtmlInCanvas/compose-webgpu.tsx b/packages/example/src/HtmlInCanvas/compose-webgpu.tsxindex 455a53e783..caf5f17430 100644--- a/packages/example/src/HtmlInCanvas/compose-webgpu.tsx+++ b/packages/example/src/HtmlInCanvas/compose-webgpu.tsx@@ -41,6 +41,14 @@ type GpuDevice = {queue: {submit(c: unknown[]): void;writeBuffer(b: GpuBuffer, offset: number, data: BufferSource): void;+ drawElementImageToTexture?(+ source: {source: Element | ElementImage},+ destination: {+ texture: GpuTexture;+ size: {width: number; height: number};+ },+ ): void;+ // Kept for Chrome builds predating drawElementImageToTexture.copyElementImageToTexture(source: Element | ElementImage,width: number,@@ -248,12 +256,19 @@ export const HtmlInCanvasComposeWebGPU: React.FC = () => {const {device, context, pipeline, texture, bindGroup, uniformBuffer} =gpu;- device.queue.copyElementImageToTexture(- elementImage,- gpu.width,- gpu.height,- {texture},- );+ if (device.queue.drawElementImageToTexture) {+ device.queue.drawElementImageToTexture(+ {source: elementImage},+ {texture, size: {width: gpu.width, height: gpu.height}},+ );+ } else {+ device.queue.copyElementImageToTexture(+ elementImage,+ gpu.width,+ gpu.height,+ {texture},+ );+ }const uniforms = new Float32Array([time, 0, gpu.width, gpu.height]);device.queue.writeBuffer(uniformBuffer, 0, uniforms);diff --git a/packages/example/src/HtmlInCanvas/docs-demo-2d-blur.tsx b/packages/example/src/HtmlInCanvas/docs-demo-2d-blur.tsxindex b1076151e2..5d6b325957 100644--- a/packages/example/src/HtmlInCanvas/docs-demo-2d-blur.tsx+++ b/packages/example/src/HtmlInCanvas/docs-demo-2d-blur.tsx@@ -36,7 +36,9 @@ export const HtmlInCanvasDocsDemo2DBlur: React.FC = () => {ctx.reset();ctx.filter = `blur(${blurPx}px)`;const transform = ctx.drawElementImage(elementImage, 0, 0);- element.style.transform = transform.toString();+ if (transform) {+ element.style.transform = transform.toString();+ }},[frame, fps],);diff --git a/packages/example/src/HtmlInCanvas/minimal-docs-webgl.tsx b/packages/example/src/HtmlInCanvas/minimal-docs-webgl.tsxindex 71ba2ef462..7cd52c968f 100644--- a/packages/example/src/HtmlInCanvas/minimal-docs-webgl.tsx+++ b/packages/example/src/HtmlInCanvas/minimal-docs-webgl.tsx@@ -19,6 +19,8 @@ type GlState = {uTex: WebGLUniformLocation | null;uTime: WebGLUniformLocation | null;texture: WebGLTexture;+ textureWidth: number;+ textureHeight: number;vao: WebGLVertexArrayObject;};@@ -117,7 +119,16 @@ export const HtmlInCanvasDocsMinimalWebGL: React.FC = () => {gl.enableVertexAttribArray(locUv);gl.vertexAttribPointer(locUv, 2, gl.FLOAT, false, 16, 8);- gpuRef.current = {gl, program, uTex, uTime, texture, vao};+ gpuRef.current = {+ gl,+ program,+ uTex,+ uTime,+ texture,+ textureWidth: 0,+ textureHeight: 0,+ vao,+ };return () => {gl.deleteProgram(program);@@ -141,14 +152,39 @@ export const HtmlInCanvasDocsMinimalWebGL: React.FC = () => {gl.activeTexture(gl.TEXTURE0);gl.bindTexture(gl.TEXTURE_2D, gpu.texture);- gl.texElementImage2D(- gl.TEXTURE_2D,- 0,- gl.RGBA,- gl.RGBA,- gl.UNSIGNED_BYTE,- elementImage,- );+ if (typeof gl.texElementSubImage2D === 'function') {+ const imageWidth = Math.max(1, Math.ceil(elementImage.width));+ const imageHeight = Math.max(1, Math.ceil(elementImage.height));+ if (+ gpu.textureWidth !== imageWidth ||+ gpu.textureHeight !== imageHeight+ ) {+ gl.texImage2D(+ gl.TEXTURE_2D,+ 0,+ gl.RGBA8,+ imageWidth,+ imageHeight,+ 0,+ gl.RGBA,+ gl.UNSIGNED_BYTE,+ null,+ );+ gpu.textureWidth = imageWidth;+ gpu.textureHeight = imageHeight;+ }++ gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage);+ } else {+ gl.texElementImage2D(+ gl.TEXTURE_2D,+ 0,+ gl.RGBA,+ gl.RGBA,+ gl.UNSIGNED_BYTE,+ elementImage,+ );+ }if (gpu.uTex) {gl.uniform1i(gpu.uTex, 0);diff --git a/packages/motion-blur/src/HtmlInCanvasMotionBlur.tsx b/packages/motion-blur/src/HtmlInCanvasMotionBlur.tsxindex 462118dba1..6cfc4efb8f 100644--- a/packages/motion-blur/src/HtmlInCanvasMotionBlur.tsx+++ b/packages/motion-blur/src/HtmlInCanvasMotionBlur.tsx@@ -204,7 +204,7 @@ const HtmlInCanvasMotionBlurInner: React.FC<: layoutCanvas.captureElementImage(sampleElement);try {const sampleTransform = context.drawElementImage(image, 0, 0);- if (index === Math.floor(actualSamples / 2)) {+ if (sampleTransform && index === Math.floor(actualSamples / 2)) {(sampleElement as HTMLElement).style.transform =sampleTransform.toString();}diff --git a/packages/skills/skills/remotion-markup/html-in-canvas.md b/packages/skills/skills/remotion-markup/html-in-canvas.mdindex d2638015e4..b78e9dd4ed 100644--- a/packages/skills/skills/remotion-markup/html-in-canvas.md+++ b/packages/skills/skills/remotion-markup/html-in-canvas.md@@ -47,7 +47,7 @@ export const MyComp = () => {## 2D effect with `onPaint`-`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM, and assign the returned transform to `element.style.transform` so DOM selection still aligns with the painted output.+`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM. Current Chromium synchronizes the element's geometry automatically; older versions return a transform that must be assigned to `element.style.transform`.```tsximport {@@ -73,7 +73,9 @@ export const Blur = () => {ctx.reset();ctx.filter = `blur(${blurPx}px)`;const transform = ctx.drawElementImage(elementImage, 0, 0);- element.style.transform = transform.toString();+ if (transform) {+ element.style.transform = transform.toString();+ }},[frame, fps],);@@ -98,28 +100,7 @@ export const Blur = () => {## WebGL effects-For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Inside `onPaint`, upload the captured DOM with `gl.texElementImage2D(...)` and draw.--```tsx-const onInit: HtmlInCanvasOnInit = useCallback(({ canvas }) => {- const gl = canvas.getContext("webgl2", { alpha: true, premultipliedAlpha: true });- if (!gl) {- throw new Error(- "WebGL2 unavailable. Try rendering with the --gl=angle option. See https://remotion.dev/docs/gl-options.",- );- }- gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);- // compile program, create texture, set up VAO...- return () => {- // delete program, texture, buffers...- };-}, []);--const onPaint: HtmlInCanvasOnPaint = useCallback(({ elementImage }) => {- gl.texElementImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, elementImage);- gl.drawArrays(gl.TRIANGLES, 0, 6);-}, []);-```+For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Allocate the texture with `gl.texImage2D(..., null)` when `elementImage` changes size. Inside `onPaint`, upload with `gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage)` and draw. Older Chromium versions only expose `gl.texElementImage2D()`; use it when the new method is absent.For a fully working minimal example, see https://github.com/remotion-dev/remotion/blob/main/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx.diff --git a/packages/transitions/src/html-in-canvas-presentation.tsx b/packages/transitions/src/html-in-canvas-presentation.tsxindex 6d8d8fb529..5f9b53e070 100644--- a/packages/transitions/src/html-in-canvas-presentation.tsx+++ b/packages/transitions/src/html-in-canvas-presentation.tsx@@ -185,13 +185,35 @@ export const HtmlInCanvasPresentation = <const canvas = canvasRef.current;if (!canvas) {- throw new Error('Canvas not found');+ return;+ }++ if (canvas.getAttribute('content') !== 'drawable') {+ canvas.setAttribute('content', 'drawable');+ }++ if (canvas.layoutSubtree !== true) {+ canvas.layoutSubtree = true;+ }++ const firstChild = canvas.firstElementChild;+ if (firstChild && !firstChild.hasAttribute('drawable')) {+ firstChild.setAttribute('drawable', '');+ }+ });++ useLayoutEffect(() => {+ if (passThrough) {+ return;}- canvas.layoutSubtree = true;+ const canvas = canvasRef.current;+ if (!canvas) {+ throw new Error('Canvas not found');+ }const onPaint = () => {- const firstChild = canvas.firstChild as HTMLElement;+ const firstChild = canvas.firstElementChild;const captureCanvas = captureCanvasRef.current;const captureContext = captureContextRef.current;diff --git a/packages/web-renderer/src/html-in-canvas.ts b/packages/web-renderer/src/html-in-canvas.tsindex 300fda3100..98e8b30a8a 100644--- a/packages/web-renderer/src/html-in-canvas.ts+++ b/packages/web-renderer/src/html-in-canvas.ts@@ -5,7 +5,7 @@ type Canvas2DWithDrawElement = CanvasRenderingContext2D & {dy: number,dwidth: number,dheight: number,- ) => DOMMatrix;+ ) => DOMMatrix | void;};type HTMLCanvasWithLayoutSubtree = HTMLCanvasElement & {@@ -26,18 +26,20 @@ export const supportsNativeHtmlInCanvas = (): boolean => {export const containsLayoutSubtreeCanvas = (element: HTMLElement): boolean => {return Array.from(element.querySelectorAll('canvas')).some(- (canvas) => (canvas as HTMLCanvasWithLayoutSubtree).layoutSubtree === true,+ (canvas) =>+ (canvas as HTMLCanvasWithLayoutSubtree).layoutSubtree === true ||+ canvas.getAttribute('content') === 'drawable',);};export type HtmlInCanvasContext = {layoutCanvas: HTMLCanvasWithLayoutSubtree;ctx: Canvas2DWithDrawElement;+ wasDrawable: boolean;};/**- * Sets up a persistent layoutsubtree canvas that wraps the scaffold div.- * The div becomes a direct child of the canvas, which is required for drawElementImage.+ * Sets up a persistent drawable canvas that wraps the scaffold div.* Must be called once before rendering begins; the canvas stays in the DOM for the* lifetime of the render.*/@@ -59,6 +61,7 @@ export const setupHtmlInCanvas = ({const layoutCanvas = document.createElement('canvas',) as HTMLCanvasWithLayoutSubtree;+ layoutCanvas.setAttribute('content', 'drawable');layoutCanvas.layoutSubtree = true;layoutCanvas.width = width;@@ -85,11 +88,13 @@ export const setupHtmlInCanvas = ({return null;}+ const wasDrawable = div.hasAttribute('drawable');+ div.setAttribute('drawable', '');wrapper.removeChild(div);layoutCanvas.appendChild(div);wrapper.appendChild(layoutCanvas);- return {layoutCanvas, ctx: maybeCtx};+ return {layoutCanvas, ctx: maybeCtx, wasDrawable};};const waitForPaint = (@@ -153,8 +158,11 @@ export const teardownHtmlInCanvas = ({wrapper: HTMLDivElement;div: HTMLDivElement;}) => {- const {layoutCanvas} = htmlInCanvasContext;+ const {layoutCanvas, wasDrawable} = htmlInCanvasContext;layoutCanvas.removeChild(div);wrapper.removeChild(layoutCanvas);wrapper.appendChild(div);+ if (!wasDrawable) {+ div.removeAttribute('drawable');+ }};diff --git a/packages/web-renderer/src/test/html-in-canvas.test.tsx b/packages/web-renderer/src/test/html-in-canvas.test.tsxindex 942f90e9ad..f3028a1af6 100644--- a/packages/web-renderer/src/test/html-in-canvas.test.tsx+++ b/packages/web-renderer/src/test/html-in-canvas.test.tsx@@ -151,6 +151,7 @@ test('does not create a nested HTML-in-canvas capture', async () => {drawElementImage: typeof drawElementImage;},layoutCanvas: outerLayoutCanvas,+ wasDrawable: false,},internalState,logLevel: 'error',