Skip to content

πŸŽ₯ Make videos programmatically with React

Adapt HTML-in-canvas to the drawable API

JonnyBurgerc8202eae50ed
diff --git a/packages/canvas-capture-extension/README.md b/packages/canvas-capture-extension/README.md
index 4e082cda75..ef3b7f818d 100644
--- a/packages/canvas-capture-extension/README.md
+++ b/packages/canvas-capture-extension/README.md
@@ -77,4 +77,4 @@ recordings. Stop the recording, then open it directly in
[remotion.dev/new](https://remotion.dev/new), inspect it in
[remotion.dev/convert](https://remotion.dev/convert), or save it to disk.
-The page contents are temporarily placed inside a `layoutSubtree` canvas while recording and restored afterward. Websites that rely on direct-child CSS selectors may look different during capture. Chrome's own pages and the Chrome Web Store do not allow extension script injection.
+The page contents are temporarily placed inside a `content="drawable"` canvas while recording and restored afterward. The extension also sets `layoutSubtree` for older Chromium versions. Websites that rely on direct-child CSS selectors may look different during capture. Chrome's own pages and the Chrome Web Store do not allow extension script injection.
diff --git a/packages/canvas-capture-extension/src/capture.ts b/packages/canvas-capture-extension/src/capture.ts
index 1517bb7892..40da973a9b 100644
--- a/packages/canvas-capture-extension/src/capture.ts
+++ b/packages/canvas-capture-extension/src/capture.ts
@@ -137,8 +137,10 @@ const wrapWholePage = (): WrappedPage => {
const minimumSize = getWholePageSize();
const canvas = document.createElement('canvas') as HtmlInCanvasElement;
const content = document.createElement('div');
+ canvas.setAttribute('content', 'drawable');
canvas.layoutSubtree = true;
canvas.setAttribute('layoutsubtree', '');
+ content.setAttribute('drawable', '');
canvas.style.display = 'block';
canvas.style.position = 'relative';
content.style.position = 'absolute';
diff --git a/packages/codex-plugin/skills/remotion-best-practices/remotion-markup/html-in-canvas.md b/packages/codex-plugin/skills/remotion-best-practices/remotion-markup/html-in-canvas.md
index d2638015e4..b78e9dd4ed 100644
--- a/packages/codex-plugin/skills/remotion-best-practices/remotion-markup/html-in-canvas.md
+++ b/packages/codex-plugin/skills/remotion-best-practices/remotion-markup/html-in-canvas.md
@@ -47,7 +47,7 @@ export const MyComp = () => {
## 2D effect with `onPaint`
-`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM, and assign the returned transform to `element.style.transform` so DOM selection still aligns with the painted output.
+`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM. Current Chromium synchronizes the element's geometry automatically; older versions return a transform that must be assigned to `element.style.transform`.
```tsx
import {
@@ -73,7 +73,9 @@ export const Blur = () => {
ctx.reset();
ctx.filter = `blur(${blurPx}px)`;
const transform = ctx.drawElementImage(elementImage, 0, 0);
- element.style.transform = transform.toString();
+ if (transform) {
+ element.style.transform = transform.toString();
+ }
},
[frame, fps],
);
@@ -98,28 +100,7 @@ export const Blur = () => {
## WebGL effects
-For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Inside `onPaint`, upload the captured DOM with `gl.texElementImage2D(...)` and draw.
-
-```tsx
-const onInit: HtmlInCanvasOnInit = useCallback(({ canvas }) => {
- const gl = canvas.getContext("webgl2", { alpha: true, premultipliedAlpha: true });
- if (!gl) {
- throw new Error(
- "WebGL2 unavailable. Try rendering with the --gl=angle option. See https://remotion.dev/docs/gl-options.",
- );
- }
- gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
- // compile program, create texture, set up VAO...
- return () => {
- // delete program, texture, buffers...
- };
-}, []);
-
-const onPaint: HtmlInCanvasOnPaint = useCallback(({ elementImage }) => {
- gl.texElementImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, elementImage);
- gl.drawArrays(gl.TRIANGLES, 0, 6);
-}, []);
-```
+For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Allocate the texture with `gl.texImage2D(..., null)` when `elementImage` changes size. Inside `onPaint`, upload with `gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage)` and draw. Older Chromium versions only expose `gl.texElementImage2D()`; use it when the new method is absent.
For a fully working minimal example, see https://github.com/remotion-dev/remotion/blob/main/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx.
diff --git a/packages/codex-plugin/skills/remotion-markup/html-in-canvas.md b/packages/codex-plugin/skills/remotion-markup/html-in-canvas.md
index d2638015e4..b78e9dd4ed 100644
--- a/packages/codex-plugin/skills/remotion-markup/html-in-canvas.md
+++ b/packages/codex-plugin/skills/remotion-markup/html-in-canvas.md
@@ -47,7 +47,7 @@ export const MyComp = () => {
## 2D effect with `onPaint`
-`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM, and assign the returned transform to `element.style.transform` so DOM selection still aligns with the painted output.
+`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM. Current Chromium synchronizes the element's geometry automatically; older versions return a transform that must be assigned to `element.style.transform`.
```tsx
import {
@@ -73,7 +73,9 @@ export const Blur = () => {
ctx.reset();
ctx.filter = `blur(${blurPx}px)`;
const transform = ctx.drawElementImage(elementImage, 0, 0);
- element.style.transform = transform.toString();
+ if (transform) {
+ element.style.transform = transform.toString();
+ }
},
[frame, fps],
);
@@ -98,28 +100,7 @@ export const Blur = () => {
## WebGL effects
-For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Inside `onPaint`, upload the captured DOM with `gl.texElementImage2D(...)` and draw.
-
-```tsx
-const onInit: HtmlInCanvasOnInit = useCallback(({ canvas }) => {
- const gl = canvas.getContext("webgl2", { alpha: true, premultipliedAlpha: true });
- if (!gl) {
- throw new Error(
- "WebGL2 unavailable. Try rendering with the --gl=angle option. See https://remotion.dev/docs/gl-options.",
- );
- }
- gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
- // compile program, create texture, set up VAO...
- return () => {
- // delete program, texture, buffers...
- };
-}, []);
-
-const onPaint: HtmlInCanvasOnPaint = useCallback(({ elementImage }) => {
- gl.texElementImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, elementImage);
- gl.drawArrays(gl.TRIANGLES, 0, 6);
-}, []);
-```
+For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Allocate the texture with `gl.texImage2D(..., null)` when `elementImage` changes size. Inside `onPaint`, upload with `gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage)` and draw. Older Chromium versions only expose `gl.texElementImage2D()`; use it when the new method is absent.
For a fully working minimal example, see https://github.com/remotion-dev/remotion/blob/main/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx.
diff --git a/packages/core/src/HtmlInCanvas.tsx b/packages/core/src/HtmlInCanvas.tsx
index cd73ba4c82..2744bb55b2 100644
--- a/packages/core/src/HtmlInCanvas.tsx
+++ b/packages/core/src/HtmlInCanvas.tsx
@@ -41,9 +41,28 @@ import {useRemotionEnvironment} from './use-remotion-environment.js';
import {withInteractivitySchema} from './with-interactivity-schema.js';
// IDL: https://github.com/WICG/html-in-canvas#idl-changes
-// WebGPU's `copyElementImageToTexture` is omitted β€” `GPUQueue` is not in
+// WebGPU's `drawElementImageToTexture` is omitted β€” `GPUQueue` is not in
// lib.dom.d.ts and would require pulling in `@webgpu/types`.
declare global {
+ interface DrawElementImageOptions {
+ preserveElementGeometry?: boolean;
+ }
+
+ interface WebGLCopyElementImageConfig {
+ sx?: GLfloat;
+ sy?: GLfloat;
+ swidth?: GLfloat;
+ sheight?: GLfloat;
+ width?: GLsizei;
+ height?: GLsizei;
+ }
+
+ interface UpdateElementGeometryOptions {
+ preserveHitTestOrder?: boolean;
+ clip?: DOMRectInit;
+ canvasTransform?: DOMMatrixInit;
+ }
+
interface ElementImage {
readonly width: number;
readonly height: number;
@@ -55,14 +74,16 @@ declare global {
element: Element | ElementImage,
dx: number,
dy: number,
- ): DOMMatrix;
+ options?: DrawElementImageOptions,
+ ): DOMMatrix | void;
drawElementImage(
element: Element | ElementImage,
dx: number,
dy: number,
dwidth: number,
dheight: number,
- ): DOMMatrix;
+ options?: DrawElementImageOptions,
+ ): DOMMatrix | void;
drawElementImage(
element: Element | ElementImage,
sx: number,
@@ -71,7 +92,8 @@ declare global {
sheight: number,
dx: number,
dy: number,
- ): DOMMatrix;
+ options?: DrawElementImageOptions,
+ ): DOMMatrix | void;
drawElementImage(
element: Element | ElementImage,
sx: number,
@@ -82,7 +104,8 @@ declare global {
dy: number,
dwidth: number,
dheight: number,
- ): DOMMatrix;
+ options?: DrawElementImageOptions,
+ ): DOMMatrix | void;
}
interface OffscreenCanvasRenderingContext2D {
@@ -90,14 +113,16 @@ declare global {
element: Element | ElementImage,
dx: number,
dy: number,
- ): DOMMatrix;
+ options?: DrawElementImageOptions,
+ ): DOMMatrix | void;
drawElementImage(
element: Element | ElementImage,
dx: number,
dy: number,
dwidth: number,
dheight: number,
- ): DOMMatrix;
+ options?: DrawElementImageOptions,
+ ): DOMMatrix | void;
drawElementImage(
element: Element | ElementImage,
sx: number,
@@ -106,7 +131,8 @@ declare global {
sheight: number,
dx: number,
dy: number,
- ): DOMMatrix;
+ options?: DrawElementImageOptions,
+ ): DOMMatrix | void;
drawElementImage(
element: Element | ElementImage,
sx: number,
@@ -117,11 +143,22 @@ declare global {
dy: number,
dwidth: number,
dheight: number,
- ): DOMMatrix;
+ options?: DrawElementImageOptions,
+ ): DOMMatrix | void;
}
// Augmenting the base interface applies to both WebGL1 and WebGL2.
interface WebGLRenderingContextBase {
+ texElementSubImage2D(
+ target: GLenum,
+ level: GLint,
+ xoffset: GLint,
+ yoffset: GLint,
+ element: Element | ElementImage,
+ config?: WebGLCopyElementImageConfig,
+ ): void;
+
+ // Older Chromium builds expose texElementImage2D during the migration.
texElementImage2D(
target: GLenum,
level: GLint,
@@ -173,15 +210,30 @@ declare global {
}
interface HTMLCanvasElement {
+ content: string;
layoutSubtree?: boolean;
onpaint: ((this: HTMLCanvasElement, ev: Event) => unknown) | null;
requestPaint?(): void;
captureElementImage(element: Element): ElementImage;
+ updateElementGeometry(
+ element: Element | ElementImage,
+ options?: UpdateElementGeometryOptions,
+ ): void;
+ clearElementGeometry(element: Element | ElementImage): void;
+ getElementTransform(element: Element): DOMMatrix;
getElementTransform(
element: Element | ElementImage,
drawTransform: DOMMatrix,
): DOMMatrix;
}
+
+ interface OffscreenCanvas {
+ updateElementGeometry(
+ element: Element | ElementImage,
+ options?: UpdateElementGeometryOptions,
+ ): void;
+ clearElementGeometry(element: Element | ElementImage): void;
+ }
}
export type HtmlInCanvasOnPaintParams = {
@@ -376,7 +428,10 @@ const defaultOnPaint = ({
ctx.reset();
const transform = ctx.drawElementImage(elementImage, 0, 0);
- element.style.transform = transform.toString();
+ if (transform) {
+ // Older Chromium requires manual geometry synchronization.
+ element.style.transform = transform.toString();
+ }
};
/* eslint-disable react/require-default-props -- optional fields mirror `<Sequence>` / canvas hooks API */
@@ -660,7 +715,9 @@ const HtmlInCanvasContent = forwardRef<
throw new Error('Canvas not found');
}
+ placeholder.setAttribute('content', 'drawable');
placeholder.layoutSubtree = true;
+ divRef.current?.setAttribute('drawable', '');
const paintTarget = usesDirectLayoutCanvas
? placeholder
diff --git a/packages/docs/components/demos/HtmlInCanvasDocsDemo2DBlur.tsx b/packages/docs/components/demos/HtmlInCanvasDocsDemo2DBlur.tsx
index 26eed98f80..d3a3a5a1ea 100644
--- a/packages/docs/components/demos/HtmlInCanvasDocsDemo2DBlur.tsx
+++ b/packages/docs/components/demos/HtmlInCanvasDocsDemo2DBlur.tsx
@@ -31,7 +31,9 @@ const HtmlInCanvasDocsDemo2DBlurInner: React.FC = () => {
ctx.reset();
ctx.filter = `blur(${blurPx}px)`;
const transform = ctx.drawElementImage(elementImage, 0, 0);
- element.style.transform = transform.toString();
+ if (transform) {
+ element.style.transform = transform.toString();
+ }
},
[frame, fps],
);
diff --git a/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx b/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx
index 2cad5c5d83..de1a684892 100644
--- a/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx
+++ b/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx
@@ -20,6 +20,8 @@ type GlState = {
uTex: WebGLUniformLocation | null;
uTime: WebGLUniformLocation | null;
texture: WebGLTexture;
+ textureWidth: number;
+ textureHeight: number;
vao: WebGLVertexArrayObject;
};
@@ -118,7 +120,16 @@ const HtmlInCanvasDocsWebGLInner: React.FC = () => {
gl.enableVertexAttribArray(locUv);
gl.vertexAttribPointer(locUv, 2, gl.FLOAT, false, 16, 8);
- gpuRef.current = {gl, program, uTex, uTime, texture, vao};
+ gpuRef.current = {
+ gl,
+ program,
+ uTex,
+ uTime,
+ texture,
+ textureWidth: 0,
+ textureHeight: 0,
+ vao,
+ };
return () => {
gl.deleteProgram(program);
@@ -142,14 +153,39 @@ const HtmlInCanvasDocsWebGLInner: React.FC = () => {
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, gpu.texture);
- gl.texElementImage2D(
- gl.TEXTURE_2D,
- 0,
- gl.RGBA,
- gl.RGBA,
- gl.UNSIGNED_BYTE,
- elementImage,
- );
+ if (typeof gl.texElementSubImage2D === 'function') {
+ const imageWidth = Math.max(1, Math.ceil(elementImage.width));
+ const imageHeight = Math.max(1, Math.ceil(elementImage.height));
+ if (
+ gpu.textureWidth !== imageWidth ||
+ gpu.textureHeight !== imageHeight
+ ) {
+ gl.texImage2D(
+ gl.TEXTURE_2D,
+ 0,
+ gl.RGBA8,
+ imageWidth,
+ imageHeight,
+ 0,
+ gl.RGBA,
+ gl.UNSIGNED_BYTE,
+ null,
+ );
+ gpu.textureWidth = imageWidth;
+ gpu.textureHeight = imageHeight;
+ }
+
+ gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage);
+ } else {
+ gl.texElementImage2D(
+ gl.TEXTURE_2D,
+ 0,
+ gl.RGBA,
+ gl.RGBA,
+ gl.UNSIGNED_BYTE,
+ elementImage,
+ );
+ }
if (gpu.uTex) {
gl.uniform1i(gpu.uTex, 0);
diff --git a/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGPU.tsx b/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGPU.tsx
index 167ff68238..662a85eb12 100644
--- a/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGPU.tsx
+++ b/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGPU.tsx
@@ -43,6 +43,14 @@ type GpuDevice = {
queue: {
submit(c: unknown[]): void;
writeBuffer(b: GpuBuffer, offset: number, data: BufferSource): void;
+ drawElementImageToTexture?(
+ source: {source: Element | ElementImage},
+ destination: {
+ texture: GpuTexture;
+ size: {width: number; height: number};
+ },
+ ): void;
+ // Kept for Chrome builds predating drawElementImageToTexture.
copyElementImageToTexture(
source: Element | ElementImage,
width: number,
@@ -247,12 +255,19 @@ const HtmlInCanvasDocsWebGPUInner: React.FC = () => {
const {device, context, pipeline, texture, bindGroup, uniformBuffer} =
gpu;
- device.queue.copyElementImageToTexture(
- elementImage,
- gpu.width,
- gpu.height,
- {texture},
- );
+ if (device.queue.drawElementImageToTexture) {
+ device.queue.drawElementImageToTexture(
+ {source: elementImage},
+ {texture, size: {width: gpu.width, height: gpu.height}},
+ );
+ } else {
+ device.queue.copyElementImageToTexture(
+ elementImage,
+ gpu.width,
+ gpu.height,
+ {texture},
+ );
+ }
const uniforms = new Float32Array([time, 0, gpu.width, gpu.height]);
device.queue.writeBuffer(uniformBuffer, 0, uniforms);
diff --git a/packages/docs/docs/html-in-canvas-guide.mdx b/packages/docs/docs/html-in-canvas-guide.mdx
index 556678bbeb..34c2d5834d 100644
--- a/packages/docs/docs/html-in-canvas-guide.mdx
+++ b/packages/docs/docs/html-in-canvas-guide.mdx
@@ -50,7 +50,9 @@ const onPaint: HtmlInCanvasOnPaint = ({canvas, element, elementImage}) => {
ctx.reset();
ctx.filter = 'blur(8px)';
const transform = ctx.drawElementImage(elementImage, 0, 0);
- element.style.transform = transform.toString();
+ if (transform) {
+ element.style.transform = transform.toString();
+ }
};
export const MyComp: React.FC = () => {
diff --git a/packages/docs/docs/remotion/html-in-canvas.mdx b/packages/docs/docs/remotion/html-in-canvas.mdx
index 574f6692a0..63f5e50b86 100644
--- a/packages/docs/docs/remotion/html-in-canvas.mdx
+++ b/packages/docs/docs/remotion/html-in-canvas.mdx
@@ -92,7 +92,9 @@ const onPaint: HtmlInCanvasOnPaint = ({canvas, element, elementImage, pixelDensi
ctx.reset();
ctx.filter = `blur(${10 * pixelDensity}px)`;
const transform = ctx.drawElementImage(elementImage, 0, 0);
- element.style.transform = transform.toString();
+ if (transform) {
+ element.style.transform = transform.toString();
+ }
};
```
@@ -108,7 +110,7 @@ You should paint to this canvas.
#### `element`
The inner [`HTMLDivElement`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLDivElement) that wraps `children`.
-You should apply the return value of [`drawElementImage`](https://github.com/WICG/html-in-canvas#2-drawelementimage-and-webglwebgpu-equivalents) to this element's `style.transform` property.
+Current Chromium synchronizes its position and hit testing when you call `drawElementImage()`. Older versions return a matrix that you should apply to `element.style.transform`.
#### `elementImage`
@@ -150,7 +152,7 @@ From <AvailableFrom v="4.0.528" inline />, it also inherits [`premountFor`](/doc
### `ref?`
You can add a [React ref](https://react.dev/learn/manipulating-the-dom-with-refs) to `<HtmlInCanvas />`.
-It is attached to the layout [`HTMLCanvasElement`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement) β€” the canvas that hosts the laid-out subtree (`layoutSubtree`).
+It is attached to the [`HTMLCanvasElement`](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement) that hosts the drawable subtree.
If you use TypeScript, type the ref with `HTMLCanvasElement`:
@@ -173,8 +175,8 @@ export const Example: React.FC = () => {
### 2D
-Call [`drawElementImage()`](https://github.com/WICG/html-in-canvas#2-drawelementimage-and-webglwebgpu-equivalents) on the 2D context to draw `elementImage` into the bitmap.
-HTML-in-canvas recommends that you use the return value and assign it to `element.style.transform`, so that the original DOM element matches the transform, so that selection still works.
+Call [`drawElementImage()`](https://github.com/WICG/html-in-canvas#idl-changes) on the 2D context to draw `elementImage` into the bitmap.
+Current Chromium updates the element's position and hit testing automatically. Older versions return a transform that must be assigned to `element.style.transform` for selection and input to follow the drawing.
For 2D, you usually do not need `onInit`.
@@ -213,7 +215,9 @@ export const HtmlInCanvas2DBlur: React.FC = () => {
ctx.reset();
ctx.filter = `blur(${blurPx * pixelDensity}px)`;
const transform = ctx.drawElementImage(elementImage, 0, 0);
- element.style.transform = transform.toString();
+ if (transform) {
+ element.style.transform = transform.toString();
+ }
},
[frame, fps],
);
@@ -242,7 +246,8 @@ export const HtmlInCanvas2DBlur: React.FC = () => {
Do all setup in [`onInit`](#oninit) such as getting the WebGL context and compiling the shader.
`onInit` must return a cleanup function that destroys the resources that were created in `onInit`.
-Use [`texElementImage2D`](https://github.com/WICG/html-in-canvas#2-drawelementimage-and-webglwebgpu-equivalents) to turn the `elementImage` into a texture.
+Use [`texElementSubImage2D`](https://github.com/WICG/html-in-canvas#idl-changes) to upload `elementImage` into a preallocated texture. Older Chromium versions expose `texElementImage2D` instead.
+WebGL does not update the DOM element's hit testing geometry automatically. If the content needs to remain interactive, call `canvas.updateElementGeometry(element, {canvasTransform})` with a transform matching where your shader draws it.
<Demo type="html-in-canvas-webgl" />
@@ -268,6 +273,8 @@ type GlState = {
uTex: WebGLUniformLocation | null;
uTime: WebGLUniformLocation | null;
texture: WebGLTexture;
+ textureWidth: number;
+ textureHeight: number;
vao: WebGLVertexArrayObject;
};
@@ -366,7 +373,16 @@ export const HtmlInCanvasDocsMinimalWebGL: React.FC = () => {
gl.enableVertexAttribArray(locUv);
gl.vertexAttribPointer(locUv, 2, gl.FLOAT, false, 16, 8);
- gpuRef.current = {gl, program, uTex, uTime, texture, vao};
+ gpuRef.current = {
+ gl,
+ program,
+ uTex,
+ uTime,
+ texture,
+ textureWidth: 0,
+ textureHeight: 0,
+ vao,
+ };
return () => {
gl.deleteProgram(program);
@@ -390,14 +406,39 @@ export const HtmlInCanvasDocsMinimalWebGL: React.FC = () => {
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, gpu.texture);
- gl.texElementImage2D(
- gl.TEXTURE_2D,
- 0,
- gl.RGBA,
- gl.RGBA,
- gl.UNSIGNED_BYTE,
- elementImage,
- );
+ if (typeof gl.texElementSubImage2D === 'function') {
+ const imageWidth = Math.max(1, Math.ceil(elementImage.width));
+ const imageHeight = Math.max(1, Math.ceil(elementImage.height));
+ if (
+ gpu.textureWidth !== imageWidth ||
+ gpu.textureHeight !== imageHeight
+ ) {
+ gl.texImage2D(
+ gl.TEXTURE_2D,
+ 0,
+ gl.RGBA8,
+ imageWidth,
+ imageHeight,
+ 0,
+ gl.RGBA,
+ gl.UNSIGNED_BYTE,
+ null,
+ );
+ gpu.textureWidth = imageWidth;
+ gpu.textureHeight = imageHeight;
+ }
+
+ gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage);
+ } else {
+ gl.texElementImage2D(
+ gl.TEXTURE_2D,
+ 0,
+ gl.RGBA,
+ gl.RGBA,
+ gl.UNSIGNED_BYTE,
+ elementImage,
+ );
+ }
if (gpu.uTex) {
gl.uniform1i(gpu.uTex, 0);
@@ -439,7 +480,8 @@ export const HtmlInCanvasDocsMinimalWebGL: React.FC = () => {
Use [`onInit`](#oninit) to get a WebGPU context, to request a GPU and to compile shaders.
`onInit` must return a cleanup function that destroys the resources that were created in `onInit`.
-In `onPaint`, draw the `elementImage` to the canvas using [`copyElementImageToTexture`](https://github.com/WICG/html-in-canvas#2-drawelementimage-and-webglwebgpu-equivalents).
+In `onPaint`, upload `elementImage` using [`drawElementImageToTexture`](https://github.com/WICG/html-in-canvas#idl-changes). Older Chromium versions expose `copyElementImageToTexture` instead.
+WebGPU also requires `canvas.updateElementGeometry(element, {canvasTransform})` when the drawn content needs hit testing.
<Demo type="html-in-canvas-webgpu" />
@@ -490,6 +532,13 @@ type GpuDevice = {
queue: {
submit(c: unknown[]): void;
writeBuffer(b: GpuBuffer, offset: number, data: BufferSource): void;
+ drawElementImageToTexture?(
+ source: {source: Element | ElementImage},
+ destination: {
+ texture: GpuTexture;
+ size: {width: number; height: number};
+ },
+ ): void;
copyElementImageToTexture(
source: Element | ElementImage,
width: number,
@@ -697,12 +746,19 @@ export const HtmlInCanvasComposeWebGPU: React.FC = () => {
const {device, context, pipeline, texture, bindGroup, uniformBuffer} =
gpu;
- device.queue.copyElementImageToTexture(
- elementImage,
- gpu.width,
- gpu.height,
- {texture},
- );
+ if (device.queue.drawElementImageToTexture) {
+ device.queue.drawElementImageToTexture(
+ {source: elementImage},
+ {texture, size: {width: gpu.width, height: gpu.height}},
+ );
+ } else {
+ device.queue.copyElementImageToTexture(
+ elementImage,
+ gpu.width,
+ gpu.height,
+ {texture},
+ );
+ }
const uniforms = new Float32Array([time, 0, gpu.width, gpu.height]);
device.queue.writeBuffer(uniformBuffer, 0, uniforms);
diff --git a/packages/example/src/HtmlInCanvas/changing-size.tsx b/packages/example/src/HtmlInCanvas/changing-size.tsx
index 94d5975095..eb19627b22 100644
--- a/packages/example/src/HtmlInCanvas/changing-size.tsx
+++ b/packages/example/src/HtmlInCanvas/changing-size.tsx
@@ -38,7 +38,9 @@ export const HtmlInCanvasChangingSize: React.FC = () => {
canvas.width,
canvas.height,
);
- element.style.transform = transform.toString();
+ if (transform) {
+ element.style.transform = transform.toString();
+ }
const {data: pixels} = ctx.getImageData(
0,
0,
diff --git a/packages/example/src/HtmlInCanvas/complex-text.tsx b/packages/example/src/HtmlInCanvas/complex-text.tsx
index 48f31ed2fd..875d2aaf5c 100644
--- a/packages/example/src/HtmlInCanvas/complex-text.tsx
+++ b/packages/example/src/HtmlInCanvas/complex-text.tsx
@@ -28,7 +28,9 @@ export const HtmlInCanvasComplexText: React.FC = () => {
ctx.rotate((15 * Math.PI) / 180);
ctx.translate(160, -40);
const transform = ctx.drawElementImage(elementImage, 0, 0);
- element.style.transform = transform.toString();
+ if (transform) {
+ element.style.transform = transform.toString();
+ }
}}
>
<div
diff --git a/packages/example/src/HtmlInCanvas/compose-webgl-crt.tsx b/packages/example/src/HtmlInCanvas/compose-webgl-crt.tsx
index 9b69f6725e..86f37484ff 100644
--- a/packages/example/src/HtmlInCanvas/compose-webgl-crt.tsx
+++ b/packages/example/src/HtmlInCanvas/compose-webgl-crt.tsx
@@ -15,6 +15,8 @@ type GpuState = {
uTime: WebGLUniformLocation | null;
uResolution: WebGLUniformLocation | null;
texture: WebGLTexture;
+ textureWidth: number;
+ textureHeight: number;
vao: WebGLVertexArrayObject;
buffer: WebGLBuffer;
};
@@ -166,6 +168,8 @@ export const HtmlInCanvasComposeWebGLCrt: React.FC = () => {
uTime,
uResolution,
texture,
+ textureWidth: 0,
+ textureHeight: 0,
vao,
buffer,
};
@@ -192,14 +196,39 @@ export const HtmlInCanvasComposeWebGLCrt: React.FC = () => {
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, gpu.texture);
- gl.texElementImage2D(
- gl.TEXTURE_2D,
- 0,
- gl.RGBA,
- gl.RGBA,
- gl.UNSIGNED_BYTE,
- elementImage,
- );
+ if (typeof gl.texElementSubImage2D === 'function') {
+ const imageWidth = Math.max(1, Math.ceil(elementImage.width));
+ const imageHeight = Math.max(1, Math.ceil(elementImage.height));
+ if (
+ gpu.textureWidth !== imageWidth ||
+ gpu.textureHeight !== imageHeight
+ ) {
+ gl.texImage2D(
+ gl.TEXTURE_2D,
+ 0,
+ gl.RGBA8,
+ imageWidth,
+ imageHeight,
+ 0,
+ gl.RGBA,
+ gl.UNSIGNED_BYTE,
+ null,
+ );
+ gpu.textureWidth = imageWidth;
+ gpu.textureHeight = imageHeight;
+ }
+
+ gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage);
+ } else {
+ gl.texElementImage2D(
+ gl.TEXTURE_2D,
+ 0,
+ gl.RGBA,
+ gl.RGBA,
+ gl.UNSIGNED_BYTE,
+ elementImage,
+ );
+ }
if (gpu.uTex) {
gl.uniform1i(gpu.uTex, 0);
diff --git a/packages/example/src/HtmlInCanvas/compose-webgl.tsx b/packages/example/src/HtmlInCanvas/compose-webgl.tsx
index 47a8e5b1f0..8f604f360d 100644
--- a/packages/example/src/HtmlInCanvas/compose-webgl.tsx
+++ b/packages/example/src/HtmlInCanvas/compose-webgl.tsx
@@ -16,6 +16,8 @@ type GpuState = {
uMatrix: WebGLUniformLocation | null;
uTex: WebGLUniformLocation | null;
texture: WebGLTexture;
+ textureWidth: number;
+ textureHeight: number;
vao: WebGLVertexArrayObject;
buffer: WebGLBuffer;
};
@@ -119,7 +121,17 @@ export const HtmlInCanvasComposeWebGL: React.FC = () => {
gl.enableVertexAttribArray(locUv);
gl.vertexAttribPointer(locUv, 2, gl.FLOAT, false, 16, 8);
- gpuRef.current = {gl, program, uMatrix, uTex, texture, vao, buffer};
+ gpuRef.current = {
+ gl,
+ program,
+ uMatrix,
+ uTex,
+ texture,
+ textureWidth: 0,
+ textureHeight: 0,
+ vao,
+ buffer,
+ };
return () => {
gl.deleteProgram(program);
@@ -147,14 +159,39 @@ export const HtmlInCanvasComposeWebGL: React.FC = () => {
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, gpu.texture);
- gl.texElementImage2D(
- gl.TEXTURE_2D,
- 0,
- gl.RGBA,
- gl.RGBA,
- gl.UNSIGNED_BYTE,
- elementImage,
- );
+ if (typeof gl.texElementSubImage2D === 'function') {
+ const imageWidth = Math.max(1, Math.ceil(elementImage.width));
+ const imageHeight = Math.max(1, Math.ceil(elementImage.height));
+ if (
+ gpu.textureWidth !== imageWidth ||
+ gpu.textureHeight !== imageHeight
+ ) {
+ gl.texImage2D(
+ gl.TEXTURE_2D,
+ 0,
+ gl.RGBA8,
+ imageWidth,
+ imageHeight,
+ 0,
+ gl.RGBA,
+ gl.UNSIGNED_BYTE,
+ null,
+ );
+ gpu.textureWidth = imageWidth;
+ gpu.textureHeight = imageHeight;
+ }
+
+ gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage);
+ } else {
+ gl.texElementImage2D(
+ gl.TEXTURE_2D,
+ 0,
+ gl.RGBA,
+ gl.RGBA,
+ gl.UNSIGNED_BYTE,
+ elementImage,
+ );
+ }
if (gpu.uTex) {
gl.uniform1i(gpu.uTex, 0);
diff --git a/packages/example/src/HtmlInCanvas/compose-webgpu.tsx b/packages/example/src/HtmlInCanvas/compose-webgpu.tsx
index 455a53e783..caf5f17430 100644
--- a/packages/example/src/HtmlInCanvas/compose-webgpu.tsx
+++ b/packages/example/src/HtmlInCanvas/compose-webgpu.tsx
@@ -41,6 +41,14 @@ type GpuDevice = {
queue: {
submit(c: unknown[]): void;
writeBuffer(b: GpuBuffer, offset: number, data: BufferSource): void;
+ drawElementImageToTexture?(
+ source: {source: Element | ElementImage},
+ destination: {
+ texture: GpuTexture;
+ size: {width: number; height: number};
+ },
+ ): void;
+ // Kept for Chrome builds predating drawElementImageToTexture.
copyElementImageToTexture(
source: Element | ElementImage,
width: number,
@@ -248,12 +256,19 @@ export const HtmlInCanvasComposeWebGPU: React.FC = () => {
const {device, context, pipeline, texture, bindGroup, uniformBuffer} =
gpu;
- device.queue.copyElementImageToTexture(
- elementImage,
- gpu.width,
- gpu.height,
- {texture},
- );
+ if (device.queue.drawElementImageToTexture) {
+ device.queue.drawElementImageToTexture(
+ {source: elementImage},
+ {texture, size: {width: gpu.width, height: gpu.height}},
+ );
+ } else {
+ device.queue.copyElementImageToTexture(
+ elementImage,
+ gpu.width,
+ gpu.height,
+ {texture},
+ );
+ }
const uniforms = new Float32Array([time, 0, gpu.width, gpu.height]);
device.queue.writeBuffer(uniformBuffer, 0, uniforms);
diff --git a/packages/example/src/HtmlInCanvas/docs-demo-2d-blur.tsx b/packages/example/src/HtmlInCanvas/docs-demo-2d-blur.tsx
index b1076151e2..5d6b325957 100644
--- a/packages/example/src/HtmlInCanvas/docs-demo-2d-blur.tsx
+++ b/packages/example/src/HtmlInCanvas/docs-demo-2d-blur.tsx
@@ -36,7 +36,9 @@ export const HtmlInCanvasDocsDemo2DBlur: React.FC = () => {
ctx.reset();
ctx.filter = `blur(${blurPx}px)`;
const transform = ctx.drawElementImage(elementImage, 0, 0);
- element.style.transform = transform.toString();
+ if (transform) {
+ element.style.transform = transform.toString();
+ }
},
[frame, fps],
);
diff --git a/packages/example/src/HtmlInCanvas/minimal-docs-webgl.tsx b/packages/example/src/HtmlInCanvas/minimal-docs-webgl.tsx
index 71ba2ef462..7cd52c968f 100644
--- a/packages/example/src/HtmlInCanvas/minimal-docs-webgl.tsx
+++ b/packages/example/src/HtmlInCanvas/minimal-docs-webgl.tsx
@@ -19,6 +19,8 @@ type GlState = {
uTex: WebGLUniformLocation | null;
uTime: WebGLUniformLocation | null;
texture: WebGLTexture;
+ textureWidth: number;
+ textureHeight: number;
vao: WebGLVertexArrayObject;
};
@@ -117,7 +119,16 @@ export const HtmlInCanvasDocsMinimalWebGL: React.FC = () => {
gl.enableVertexAttribArray(locUv);
gl.vertexAttribPointer(locUv, 2, gl.FLOAT, false, 16, 8);
- gpuRef.current = {gl, program, uTex, uTime, texture, vao};
+ gpuRef.current = {
+ gl,
+ program,
+ uTex,
+ uTime,
+ texture,
+ textureWidth: 0,
+ textureHeight: 0,
+ vao,
+ };
return () => {
gl.deleteProgram(program);
@@ -141,14 +152,39 @@ export const HtmlInCanvasDocsMinimalWebGL: React.FC = () => {
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, gpu.texture);
- gl.texElementImage2D(
- gl.TEXTURE_2D,
- 0,
- gl.RGBA,
- gl.RGBA,
- gl.UNSIGNED_BYTE,
- elementImage,
- );
+ if (typeof gl.texElementSubImage2D === 'function') {
+ const imageWidth = Math.max(1, Math.ceil(elementImage.width));
+ const imageHeight = Math.max(1, Math.ceil(elementImage.height));
+ if (
+ gpu.textureWidth !== imageWidth ||
+ gpu.textureHeight !== imageHeight
+ ) {
+ gl.texImage2D(
+ gl.TEXTURE_2D,
+ 0,
+ gl.RGBA8,
+ imageWidth,
+ imageHeight,
+ 0,
+ gl.RGBA,
+ gl.UNSIGNED_BYTE,
+ null,
+ );
+ gpu.textureWidth = imageWidth;
+ gpu.textureHeight = imageHeight;
+ }
+
+ gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage);
+ } else {
+ gl.texElementImage2D(
+ gl.TEXTURE_2D,
+ 0,
+ gl.RGBA,
+ gl.RGBA,
+ gl.UNSIGNED_BYTE,
+ elementImage,
+ );
+ }
if (gpu.uTex) {
gl.uniform1i(gpu.uTex, 0);
diff --git a/packages/motion-blur/src/HtmlInCanvasMotionBlur.tsx b/packages/motion-blur/src/HtmlInCanvasMotionBlur.tsx
index 462118dba1..6cfc4efb8f 100644
--- a/packages/motion-blur/src/HtmlInCanvasMotionBlur.tsx
+++ b/packages/motion-blur/src/HtmlInCanvasMotionBlur.tsx
@@ -204,7 +204,7 @@ const HtmlInCanvasMotionBlurInner: React.FC<
: layoutCanvas.captureElementImage(sampleElement);
try {
const sampleTransform = context.drawElementImage(image, 0, 0);
- if (index === Math.floor(actualSamples / 2)) {
+ if (sampleTransform && index === Math.floor(actualSamples / 2)) {
(sampleElement as HTMLElement).style.transform =
sampleTransform.toString();
}
diff --git a/packages/skills/skills/remotion-markup/html-in-canvas.md b/packages/skills/skills/remotion-markup/html-in-canvas.md
index d2638015e4..b78e9dd4ed 100644
--- a/packages/skills/skills/remotion-markup/html-in-canvas.md
+++ b/packages/skills/skills/remotion-markup/html-in-canvas.md
@@ -47,7 +47,7 @@ export const MyComp = () => {
## 2D effect with `onPaint`
-`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM, and assign the returned transform to `element.style.transform` so DOM selection still aligns with the painted output.
+`onPaint` runs whenever the content updates. Call `ctx.drawElementImage(elementImage, 0, 0)` to draw the captured DOM. Current Chromium synchronizes the element's geometry automatically; older versions return a transform that must be assigned to `element.style.transform`.
```tsx
import {
@@ -73,7 +73,9 @@ export const Blur = () => {
ctx.reset();
ctx.filter = `blur(${blurPx}px)`;
const transform = ctx.drawElementImage(elementImage, 0, 0);
- element.style.transform = transform.toString();
+ if (transform) {
+ element.style.transform = transform.toString();
+ }
},
[frame, fps],
);
@@ -98,28 +100,7 @@ export const Blur = () => {
## WebGL effects
-For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Inside `onPaint`, upload the captured DOM with `gl.texElementImage2D(...)` and draw.
-
-```tsx
-const onInit: HtmlInCanvasOnInit = useCallback(({ canvas }) => {
- const gl = canvas.getContext("webgl2", { alpha: true, premultipliedAlpha: true });
- if (!gl) {
- throw new Error(
- "WebGL2 unavailable. Try rendering with the --gl=angle option. See https://remotion.dev/docs/gl-options.",
- );
- }
- gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
- // compile program, create texture, set up VAO...
- return () => {
- // delete program, texture, buffers...
- };
-}, []);
-
-const onPaint: HtmlInCanvasOnPaint = useCallback(({ elementImage }) => {
- gl.texElementImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, elementImage);
- gl.drawArrays(gl.TRIANGLES, 0, 6);
-}, []);
-```
+For WebGL, set up the context, program, and texture in `onInit` and return a cleanup function. Allocate the texture with `gl.texImage2D(..., null)` when `elementImage` changes size. Inside `onPaint`, upload with `gl.texElementSubImage2D(gl.TEXTURE_2D, 0, 0, 0, elementImage)` and draw. Older Chromium versions only expose `gl.texElementImage2D()`; use it when the new method is absent.
For a fully working minimal example, see https://github.com/remotion-dev/remotion/blob/main/packages/docs/components/demos/HtmlInCanvasDocsDemoWebGL.tsx.
diff --git a/packages/transitions/src/html-in-canvas-presentation.tsx b/packages/transitions/src/html-in-canvas-presentation.tsx
index 6d8d8fb529..5f9b53e070 100644
--- a/packages/transitions/src/html-in-canvas-presentation.tsx
+++ b/packages/transitions/src/html-in-canvas-presentation.tsx
@@ -185,13 +185,35 @@ export const HtmlInCanvasPresentation = <
const canvas = canvasRef.current;
if (!canvas) {
- throw new Error('Canvas not found');
+ return;
+ }
+
+ if (canvas.getAttribute('content') !== 'drawable') {
+ canvas.setAttribute('content', 'drawable');
+ }
+
+ if (canvas.layoutSubtree !== true) {
+ canvas.layoutSubtree = true;
+ }
+
+ const firstChild = canvas.firstElementChild;
+ if (firstChild && !firstChild.hasAttribute('drawable')) {
+ firstChild.setAttribute('drawable', '');
+ }
+ });
+
+ useLayoutEffect(() => {
+ if (passThrough) {
+ return;
}
- canvas.layoutSubtree = true;
+ const canvas = canvasRef.current;
+ if (!canvas) {
+ throw new Error('Canvas not found');
+ }
const onPaint = () => {
- const firstChild = canvas.firstChild as HTMLElement;
+ const firstChild = canvas.firstElementChild;
const captureCanvas = captureCanvasRef.current;
const captureContext = captureContextRef.current;
diff --git a/packages/web-renderer/src/html-in-canvas.ts b/packages/web-renderer/src/html-in-canvas.ts
index 300fda3100..98e8b30a8a 100644
--- a/packages/web-renderer/src/html-in-canvas.ts
+++ b/packages/web-renderer/src/html-in-canvas.ts
@@ -5,7 +5,7 @@ type Canvas2DWithDrawElement = CanvasRenderingContext2D & {
dy: number,
dwidth: number,
dheight: number,
- ) => DOMMatrix;
+ ) => DOMMatrix | void;
};
type HTMLCanvasWithLayoutSubtree = HTMLCanvasElement & {
@@ -26,18 +26,20 @@ export const supportsNativeHtmlInCanvas = (): boolean => {
export const containsLayoutSubtreeCanvas = (element: HTMLElement): boolean => {
return Array.from(element.querySelectorAll('canvas')).some(
- (canvas) => (canvas as HTMLCanvasWithLayoutSubtree).layoutSubtree === true,
+ (canvas) =>
+ (canvas as HTMLCanvasWithLayoutSubtree).layoutSubtree === true ||
+ canvas.getAttribute('content') === 'drawable',
);
};
export type HtmlInCanvasContext = {
layoutCanvas: HTMLCanvasWithLayoutSubtree;
ctx: Canvas2DWithDrawElement;
+ wasDrawable: boolean;
};
/**
- * Sets up a persistent layoutsubtree canvas that wraps the scaffold div.
- * The div becomes a direct child of the canvas, which is required for drawElementImage.
+ * Sets up a persistent drawable canvas that wraps the scaffold div.
* Must be called once before rendering begins; the canvas stays in the DOM for the
* lifetime of the render.
*/
@@ -59,6 +61,7 @@ export const setupHtmlInCanvas = ({
const layoutCanvas = document.createElement(
'canvas',
) as HTMLCanvasWithLayoutSubtree;
+ layoutCanvas.setAttribute('content', 'drawable');
layoutCanvas.layoutSubtree = true;
layoutCanvas.width = width;
@@ -85,11 +88,13 @@ export const setupHtmlInCanvas = ({
return null;
}
+ const wasDrawable = div.hasAttribute('drawable');
+ div.setAttribute('drawable', '');
wrapper.removeChild(div);
layoutCanvas.appendChild(div);
wrapper.appendChild(layoutCanvas);
- return {layoutCanvas, ctx: maybeCtx};
+ return {layoutCanvas, ctx: maybeCtx, wasDrawable};
};
const waitForPaint = (
@@ -153,8 +158,11 @@ export const teardownHtmlInCanvas = ({
wrapper: HTMLDivElement;
div: HTMLDivElement;
}) => {
- const {layoutCanvas} = htmlInCanvasContext;
+ const {layoutCanvas, wasDrawable} = htmlInCanvasContext;
layoutCanvas.removeChild(div);
wrapper.removeChild(layoutCanvas);
wrapper.appendChild(div);
+ if (!wasDrawable) {
+ div.removeAttribute('drawable');
+ }
};
diff --git a/packages/web-renderer/src/test/html-in-canvas.test.tsx b/packages/web-renderer/src/test/html-in-canvas.test.tsx
index 942f90e9ad..f3028a1af6 100644
--- a/packages/web-renderer/src/test/html-in-canvas.test.tsx
+++ b/packages/web-renderer/src/test/html-in-canvas.test.tsx
@@ -151,6 +151,7 @@ test('does not create a nested HTML-in-canvas capture', async () => {
drawElementImage: typeof drawElementImage;
},
layoutCanvas: outerLayoutCanvas,
+ wasDrawable: false,
},
internalState,
logLevel: 'error',