Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
/**
* Patches GPU texture-upload paths so that video-backed effects work in both
* preview and render mode — for WebGPU (`GPUQueue.copyExternalImageToTexture`)
* and WebGL (`texImage2D` / `texSubImage2D`).
*
* During render, the engine's video-frame injector replaces each `<video>`
* with a pre-decoded `<img class="__render_frame__">` sibling. Chrome's
* headless compositor can't supply decoded frames from the native `<video>`
* element to the GPU, so uploading a `<video>` directly fails (WebGPU throws
* "Browser fails extracting valid resource from external image"; WebGL uploads
* a black/stale frame). These patches transparently substitute the decoded
* render-frame `<img>` as the upload source. In preview mode (no render-frame
* sibling), the original `<video>` path is used unchanged.
*/
import { findInjectedRenderFrame } from "../renderFrameSibling.js";
import { isImageElement, isVideoElement } from "../domRealm";
/**
* Resolve the decoded render-frame `<img>` for a source `<video>`, if the
* engine has injected one and it has decoded pixels. Returns null in preview
* mode or before the frame is decoded, so callers fall back to the video.
*
* The injector inserts the `<img>` as the video's immediate next sibling and
* also gives it the id `__render_frame_<renderId>__`; we check the sibling
* first (cheap) and fall back to an id lookup in case a node was inserted
* between them.
*/
function resolveRenderFrameImage(video: HTMLVideoElement): HTMLImageElement | null {
const sibling = video.nextElementSibling;
if (
isImageElement(sibling) &&
sibling.classList.contains("__render_frame__") &&
sibling.complete &&
sibling.naturalWidth > 0
) {
return sibling;
}
const byId = findInjectedRenderFrame(video);
if (byId && byId.complete && byId.naturalWidth > 0) {
return byId;
}
return null;
}
export function patchVideoTextureCompat(): void {
const GPUQueueCtor = (globalThis as Record<string, unknown>).GPUQueue as
| { prototype: Record<string, unknown> }
| undefined;
if (!GPUQueueCtor?.prototype?.copyExternalImageToTexture) return;
const orig = GPUQueueCtor.prototype.copyExternalImageToTexture as (
source: unknown,
destination: unknown,
copySize: unknown,
) => void;
GPUQueueCtor.prototype.copyExternalImageToTexture = function (
source: Record<string, unknown>,
destination: unknown,
copySize: unknown,
) {
if (isVideoElement(source?.source)) {
const img = resolveRenderFrameImage(source.source);
if (img) {
return orig.call(this, { ...source, source: img }, destination, copySize);
}
}
return orig.call(this, source, destination, copySize);
};
}
/**
* WebGL analog of {@link patchVideoTextureCompat}. Patches `texImage2D` and
* `texSubImage2D` on both `WebGL2RenderingContext` and `WebGLRenderingContext`
* so that when a `<video>` is passed as the texture source (the last argument
* in the DOM-source overloads), the decoded render-frame `<img>` is uploaded
* instead during render. Numeric/`ArrayBufferView` overloads are untouched —
* only a trailing `HTMLVideoElement` argument is substituted.
*/
export function patchWebGLVideoTextureCompat(): void {
const ctors = [
(globalThis as Record<string, unknown>).WebGL2RenderingContext,
(globalThis as Record<string, unknown>).WebGLRenderingContext,
] as Array<{ prototype: Record<string, unknown> } | undefined>;
const methods = ["texImage2D", "texSubImage2D"] as const;
for (const ctor of ctors) {
const proto = ctor?.prototype;
if (!proto) continue;
for (const method of methods) {
const orig = proto[method] as ((...args: unknown[]) => unknown) & {
__hfVideoPatched?: boolean;
};
if (typeof orig !== "function" || orig.__hfVideoPatched) continue;
const patched = function (this: unknown, ...args: unknown[]) {
const lastIndex = args.length - 1;
const last = args[lastIndex];
if (isVideoElement(last)) {
const img = resolveRenderFrameImage(last);
if (img) args[lastIndex] = img;
}
return orig.apply(this, args);
} as ((...args: unknown[]) => unknown) & { __hfVideoPatched?: boolean };
patched.__hfVideoPatched = true;
proto[method] = patched;
}
}
}