Skip to content

🎥 Make videos programmatically with React

`@remotion/video-matting`: Add Node.js support for `separateVideoLayers()`

Jonny Burgerff38ef57502f
diff --git a/bun.lock b/bun.lock
index 4b566665f0..84fdebc54c 100644
--- a/bun.lock
+++ b/bun.lock
@@ -565,6 +565,7 @@
"@docusaurus/preset-classic": "3.10.1",
"@docusaurus/theme-common": "3.10.1",
"@docusaurus/types": "3.10.1",
+ "@huggingface/transformers": "catalog:",
"@mdx-js/react": "3.0.1",
"@mediabunny/ac3": "catalog:",
"@mediabunny/dts": "catalog:",
@@ -2815,9 +2816,11 @@
"version": "4.0.527",
"dependencies": {
"mediabunny": "catalog:",
+ "onnxruntime-node": "catalog:",
},
"devDependencies": {
"@huggingface/transformers": "catalog:",
+ "@mediabunny/server": "catalog:",
"@remotion/eslint-config-internal": "workspace:*",
"@typescript/native-preview": "catalog:",
"eslint": "catalog:",
diff --git a/packages/docs/docs/video-matting/can-use-video-matting.mdx b/packages/docs/docs/video-matting/can-use-video-matting.mdx
index 742633af9c..42a4c2b386 100644
--- a/packages/docs/docs/video-matting/can-use-video-matting.mdx
+++ b/packages/docs/docs/video-matting/can-use-video-matting.mdx
@@ -6,7 +6,7 @@ crumb: '@remotion/video-matting'
# canUseVideoMatting()<AvailableFrom v="4.0.523" />
-Checks whether a video matting model can run in the current browser.
+Checks whether a video matting model can run in the current browser or Node.js environment.
This function checks WebGPU and model-specific requirements. It does not inspect an input file or check video decoding and encoding support.
@@ -42,6 +42,8 @@ When `supported` is `false`, one of:
- `webgpu-requires-secure-context`
- `shader-f16-unavailable`
+In Node.js, failed native WebGPU probes return `webgpu-unavailable` with the ONNX Runtime error in `detailedReason`.
+
`shader-f16-unavailable` applies to models such as `ben2-base` that require 16-bit shader support.
### detailedReason?
@@ -50,7 +52,9 @@ When `supported` is `false`, a human-readable explanation of the reason.
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.
+
+<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/video-matting/dispose-video-matting-model.mdx b/packages/docs/docs/video-matting/dispose-video-matting-model.mdx
index 3e7adf5573..63b54209a3 100644
--- a/packages/docs/docs/video-matting/dispose-video-matting-model.mdx
+++ b/packages/docs/docs/video-matting/dispose-video-matting-model.mdx
@@ -6,7 +6,7 @@ crumb: '@remotion/video-matting'
# disposeVideoMattingModel()<AvailableFrom v="4.0.523" />
-Releases the memory used by initialized video matting models. Persistent browser-cached model files are not deleted.
+Releases the memory used by initialized video matting models. Persistent cached model files are not deleted.
```ts twoslash title="dispose.ts"
import {disposeVideoMattingModel} from '@remotion/video-matting';
@@ -28,7 +28,9 @@ A `Promise<void>` that resolves after the selected models have been disposed.
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.
+
+<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/video-matting/download-video-matting-model.mdx b/packages/docs/docs/video-matting/download-video-matting-model.mdx
index 39515d2c52..ba77101ddd 100644
--- a/packages/docs/docs/video-matting/download-video-matting-model.mdx
+++ b/packages/docs/docs/video-matting/download-video-matting-model.mdx
@@ -6,9 +6,9 @@ crumb: '@remotion/video-matting'
# downloadVideoMattingModel()<AvailableFrom v="4.0.528" />
-Downloads a video matting model from `remotion.media` into the browser cache without initializing WebGPU. Call [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model) or [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) afterwards.
+Downloads a video matting model from `remotion.media` into the browser or Node.js filesystem cache without initializing WebGPU. Call [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model) or [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) afterwards.
-The browser Cache API or a Transformers.js custom cache must be enabled.
+The browser Cache API, Node.js filesystem cache, or a Transformers.js custom cache must be enabled.
```ts twoslash title="download-model.ts"
import {downloadVideoMattingModel} from '@remotion/video-matting';
@@ -55,7 +55,9 @@ Estimated total model size in bytes.
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.
+
+<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/video-matting/index.mdx b/packages/docs/docs/video-matting/index.mdx
index 843e9e4c95..5da68b178b 100644
--- a/packages/docs/docs/video-matting/index.mdx
+++ b/packages/docs/docs/video-matting/index.mdx
@@ -6,7 +6,7 @@ crumb: 'Separate a video into layers'
# @remotion/video-matting<AvailableFrom v="4.0.523" />
-Separate a video into an opaque base layer and a foreground layer with alpha, locally in the browser using WebGPU.
+Separate a video into an opaque base layer and a foreground layer with alpha, locally in the browser or [Node.js](/docs/video-matting/node) using WebGPU.
import {TableOfContents} from './table-of-contents';
diff --git a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx
index 279ad7da9c..3ae61ff002 100644
--- a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx
+++ b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx
@@ -6,7 +6,7 @@ crumb: '@remotion/video-matting'
# isVideoMattingModelCached()<AvailableFrom v="4.0.523" />
-Checks whether all files hosted on `remotion.media` for a video matting model are present in the browser cache. The check does not download or initialize the model.
+Checks whether all files hosted on `remotion.media` for a video matting model are present in the browser or Node.js filesystem cache. The check does not download or initialize the model.
```ts twoslash title="model-cache.ts"
import {isVideoMattingModelCached} from '@remotion/video-matting';
@@ -28,7 +28,9 @@ A `Promise<boolean>` resolving to `true` if every required model file is cached.
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.
+
+<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/video-matting/load-video-matting-model.mdx b/packages/docs/docs/video-matting/load-video-matting-model.mdx
index fe5350f7f4..c008c005b9 100644
--- a/packages/docs/docs/video-matting/load-video-matting-model.mdx
+++ b/packages/docs/docs/video-matting/load-video-matting-model.mdx
@@ -71,7 +71,9 @@ Call [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-mo
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.
+
+<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/video-matting/node.mdx b/packages/docs/docs/video-matting/node.mdx
new file mode 100644
index 0000000000..6b6cb27448
--- /dev/null
+++ b/packages/docs/docs/video-matting/node.mdx
@@ -0,0 +1,84 @@
+---
+image: /generated/articles-docs-video-matting-node.png
+title: 'Using @remotion/video-matting in Node.js'
+crumb: '@remotion/video-matting'
+---
+
+# Using @remotion/video-matting in Node.js<AvailableFrom v="4.0.528" />
+
+You can use `@remotion/video-matting` in Node.js to separate a local video into an opaque base and a transparent foreground.
+
+Register [`@mediabunny/server`](https://mediabunny.dev/guide/extensions/server) to enable video decoding, VP9 encoding with alpha, and audio processing. A browser or canvas polyfill is not required.
+
+## Installation
+
+<Installation pkg="@remotion/video-matting @huggingface/transformers mediabunny @mediabunny/server" />
+
+## Example
+
+```ts twoslash title="separate-video.ts"
+import {registerMediabunnyServer} from '@mediabunny/server';
+import {
+ canUseVideoMatting,
+ downloadVideoMattingModel,
+ loadVideoMattingModel,
+ separateVideoLayers,
+} from '@remotion/video-matting';
+import {writeFile} from 'node:fs/promises';
+
+registerMediabunnyServer();
+
+const model = 'modnet';
+const support = await canUseVideoMatting({model});
+if (!support.supported) {
+ throw new Error(support.detailedReason);
+}
+
+await downloadVideoMattingModel({model});
+
+// In long-running apps, put this code in an async function so resources
+// are released when that function finishes.
+await using modelHandle = await loadVideoMattingModel({model});
+await using layers = await separateVideoLayers({
+ src: './input.mp4',
+ model,
+ audio: 'base',
+ onProgress: ({processedFrames}) => console.log(processedFrames),
+});
+
+await writeFile(
+ 'base.webm',
+ new Uint8Array(await (await layers.base.getBlob()).arrayBuffer()),
+);
+await writeFile(
+ 'foreground.webm',
+ new Uint8Array(await (await layers.foreground.getBlob()).arrayBuffer()),
+);
+```
+
+[`separateVideoLayers()`](/docs/video-matting/separate-video-layers) accepts local file paths, `file:` URLs, remote URLs, and `Blob` inputs. Outputs are held in memory by default. Use `outputWritable` to stream large outputs; handle each chunk's `position` because writes are not necessarily append-only.
+
+## Model cache
+
+Model files use the Transformers.js filesystem cache. To choose the directory, set `env.cacheDir` before downloading or loading models:
+
+```ts twoslash title="model-cache.ts"
+import {env} from '@huggingface/transformers';
+import {resolve} from 'node:path';
+
+env.cacheDir = resolve('.cache/video-matting');
+```
+
+[`removeVideoMattingModel()`](/docs/video-matting/remove-video-matting-model) removes cached files. Disposing a loaded model releases its memory and keeps the downloaded files.
+
+## Requirements
+
+A compatible GPU is required. Transformers.js uses ONNX Runtime's native WebGPU provider in Node.js. [`canUseVideoMatting()`](/docs/video-matting/can-use-video-matting) runs a tiny inference to check support, including fp16 computation for `ben2-base`.
+
+The native ONNX Runtime dependency does not support Linux arm64. GPU availability also depends on the host and its drivers; a typical serverless environment does not provide a GPU.
+
+## See also
+
+- [`separateVideoLayers()`](/docs/video-matting/separate-video-layers)
+- [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model)
+- [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model)
diff --git a/packages/docs/docs/video-matting/remove-video-matting-model.mdx b/packages/docs/docs/video-matting/remove-video-matting-model.mdx
index 7fedf0bdac..1f4b08d9cc 100644
--- a/packages/docs/docs/video-matting/remove-video-matting-model.mdx
+++ b/packages/docs/docs/video-matting/remove-video-matting-model.mdx
@@ -6,7 +6,7 @@ crumb: '@remotion/video-matting'
# removeVideoMattingModel()<AvailableFrom v="4.0.523" />
-Removes a video matting model from memory and the persistent browser cache.
+Removes a video matting model from memory and the persistent model cache.
If the model is still loading or separating a video, removal waits for the operation to finish.
@@ -24,7 +24,9 @@ One of the names returned by [`getAvailableModels()`](/docs/video-matting/get-av
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.
+
+<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/video-matting/separate-video-layers.mdx b/packages/docs/docs/video-matting/separate-video-layers.mdx
index bbec704dc6..26a40ac3a2 100644
--- a/packages/docs/docs/video-matting/separate-video-layers.mdx
+++ b/packages/docs/docs/video-matting/separate-video-layers.mdx
@@ -35,6 +35,8 @@ Place inserted content between the base and foreground. The base remains opaque
A `string`, `URL`, or `Blob` containing the input video. A `File` may be passed because it extends `Blob`.
+In [Node.js](/docs/video-matting/node), strings can also be local file paths, and `file:` URLs are supported.
+
### model?
The video matting model to use. Default: `modnet`.
@@ -220,7 +222,9 @@ Both layers are WebM files encoded with VP9. The foreground preserves alpha. Aud
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.
+
+<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/package.json b/packages/docs/package.json
index 208b3fd0f4..8db60f742f 100644
--- a/packages/docs/package.json
+++ b/packages/docs/package.json
@@ -24,6 +24,7 @@
"upload-element-preview": "bun run upload-element-preview.ts"
},
"dependencies": {
+ "@huggingface/transformers": "catalog:",
"@remotion/studio-protocol": "workspace:*",
"@aws-sdk/s3-request-presigner": "catalog:",
"@babel/parser": "7.24.1",
diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.ts
index 316478653c..3b9a05f216 100644
--- a/packages/docs/sidebars.ts
+++ b/packages/docs/sidebars.ts
@@ -960,6 +960,11 @@ const sidebars: SidebarsConfig = {
id: 'video-matting/index',
},
items: [
+ {
+ type: 'doc',
+ id: 'video-matting/node',
+ label: 'Usage in Node.js',
+ },
'video-matting/can-use-video-matting',
'video-matting/get-available-models',
'video-matting/is-video-matting-model-cached',
diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.ts
index b3e623ceef..82efe05125 100644
--- a/packages/docs/src/data/articles.ts
+++ b/packages/docs/src/data/articles.ts
@@ -8958,6 +8958,15 @@ export const articles = [
noAi: false,
slug: 'video-matting/load-video-matting-model',
},
+ {
+ id: 'video-matting/node',
+ title: 'Using @remotion/video-matting in Node.js',
+ relativePath: 'docs/video-matting/node.mdx',
+ compId: 'articles-docs-video-matting-node',
+ crumb: '@remotion/video-matting',
+ noAi: false,
+ slug: 'video-matting/node',
+ },
{
id: 'video-matting/remove-video-matting-model',
title: 'removeVideoMattingModel()',
diff --git a/packages/docs/static/generated/articles-docs-video-matting-node.png b/packages/docs/static/generated/articles-docs-video-matting-node.png
new file mode 100644
index 0000000000..5613af2f7b
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-node.png differ
diff --git a/packages/video-matting/README.md b/packages/video-matting/README.md
index 312b6d5931..bb7676e785 100644
--- a/packages/video-matting/README.md
+++ b/packages/video-matting/README.md
@@ -1,6 +1,6 @@
# @remotion/video-matting
-Separate a video into a base layer and transparent foreground in the browser
+Separate a video into a base layer and transparent foreground in the browser and Node.js
[![NPM Downloads](https://img.shields.io/npm/dm/@remotion/video-matting.svg?style=flat&color=black&label=Downloads)](https://npmcharts.com/compare/@remotion/video-matting?minimal=true)
diff --git a/packages/video-matting/package.json b/packages/video-matting/package.json
index 1a4dee924e..9315d6c8c7 100644
--- a/packages/video-matting/package.json
+++ b/packages/video-matting/package.json
@@ -1,7 +1,7 @@
{
"name": "@remotion/video-matting",
"version": "4.0.527",
- "description": "Separate a video into a base layer and transparent foreground in the browser",
+ "description": "Separate a video into a base layer and transparent foreground in the browser and Node.js",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"module": "dist/esm/index.mjs",
@@ -24,7 +24,8 @@
"url": "https://github.com/remotion-dev/remotion/issues"
},
"dependencies": {
- "mediabunny": "catalog:"
+ "mediabunny": "catalog:",
+ "onnxruntime-node": "catalog:"
},
"peerDependencies": {
"@huggingface/transformers": "4.2.0"
@@ -39,6 +40,7 @@
},
"devDependencies": {
"@huggingface/transformers": "catalog:",
+ "@mediabunny/server": "catalog:",
"@remotion/eslint-config-internal": "workspace:*",
"@typescript/native-preview": "catalog:",
"eslint": "catalog:"
diff --git a/packages/video-matting/src/can-use-video-matting.ts b/packages/video-matting/src/can-use-video-matting.ts
index 1f8f3a8020..d02f02fd8f 100644
--- a/packages/video-matting/src/can-use-video-matting.ts
+++ b/packages/video-matting/src/can-use-video-matting.ts
@@ -32,12 +32,31 @@ export const canUseVideoMatting = async ({
}: CanUseVideoMattingOptions = {}): Promise<CanUseVideoMattingResult> => {
const modelInfo = getVideoMattingModelInfo(model);
+ if (
+ typeof window === 'undefined' &&
+ typeof process !== 'undefined' &&
+ process.release?.name === 'node'
+ ) {
+ try {
+ const {probeNodeWebGpu} = await import('./probe-node-webgpu');
+ await probeNodeWebGpu({requiresShaderF16: modelInfo.requiresShaderF16});
+ return {supported: true};
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ return {
+ supported: false,
+ reason: VideoMattingUnsupportedReason.WebGpuUnavailable,
+ detailedReason: `ONNX Runtime could not run WebGPU inference for "${model}": ${message}`,
+ };
+ }
+ }
+
if (typeof window === 'undefined' && typeof OffscreenCanvas === 'undefined') {
return {
supported: false,
reason: VideoMattingUnsupportedReason.WindowUndefined,
detailedReason:
- 'No browser window or worker canvas environment is available. @remotion/video-matting is intended for browser environments.',
+ 'No browser, worker canvas, or Node.js environment is available.',
};
}
diff --git a/packages/video-matting/src/create-video-matting-frames.ts b/packages/video-matting/src/create-video-matting-frames.ts
new file mode 100644
index 0000000000..5847fcf409
--- /dev/null
+++ b/packages/video-matting/src/create-video-matting-frames.ts
@@ -0,0 +1,127 @@
+import {CanvasSink, CanvasSource} from 'mediabunny';
+import type {InputVideoTrack, Quality, VideoSource} from 'mediabunny';
+import type {
+ VideoMattingImageSource,
+ VideoMattingPipelineResult,
+} from './load-video-matting-model';
+import {
+ createVideoMattingCanvas,
+ drawForegroundFrame,
+ drawOpaqueBaseFrame,
+ getVideoMattingCanvasContext,
+} from './video-matting-canvas';
+
+export type VideoMattingFrame = {
+ image: VideoMattingImageSource;
+ timestamp: number;
+ duration: number;
+};
+
+export type VideoMattingFramesOptions = {
+ videoTrack: InputVideoTrack;
+ width: number;
+ height: number;
+ videoQuality: Quality;
+ keyframeIntervalInSeconds: number;
+ videoStartTimestamp: number;
+ videoEndTimestamp: number;
+};
+
+export type VideoMattingFrames = {
+ frames: AsyncGenerator<VideoMattingFrame, void, unknown>;
+ baseSource: VideoSource;
+ foregroundSource: VideoSource;
+ addFrame: (options: {
+ frame: VideoMattingFrame;
+ foreground: VideoMattingPipelineResult;
+ timestamp: number;
+ duration: number;
+ }) => Promise<void>;
+};
+
+export const createVideoMattingFrames = async (
+ options: VideoMattingFramesOptions,
+): Promise<VideoMattingFrames> => {
+ if (
+ typeof window === 'undefined' &&
+ typeof process !== 'undefined' &&
+ process.release?.name === 'node'
+ ) {
+ const {createNodeVideoMattingFrames} =
+ await import('./node-video-matting-frames');
+ return createNodeVideoMattingFrames(options);
+ }
+
+ const {
+ videoTrack,
+ width,
+ height,
+ videoQuality,
+ keyframeIntervalInSeconds,
+ videoStartTimestamp,
+ videoEndTimestamp,
+ } = options;
+ const sink = new CanvasSink(videoTrack, {
+ alpha: true,
+ width,
+ height,
+ fit: 'fill',
+ poolSize: 1,
+ });
+ const baseCanvas = createVideoMattingCanvas({width, height});
+ const foregroundCanvas = createVideoMattingCanvas({width, height});
+ const baseContext = getVideoMattingCanvasContext(baseCanvas);
+ const foregroundContext = getVideoMattingCanvasContext(foregroundCanvas);
+ const baseSource = new CanvasSource(baseCanvas, {
+ codec: 'vp9',
+ quality: videoQuality,
+ keyFrameInterval: keyframeIntervalInSeconds,
+ alpha: 'discard',
+ });
+ const foregroundSource = new CanvasSource(foregroundCanvas, {
+ codec: 'vp9',
+ quality: videoQuality,
+ keyFrameInterval: keyframeIntervalInSeconds,
+ alpha: 'keep',
+ });
+
+ return {
+ frames: (async function* () {
+ for await (const frame of sink.canvases(
+ videoStartTimestamp,
+ videoEndTimestamp,
+ )) {
+ yield {
+ image: frame.canvas,
+ timestamp: frame.timestamp,
+ duration: frame.duration,
+ };
+ }
+ })(),
+ baseSource,
+ foregroundSource,
+ addFrame: async ({frame, foreground, timestamp, duration}) => {
+ if ('data' in frame.image) {
+ throw new Error('Expected a canvas frame.');
+ }
+
+ drawOpaqueBaseFrame({
+ context: baseContext,
+ source: frame.image,
+ width,
+ height,
+ });
+ drawForegroundFrame({
+ context: foregroundContext,
+ result: foreground,
+ source: frame.image,
+ targetWidth: width,
+ targetHeight: height,
+ });
+ await Promise.all([
+ baseSource.add(timestamp, duration),
+ foregroundSource.add(timestamp, duration),
+ ]);
+ },
+ };
+};
diff --git a/packages/video-matting/src/download-video-matting-model.ts b/packages/video-matting/src/download-video-matting-model.ts
index b5de024201..638230f828 100644
--- a/packages/video-matting/src/download-video-matting-model.ts
+++ b/packages/video-matting/src/download-video-matting-model.ts
@@ -1,3 +1,7 @@
+import type * as NodeCrypto from 'node:crypto';
+import type * as NodeFsPromises from 'node:fs/promises';
+import type * as NodePath from 'node:path';
+import {importNodeModule} from './import-node-module';
import {
getHostedVideoMattingModelId,
getVideoMattingModelInfo,
@@ -34,9 +38,10 @@ export const downloadVideoMattingModel = ({
: env.useBrowserCache && typeof caches !== 'undefined'
? await caches.open(env.cacheKey)
: null;
- if (cache === null) {
+ const useFileCache = cache === null && env.useFSCache;
+ if (cache === null && !useFileCache) {
throw new Error(
- 'A Transformers.js browser or custom cache is required to download a video matting model.',
+ 'A Transformers.js model cache is required to download a video matting model.',
);
}
@@ -77,32 +82,75 @@ export const downloadVideoMattingModel = ({
}
const url = `${env.remoteHost}${env.remotePathTemplate.replace('{model}', hostedModelId)}${file}`;
- // Transformers.js 4.2.0 uses the remote URL as the browser cache key.
+ // Transformers.js 4.2.0 keys browser caches by URL and its FileCache by model/file.
+ const cacheKey = cache === null ? `${hostedModelId}/${file}` : url;
const response = await env.fetch(url);
if (!response.ok || response.body === null) {
throw new Error(`Could not download ${url}: HTTP ${response.status}.`);
}
const reader = response.body.getReader();
- const stream = new ReadableStream<Uint8Array>({
- async pull(controller) {
- try {
+ if (cache !== null) {
+ const stream = new ReadableStream<Uint8Array>({
+ async pull(controller) {
+ try {
+ const {done, value} = await reader.read();
+ if (done) {
+ controller.close();
+ return;
+ }
+
+ loadedBytes += value.byteLength;
+ emitProgress(file);
+ controller.enqueue(value);
+ } catch (error) {
+ controller.error(error);
+ }
+ },
+ cancel: () => reader.cancel(),
+ });
+ await cache.put(
+ cacheKey,
+ new Response(stream, {headers: response.headers}),
+ );
+ } else {
+ // Match Transformers.js FileCache's model/file layout without loading ONNX.
+ const [{mkdir, open, rename, unlink}, {join, dirname}, {randomUUID}] =
+ await Promise.all([
+ importNodeModule<typeof NodeFsPromises>('node:fs/promises'),
+ importNodeModule<typeof NodePath>('node:path'),
+ importNodeModule<typeof NodeCrypto>('node:crypto'),
+ ]);
+ if (env.cacheDir === null) {
+ throw new Error('Transformers.js env.cacheDir must be set.');
+ }
+
+ const destination = join(env.cacheDir, cacheKey);
+ const temporary = `${destination}.tmp.${randomUUID()}`;
+ await mkdir(dirname(destination), {recursive: true});
+ const handle = await open(temporary, 'wx');
+ try {
+ while (true) {
const {done, value} = await reader.read();
if (done) {
- controller.close();
- return;
+ break;
}
+ await handle.writeFile(value);
loadedBytes += value.byteLength;
emitProgress(file);
- controller.enqueue(value);
- } catch (error) {
- controller.error(error);
}
- },
- cancel: () => reader.cancel(),
- });
- await cache.put(url, new Response(stream, {headers: response.headers}));
+
+ await handle.close();
+ await rename(temporary, destination);
+ } catch (error) {
+ await Promise.allSettled([handle.close(), reader.cancel(error)]);
+ await unlink(temporary).catch(() => undefined);
+ throw error;
+ } finally {
+ reader.releaseLock();
+ }
+ }
}
if (
diff --git a/packages/video-matting/src/import-node-module.ts b/packages/video-matting/src/import-node-module.ts
new file mode 100644
index 0000000000..57f7c7d23a
--- /dev/null
+++ b/packages/video-matting/src/import-node-module.ts
@@ -0,0 +1,10 @@
+// Called only in Node.js branches. An indirect import keeps native Node modules
+// out of browser bundles without making bundlers resolve a variable import.
+export const importNodeModule = <T>(specifier: string): Promise<T> => {
+ // eslint-disable-next-line no-new-func
+ const dynamicImport = new Function(
+ 'specifier',
+ 'return import(specifier)',
+ ) as (module: string) => Promise<T>;
+ return dynamicImport(specifier);
+};
diff --git a/packages/video-matting/src/load-video-matting-model.ts b/packages/video-matting/src/load-video-matting-model.ts
index 0ad40d953a..f0f12924a2 100644
--- a/packages/video-matting/src/load-video-matting-model.ts
+++ b/packages/video-matting/src/load-video-matting-model.ts
@@ -1,4 +1,5 @@
import './symbol-async-dispose';
+import type {RawImage} from '@huggingface/transformers';
import {
getHostedVideoMattingModelId,
getVideoMattingModelInfo,
@@ -18,7 +19,10 @@ export type OnVideoMattingModelLoadProgress = (
progress: VideoMattingModelLoadProgress,
) => void;
-export type VideoMattingImageSource = HTMLCanvasElement | OffscreenCanvas;
+export type VideoMattingImageSource =
+ | HTMLCanvasElement
+ | OffscreenCanvas
+ | VideoMattingPipelineResult;
export type VideoMattingPipelineResult = {
data: Uint8ClampedArray<ArrayBuffer>;
@@ -39,7 +43,9 @@ type TransformerPipelineResult = {
};
type TransformerPipeline = {
- (image: VideoMattingImageSource): Promise<TransformerPipelineResult>;
+ (
+ image: HTMLCanvasElement | OffscreenCanvas | RawImage,
+ ): Promise<TransformerPipelineResult>;
dispose: () => Promise<void>;
};
@@ -164,6 +170,7 @@ const getOrCreateVideoMattingPipeline = ({
AutoProcessor,
BackgroundRemovalPipeline,
ModelRegistry,
+ RawImage,
}) => {
const hostedModelId = getHostedVideoMattingModelId(model);
if (
@@ -219,7 +226,16 @@ const getOrCreateVideoMattingPipeline = ({
const loadedPipeline: LoadedPipeline = {
run: async (image) => {
- const result = await loadedTransformerPipeline(image);
+ const result = await loadedTransformerPipeline(
+ 'data' in image
+ ? new RawImage(
+ image.data,
+ image.width,
+ image.height,
+ image.channels,
+ )
+ : image,
+ );
if (result.channels !== 4) {
throw new Error(
`The video matting model "${model}" returned ${result.channels} channels instead of RGBA.`,
diff --git a/packages/video-matting/src/node-video-matting-frames.ts b/packages/video-matting/src/node-video-matting-frames.ts
new file mode 100644
index 0000000000..c638b28e2c
--- /dev/null
+++ b/packages/video-matting/src/node-video-matting-frames.ts
@@ -0,0 +1,136 @@
+import {VideoSample, VideoSampleSink, VideoSampleSource} from 'mediabunny';
+import type {
+ VideoMattingFrames,
+ VideoMattingFramesOptions,
+} from './create-video-matting-frames';
+
+export const createNodeVideoMattingFrames = ({
+ videoTrack,
+ width,
+ height,
+ videoQuality,
+ keyframeIntervalInSeconds,
+ videoStartTimestamp,
+ videoEndTimestamp,
+}: VideoMattingFramesOptions): VideoMattingFrames => {
+ const sink = new VideoSampleSink(videoTrack);
+ const baseSource = new VideoSampleSource({
+ codec: 'vp9',
+ quality: videoQuality,
+ keyFrameInterval: keyframeIntervalInSeconds,
+ alpha: 'discard',
+ });
+ const foregroundSource = new VideoSampleSource({
+ codec: 'vp9',
+ quality: videoQuality,
+ keyFrameInterval: keyframeIntervalInSeconds,
+ alpha: 'keep',
+ });
+
+ return {
+ frames: (async function* () {
+ for await (const sample of sink.samples(
+ videoStartTimestamp,
+ videoEndTimestamp,
+ )) {
+ const data = new Uint8ClampedArray(width * height * 4);
+ try {
+ // Apply rotation and pixel aspect ratio before passing pixels to the model.
+ const transformed = await sample.transform({
+ width,
+ height,
+ fit: 'fill',
+ });
+ try {
+ await transformed.copyTo(data, {format: 'RGBA'});
+ } finally {
+ transformed.close();
+ }
+ } finally {
+ sample.close();
+ }
+
+ yield {
+ image: {data, width, height, channels: 4},
+ timestamp: sample.timestamp,
+ duration: sample.duration,
+ };
+ }
+ })(),
+ baseSource,
+ foregroundSource,
+ addFrame: async ({frame, foreground, timestamp, duration}) => {
+ if (!('data' in frame.image)) {
+ throw new Error('Expected an RGBA frame.');
+ }
+
+ const baseData = frame.image.data.slice();
+ let foregroundData: Uint8ClampedArray<ArrayBuffer>;
+ if (foreground.width === width && foreground.height === height) {
+ foregroundData = foreground.data.slice();
+ } else {
+ foregroundData = new Uint8ClampedArray(width * height * 4);
+ const foregroundSample = new VideoSample(foreground.data, {
+ format: 'RGBA',
+ codedWidth: foreground.width,
+ codedHeight: foreground.height,
+ timestamp,
+ duration,
+ });
+ try {
+ const resized = await foregroundSample.transform({
+ width,
+ height,
+ fit: 'fill',
+ });
+ try {
+ await resized.copyTo(foregroundData, {format: 'RGBA'});
+ } finally {
+ resized.close();
+ }
+ } finally {
+ foregroundSample.close();
+ }
+ }
+
+ for (let i = 0; i < baseData.length; i += 4) {
+ const alpha = baseData[i + 3]! / 255;
+ // Match the browser's opaque black base and destination-in foreground.
+ baseData[i] = Math.round(baseData[i]! * alpha);
+ baseData[i + 1] = Math.round(baseData[i + 1]! * alpha);
+ baseData[i + 2] = Math.round(baseData[i + 2]! * alpha);
+ baseData[i + 3] = 255;
+ foregroundData[i + 3] = Math.round(foregroundData[i + 3]! * alpha);
+ }
+
+ const base = new VideoSample(baseData, {
+ format: 'RGBA',
+ codedWidth: width,
+ codedHeight: height,
+ timestamp,
+ duration,
+ });
+ const foregroundFrame = new VideoSample(foregroundData, {
+ format: 'RGBA',
+ codedWidth: width,
+ codedHeight: height,
+ timestamp,
+ duration,
+ });
+ try {
+ const results = await Promise.allSettled([
+ baseSource.add(base),
+ foregroundSource.add(foregroundFrame),
+ ]);
+ for (const result of results) {
+ if (result.status === 'rejected') {
+ throw result.reason;
+ }
+ }
+ } finally {
+ base.close();
+ foregroundFrame.close();
+ }
+ },
+ };
+};
diff --git a/packages/video-matting/src/probe-node-webgpu.ts b/packages/video-matting/src/probe-node-webgpu.ts
new file mode 100644
index 0000000000..e986340e52
--- /dev/null
+++ b/packages/video-matting/src/probe-node-webgpu.ts
@@ -0,0 +1,47 @@
+import type * as OnnxRuntimeNode from 'onnxruntime-node';
+import {importNodeModule} from './import-node-module';
+
+// Single-element ONNX Mul graphs (Y = X * X), opset 13. The fp16 graph casts
+// float32 inputs to float16 for multiplication, then back for the Node binding.
+const webGpuProbeModel = new Uint8Array([
+ 0x08, 0x08, 0x3a, 0x40, 0x0a, 0x0e, 0x0a, 0x01, 0x58, 0x0a, 0x01, 0x58, 0x12,
+ 0x01, 0x59, 0x22, 0x03, 0x4d, 0x75, 0x6c, 0x12, 0x0c, 0x77, 0x65, 0x62, 0x67,
+ 0x70, 0x75, 0x2d, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x5a, 0x0f, 0x0a, 0x01, 0x58,
+ 0x12, 0x0a, 0x0a, 0x08, 0x08, 0x01, 0x12, 0x04, 0x0a, 0x02, 0x08, 0x01, 0x62,
+ 0x0f, 0x0a, 0x01, 0x59, 0x12, 0x0a, 0x0a, 0x08, 0x08, 0x01, 0x12, 0x04, 0x0a,
+ 0x02, 0x08, 0x01, 0x42, 0x02, 0x10, 0x0d,
+]);
+
+const webGpuFp16ProbeModel = new Uint8Array([
+ 0x08, 0x08, 0x3a, 0x72, 0x0a, 0x17, 0x0a, 0x01, 0x58, 0x12, 0x01, 0x48, 0x22,
+ 0x04, 0x43, 0x61, 0x73, 0x74, 0x2a, 0x09, 0x0a, 0x02, 0x74, 0x6f, 0x18, 0x0a,
+ 0xa0, 0x01, 0x02, 0x0a, 0x0e, 0x0a, 0x01, 0x48, 0x0a, 0x01, 0x48, 0x12, 0x01,
+ 0x53, 0x22, 0x03, 0x4d, 0x75, 0x6c, 0x0a, 0x17, 0x0a, 0x01, 0x53, 0x12, 0x01,
+ 0x59, 0x22, 0x04, 0x43, 0x61, 0x73, 0x74, 0x2a, 0x09, 0x0a, 0x02, 0x74, 0x6f,
+ 0x18, 0x01, 0xa0, 0x01, 0x02, 0x12, 0x0c, 0x77, 0x65, 0x62, 0x67, 0x70, 0x75,
+ 0x2d, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x5a, 0x0f, 0x0a, 0x01, 0x58, 0x12, 0x0a,
+ 0x0a, 0x08, 0x08, 0x01, 0x12, 0x04, 0x0a, 0x02, 0x08, 0x01, 0x62, 0x0f, 0x0a,
+ 0x01, 0x59, 0x12, 0x0a, 0x0a, 0x08, 0x08, 0x01, 0x12, 0x04, 0x0a, 0x02, 0x08,
+ 0x01, 0x42, 0x02, 0x10, 0x0d,
+]);
+
+export const probeNodeWebGpu = async ({
+ requiresShaderF16,
+}: {
+ requiresShaderF16: boolean;
+}): Promise<void> => {
+ const {InferenceSession, Tensor} =
+ await importNodeModule<typeof OnnxRuntimeNode>('onnxruntime-node');
+ const session = await InferenceSession.create(
+ requiresShaderF16 ? webGpuFp16ProbeModel : webGpuProbeModel,
+ {
+ executionProviders: ['webgpu'],
+ extra: {session: {disable_cpu_ep_fallback: '1'}},
+ },
+ );
+ try {
+ await session.run({X: new Tensor('float32', new Float32Array([1]), [1])});
+ } finally {
+ await session.release();
+ }
+};
diff --git a/packages/video-matting/src/separate-video-layers.ts b/packages/video-matting/src/separate-video-layers.ts
index 1b7063746e..822266e409 100644
--- a/packages/video-matting/src/separate-video-layers.ts
+++ b/packages/video-matting/src/separate-video-layers.ts
@@ -1,8 +1,8 @@
+import type * as NodeUrl from 'node:url';
import {
ALL_FORMATS,
BlobSource,
- CanvasSink,
- CanvasSource,
+ FilePathSource,
Input,
type InputVideoTrack,
type Quality,
@@ -11,6 +11,11 @@ import {
canEncodeVideo,
} from 'mediabunny';
import {createVideoLayerOutput} from './create-video-layer-output';
+import {
+ createVideoMattingFrames,
+ type VideoMattingFrames,
+} from './create-video-matting-frames';
+import {importNodeModule} from './import-node-module';
import {
type OnVideoMattingModelLoadProgress,
withLoadedVideoMattingPipeline,
@@ -18,12 +23,6 @@ import {
import {getVideoMattingModelInfo, type VideoMattingModel} from './models';
import type {VideoLayerOutput, VideoLayerOutputOptions} from './output-target';
import {prepareAudio, type PreparedVideoMattingAudio} from './prepare-audio';
-import {
- createVideoMattingCanvas,
- drawForegroundFrame,
- drawOpaqueBaseFrame,
- getVideoMattingCanvasContext,
-} from './video-matting-canvas';
import {
resolveVideoMattingQuality,
type VideoMattingBitrate,
@@ -233,7 +232,32 @@ const validateOptions = (options: SeparateVideoLayersOptions) => {
}
};
-const makeInput = (src: string | URL | Blob): Input => {
+const makeInput = async (src: string | URL | Blob): Promise<Input> => {
+ if (
+ typeof window === 'undefined' &&
+ typeof process !== 'undefined' &&
+ process.release?.name === 'node' &&
+ (typeof src === 'string' || src instanceof URL)
+ ) {
+ const path = String(src);
+ if (path.startsWith('file:')) {
+ const {fileURLToPath} =
+ await importNodeModule<typeof NodeUrl>('node:url');
+ return new Input({
+ formats: ALL_FORMATS,
+ source: new FilePathSource(fileURLToPath(path)),
+ });
+ }
+
+ // Treat Windows drive letters as paths, while preserving HTTP/data URLs.
+ if (/^[a-z]:[/\\]/i.test(path) || !/^[a-z][a-z\d+.-]*:/i.test(path)) {
+ return new Input({
+ formats: ALL_FORMATS,
+ source: new FilePathSource(path),
+ });
+ }
+ }
+
const source =
typeof src === 'string' || src instanceof URL
? new UrlSource(src)
@@ -291,7 +315,7 @@ const probeVideoInput = async ({
]);
if (!canEncodeBase || !canEncodeForeground) {
throw new Error(
- 'This browser cannot encode the VP9 video streams required for video layer separation.',
+ 'This environment cannot encode the VP9 video streams required for video layer separation.',
);
}
@@ -313,11 +337,12 @@ export const separateVideoLayers = async (
options.audioBitrate ?? 'medium',
);
const keyframeIntervalInSeconds = options.keyframeIntervalInSeconds ?? 1;
- const input = makeInput(options.src);
+ const input = await makeInput(options.src);
const onInputAbort = () => input.dispose();
options.signal?.addEventListener('abort', onInputAbort, {once: true});
try {
+ throwIfAborted(options.signal);
const {videoTrack, width, height} = await probeVideoInput({
input,
videoQuality,
@@ -342,23 +367,13 @@ export const separateVideoLayers = async (
signal: options.signal ?? null,
run: async (pipeline) => {
throwIfAborted(options.signal);
- let iterator: AsyncGenerator<
- {
- canvas: HTMLCanvasElement | OffscreenCanvas;
- timestamp: number;
- duration: number;
- },
- void,
- unknown
- > | null = null;
+ let videoFrames: VideoMattingFrames | null = null;
let baseOutput: Awaited<
ReturnType<typeof createVideoLayerOutput<WebMOutputFormat>>
> | null = null;
let foregroundOutput: Awaited<
ReturnType<typeof createVideoLayerOutput<WebMOutputFormat>>
> | null = null;
- let baseVideoSource: CanvasSource | null = null;
- let foregroundVideoSource: CanvasSource | null = null;
let audioWriter: PreparedVideoMattingAudio | null = null;
let completed = false;
let abortCleanupPromise: Promise<void> | null = null;
@@ -382,17 +397,16 @@ export const separateVideoLayers = async (
options.signal?.addEventListener('abort', onAbort, {once: true});
try {
- const canvasSink = new CanvasSink(videoTrack, {
- alpha: true,
+ videoFrames = await createVideoMattingFrames({
+ videoTrack,
width,
height,
- fit: 'fill',
- poolSize: 1,
- });
- iterator = canvasSink.canvases(
+ videoQuality,
+ keyframeIntervalInSeconds,
videoStartTimestamp,
videoEndTimestamp,
- );
+ });
+ const iterator = videoFrames.frames;
let nextFrame = await iterator.next();
if (nextFrame.done) {
throw new Error(
@@ -402,12 +416,6 @@ export const separateVideoLayers = async (
throwIfAborted(options.signal);
- const baseCanvas = createVideoMattingCanvas({width, height});
- const foregroundCanvas = createVideoMattingCanvas({width, height});
- const baseContext = getVideoMattingCanvasContext(baseCanvas);
- const foregroundContext =
- getVideoMattingCanvasContext(foregroundCanvas);
-
baseOutput = await createVideoLayerOutput({
format: new WebMOutputFormat(),
options: options.outputs?.base,
@@ -419,20 +427,8 @@ export const separateVideoLayers = async (
});
throwIfAborted(options.signal);
- baseVideoSource = new CanvasSource(baseCanvas, {
- codec: 'vp9',
- quality: videoQuality,
- keyFrameInterval: keyframeIntervalInSeconds,
- alpha: 'discard',
- });
- foregroundVideoSource = new CanvasSource(foregroundCanvas, {
- codec: 'vp9',
- quality: videoQuality,
- keyFrameInterval: keyframeIntervalInSeconds,
- alpha: 'keep',
- });
- baseOutput.output.addVideoTrack(baseVideoSource);
- foregroundOutput.output.addVideoTrack(foregroundVideoSource);
+ baseOutput.output.addVideoTrack(videoFrames.baseSource);
+ foregroundOutput.output.addVideoTrack(videoFrames.foregroundSource);
audioWriter = await prepareAudio({
input,
@@ -478,26 +474,15 @@ export const separateVideoLayers = async (
continue;
}
- const foregroundFrame = await pipeline(frame.canvas);
+ const foregroundFrame = await pipeline(frame.image);
throwIfAborted(options.signal);
- drawOpaqueBaseFrame({
- context: baseContext,
- source: frame.canvas,
- width,
- height,
- });
- drawForegroundFrame({
- context: foregroundContext,
- result: foregroundFrame,
- source: frame.canvas,
- targetWidth: width,
- targetHeight: height,
- });
-
await Promise.all([
- baseVideoSource.add(timing.timestamp, timing.duration),
- foregroundVideoSource.add(timing.timestamp, timing.duration),
+ videoFrames.addFrame({
+ frame,
+ foreground: foregroundFrame,
+ ...timing,
+ }),
audioWriter.writeAudioUntil(timing.timestamp + timing.duration),
]);
@@ -529,8 +514,8 @@ export const separateVideoLayers = async (
durationInSeconds,
});
await audioWriter.finishAudio();
- baseVideoSource.close();
- foregroundVideoSource.close();
+ videoFrames.baseSource.close();
+ videoFrames.foregroundSource.close();
const [base, foreground] = await Promise.all([
baseOutput.finalize(),
foregroundOutput.finalize(),
@@ -545,6 +530,7 @@ export const separateVideoLayers = async (
width,
height,
durationInSeconds,
+ processedFrames,
} as SeparateVideoLayersResult;
Object.defineProperty(separated, Symbol.asyncDispose, {
enumerable: false,
@@ -566,7 +552,7 @@ export const separateVideoLayers = async (
await abortCleanupPromise;
if (!completed) {
try {
- await iterator?.return();
+ await videoFrames?.frames.return();
} catch {
// Cleanup must not replace the operation's original error.
}
diff --git a/packages/video-matting/src/test/model-runtime.test.ts b/packages/video-matting/src/test/model-runtime.test.ts
index 100277435b..c17d32e411 100644
--- a/packages/video-matting/src/test/model-runtime.test.ts
+++ b/packages/video-matting/src/test/model-runtime.test.ts
@@ -1,4 +1,9 @@
import {beforeEach, expect, mock, test} from 'bun:test';
+import {existsSync} from 'node:fs';
+import {mkdtemp, readdir, readFile, rm, writeFile} from 'node:fs/promises';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {pathToFileURL} from 'node:url';
let initializationCount = 0;
let initializationStarted: ((count: number) => void) | null = null;
@@ -37,6 +42,8 @@ const originalTransformersEnvironment = {
useCustomCache: true,
customCache,
useBrowserCache: false,
+ useFSCache: false,
+ cacheDir: null as string | null,
cacheKey: 'transformers-cache',
fetch: (url: string) => {
requestedFiles.push(url);
@@ -56,11 +63,39 @@ Object.defineProperty(
{value: preexistingWhisperModelHostState},
);
+class RawImage {
+ public data: Uint8ClampedArray;
+ public width: number;
+ public height: number;
+ public channels: number;
+
+ constructor(
+ data: Uint8ClampedArray,
+ width: number,
+ height: number,
+ channels: number,
+ ) {
+ this.data = data;
+ this.width = width;
+ this.height = height;
+ this.channels = channels;
+ }
+}
+
function BackgroundRemovalPipeline() {
livePipelineCount++;
return Object.assign(
- () => {
+ (image: unknown) => {
pipelineRunCount++;
+ if (image instanceof RawImage) {
+ const data = image.data.slice();
+ for (let i = 3; i < data.length; i += 4) {
+ data[i] = 128;
+ }
+
+ return Promise.resolve({...image, data});
+ }
+
return Promise.resolve({
data: new Uint8ClampedArray([10, 20, 30, 128]),
width: 1,
@@ -112,13 +147,18 @@ mock.module('@huggingface/transformers', () => ({
},
},
BackgroundRemovalPipeline,
+ RawImage,
ModelRegistry: {
is_pipeline_cached_files: () => {
const files = ['config.json', 'onnx/model.onnx'].map((file) => ({
file,
- cached: downloadedFiles.has(
- `https://remotion.media/models/modnet-v1/${file}`,
- ),
+ cached: transformersEnvironment.useFSCache
+ ? existsSync(
+ join(transformersEnvironment.cacheDir!, 'modnet-v1', file),
+ )
+ : downloadedFiles.has(
+ `https://remotion.media/models/modnet-v1/${file}`,
+ ),
}));
return Promise.resolve({
files,
@@ -131,6 +171,13 @@ mock.module('@huggingface/transformers', () => ({
options: Record<string, unknown>,
) => {
cacheClear = {task, modelId, options};
+ if (transformersEnvironment.useFSCache) {
+ return rm(join(transformersEnvironment.cacheDir!, modelId), {
+ recursive: true,
+ force: true,
+ });
+ }
+
return Promise.resolve();
},
is_pipeline_cached: (
@@ -144,7 +191,15 @@ mock.module('@huggingface/transformers', () => ({
remotePathTemplate: transformersEnvironment.remotePathTemplate,
};
return Promise.resolve(
- checkDownloadedFiles ? downloadedFiles.size === 2 : true,
+ transformersEnvironment.useFSCache
+ ? ['config.json', 'onnx/model.onnx'].every((file) =>
+ existsSync(
+ join(transformersEnvironment.cacheDir!, modelId, file),
+ ),
+ )
+ : checkDownloadedFiles
+ ? downloadedFiles.size === 2
+ : true,
);
},
},
@@ -425,3 +480,189 @@ test('coordinates the shared Transformers environment across model loads', async
expect(transformersEnvironment).toEqual(originalTransformersEnvironment);
await disposeVideoMattingModel();
});
+
+// Keep the neural network deterministic; decoding, encoding, pixels, storage,
+// cancellation, and the public package workflow all use the real implementation.
+test('downloads to disk and separates local video with alpha, audio, and cancellation', async () => {
+ const {registerMediabunnyServer} = await import('@mediabunny/server');
+ const {
+ ALL_FORMATS,
+ AudioSample,
+ AudioSampleSink,
+ AudioSampleSource,
+ BlobSource,
+ BufferTarget,
+ Input,
+ Output,
+ VideoSample,
+ VideoSampleSink,
+ VideoSampleSource,
+ WebMOutputFormat,
+ } = await import('mediabunny');
+ const {
+ downloadVideoMattingModel,
+ isVideoMattingModelCached,
+ loadVideoMattingModel,
+ removeVideoMattingModel,
+ separateVideoLayers,
+ } = await import('../index');
+ registerMediabunnyServer({hardwareContext: null});
+ const directory = await mkdtemp(join(tmpdir(), 'video-matting-node-'));
+ transformersEnvironment.useCustomCache = false;
+ transformersEnvironment.useFSCache = true;
+ transformersEnvironment.cacheDir = join(directory, 'models');
+ try {
+ expect(await isVideoMattingModelCached({model: 'modnet'})).toBe(false);
+ expect(await downloadVideoMattingModel({model: 'modnet'})).toEqual({
+ alreadyDownloaded: false,
+ });
+ expect(
+ await readFile(
+ join(directory, 'models/modnet-v1/onnx/model.onnx'),
+ 'utf8',
+ ),
+ ).toBe('https://remotion.media/models/modnet-v1/onnx/model.onnx');
+ expect(await downloadVideoMattingModel({model: 'modnet'})).toEqual({
+ alreadyDownloaded: true,
+ });
+ expect(requestedFiles).toHaveLength(2);
+ expect(initializationCount).toBe(0);
+
+ const output = new Output({
+ format: new WebMOutputFormat(),
+ target: new BufferTarget(),
+ });
+ const video = new VideoSampleSource({
+ codec: 'vp9',
+ alpha: 'keep',
+ bitrate: 1_000_000,
+ });
+ const audio = new AudioSampleSource({codec: 'opus', bitrate: 64_000});
+ output.addVideoTrack(video);
+ output.addAudioTrack(audio);
+ await output.start();
+ for (let i = 0; i < 3; i++) {
+ const data = new Uint8Array(32 * 16 * 4);
+ for (let j = 0; j < data.length; j += 4) {
+ data[j] = 255;
+ data[j + 3] = 128;
+ }
+
+ using sample = new VideoSample(data, {
+ format: 'RGBA',
+ codedWidth: 32,
+ codedHeight: 16,
+ timestamp: i / 10,
+ duration: 0.1,
+ });
+ using sound = new AudioSample({
+ format: 'f32',
+ sampleRate: 48_000,
+ numberOfChannels: 1,
+ timestamp: i / 10,
+ data: new Float32Array(4800).fill(0.1),
+ });
+ await Promise.all([video.add(sample), audio.add(sound)]);
+ }
+
+ video.close();
+ audio.close();
+ await output.finalize();
+ const src = join(directory, 'input.webm');
+ await writeFile(src, new Uint8Array(output.target.buffer!));
+ await using modelHandle = await loadVideoMattingModel({model: 'modnet'});
+ expect(modelHandle.alreadyLoaded).toBe(false);
+
+ const controller = new AbortController();
+ await expect(
+ separateVideoLayers({
+ src,
+ signal: controller.signal,
+ onProgress: ({processedFrames}) => {
+ if (processedFrames === 1) {
+ controller.abort(new Error('stop after first frame'));
+ }
+ },
+ }),
+ ).rejects.toThrow('stop after first frame');
+
+ await using layers = await separateVideoLayers({
+ src: pathToFileURL(src),
+ audio: 'both',
+ audioBitrate: 'medium',
+ });
+ expect(layers).toMatchObject({
+ width: 32,
+ height: 16,
+ processedFrames: 3,
+ durationInSeconds: 0.3,
+ });
+ for (const [layer, expectedRed, expectedAlpha] of [
+ [layers.base, 128, 255],
+ [layers.foreground, 255, 64],
+ ] as const) {
+ using input = new Input({
+ formats: ALL_FORMATS,
+ source: new BlobSource(await layer.getBlob()),
+ });
+ const track = (await input.getPrimaryVideoTrack())!;
+ expect(await track.getCodec()).toBe('vp9');
+ expect(await track.getDisplayWidth()).toBe(32);
+ expect(await track.getDisplayHeight()).toBe(16);
+ const timestamps: number[] = [];
+ for await (const frame of new VideoSampleSink(track).samples()) {
+ try {
+ const data = new Uint8Array(16 * 32 * 4);
+ await frame.copyTo(data, {format: 'RGBA'});
+ expect(Math.abs(data[0]! - expectedRed)).toBeLessThan(5);
+ expect(Math.abs(data[3]! - expectedAlpha)).toBeLessThan(3);
+ timestamps.push(frame.timestamp);
+ } finally {
+ frame.close();
+ }
+ }
+
+ expect(timestamps).toEqual([0, 0.1, 0.2]);
+ const audioTrack = (await input.getPrimaryAudioTrack())!;
+ expect(await audioTrack.getCodec()).toBe('opus');
+ using decodedAudio = await new AudioSampleSink(audioTrack).getSample(0);
+ expect(decodedAudio!.numberOfFrames).toBeGreaterThan(0);
+ }
+
+ await removeVideoMattingModel({model: 'modnet'});
+ expect(await isVideoMattingModelCached({model: 'modnet'})).toBe(false);
+ expect(livePipelineCount).toBe(0);
+ } finally {
+ await rm(directory, {recursive: true, force: true});
+ }
+}, 30_000);
+
+test('an interrupted filesystem download can be retried without leaving partial model files', async () => {
+ const {downloadVideoMattingModel} = await import('../index');
+ const directory = await mkdtemp(join(tmpdir(), 'video-matting-download-'));
+ transformersEnvironment.useCustomCache = false;
+ transformersEnvironment.useFSCache = true;
+ transformersEnvironment.cacheDir = directory;
+ transformersEnvironment.fetch = () =>
+ Promise.resolve(
+ new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.error(new Error('download interrupted'));
+ },
+ }),
+ ),
+ );
+ try {
+ await expect(downloadVideoMattingModel({model: 'modnet'})).rejects.toThrow(
+ 'download interrupted',
+ );
+ expect(await readdir(join(directory, 'modnet-v1'))).toEqual([]);
+ transformersEnvironment.fetch = originalTransformersEnvironment.fetch;
+ expect(await downloadVideoMattingModel({model: 'modnet'})).toEqual({
+ alreadyDownloaded: false,
+ });
+ } finally {
+ await rm(directory, {recursive: true, force: true});
+ }
+});
diff --git a/packages/video-matting/src/test/models.test.ts b/packages/video-matting/src/test/models.test.ts
index eae26adf9e..b3929fa91a 100644
--- a/packages/video-matting/src/test/models.test.ts
+++ b/packages/video-matting/src/test/models.test.ts
@@ -1,4 +1,4 @@
-import {expect, test} from 'bun:test';
+import {expect, mock, test} from 'bun:test';
import {
canUseVideoMatting,
VideoMattingUnsupportedReason,
@@ -76,3 +76,37 @@ test('requires shader-f16 only for the fp16 BEN2 model', async () => {
}
}
});
+
+test('checks native WebGPU inference and reports failures without requiring browser globals', async () => {
+ let failure: Error | null = null;
+ let releases = 0;
+ mock.module('onnxruntime-node', () => ({
+ Tensor: class {},
+ InferenceSession: {
+ create: () =>
+ Promise.resolve({
+ run: () =>
+ failure === null ? Promise.resolve({}) : Promise.reject(failure),
+ release: () => {
+ releases++;
+ return Promise.resolve();
+ },
+ }),
+ },
+ }));
+
+ expect(await canUseVideoMatting({model: 'modnet'})).toEqual({
+ supported: true,
+ });
+ expect(await canUseVideoMatting({model: 'ben2-base'})).toEqual({
+ supported: true,
+ });
+ failure = new Error('No supported adapter');
+ expect(await canUseVideoMatting()).toEqual({
+ supported: false,
+ reason: VideoMattingUnsupportedReason.WebGpuUnavailable,
+ detailedReason:
+ 'ONNX Runtime could not run WebGPU inference for "modnet": No supported adapter',
+ });
+ expect(releases).toBe(3);
+});
diff --git a/packages/video-matting/src/with-remotion-model-host.ts b/packages/video-matting/src/with-remotion-model-host.ts
index 2d92551193..a74ac39abf 100644
--- a/packages/video-matting/src/with-remotion-model-host.ts
+++ b/packages/video-matting/src/with-remotion-model-host.ts
@@ -4,6 +4,7 @@ import type {
BackgroundRemovalPipeline,
env,
ModelRegistry,
+ RawImage,
} from '@huggingface/transformers';
const REMOTION_MODEL_HOST = 'https://remotion.media/';
@@ -21,6 +22,7 @@ type TransformersModule = {
readonly BackgroundRemovalPipeline: typeof BackgroundRemovalPipeline;
readonly env: typeof env;
readonly ModelRegistry: typeof ModelRegistry;
+ readonly RawImage: typeof RawImage;
};
type ModelHostState = {