🎥 Make videos programmatically with React
`@remotion/video-matting`: Add Node.js support for `separateVideoLayers()`
Jonny Burgerff38ef57502f
diff --git a/bun.lock b/bun.lockindex 4b566665f0..84fdebc54c 100644--- a/bun.lock+++ b/bun.lock@@ -565,6 +565,7 @@"@docusaurus/preset-classic": "3.10.1","@docusaurus/theme-common": "3.10.1","@docusaurus/types": "3.10.1",+ "@huggingface/transformers": "catalog:","@mdx-js/react": "3.0.1","@mediabunny/ac3": "catalog:","@mediabunny/dts": "catalog:",@@ -2815,9 +2816,11 @@"version": "4.0.527","dependencies": {"mediabunny": "catalog:",+ "onnxruntime-node": "catalog:",},"devDependencies": {"@huggingface/transformers": "catalog:",+ "@mediabunny/server": "catalog:","@remotion/eslint-config-internal": "workspace:*","@typescript/native-preview": "catalog:","eslint": "catalog:",diff --git a/packages/docs/docs/video-matting/can-use-video-matting.mdx b/packages/docs/docs/video-matting/can-use-video-matting.mdxindex 742633af9c..42a4c2b386 100644--- a/packages/docs/docs/video-matting/can-use-video-matting.mdx+++ b/packages/docs/docs/video-matting/can-use-video-matting.mdx@@ -6,7 +6,7 @@ crumb: '@remotion/video-matting'# canUseVideoMatting()<AvailableFrom v="4.0.523" />-Checks whether a video matting model can run in the current browser.+Checks whether a video matting model can run in the current browser or Node.js environment.This function checks WebGPU and model-specific requirements. It does not inspect an input file or check video decoding and encoding support.@@ -42,6 +42,8 @@ When `supported` is `false`, one of:- `webgpu-requires-secure-context`- `shader-f16-unavailable`+In Node.js, failed native WebGPU probes return `webgpu-unavailable` with the ONNX Runtime error in `detailedReason`.+`shader-f16-unavailable` applies to models such as `ben2-base` that require 16-bit shader support.### detailedReason?@@ -50,7 +52,9 @@ When `supported` is `false`, a human-readable explanation of the reason.## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.++<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/video-matting/dispose-video-matting-model.mdx b/packages/docs/docs/video-matting/dispose-video-matting-model.mdxindex 3e7adf5573..63b54209a3 100644--- a/packages/docs/docs/video-matting/dispose-video-matting-model.mdx+++ b/packages/docs/docs/video-matting/dispose-video-matting-model.mdx@@ -6,7 +6,7 @@ crumb: '@remotion/video-matting'# disposeVideoMattingModel()<AvailableFrom v="4.0.523" />-Releases the memory used by initialized video matting models. Persistent browser-cached model files are not deleted.+Releases the memory used by initialized video matting models. Persistent cached model files are not deleted.```ts twoslash title="dispose.ts"import {disposeVideoMattingModel} from '@remotion/video-matting';@@ -28,7 +28,9 @@ A `Promise<void>` that resolves after the selected models have been disposed.## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.++<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/video-matting/download-video-matting-model.mdx b/packages/docs/docs/video-matting/download-video-matting-model.mdxindex 39515d2c52..ba77101ddd 100644--- a/packages/docs/docs/video-matting/download-video-matting-model.mdx+++ b/packages/docs/docs/video-matting/download-video-matting-model.mdx@@ -6,9 +6,9 @@ crumb: '@remotion/video-matting'# downloadVideoMattingModel()<AvailableFrom v="4.0.528" />-Downloads a video matting model from `remotion.media` into the browser cache without initializing WebGPU. Call [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model) or [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) afterwards.+Downloads a video matting model from `remotion.media` into the browser or Node.js filesystem cache without initializing WebGPU. Call [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model) or [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) afterwards.-The browser Cache API or a Transformers.js custom cache must be enabled.+The browser Cache API, Node.js filesystem cache, or a Transformers.js custom cache must be enabled.```ts twoslash title="download-model.ts"import {downloadVideoMattingModel} from '@remotion/video-matting';@@ -55,7 +55,9 @@ Estimated total model size in bytes.## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.++<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/video-matting/index.mdx b/packages/docs/docs/video-matting/index.mdxindex 843e9e4c95..5da68b178b 100644--- a/packages/docs/docs/video-matting/index.mdx+++ b/packages/docs/docs/video-matting/index.mdx@@ -6,7 +6,7 @@ crumb: 'Separate a video into layers'# @remotion/video-matting<AvailableFrom v="4.0.523" />-Separate a video into an opaque base layer and a foreground layer with alpha, locally in the browser using WebGPU.+Separate a video into an opaque base layer and a foreground layer with alpha, locally in the browser or [Node.js](/docs/video-matting/node) using WebGPU.import {TableOfContents} from './table-of-contents';diff --git a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdxindex 279ad7da9c..3ae61ff002 100644--- a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx+++ b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx@@ -6,7 +6,7 @@ crumb: '@remotion/video-matting'# isVideoMattingModelCached()<AvailableFrom v="4.0.523" />-Checks whether all files hosted on `remotion.media` for a video matting model are present in the browser cache. The check does not download or initialize the model.+Checks whether all files hosted on `remotion.media` for a video matting model are present in the browser or Node.js filesystem cache. The check does not download or initialize the model.```ts twoslash title="model-cache.ts"import {isVideoMattingModelCached} from '@remotion/video-matting';@@ -28,7 +28,9 @@ A `Promise<boolean>` resolving to `true` if every required model file is cached.## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.++<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/video-matting/load-video-matting-model.mdx b/packages/docs/docs/video-matting/load-video-matting-model.mdxindex fe5350f7f4..c008c005b9 100644--- a/packages/docs/docs/video-matting/load-video-matting-model.mdx+++ b/packages/docs/docs/video-matting/load-video-matting-model.mdx@@ -71,7 +71,9 @@ Call [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-mo## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.++<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/video-matting/node.mdx b/packages/docs/docs/video-matting/node.mdxnew file mode 100644index 0000000000..6b6cb27448--- /dev/null+++ b/packages/docs/docs/video-matting/node.mdx@@ -0,0 +1,84 @@+---+image: /generated/articles-docs-video-matting-node.png+title: 'Using @remotion/video-matting in Node.js'+crumb: '@remotion/video-matting'+---++# Using @remotion/video-matting in Node.js<AvailableFrom v="4.0.528" />++You can use `@remotion/video-matting` in Node.js to separate a local video into an opaque base and a transparent foreground.++Register [`@mediabunny/server`](https://mediabunny.dev/guide/extensions/server) to enable video decoding, VP9 encoding with alpha, and audio processing. A browser or canvas polyfill is not required.++## Installation++<Installation pkg="@remotion/video-matting @huggingface/transformers mediabunny @mediabunny/server" />++## Example++```ts twoslash title="separate-video.ts"+import {registerMediabunnyServer} from '@mediabunny/server';+import {+ canUseVideoMatting,+ downloadVideoMattingModel,+ loadVideoMattingModel,+ separateVideoLayers,+} from '@remotion/video-matting';+import {writeFile} from 'node:fs/promises';++registerMediabunnyServer();++const model = 'modnet';+const support = await canUseVideoMatting({model});+if (!support.supported) {+ throw new Error(support.detailedReason);+}++await downloadVideoMattingModel({model});++// In long-running apps, put this code in an async function so resources+// are released when that function finishes.+await using modelHandle = await loadVideoMattingModel({model});+await using layers = await separateVideoLayers({+ src: './input.mp4',+ model,+ audio: 'base',+ onProgress: ({processedFrames}) => console.log(processedFrames),+});++await writeFile(+ 'base.webm',+ new Uint8Array(await (await layers.base.getBlob()).arrayBuffer()),+);+await writeFile(+ 'foreground.webm',+ new Uint8Array(await (await layers.foreground.getBlob()).arrayBuffer()),+);+```++[`separateVideoLayers()`](/docs/video-matting/separate-video-layers) accepts local file paths, `file:` URLs, remote URLs, and `Blob` inputs. Outputs are held in memory by default. Use `outputWritable` to stream large outputs; handle each chunk's `position` because writes are not necessarily append-only.++## Model cache++Model files use the Transformers.js filesystem cache. To choose the directory, set `env.cacheDir` before downloading or loading models:++```ts twoslash title="model-cache.ts"+import {env} from '@huggingface/transformers';+import {resolve} from 'node:path';++env.cacheDir = resolve('.cache/video-matting');+```++[`removeVideoMattingModel()`](/docs/video-matting/remove-video-matting-model) removes cached files. Disposing a loaded model releases its memory and keeps the downloaded files.++## Requirements++A compatible GPU is required. Transformers.js uses ONNX Runtime's native WebGPU provider in Node.js. [`canUseVideoMatting()`](/docs/video-matting/can-use-video-matting) runs a tiny inference to check support, including fp16 computation for `ben2-base`.++The native ONNX Runtime dependency does not support Linux arm64. GPU availability also depends on the host and its drivers; a typical serverless environment does not provide a GPU.++## See also++- [`separateVideoLayers()`](/docs/video-matting/separate-video-layers)+- [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model)+- [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model)diff --git a/packages/docs/docs/video-matting/remove-video-matting-model.mdx b/packages/docs/docs/video-matting/remove-video-matting-model.mdxindex 7fedf0bdac..1f4b08d9cc 100644--- a/packages/docs/docs/video-matting/remove-video-matting-model.mdx+++ b/packages/docs/docs/video-matting/remove-video-matting-model.mdx@@ -6,7 +6,7 @@ crumb: '@remotion/video-matting'# removeVideoMattingModel()<AvailableFrom v="4.0.523" />-Removes a video matting model from memory and the persistent browser cache.+Removes a video matting model from memory and the persistent model cache.If the model is still loading or separating a video, removal waits for the operation to finish.@@ -24,7 +24,9 @@ One of the names returned by [`getAvailableModels()`](/docs/video-matting/get-av## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.++<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/video-matting/separate-video-layers.mdx b/packages/docs/docs/video-matting/separate-video-layers.mdxindex bbec704dc6..26a40ac3a2 100644--- a/packages/docs/docs/video-matting/separate-video-layers.mdx+++ b/packages/docs/docs/video-matting/separate-video-layers.mdx@@ -35,6 +35,8 @@ Place inserted content between the base and foreground. The base remains opaqueA `string`, `URL`, or `Blob` containing the input video. A `File` may be passed because it extends `Blob`.+In [Node.js](/docs/video-matting/node), strings can also be local file paths, and `file:` URLs are supported.+### model?The video matting model to use. Default: `modnet`.@@ -220,7 +222,9 @@ Both layers are WebM files encoded with VP9. The foreground preserves alpha. Aud## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+[Node.js usage](/docs/video-matting/node) is available from <AvailableFrom v="4.0.528" inline />.++<CompatibilityTable chrome firefox safari nodejs bun="" serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/package.json b/packages/docs/package.jsonindex 208b3fd0f4..8db60f742f 100644--- a/packages/docs/package.json+++ b/packages/docs/package.json@@ -24,6 +24,7 @@"upload-element-preview": "bun run upload-element-preview.ts"},"dependencies": {+ "@huggingface/transformers": "catalog:","@remotion/studio-protocol": "workspace:*","@aws-sdk/s3-request-presigner": "catalog:","@babel/parser": "7.24.1",diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.tsindex 316478653c..3b9a05f216 100644--- a/packages/docs/sidebars.ts+++ b/packages/docs/sidebars.ts@@ -960,6 +960,11 @@ const sidebars: SidebarsConfig = {id: 'video-matting/index',},items: [+ {+ type: 'doc',+ id: 'video-matting/node',+ label: 'Usage in Node.js',+ },'video-matting/can-use-video-matting','video-matting/get-available-models','video-matting/is-video-matting-model-cached',diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.tsindex b3e623ceef..82efe05125 100644--- a/packages/docs/src/data/articles.ts+++ b/packages/docs/src/data/articles.ts@@ -8958,6 +8958,15 @@ export const articles = [noAi: false,slug: 'video-matting/load-video-matting-model',},+ {+ id: 'video-matting/node',+ title: 'Using @remotion/video-matting in Node.js',+ relativePath: 'docs/video-matting/node.mdx',+ compId: 'articles-docs-video-matting-node',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/node',+ },{id: 'video-matting/remove-video-matting-model',title: 'removeVideoMattingModel()',diff --git a/packages/docs/static/generated/articles-docs-video-matting-node.png b/packages/docs/static/generated/articles-docs-video-matting-node.pngnew file mode 100644index 0000000000..5613af2f7bBinary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-node.png differdiff --git a/packages/video-matting/README.md b/packages/video-matting/README.mdindex 312b6d5931..bb7676e785 100644--- a/packages/video-matting/README.md+++ b/packages/video-matting/README.md@@ -1,6 +1,6 @@# @remotion/video-matting-Separate a video into a base layer and transparent foreground in the browser+Separate a video into a base layer and transparent foreground in the browser and Node.js[](https://npmcharts.com/compare/@remotion/video-matting?minimal=true)diff --git a/packages/video-matting/package.json b/packages/video-matting/package.jsonindex 1a4dee924e..9315d6c8c7 100644--- a/packages/video-matting/package.json+++ b/packages/video-matting/package.json@@ -1,7 +1,7 @@{"name": "@remotion/video-matting","version": "4.0.527",- "description": "Separate a video into a base layer and transparent foreground in the browser",+ "description": "Separate a video into a base layer and transparent foreground in the browser and Node.js","main": "dist/index.js","types": "dist/index.d.ts","module": "dist/esm/index.mjs",@@ -24,7 +24,8 @@"url": "https://github.com/remotion-dev/remotion/issues"},"dependencies": {- "mediabunny": "catalog:"+ "mediabunny": "catalog:",+ "onnxruntime-node": "catalog:"},"peerDependencies": {"@huggingface/transformers": "4.2.0"@@ -39,6 +40,7 @@},"devDependencies": {"@huggingface/transformers": "catalog:",+ "@mediabunny/server": "catalog:","@remotion/eslint-config-internal": "workspace:*","@typescript/native-preview": "catalog:","eslint": "catalog:"diff --git a/packages/video-matting/src/can-use-video-matting.ts b/packages/video-matting/src/can-use-video-matting.tsindex 1f8f3a8020..d02f02fd8f 100644--- a/packages/video-matting/src/can-use-video-matting.ts+++ b/packages/video-matting/src/can-use-video-matting.ts@@ -32,12 +32,31 @@ export const canUseVideoMatting = async ({}: CanUseVideoMattingOptions = {}): Promise<CanUseVideoMattingResult> => {const modelInfo = getVideoMattingModelInfo(model);+ if (+ typeof window === 'undefined' &&+ typeof process !== 'undefined' &&+ process.release?.name === 'node'+ ) {+ try {+ const {probeNodeWebGpu} = await import('./probe-node-webgpu');+ await probeNodeWebGpu({requiresShaderF16: modelInfo.requiresShaderF16});+ return {supported: true};+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error);+ return {+ supported: false,+ reason: VideoMattingUnsupportedReason.WebGpuUnavailable,+ detailedReason: `ONNX Runtime could not run WebGPU inference for "${model}": ${message}`,+ };+ }+ }+if (typeof window === 'undefined' && typeof OffscreenCanvas === 'undefined') {return {supported: false,reason: VideoMattingUnsupportedReason.WindowUndefined,detailedReason:- 'No browser window or worker canvas environment is available. @remotion/video-matting is intended for browser environments.',+ 'No browser, worker canvas, or Node.js environment is available.',};}diff --git a/packages/video-matting/src/create-video-matting-frames.ts b/packages/video-matting/src/create-video-matting-frames.tsnew file mode 100644index 0000000000..5847fcf409--- /dev/null+++ b/packages/video-matting/src/create-video-matting-frames.ts@@ -0,0 +1,127 @@+import {CanvasSink, CanvasSource} from 'mediabunny';+import type {InputVideoTrack, Quality, VideoSource} from 'mediabunny';+import type {+ VideoMattingImageSource,+ VideoMattingPipelineResult,+} from './load-video-matting-model';+import {+ createVideoMattingCanvas,+ drawForegroundFrame,+ drawOpaqueBaseFrame,+ getVideoMattingCanvasContext,+} from './video-matting-canvas';++export type VideoMattingFrame = {+ image: VideoMattingImageSource;+ timestamp: number;+ duration: number;+};++export type VideoMattingFramesOptions = {+ videoTrack: InputVideoTrack;+ width: number;+ height: number;+ videoQuality: Quality;+ keyframeIntervalInSeconds: number;+ videoStartTimestamp: number;+ videoEndTimestamp: number;+};++export type VideoMattingFrames = {+ frames: AsyncGenerator<VideoMattingFrame, void, unknown>;+ baseSource: VideoSource;+ foregroundSource: VideoSource;+ addFrame: (options: {+ frame: VideoMattingFrame;+ foreground: VideoMattingPipelineResult;+ timestamp: number;+ duration: number;+ }) => Promise<void>;+};++export const createVideoMattingFrames = async (+ options: VideoMattingFramesOptions,+): Promise<VideoMattingFrames> => {+ if (+ typeof window === 'undefined' &&+ typeof process !== 'undefined' &&+ process.release?.name === 'node'+ ) {+ const {createNodeVideoMattingFrames} =+ await import('./node-video-matting-frames');+ return createNodeVideoMattingFrames(options);+ }++ const {+ videoTrack,+ width,+ height,+ videoQuality,+ keyframeIntervalInSeconds,+ videoStartTimestamp,+ videoEndTimestamp,+ } = options;+ const sink = new CanvasSink(videoTrack, {+ alpha: true,+ width,+ height,+ fit: 'fill',+ poolSize: 1,+ });+ const baseCanvas = createVideoMattingCanvas({width, height});+ const foregroundCanvas = createVideoMattingCanvas({width, height});+ const baseContext = getVideoMattingCanvasContext(baseCanvas);+ const foregroundContext = getVideoMattingCanvasContext(foregroundCanvas);+ const baseSource = new CanvasSource(baseCanvas, {+ codec: 'vp9',+ quality: videoQuality,+ keyFrameInterval: keyframeIntervalInSeconds,+ alpha: 'discard',+ });+ const foregroundSource = new CanvasSource(foregroundCanvas, {+ codec: 'vp9',+ quality: videoQuality,+ keyFrameInterval: keyframeIntervalInSeconds,+ alpha: 'keep',+ });++ return {+ frames: (async function* () {+ for await (const frame of sink.canvases(+ videoStartTimestamp,+ videoEndTimestamp,+ )) {+ yield {+ image: frame.canvas,+ timestamp: frame.timestamp,+ duration: frame.duration,+ };+ }+ })(),+ baseSource,+ foregroundSource,+ addFrame: async ({frame, foreground, timestamp, duration}) => {+ if ('data' in frame.image) {+ throw new Error('Expected a canvas frame.');+ }++ drawOpaqueBaseFrame({+ context: baseContext,+ source: frame.image,+ width,+ height,+ });+ drawForegroundFrame({+ context: foregroundContext,+ result: foreground,+ source: frame.image,+ targetWidth: width,+ targetHeight: height,+ });+ await Promise.all([+ baseSource.add(timestamp, duration),+ foregroundSource.add(timestamp, duration),+ ]);+ },+ };+};diff --git a/packages/video-matting/src/download-video-matting-model.ts b/packages/video-matting/src/download-video-matting-model.tsindex b5de024201..638230f828 100644--- a/packages/video-matting/src/download-video-matting-model.ts+++ b/packages/video-matting/src/download-video-matting-model.ts@@ -1,3 +1,7 @@+import type * as NodeCrypto from 'node:crypto';+import type * as NodeFsPromises from 'node:fs/promises';+import type * as NodePath from 'node:path';+import {importNodeModule} from './import-node-module';import {getHostedVideoMattingModelId,getVideoMattingModelInfo,@@ -34,9 +38,10 @@ export const downloadVideoMattingModel = ({: env.useBrowserCache && typeof caches !== 'undefined'? await caches.open(env.cacheKey): null;- if (cache === null) {+ const useFileCache = cache === null && env.useFSCache;+ if (cache === null && !useFileCache) {throw new Error(- 'A Transformers.js browser or custom cache is required to download a video matting model.',+ 'A Transformers.js model cache is required to download a video matting model.',);}@@ -77,32 +82,75 @@ export const downloadVideoMattingModel = ({}const url = `${env.remoteHost}${env.remotePathTemplate.replace('{model}', hostedModelId)}${file}`;- // Transformers.js 4.2.0 uses the remote URL as the browser cache key.+ // Transformers.js 4.2.0 keys browser caches by URL and its FileCache by model/file.+ const cacheKey = cache === null ? `${hostedModelId}/${file}` : url;const response = await env.fetch(url);if (!response.ok || response.body === null) {throw new Error(`Could not download ${url}: HTTP ${response.status}.`);}const reader = response.body.getReader();- const stream = new ReadableStream<Uint8Array>({- async pull(controller) {- try {+ if (cache !== null) {+ const stream = new ReadableStream<Uint8Array>({+ async pull(controller) {+ try {+ const {done, value} = await reader.read();+ if (done) {+ controller.close();+ return;+ }++ loadedBytes += value.byteLength;+ emitProgress(file);+ controller.enqueue(value);+ } catch (error) {+ controller.error(error);+ }+ },+ cancel: () => reader.cancel(),+ });+ await cache.put(+ cacheKey,+ new Response(stream, {headers: response.headers}),+ );+ } else {+ // Match Transformers.js FileCache's model/file layout without loading ONNX.+ const [{mkdir, open, rename, unlink}, {join, dirname}, {randomUUID}] =+ await Promise.all([+ importNodeModule<typeof NodeFsPromises>('node:fs/promises'),+ importNodeModule<typeof NodePath>('node:path'),+ importNodeModule<typeof NodeCrypto>('node:crypto'),+ ]);+ if (env.cacheDir === null) {+ throw new Error('Transformers.js env.cacheDir must be set.');+ }++ const destination = join(env.cacheDir, cacheKey);+ const temporary = `${destination}.tmp.${randomUUID()}`;+ await mkdir(dirname(destination), {recursive: true});+ const handle = await open(temporary, 'wx');+ try {+ while (true) {const {done, value} = await reader.read();if (done) {- controller.close();- return;+ break;}+ await handle.writeFile(value);loadedBytes += value.byteLength;emitProgress(file);- controller.enqueue(value);- } catch (error) {- controller.error(error);}- },- cancel: () => reader.cancel(),- });- await cache.put(url, new Response(stream, {headers: response.headers}));++ await handle.close();+ await rename(temporary, destination);+ } catch (error) {+ await Promise.allSettled([handle.close(), reader.cancel(error)]);+ await unlink(temporary).catch(() => undefined);+ throw error;+ } finally {+ reader.releaseLock();+ }+ }}if (diff --git a/packages/video-matting/src/import-node-module.ts b/packages/video-matting/src/import-node-module.tsnew file mode 100644index 0000000000..57f7c7d23a--- /dev/null+++ b/packages/video-matting/src/import-node-module.ts@@ -0,0 +1,10 @@+// Called only in Node.js branches. An indirect import keeps native Node modules+// out of browser bundles without making bundlers resolve a variable import.+export const importNodeModule = <T>(specifier: string): Promise<T> => {+ // eslint-disable-next-line no-new-func+ const dynamicImport = new Function(+ 'specifier',+ 'return import(specifier)',+ ) as (module: string) => Promise<T>;+ return dynamicImport(specifier);+};diff --git a/packages/video-matting/src/load-video-matting-model.ts b/packages/video-matting/src/load-video-matting-model.tsindex 0ad40d953a..f0f12924a2 100644--- a/packages/video-matting/src/load-video-matting-model.ts+++ b/packages/video-matting/src/load-video-matting-model.ts@@ -1,4 +1,5 @@import './symbol-async-dispose';+import type {RawImage} from '@huggingface/transformers';import {getHostedVideoMattingModelId,getVideoMattingModelInfo,@@ -18,7 +19,10 @@ export type OnVideoMattingModelLoadProgress = (progress: VideoMattingModelLoadProgress,) => void;-export type VideoMattingImageSource = HTMLCanvasElement | OffscreenCanvas;+export type VideoMattingImageSource =+ | HTMLCanvasElement+ | OffscreenCanvas+ | VideoMattingPipelineResult;export type VideoMattingPipelineResult = {data: Uint8ClampedArray<ArrayBuffer>;@@ -39,7 +43,9 @@ type TransformerPipelineResult = {};type TransformerPipeline = {- (image: VideoMattingImageSource): Promise<TransformerPipelineResult>;+ (+ image: HTMLCanvasElement | OffscreenCanvas | RawImage,+ ): Promise<TransformerPipelineResult>;dispose: () => Promise<void>;};@@ -164,6 +170,7 @@ const getOrCreateVideoMattingPipeline = ({AutoProcessor,BackgroundRemovalPipeline,ModelRegistry,+ RawImage,}) => {const hostedModelId = getHostedVideoMattingModelId(model);if (@@ -219,7 +226,16 @@ const getOrCreateVideoMattingPipeline = ({const loadedPipeline: LoadedPipeline = {run: async (image) => {- const result = await loadedTransformerPipeline(image);+ const result = await loadedTransformerPipeline(+ 'data' in image+ ? new RawImage(+ image.data,+ image.width,+ image.height,+ image.channels,+ )+ : image,+ );if (result.channels !== 4) {throw new Error(`The video matting model "${model}" returned ${result.channels} channels instead of RGBA.`,diff --git a/packages/video-matting/src/node-video-matting-frames.ts b/packages/video-matting/src/node-video-matting-frames.tsnew file mode 100644index 0000000000..c638b28e2c--- /dev/null+++ b/packages/video-matting/src/node-video-matting-frames.ts@@ -0,0 +1,136 @@+import {VideoSample, VideoSampleSink, VideoSampleSource} from 'mediabunny';+import type {+ VideoMattingFrames,+ VideoMattingFramesOptions,+} from './create-video-matting-frames';++export const createNodeVideoMattingFrames = ({+ videoTrack,+ width,+ height,+ videoQuality,+ keyframeIntervalInSeconds,+ videoStartTimestamp,+ videoEndTimestamp,+}: VideoMattingFramesOptions): VideoMattingFrames => {+ const sink = new VideoSampleSink(videoTrack);+ const baseSource = new VideoSampleSource({+ codec: 'vp9',+ quality: videoQuality,+ keyFrameInterval: keyframeIntervalInSeconds,+ alpha: 'discard',+ });+ const foregroundSource = new VideoSampleSource({+ codec: 'vp9',+ quality: videoQuality,+ keyFrameInterval: keyframeIntervalInSeconds,+ alpha: 'keep',+ });++ return {+ frames: (async function* () {+ for await (const sample of sink.samples(+ videoStartTimestamp,+ videoEndTimestamp,+ )) {+ const data = new Uint8ClampedArray(width * height * 4);+ try {+ // Apply rotation and pixel aspect ratio before passing pixels to the model.+ const transformed = await sample.transform({+ width,+ height,+ fit: 'fill',+ });+ try {+ await transformed.copyTo(data, {format: 'RGBA'});+ } finally {+ transformed.close();+ }+ } finally {+ sample.close();+ }++ yield {+ image: {data, width, height, channels: 4},+ timestamp: sample.timestamp,+ duration: sample.duration,+ };+ }+ })(),+ baseSource,+ foregroundSource,+ addFrame: async ({frame, foreground, timestamp, duration}) => {+ if (!('data' in frame.image)) {+ throw new Error('Expected an RGBA frame.');+ }++ const baseData = frame.image.data.slice();+ let foregroundData: Uint8ClampedArray<ArrayBuffer>;+ if (foreground.width === width && foreground.height === height) {+ foregroundData = foreground.data.slice();+ } else {+ foregroundData = new Uint8ClampedArray(width * height * 4);+ const foregroundSample = new VideoSample(foreground.data, {+ format: 'RGBA',+ codedWidth: foreground.width,+ codedHeight: foreground.height,+ timestamp,+ duration,+ });+ try {+ const resized = await foregroundSample.transform({+ width,+ height,+ fit: 'fill',+ });+ try {+ await resized.copyTo(foregroundData, {format: 'RGBA'});+ } finally {+ resized.close();+ }+ } finally {+ foregroundSample.close();+ }+ }++ for (let i = 0; i < baseData.length; i += 4) {+ const alpha = baseData[i + 3]! / 255;+ // Match the browser's opaque black base and destination-in foreground.+ baseData[i] = Math.round(baseData[i]! * alpha);+ baseData[i + 1] = Math.round(baseData[i + 1]! * alpha);+ baseData[i + 2] = Math.round(baseData[i + 2]! * alpha);+ baseData[i + 3] = 255;+ foregroundData[i + 3] = Math.round(foregroundData[i + 3]! * alpha);+ }++ const base = new VideoSample(baseData, {+ format: 'RGBA',+ codedWidth: width,+ codedHeight: height,+ timestamp,+ duration,+ });+ const foregroundFrame = new VideoSample(foregroundData, {+ format: 'RGBA',+ codedWidth: width,+ codedHeight: height,+ timestamp,+ duration,+ });+ try {+ const results = await Promise.allSettled([+ baseSource.add(base),+ foregroundSource.add(foregroundFrame),+ ]);+ for (const result of results) {+ if (result.status === 'rejected') {+ throw result.reason;+ }+ }+ } finally {+ base.close();+ foregroundFrame.close();+ }+ },+ };+};diff --git a/packages/video-matting/src/probe-node-webgpu.ts b/packages/video-matting/src/probe-node-webgpu.tsnew file mode 100644index 0000000000..e986340e52--- /dev/null+++ b/packages/video-matting/src/probe-node-webgpu.ts@@ -0,0 +1,47 @@+import type * as OnnxRuntimeNode from 'onnxruntime-node';+import {importNodeModule} from './import-node-module';++// Single-element ONNX Mul graphs (Y = X * X), opset 13. The fp16 graph casts+// float32 inputs to float16 for multiplication, then back for the Node binding.+const webGpuProbeModel = new Uint8Array([+ 0x08, 0x08, 0x3a, 0x40, 0x0a, 0x0e, 0x0a, 0x01, 0x58, 0x0a, 0x01, 0x58, 0x12,+ 0x01, 0x59, 0x22, 0x03, 0x4d, 0x75, 0x6c, 0x12, 0x0c, 0x77, 0x65, 0x62, 0x67,+ 0x70, 0x75, 0x2d, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x5a, 0x0f, 0x0a, 0x01, 0x58,+ 0x12, 0x0a, 0x0a, 0x08, 0x08, 0x01, 0x12, 0x04, 0x0a, 0x02, 0x08, 0x01, 0x62,+ 0x0f, 0x0a, 0x01, 0x59, 0x12, 0x0a, 0x0a, 0x08, 0x08, 0x01, 0x12, 0x04, 0x0a,+ 0x02, 0x08, 0x01, 0x42, 0x02, 0x10, 0x0d,+]);++const webGpuFp16ProbeModel = new Uint8Array([+ 0x08, 0x08, 0x3a, 0x72, 0x0a, 0x17, 0x0a, 0x01, 0x58, 0x12, 0x01, 0x48, 0x22,+ 0x04, 0x43, 0x61, 0x73, 0x74, 0x2a, 0x09, 0x0a, 0x02, 0x74, 0x6f, 0x18, 0x0a,+ 0xa0, 0x01, 0x02, 0x0a, 0x0e, 0x0a, 0x01, 0x48, 0x0a, 0x01, 0x48, 0x12, 0x01,+ 0x53, 0x22, 0x03, 0x4d, 0x75, 0x6c, 0x0a, 0x17, 0x0a, 0x01, 0x53, 0x12, 0x01,+ 0x59, 0x22, 0x04, 0x43, 0x61, 0x73, 0x74, 0x2a, 0x09, 0x0a, 0x02, 0x74, 0x6f,+ 0x18, 0x01, 0xa0, 0x01, 0x02, 0x12, 0x0c, 0x77, 0x65, 0x62, 0x67, 0x70, 0x75,+ 0x2d, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x5a, 0x0f, 0x0a, 0x01, 0x58, 0x12, 0x0a,+ 0x0a, 0x08, 0x08, 0x01, 0x12, 0x04, 0x0a, 0x02, 0x08, 0x01, 0x62, 0x0f, 0x0a,+ 0x01, 0x59, 0x12, 0x0a, 0x0a, 0x08, 0x08, 0x01, 0x12, 0x04, 0x0a, 0x02, 0x08,+ 0x01, 0x42, 0x02, 0x10, 0x0d,+]);++export const probeNodeWebGpu = async ({+ requiresShaderF16,+}: {+ requiresShaderF16: boolean;+}): Promise<void> => {+ const {InferenceSession, Tensor} =+ await importNodeModule<typeof OnnxRuntimeNode>('onnxruntime-node');+ const session = await InferenceSession.create(+ requiresShaderF16 ? webGpuFp16ProbeModel : webGpuProbeModel,+ {+ executionProviders: ['webgpu'],+ extra: {session: {disable_cpu_ep_fallback: '1'}},+ },+ );+ try {+ await session.run({X: new Tensor('float32', new Float32Array([1]), [1])});+ } finally {+ await session.release();+ }+};diff --git a/packages/video-matting/src/separate-video-layers.ts b/packages/video-matting/src/separate-video-layers.tsindex 1b7063746e..822266e409 100644--- a/packages/video-matting/src/separate-video-layers.ts+++ b/packages/video-matting/src/separate-video-layers.ts@@ -1,8 +1,8 @@+import type * as NodeUrl from 'node:url';import {ALL_FORMATS,BlobSource,- CanvasSink,- CanvasSource,+ FilePathSource,Input,type InputVideoTrack,type Quality,@@ -11,6 +11,11 @@ import {canEncodeVideo,} from 'mediabunny';import {createVideoLayerOutput} from './create-video-layer-output';+import {+ createVideoMattingFrames,+ type VideoMattingFrames,+} from './create-video-matting-frames';+import {importNodeModule} from './import-node-module';import {type OnVideoMattingModelLoadProgress,withLoadedVideoMattingPipeline,@@ -18,12 +23,6 @@ import {import {getVideoMattingModelInfo, type VideoMattingModel} from './models';import type {VideoLayerOutput, VideoLayerOutputOptions} from './output-target';import {prepareAudio, type PreparedVideoMattingAudio} from './prepare-audio';-import {- createVideoMattingCanvas,- drawForegroundFrame,- drawOpaqueBaseFrame,- getVideoMattingCanvasContext,-} from './video-matting-canvas';import {resolveVideoMattingQuality,type VideoMattingBitrate,@@ -233,7 +232,32 @@ const validateOptions = (options: SeparateVideoLayersOptions) => {}};-const makeInput = (src: string | URL | Blob): Input => {+const makeInput = async (src: string | URL | Blob): Promise<Input> => {+ if (+ typeof window === 'undefined' &&+ typeof process !== 'undefined' &&+ process.release?.name === 'node' &&+ (typeof src === 'string' || src instanceof URL)+ ) {+ const path = String(src);+ if (path.startsWith('file:')) {+ const {fileURLToPath} =+ await importNodeModule<typeof NodeUrl>('node:url');+ return new Input({+ formats: ALL_FORMATS,+ source: new FilePathSource(fileURLToPath(path)),+ });+ }++ // Treat Windows drive letters as paths, while preserving HTTP/data URLs.+ if (/^[a-z]:[/\\]/i.test(path) || !/^[a-z][a-z\d+.-]*:/i.test(path)) {+ return new Input({+ formats: ALL_FORMATS,+ source: new FilePathSource(path),+ });+ }+ }+const source =typeof src === 'string' || src instanceof URL? new UrlSource(src)@@ -291,7 +315,7 @@ const probeVideoInput = async ({]);if (!canEncodeBase || !canEncodeForeground) {throw new Error(- 'This browser cannot encode the VP9 video streams required for video layer separation.',+ 'This environment cannot encode the VP9 video streams required for video layer separation.',);}@@ -313,11 +337,12 @@ export const separateVideoLayers = async (options.audioBitrate ?? 'medium',);const keyframeIntervalInSeconds = options.keyframeIntervalInSeconds ?? 1;- const input = makeInput(options.src);+ const input = await makeInput(options.src);const onInputAbort = () => input.dispose();options.signal?.addEventListener('abort', onInputAbort, {once: true});try {+ throwIfAborted(options.signal);const {videoTrack, width, height} = await probeVideoInput({input,videoQuality,@@ -342,23 +367,13 @@ export const separateVideoLayers = async (signal: options.signal ?? null,run: async (pipeline) => {throwIfAborted(options.signal);- let iterator: AsyncGenerator<- {- canvas: HTMLCanvasElement | OffscreenCanvas;- timestamp: number;- duration: number;- },- void,- unknown- > | null = null;+ let videoFrames: VideoMattingFrames | null = null;let baseOutput: Awaited<ReturnType<typeof createVideoLayerOutput<WebMOutputFormat>>> | null = null;let foregroundOutput: Awaited<ReturnType<typeof createVideoLayerOutput<WebMOutputFormat>>> | null = null;- let baseVideoSource: CanvasSource | null = null;- let foregroundVideoSource: CanvasSource | null = null;let audioWriter: PreparedVideoMattingAudio | null = null;let completed = false;let abortCleanupPromise: Promise<void> | null = null;@@ -382,17 +397,16 @@ export const separateVideoLayers = async (options.signal?.addEventListener('abort', onAbort, {once: true});try {- const canvasSink = new CanvasSink(videoTrack, {- alpha: true,+ videoFrames = await createVideoMattingFrames({+ videoTrack,width,height,- fit: 'fill',- poolSize: 1,- });- iterator = canvasSink.canvases(+ videoQuality,+ keyframeIntervalInSeconds,videoStartTimestamp,videoEndTimestamp,- );+ });+ const iterator = videoFrames.frames;let nextFrame = await iterator.next();if (nextFrame.done) {throw new Error(@@ -402,12 +416,6 @@ export const separateVideoLayers = async (throwIfAborted(options.signal);- const baseCanvas = createVideoMattingCanvas({width, height});- const foregroundCanvas = createVideoMattingCanvas({width, height});- const baseContext = getVideoMattingCanvasContext(baseCanvas);- const foregroundContext =- getVideoMattingCanvasContext(foregroundCanvas);-baseOutput = await createVideoLayerOutput({format: new WebMOutputFormat(),options: options.outputs?.base,@@ -419,20 +427,8 @@ export const separateVideoLayers = async (});throwIfAborted(options.signal);- baseVideoSource = new CanvasSource(baseCanvas, {- codec: 'vp9',- quality: videoQuality,- keyFrameInterval: keyframeIntervalInSeconds,- alpha: 'discard',- });- foregroundVideoSource = new CanvasSource(foregroundCanvas, {- codec: 'vp9',- quality: videoQuality,- keyFrameInterval: keyframeIntervalInSeconds,- alpha: 'keep',- });- baseOutput.output.addVideoTrack(baseVideoSource);- foregroundOutput.output.addVideoTrack(foregroundVideoSource);+ baseOutput.output.addVideoTrack(videoFrames.baseSource);+ foregroundOutput.output.addVideoTrack(videoFrames.foregroundSource);audioWriter = await prepareAudio({input,@@ -478,26 +474,15 @@ export const separateVideoLayers = async (continue;}- const foregroundFrame = await pipeline(frame.canvas);+ const foregroundFrame = await pipeline(frame.image);throwIfAborted(options.signal);- drawOpaqueBaseFrame({- context: baseContext,- source: frame.canvas,- width,- height,- });- drawForegroundFrame({- context: foregroundContext,- result: foregroundFrame,- source: frame.canvas,- targetWidth: width,- targetHeight: height,- });-await Promise.all([- baseVideoSource.add(timing.timestamp, timing.duration),- foregroundVideoSource.add(timing.timestamp, timing.duration),+ videoFrames.addFrame({+ frame,+ foreground: foregroundFrame,+ ...timing,+ }),audioWriter.writeAudioUntil(timing.timestamp + timing.duration),]);@@ -529,8 +514,8 @@ export const separateVideoLayers = async (durationInSeconds,});await audioWriter.finishAudio();- baseVideoSource.close();- foregroundVideoSource.close();+ videoFrames.baseSource.close();+ videoFrames.foregroundSource.close();const [base, foreground] = await Promise.all([baseOutput.finalize(),foregroundOutput.finalize(),@@ -545,6 +530,7 @@ export const separateVideoLayers = async (width,height,durationInSeconds,+ processedFrames,} as SeparateVideoLayersResult;Object.defineProperty(separated, Symbol.asyncDispose, {enumerable: false,@@ -566,7 +552,7 @@ export const separateVideoLayers = async (await abortCleanupPromise;if (!completed) {try {- await iterator?.return();+ await videoFrames?.frames.return();} catch {// Cleanup must not replace the operation's original error.}diff --git a/packages/video-matting/src/test/model-runtime.test.ts b/packages/video-matting/src/test/model-runtime.test.tsindex 100277435b..c17d32e411 100644--- a/packages/video-matting/src/test/model-runtime.test.ts+++ b/packages/video-matting/src/test/model-runtime.test.ts@@ -1,4 +1,9 @@import {beforeEach, expect, mock, test} from 'bun:test';+import {existsSync} from 'node:fs';+import {mkdtemp, readdir, readFile, rm, writeFile} from 'node:fs/promises';+import {tmpdir} from 'node:os';+import {join} from 'node:path';+import {pathToFileURL} from 'node:url';let initializationCount = 0;let initializationStarted: ((count: number) => void) | null = null;@@ -37,6 +42,8 @@ const originalTransformersEnvironment = {useCustomCache: true,customCache,useBrowserCache: false,+ useFSCache: false,+ cacheDir: null as string | null,cacheKey: 'transformers-cache',fetch: (url: string) => {requestedFiles.push(url);@@ -56,11 +63,39 @@ Object.defineProperty({value: preexistingWhisperModelHostState},);+class RawImage {+ public data: Uint8ClampedArray;+ public width: number;+ public height: number;+ public channels: number;++ constructor(+ data: Uint8ClampedArray,+ width: number,+ height: number,+ channels: number,+ ) {+ this.data = data;+ this.width = width;+ this.height = height;+ this.channels = channels;+ }+}+function BackgroundRemovalPipeline() {livePipelineCount++;return Object.assign(- () => {+ (image: unknown) => {pipelineRunCount++;+ if (image instanceof RawImage) {+ const data = image.data.slice();+ for (let i = 3; i < data.length; i += 4) {+ data[i] = 128;+ }++ return Promise.resolve({...image, data});+ }+return Promise.resolve({data: new Uint8ClampedArray([10, 20, 30, 128]),width: 1,@@ -112,13 +147,18 @@ mock.module('@huggingface/transformers', () => ({},},BackgroundRemovalPipeline,+ RawImage,ModelRegistry: {is_pipeline_cached_files: () => {const files = ['config.json', 'onnx/model.onnx'].map((file) => ({file,- cached: downloadedFiles.has(- `https://remotion.media/models/modnet-v1/${file}`,- ),+ cached: transformersEnvironment.useFSCache+ ? existsSync(+ join(transformersEnvironment.cacheDir!, 'modnet-v1', file),+ )+ : downloadedFiles.has(+ `https://remotion.media/models/modnet-v1/${file}`,+ ),}));return Promise.resolve({files,@@ -131,6 +171,13 @@ mock.module('@huggingface/transformers', () => ({options: Record<string, unknown>,) => {cacheClear = {task, modelId, options};+ if (transformersEnvironment.useFSCache) {+ return rm(join(transformersEnvironment.cacheDir!, modelId), {+ recursive: true,+ force: true,+ });+ }+return Promise.resolve();},is_pipeline_cached: (@@ -144,7 +191,15 @@ mock.module('@huggingface/transformers', () => ({remotePathTemplate: transformersEnvironment.remotePathTemplate,};return Promise.resolve(- checkDownloadedFiles ? downloadedFiles.size === 2 : true,+ transformersEnvironment.useFSCache+ ? ['config.json', 'onnx/model.onnx'].every((file) =>+ existsSync(+ join(transformersEnvironment.cacheDir!, modelId, file),+ ),+ )+ : checkDownloadedFiles+ ? downloadedFiles.size === 2+ : true,);},},@@ -425,3 +480,189 @@ test('coordinates the shared Transformers environment across model loads', asyncexpect(transformersEnvironment).toEqual(originalTransformersEnvironment);await disposeVideoMattingModel();});++// Keep the neural network deterministic; decoding, encoding, pixels, storage,+// cancellation, and the public package workflow all use the real implementation.+test('downloads to disk and separates local video with alpha, audio, and cancellation', async () => {+ const {registerMediabunnyServer} = await import('@mediabunny/server');+ const {+ ALL_FORMATS,+ AudioSample,+ AudioSampleSink,+ AudioSampleSource,+ BlobSource,+ BufferTarget,+ Input,+ Output,+ VideoSample,+ VideoSampleSink,+ VideoSampleSource,+ WebMOutputFormat,+ } = await import('mediabunny');+ const {+ downloadVideoMattingModel,+ isVideoMattingModelCached,+ loadVideoMattingModel,+ removeVideoMattingModel,+ separateVideoLayers,+ } = await import('../index');+ registerMediabunnyServer({hardwareContext: null});+ const directory = await mkdtemp(join(tmpdir(), 'video-matting-node-'));+ transformersEnvironment.useCustomCache = false;+ transformersEnvironment.useFSCache = true;+ transformersEnvironment.cacheDir = join(directory, 'models');+ try {+ expect(await isVideoMattingModelCached({model: 'modnet'})).toBe(false);+ expect(await downloadVideoMattingModel({model: 'modnet'})).toEqual({+ alreadyDownloaded: false,+ });+ expect(+ await readFile(+ join(directory, 'models/modnet-v1/onnx/model.onnx'),+ 'utf8',+ ),+ ).toBe('https://remotion.media/models/modnet-v1/onnx/model.onnx');+ expect(await downloadVideoMattingModel({model: 'modnet'})).toEqual({+ alreadyDownloaded: true,+ });+ expect(requestedFiles).toHaveLength(2);+ expect(initializationCount).toBe(0);++ const output = new Output({+ format: new WebMOutputFormat(),+ target: new BufferTarget(),+ });+ const video = new VideoSampleSource({+ codec: 'vp9',+ alpha: 'keep',+ bitrate: 1_000_000,+ });+ const audio = new AudioSampleSource({codec: 'opus', bitrate: 64_000});+ output.addVideoTrack(video);+ output.addAudioTrack(audio);+ await output.start();+ for (let i = 0; i < 3; i++) {+ const data = new Uint8Array(32 * 16 * 4);+ for (let j = 0; j < data.length; j += 4) {+ data[j] = 255;+ data[j + 3] = 128;+ }++ using sample = new VideoSample(data, {+ format: 'RGBA',+ codedWidth: 32,+ codedHeight: 16,+ timestamp: i / 10,+ duration: 0.1,+ });+ using sound = new AudioSample({+ format: 'f32',+ sampleRate: 48_000,+ numberOfChannels: 1,+ timestamp: i / 10,+ data: new Float32Array(4800).fill(0.1),+ });+ await Promise.all([video.add(sample), audio.add(sound)]);+ }++ video.close();+ audio.close();+ await output.finalize();+ const src = join(directory, 'input.webm');+ await writeFile(src, new Uint8Array(output.target.buffer!));+ await using modelHandle = await loadVideoMattingModel({model: 'modnet'});+ expect(modelHandle.alreadyLoaded).toBe(false);++ const controller = new AbortController();+ await expect(+ separateVideoLayers({+ src,+ signal: controller.signal,+ onProgress: ({processedFrames}) => {+ if (processedFrames === 1) {+ controller.abort(new Error('stop after first frame'));+ }+ },+ }),+ ).rejects.toThrow('stop after first frame');++ await using layers = await separateVideoLayers({+ src: pathToFileURL(src),+ audio: 'both',+ audioBitrate: 'medium',+ });+ expect(layers).toMatchObject({+ width: 32,+ height: 16,+ processedFrames: 3,+ durationInSeconds: 0.3,+ });+ for (const [layer, expectedRed, expectedAlpha] of [+ [layers.base, 128, 255],+ [layers.foreground, 255, 64],+ ] as const) {+ using input = new Input({+ formats: ALL_FORMATS,+ source: new BlobSource(await layer.getBlob()),+ });+ const track = (await input.getPrimaryVideoTrack())!;+ expect(await track.getCodec()).toBe('vp9');+ expect(await track.getDisplayWidth()).toBe(32);+ expect(await track.getDisplayHeight()).toBe(16);+ const timestamps: number[] = [];+ for await (const frame of new VideoSampleSink(track).samples()) {+ try {+ const data = new Uint8Array(16 * 32 * 4);+ await frame.copyTo(data, {format: 'RGBA'});+ expect(Math.abs(data[0]! - expectedRed)).toBeLessThan(5);+ expect(Math.abs(data[3]! - expectedAlpha)).toBeLessThan(3);+ timestamps.push(frame.timestamp);+ } finally {+ frame.close();+ }+ }++ expect(timestamps).toEqual([0, 0.1, 0.2]);+ const audioTrack = (await input.getPrimaryAudioTrack())!;+ expect(await audioTrack.getCodec()).toBe('opus');+ using decodedAudio = await new AudioSampleSink(audioTrack).getSample(0);+ expect(decodedAudio!.numberOfFrames).toBeGreaterThan(0);+ }++ await removeVideoMattingModel({model: 'modnet'});+ expect(await isVideoMattingModelCached({model: 'modnet'})).toBe(false);+ expect(livePipelineCount).toBe(0);+ } finally {+ await rm(directory, {recursive: true, force: true});+ }+}, 30_000);++test('an interrupted filesystem download can be retried without leaving partial model files', async () => {+ const {downloadVideoMattingModel} = await import('../index');+ const directory = await mkdtemp(join(tmpdir(), 'video-matting-download-'));+ transformersEnvironment.useCustomCache = false;+ transformersEnvironment.useFSCache = true;+ transformersEnvironment.cacheDir = directory;+ transformersEnvironment.fetch = () =>+ Promise.resolve(+ new Response(+ new ReadableStream({+ start(controller) {+ controller.error(new Error('download interrupted'));+ },+ }),+ ),+ );+ try {+ await expect(downloadVideoMattingModel({model: 'modnet'})).rejects.toThrow(+ 'download interrupted',+ );+ expect(await readdir(join(directory, 'modnet-v1'))).toEqual([]);+ transformersEnvironment.fetch = originalTransformersEnvironment.fetch;+ expect(await downloadVideoMattingModel({model: 'modnet'})).toEqual({+ alreadyDownloaded: false,+ });+ } finally {+ await rm(directory, {recursive: true, force: true});+ }+});diff --git a/packages/video-matting/src/test/models.test.ts b/packages/video-matting/src/test/models.test.tsindex eae26adf9e..b3929fa91a 100644--- a/packages/video-matting/src/test/models.test.ts+++ b/packages/video-matting/src/test/models.test.ts@@ -1,4 +1,4 @@-import {expect, test} from 'bun:test';+import {expect, mock, test} from 'bun:test';import {canUseVideoMatting,VideoMattingUnsupportedReason,@@ -76,3 +76,37 @@ test('requires shader-f16 only for the fp16 BEN2 model', async () => {}}});++test('checks native WebGPU inference and reports failures without requiring browser globals', async () => {+ let failure: Error | null = null;+ let releases = 0;+ mock.module('onnxruntime-node', () => ({+ Tensor: class {},+ InferenceSession: {+ create: () =>+ Promise.resolve({+ run: () =>+ failure === null ? Promise.resolve({}) : Promise.reject(failure),+ release: () => {+ releases++;+ return Promise.resolve();+ },+ }),+ },+ }));++ expect(await canUseVideoMatting({model: 'modnet'})).toEqual({+ supported: true,+ });+ expect(await canUseVideoMatting({model: 'ben2-base'})).toEqual({+ supported: true,+ });+ failure = new Error('No supported adapter');+ expect(await canUseVideoMatting()).toEqual({+ supported: false,+ reason: VideoMattingUnsupportedReason.WebGpuUnavailable,+ detailedReason:+ 'ONNX Runtime could not run WebGPU inference for "modnet": No supported adapter',+ });+ expect(releases).toBe(3);+});diff --git a/packages/video-matting/src/with-remotion-model-host.ts b/packages/video-matting/src/with-remotion-model-host.tsindex 2d92551193..a74ac39abf 100644--- a/packages/video-matting/src/with-remotion-model-host.ts+++ b/packages/video-matting/src/with-remotion-model-host.ts@@ -4,6 +4,7 @@ import type {BackgroundRemovalPipeline,env,ModelRegistry,+ RawImage,} from '@huggingface/transformers';const REMOTION_MODEL_HOST = 'https://remotion.media/';@@ -21,6 +22,7 @@ type TransformersModule = {readonly BackgroundRemovalPipeline: typeof BackgroundRemovalPipeline;readonly env: typeof env;readonly ModelRegistry: typeof ModelRegistry;+ readonly RawImage: typeof RawImage;};type ModelHostState = {