🎥 Make videos programmatically with React
Add browser video matting package
JonnyBurgerfb124b32e895
diff --git a/bun.lock b/bun.lockindex b5c3066ef8..3aeb53def1 100644--- a/bun.lock+++ b/bun.lock@@ -583,6 +583,7 @@"@remotion/three": "workspace:*","@remotion/transitions": "workspace:*","@remotion/vercel": "workspace:*",+ "@remotion/video-matting": "workspace:*","@remotion/web-renderer": "workspace:*","@remotion/webcodecs": "workspace:*","@remotion/whisper-web": "workspace:*",@@ -830,6 +831,7 @@"@remotion/tailwind-v4": "workspace:*","@remotion/three": "workspace:*","@remotion/transitions": "workspace:*",+ "@remotion/video-matting": "workspace:*","@remotion/webcodecs": "workspace:*","@remotion/whisper-web": "workspace:*","@remotion/whisper-webgpu": "workspace:*",@@ -2766,6 +2768,19 @@"@vercel/sandbox": ">=1.0.0",},},+ "packages/video-matting": {+ "name": "@remotion/video-matting",+ "version": "4.0.521",+ "dependencies": {+ "@huggingface/transformers": "catalog:",+ "mediabunny": "catalog:",+ },+ "devDependencies": {+ "@remotion/eslint-config-internal": "workspace:*",+ "@typescript/native-preview": "catalog:",+ "eslint": "catalog:",+ },+ },"packages/web-renderer": {"name": "@remotion/web-renderer","version": "4.0.521",@@ -4639,6 +4654,8 @@"@remotion/vercel": ["@remotion/vercel@workspace:packages/vercel"],+ "@remotion/video-matting": ["@remotion/video-matting@workspace:packages/video-matting"],+"@remotion/web-renderer": ["@remotion/web-renderer@workspace:packages/web-renderer"],"@remotion/webcodecs": ["@remotion/webcodecs@workspace:packages/webcodecs"],diff --git a/packages/STATS.md b/packages/STATS.mdindex 21bcd418a2..b06b1f2ffa 100644--- a/packages/STATS.md+++ b/packages/STATS.md@@ -1,4 +1,5 @@# Download statistics+Monthly downloads of Remotion packages[](https://npmcharts.com/compare/@remotion/animated-emoji?minimal=true)[](https://npmcharts.com/compare/@remotion/animation-utils?minimal=true)@@ -72,9 +73,10 @@ Monthly downloads of Remotion packages[](https://npmcharts.com/compare/@remotion/timeline-utils?minimal=true)[](https://npmcharts.com/compare/@remotion/transitions?minimal=true)[](https://npmcharts.com/compare/@remotion/vercel?minimal=true)+[](https://npmcharts.com/compare/@remotion/video-matting?minimal=true)[](https://npmcharts.com/compare/@remotion/web-renderer?minimal=true)[](https://npmcharts.com/compare/@remotion/webcodecs?minimal=true)[](https://npmcharts.com/compare/@remotion/whisper-web?minimal=true)[](https://npmcharts.com/compare/@remotion/whisper-webgpu?minimal=true)[](https://npmcharts.com/compare/@remotion/zod-types?minimal=true)-[](https://npmcharts.com/compare/@remotion/zod-types-v3?minimal=true)+[](https://npmcharts.com/compare/@remotion/zod-types-v3?minimal=true)diff --git a/packages/cli/src/list-of-remotion-packages.ts b/packages/cli/src/list-of-remotion-packages.tsindex 9e6a708d75..b3de971a35 100644--- a/packages/cli/src/list-of-remotion-packages.ts+++ b/packages/cli/src/list-of-remotion-packages.ts@@ -93,6 +93,7 @@ export const listOfRemotionPackages = ['@remotion/example-videos','@remotion/whisper-web','@remotion/whisper-webgpu',+ '@remotion/video-matting','@remotion/media','@remotion/remotion-media','@remotion/web-renderer',diff --git a/packages/create-video/src/list-of-remotion-packages.ts b/packages/create-video/src/list-of-remotion-packages.tsindex 9e6a708d75..b3de971a35 100644--- a/packages/create-video/src/list-of-remotion-packages.ts+++ b/packages/create-video/src/list-of-remotion-packages.ts@@ -93,6 +93,7 @@ export const listOfRemotionPackages = ['@remotion/example-videos','@remotion/whisper-web','@remotion/whisper-webgpu',+ '@remotion/video-matting','@remotion/media','@remotion/remotion-media','@remotion/web-renderer',diff --git a/packages/docs/components/TableOfContents/api.tsx b/packages/docs/components/TableOfContents/api.tsxindex 55d4892cb9..03cfc9adcd 100644--- a/packages/docs/components/TableOfContents/api.tsx+++ b/packages/docs/components/TableOfContents/api.tsx@@ -41,6 +41,7 @@ import {TableOfContents as TailwindTableOfContents} from '../../docs/tailwind/Taimport {TableOfContents as ThreeTableOfContents} from '../../docs/three/TableOfContents';import {TableOfContents as TransitionsTableOfContents} from '../../docs/transitions/table-of-contents';import {TableOfContents as VercelTableOfContents} from '../../docs/vercel/table-of-contents';+import {TableOfContents as VideoMattingTableOfContents} from '../../docs/video-matting/table-of-contents';import {TableOfContents as WebcodecsTableOfContents} from '../../docs/webcodecs/TableOfContents';import {TableOfContents as WhisperWebGpuTableOfContents} from '../../docs/whisper-webgpu/table-of-contents';import {ZodTypesTableOfContents} from '../../docs/zod-types/TableOfContents';@@ -196,6 +197,9 @@ export const TableOfContents: React.FC = () => {<h2>@remotion/vercel</h2><p>Render videos on Vercel Sandbox</p><VercelTableOfContents />+ <h2>@remotion/video-matting</h2>+ <p>Separate video into base and foreground layers</p>+ <VideoMattingTableOfContents /><h2>@remotion/webcodecs</h2><p>Converting media using WebCodecs</p><WebcodecsTableOfContents />diff --git a/packages/docs/docs/video-matting/can-use-video-matting.mdx b/packages/docs/docs/video-matting/can-use-video-matting.mdxnew file mode 100644index 0000000000..91f52b3472--- /dev/null+++ b/packages/docs/docs/video-matting/can-use-video-matting.mdx@@ -0,0 +1,58 @@+---+image: /generated/articles-docs-video-matting-can-use-video-matting.png+title: canUseVideoMatting()+crumb: '@remotion/video-matting'+---++# canUseVideoMatting()<AvailableFrom v="4.0.522" />++Checks whether a video matting model can run in the current browser.++This function checks WebGPU and model-specific requirements. It does not inspect an input file or check video decoding and encoding support.++```ts twoslash title="support.ts"+import {canUseVideoMatting} from '@remotion/video-matting';++const result = await canUseVideoMatting({model: 'modnet'});+if (!result.supported) {+ throw new Error(result.detailedReason);+}+```++## Options++### model?++The model whose requirements should be checked. Default: `modnet`.++## Return value++A promise resolving to a discriminated result with these properties:++### supported++`true` if the selected model is supported and `false` otherwise.++### reason?++When `supported` is `false`, one of:++- `window-undefined`+- `webgpu-unavailable`+- `webgpu-requires-secure-context`+- `shader-f16-unavailable`++`shader-f16-unavailable` applies to models such as `ben2-base` that require 16-bit shader support.++### detailedReason?++When `supported` is `false`, a human-readable explanation of the reason.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`separateVideoLayers()`](/docs/video-matting/separate-video-layers)+- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)diff --git a/packages/docs/docs/video-matting/clear-stale-video-matting-models.mdx b/packages/docs/docs/video-matting/clear-stale-video-matting-models.mdxnew file mode 100644index 0000000000..a071f516a8--- /dev/null+++ b/packages/docs/docs/video-matting/clear-stale-video-matting-models.mdx@@ -0,0 +1,32 @@+---+image: /generated/articles-docs-video-matting-clear-stale-video-matting-models.png+title: clearStaleVideoMattingModels()+crumb: '@remotion/video-matting'+---++# clearStaleVideoMattingModels()<AvailableFrom v="4.0.522" />++Removes cached model files that are no longer returned by [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models).++There are no stale models in the current version, so the function resolves without changing the cache.++Call this function after upgrading `@remotion/video-matting` to remove files for models discontinued by a newer version.++```ts twoslash title="clear-stale-models.ts"+import {clearStaleVideoMattingModels} from '@remotion/video-matting';++await clearStaleVideoMattingModels();+```++## Return value++A `Promise<void>` that resolves after stale model files have been removed.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models)+- [`isVideoMattingModelCached()`](/docs/video-matting/is-video-matting-model-cached)diff --git a/packages/docs/docs/video-matting/dispose-video-matting-model.mdx b/packages/docs/docs/video-matting/dispose-video-matting-model.mdxnew file mode 100644index 0000000000..54e611a682--- /dev/null+++ b/packages/docs/docs/video-matting/dispose-video-matting-model.mdx@@ -0,0 +1,36 @@+---+image: /generated/articles-docs-video-matting-dispose-video-matting-model.png+title: disposeVideoMattingModel()+crumb: '@remotion/video-matting'+---++# disposeVideoMattingModel()<AvailableFrom v="4.0.522" />++Releases the memory used by initialized video matting models. Persistent browser-cached model files are not deleted.++```ts twoslash title="dispose.ts"+import {disposeVideoMattingModel} from '@remotion/video-matting';++await disposeVideoMattingModel({model: 'modnet'});+```++## Options++### model?++Only dispose this model. Default: all loaded video matting models.++If the model is in use, disposal waits until active separations using it have finished.++## Return value++A `Promise<void>` that resolves after the selected models have been disposed.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)+- [`isVideoMattingModelCached()`](/docs/video-matting/is-video-matting-model-cached)diff --git a/packages/docs/docs/video-matting/get-available-video-matting-models.mdx b/packages/docs/docs/video-matting/get-available-video-matting-models.mdxnew file mode 100644index 0000000000..48c0edb18a--- /dev/null+++ b/packages/docs/docs/video-matting/get-available-video-matting-models.mdx@@ -0,0 +1,58 @@+---+image: /generated/articles-docs-video-matting-get-available-video-matting-models.png+title: getAvailableVideoMattingModels()+crumb: '@remotion/video-matting'+---++# getAvailableVideoMattingModels()<AvailableFrom v="4.0.522" />++Returns metadata for the models supported by [`separateVideoLayers()`](/docs/video-matting/separate-video-layers).++```ts twoslash title="models.ts"+import {getAvailableVideoMattingModels} from '@remotion/video-matting';++const models = getAvailableVideoMattingModels();+console.log(models);+```++| Name | Purpose | Status |+| ----------- | --------------------------- | -------------------------------- |+| `modnet` | People | Default |+| `ben2-base` | General foreground subjects | Experimental and higher-resource |++The [`modnet` model](https://huggingface.co/Xenova/modnet) is Apache-2.0 licensed. The [`ben2-base` model](https://huggingface.co/onnx-community/BEN2-ONNX) is the MIT-licensed BEN2 Base export.++Each model contains these properties:++### name++The model name accepted by APIs in this package.++### modelId++The Hugging Face model identifier.++### purpose++A short description of the subjects the model is intended to separate.++### experimental++Whether the model is experimental.++### webGpuDownloadSize++The estimated WebGPU model download size in bytes. Browser caches and supporting model files may add a small amount.++Models process each frame independently and may show temporal flicker. The set of available models may change between package versions.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`VIDEO_MATTING_MODELS`](/docs/video-matting/video-matting-models)+- [`clearStaleVideoMattingModels()`](/docs/video-matting/clear-stale-video-matting-models)+- [`isVideoMattingModelCached()`](/docs/video-matting/is-video-matting-model-cached)+- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)diff --git a/packages/docs/docs/video-matting/index.mdx b/packages/docs/docs/video-matting/index.mdxnew file mode 100644index 0000000000..058d9766b2--- /dev/null+++ b/packages/docs/docs/video-matting/index.mdx@@ -0,0 +1,64 @@+---+image: /generated/articles-docs-video-matting-index.png+title: '@remotion/video-matting'+crumb: 'Separate a video into layers'+---++# @remotion/video-matting<AvailableFrom v="4.0.522" />++Separate a video into an opaque base layer and a foreground layer with alpha, locally in the browser using WebGPU.++import {TableOfContents} from './table-of-contents';++## Installation++<Installation pkg="@remotion/video-matting" />++## Example++```ts twoslash title="separate-video.ts"+import {separateVideoLayers} from '@remotion/video-matting';++const file = new File([], 'input.mp4');++const {base, foreground} = await separateVideoLayers({+ src: file,+});++const baseBlob = await base.getBlob();+const foregroundBlob = await foreground.getBlob();++await Promise.all([base.dispose(), foreground.dispose()]);+```++Call `dispose()` after retrieving the blobs to release package-owned temporary storage.++The model is downloaded on the first call and cached by the browser. Call [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model) first to display download progress.++## Layer order++The base is an opaque copy of the input video. The foreground contains the detected subject and transparent pixels everywhere else.++Stack the layers in this order:++1. Base+2. Inserted content, such as text or graphics+3. Foreground++The base intentionally still contains the subject. The foreground draws it over the inserted content, making the content appear behind the subject. Video matting does not reconstruct scenery hidden by the subject.++Audio is included in the base by default and omitted from the foreground, so stacking both layers plays it once.++## APIs++<TableOfContents />++## Requirements++WebGPU requires HTTPS in production or `localhost` during development. Use [`canUseVideoMatting()`](/docs/video-matting/can-use-video-matting) to check whether the selected model is supported.++The browser must also be able to decode the input video and encode VP9 with alpha through WebCodecs. [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) checks these media capabilities for the selected input.++## License++The package is MIT licensed. Downloaded model weights retain the licenses listed under [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models).diff --git a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdxnew file mode 100644index 0000000000..80523d45e7--- /dev/null+++ b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx@@ -0,0 +1,36 @@+---+image: /generated/articles-docs-video-matting-is-video-matting-model-cached.png+title: isVideoMattingModelCached()+crumb: '@remotion/video-matting'+---++# isVideoMattingModelCached()<AvailableFrom v="4.0.522" />++Checks whether all files needed for a video matting model are present in the browser cache. The check does not download or initialize the model.++```ts twoslash title="model-cache.ts"+import {isVideoMattingModelCached} from '@remotion/video-matting';++const downloaded = await isVideoMattingModelCached({+ model: 'modnet',+});+```++## Options++### model++One of the names returned by [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models).++## Return value++A `Promise<boolean>` resolving to `true` if every required model file is cached.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models)+- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)diff --git a/packages/docs/docs/video-matting/load-video-matting-model.mdx b/packages/docs/docs/video-matting/load-video-matting-model.mdxnew file mode 100644index 0000000000..d66d3af9f9--- /dev/null+++ b/packages/docs/docs/video-matting/load-video-matting-model.mdx@@ -0,0 +1,74 @@+---+image: /generated/articles-docs-video-matting-load-video-matting-model.png+title: loadVideoMattingModel()+crumb: '@remotion/video-matting'+---++# loadVideoMattingModel()<AvailableFrom v="4.0.522" />++Downloads and initializes a video matting model. Calling it before [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) lets you show model-loading progress.++```ts twoslash title="load-model.ts"+import {loadVideoMattingModel} from '@remotion/video-matting';++const result = await loadVideoMattingModel({+ model: 'modnet',+ onProgress: ({progress}) => {+ console.log(progress);+ },+});++console.log(result.alreadyLoaded);+```++## Options++### model++One of the names returned by [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models).++### onProgress?++Called while model files are resolved and downloaded. Default: no callback.++The callback receives these properties:++#### status++The current model-loading status.++#### file++The model file associated with the update, or `null` for aggregate progress.++#### progress++Download progress between `0` and `1`, or `null` when it cannot be calculated.++#### loadedBytes++The number of bytes downloaded, or `null` when it is unknown.++#### totalBytes++The total number of bytes, or `null` when it is unknown.++## Return value++Returns an object with this property:++### alreadyLoaded++Whether the same model was already initialized in memory.++The browser manages the persistent [Transformers.js](https://huggingface.co/docs/transformers.js) cache. This function also retains the initialized model in memory until [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model) is called or the page closes.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model)+- [`isVideoMattingModelCached()`](/docs/video-matting/is-video-matting-model-cached)+- [`separateVideoLayers()`](/docs/video-matting/separate-video-layers)diff --git a/packages/docs/docs/video-matting/separate-video-layers.mdx b/packages/docs/docs/video-matting/separate-video-layers.mdxnew file mode 100644index 0000000000..a2806d6fa9--- /dev/null+++ b/packages/docs/docs/video-matting/separate-video-layers.mdx@@ -0,0 +1,218 @@+---+image: /generated/articles-docs-video-matting-separate-video-layers.png+title: separateVideoLayers()+crumb: '@remotion/video-matting'+---++# separateVideoLayers()<AvailableFrom v="4.0.522" />++Separates a video into an opaque base WebM and a foreground WebM with alpha.++```ts twoslash title="separate-video.ts"+import {separateVideoLayers} from '@remotion/video-matting';++const file = new File([], 'input.mp4');++const result = await separateVideoLayers({+ src: file,+ model: 'modnet',+ audio: 'base',+ onProgress: (progress) => {+ console.log(progress);+ },+});++const baseBlob = await result.base.getBlob();+const foregroundBlob = await result.foreground.getBlob();++await Promise.all([result.base.dispose(), result.foreground.dispose()]);+```++Place inserted content between the base and foreground. The base remains opaque and includes the original subject; the foreground repeats the subject with alpha so it appears in front of the inserted content.++## Options++### src++A `string`, `URL`, or `Blob` containing the input video. A `File` may be passed because it extends `Blob`.++### model?++The video matting model to use. Default: `modnet`.++`modnet` is optimized for people. `ben2-base` supports more general foreground subjects, but is experimental and requires substantially more memory and a WebGPU adapter with `shader-f16` support.++Use [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models) to inspect the available models.++### audio?++Controls which output receives the input video's primary audio track. Default: `base`.++- `base`: Include audio only in the base.+- `foreground`: Include audio only in the foreground.+- `both`: Include audio in both files. Playing both unmuted will duplicate the audio.+- `none`: Produce two silent files.++If the input has no audio, both outputs are silent.++### outputs?++Configures the destination of each output independently. Default: both layers use `web-fs` when available and fall back to `arraybuffer`.++Each value has the exported `VideoLayerOutputOptions` type.++#### base?++Output options for the opaque base layer. Default: automatic target selection.++#### foreground?++Output options for the foreground layer with alpha. Default: automatic target selection.++#### base.outputTarget?++Set to `arraybuffer` to hold the base in memory, or `web-fs` to store it in the browser's origin-private file system. Default: automatically use `web-fs` when available and otherwise use `arraybuffer`.++With `web-fs`, encoding is streamed to storage. Calling `base.getBlob()` materializes an independent Blob in memory so it remains usable after `base.dispose()`. For outputs that should never be materialized in memory, use `base.outputWritable`.++Cannot be combined with `base.outputWritable`.++#### base.outputWritable?++A `WritableStream<StreamTargetChunk>` receiving the encoded base. [`StreamTargetChunk`](https://mediabunny.dev/api/StreamTargetChunk) is a Mediabunny primitive. Writes may target arbitrary byte positions through `chunk.position`; they are not guaranteed to be append-only. The stream is closed after the output is finalized.++When `outputs.base.outputWritable` is set, `base.getBlob()` rejects because the encoded bytes were sent to the stream. Cannot be combined with `outputs.base.outputTarget`. Default: no custom stream.++#### foreground.outputTarget?++Set to `arraybuffer` to hold the foreground in memory, or `web-fs` to store it in the browser's origin-private file system. Default: automatically use `web-fs` when available and otherwise use `arraybuffer`.++With `web-fs`, encoding is streamed to storage. Calling `foreground.getBlob()` materializes an independent Blob in memory so it remains usable after `foreground.dispose()`. For outputs that should never be materialized in memory, use `foreground.outputWritable`.++Cannot be combined with `foreground.outputWritable`.++#### foreground.outputWritable?++A `WritableStream<StreamTargetChunk>` receiving the encoded foreground. [`StreamTargetChunk`](https://mediabunny.dev/api/StreamTargetChunk) is a Mediabunny primitive. Writes may target arbitrary byte positions through `chunk.position`; they are not guaranteed to be append-only. The stream is closed after the output is finalized.++When `outputs.foreground.outputWritable` is set, `foreground.getBlob()` rejects because the encoded bytes were sent to the stream. Cannot be combined with `outputs.foreground.outputTarget`. Default: no custom stream.++### videoBitrate?++The VP9 bitrate for both video layers, either a positive integer in bits per second or `very-low`, `low`, `medium`, `high`, or `very-high`. Default: `very-high`.++Higher bitrates preserve fine alpha edges more accurately and produce larger files.++### audioBitrate?++The Opus bitrate, either a positive integer in bits per second or `very-low`, `low`, `medium`, `high`, or `very-high`.++When omitted, compatible Opus audio is copied without re-encoding. Audio that cannot be copied is encoded at `medium` quality. Setting this option forces Opus encoding at the selected bitrate or quality.++This option has no effect when `audio` is `none` or the input has no audio.++### keyframeIntervalInSeconds?++Maximum interval between keyframes in both outputs. Must be a positive finite number. Default: `1`.++### signal?++An `AbortSignal` used to cancel processing. Default: no cancellation signal.++Aborting rejects the promise and cancels both outputs. Model inference may finish the current frame before cancellation completes.++### onModelLoadProgress?++Called while model files are resolved and downloaded. Default: no callback.++To load the model separately, use [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model).++### onProgress?++Called while the video is processed and the outputs are finalized. Default: no callback.++During processing, the callback receives:++#### stage++`processing` while frames are being separated, followed by `finalizing` while the WebM files are finalized.++#### progress++A number between `0` and `1` during processing, and `null` while finalizing.++#### processedFrames++The number of video frames processed so far.++#### processedDurationInSeconds++The input media time processed so far, in seconds.++#### durationInSeconds++The total video duration in seconds.++## Return value++Returns a promise resolving to an object with these properties:++Both `base` and `foreground` have the exported `VideoLayerOutput` type.++### base++The opaque base output.++#### base.getBlob()++Returns a promise resolving to a `Blob`, unless `outputs.base.outputWritable` was used.++#### base.dispose()++Releases temporary storage owned by the base output. Call it after `base.getBlob()` has resolved. The method is idempotent and has no effect for `arraybuffer` or custom writable outputs.++### foreground++The foreground output with alpha.++#### foreground.getBlob()++Returns a promise resolving to a `Blob`, unless `outputs.foreground.outputWritable` was used.++#### foreground.dispose()++Releases temporary storage owned by the foreground output. Call it after `foreground.getBlob()` has resolved. The method is idempotent and has no effect for `arraybuffer` or custom writable outputs.++### model++The model used for separation.++### width++The output width in pixels.++### height++The output height in pixels.++### durationInSeconds++The duration of the processed video in seconds.++### processedFrames++The number of video frames processed.++## Output format++Both layers are WebM files encoded with VP9. The foreground preserves alpha. Audio is encoded as Opus when included.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models)+- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)+- [`canUseVideoMatting()`](/docs/video-matting/can-use-video-matting)diff --git a/packages/docs/docs/video-matting/table-of-contents.tsx b/packages/docs/docs/video-matting/table-of-contents.tsxnew file mode 100644index 0000000000..687300c55e--- /dev/null+++ b/packages/docs/docs/video-matting/table-of-contents.tsx@@ -0,0 +1,42 @@+import React from 'react';+import {Grid} from '../../components/TableOfContents/Grid';+import {TOCItem} from '../../components/TableOfContents/TOCItem';++export const TableOfContents: React.FC = () => {+ return (+ <Grid>+ <TOCItem link="/docs/video-matting/can-use-video-matting">+ <strong>canUseVideoMatting()</strong>+ <div>Check whether a model is supported</div>+ </TOCItem>+ <TOCItem link="/docs/video-matting/get-available-video-matting-models">+ <strong>getAvailableVideoMattingModels()</strong>+ <div>List models and their download sizes</div>+ </TOCItem>+ <TOCItem link="/docs/video-matting/video-matting-models">+ <strong>VIDEO_MATTING_MODELS</strong>+ <div>Model names supported by the package</div>+ </TOCItem>+ <TOCItem link="/docs/video-matting/clear-stale-video-matting-models">+ <strong>clearStaleVideoMattingModels()</strong>+ <div>Remove models discontinued by newer versions</div>+ </TOCItem>+ <TOCItem link="/docs/video-matting/is-video-matting-model-cached">+ <strong>isVideoMattingModelCached()</strong>+ <div>Check whether a model is downloaded</div>+ </TOCItem>+ <TOCItem link="/docs/video-matting/load-video-matting-model">+ <strong>loadVideoMattingModel()</strong>+ <div>Download and initialize a model</div>+ </TOCItem>+ <TOCItem link="/docs/video-matting/separate-video-layers">+ <strong>separateVideoLayers()</strong>+ <div>Create base and foreground WebM layers</div>+ </TOCItem>+ <TOCItem link="/docs/video-matting/dispose-video-matting-model">+ <strong>disposeVideoMattingModel()</strong>+ <div>Release model memory</div>+ </TOCItem>+ </Grid>+ );+};diff --git a/packages/docs/docs/video-matting/video-matting-models.mdx b/packages/docs/docs/video-matting/video-matting-models.mdxnew file mode 100644index 0000000000..36f3cae8d7--- /dev/null+++ b/packages/docs/docs/video-matting/video-matting-models.mdx@@ -0,0 +1,27 @@+---+image: /generated/articles-docs-video-matting-video-matting-models.png+title: VIDEO_MATTING_MODELS+crumb: '@remotion/video-matting'+---++# VIDEO_MATTING_MODELS<AvailableFrom v="4.0.522" />++A readonly array containing the model names supported by `@remotion/video-matting`.++```ts twoslash title="models.ts"+import {VIDEO_MATTING_MODELS} from '@remotion/video-matting';++console.log(VIDEO_MATTING_MODELS);+// ['modnet', 'ben2-base']+```++Use [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models) to retrieve model metadata such as purpose, download size, and whether a model is experimental.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models)+- [`separateVideoLayers()`](/docs/video-matting/separate-video-layers)diff --git a/packages/docs/package.json b/packages/docs/package.jsonindex bb45cb6162..fe479690ba 100644--- a/packages/docs/package.json+++ b/packages/docs/package.json@@ -97,6 +97,7 @@"@remotion/three": "workspace:*","@remotion/transitions": "workspace:*","@remotion/vercel": "workspace:*",+ "@remotion/video-matting": "workspace:*","@remotion/web-renderer": "workspace:*","@remotion/webcodecs": "workspace:*","@remotion/whisper-web": "workspace:*",diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.tsindex 07c745d9c8..439d3ee5d6 100644--- a/packages/docs/sidebars.ts+++ b/packages/docs/sidebars.ts@@ -944,6 +944,24 @@ const sidebars: SidebarsConfig = {'vercel/vercel-types',],},+ {+ type: 'category',+ label: '@remotion/video-matting',+ link: {+ type: 'doc',+ id: 'video-matting/index',+ },+ items: [+ 'video-matting/can-use-video-matting',+ 'video-matting/get-available-video-matting-models',+ 'video-matting/video-matting-models',+ 'video-matting/clear-stale-video-matting-models',+ 'video-matting/is-video-matting-model-cached',+ 'video-matting/load-video-matting-model',+ 'video-matting/separate-video-layers',+ 'video-matting/dispose-video-matting-model',+ ],+ },{type: 'category',label: '@remotion/web-renderer',diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.tsindex f6cc3609e5..3a94011b6c 100644--- a/packages/docs/src/data/articles.ts+++ b/packages/docs/src/data/articles.ts@@ -8453,6 +8453,87 @@ export const articles = [noAi: false,slug: 'version',},+ {+ id: 'video-matting/can-use-video-matting',+ title: 'canUseVideoMatting()',+ relativePath: 'docs/video-matting/can-use-video-matting.mdx',+ compId: 'articles-docs-video-matting-can-use-video-matting',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/can-use-video-matting',+ },+ {+ id: 'video-matting/clear-stale-video-matting-models',+ title: 'clearStaleVideoMattingModels()',+ relativePath: 'docs/video-matting/clear-stale-video-matting-models.mdx',+ compId: 'articles-docs-video-matting-clear-stale-video-matting-models',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/clear-stale-video-matting-models',+ },+ {+ id: 'video-matting/dispose-video-matting-model',+ title: 'disposeVideoMattingModel()',+ relativePath: 'docs/video-matting/dispose-video-matting-model.mdx',+ compId: 'articles-docs-video-matting-dispose-video-matting-model',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/dispose-video-matting-model',+ },+ {+ id: 'video-matting/get-available-video-matting-models',+ title: 'getAvailableVideoMattingModels()',+ relativePath: 'docs/video-matting/get-available-video-matting-models.mdx',+ compId: 'articles-docs-video-matting-get-available-video-matting-models',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/get-available-video-matting-models',+ },+ {+ id: 'video-matting/index',+ title: '@remotion/video-matting',+ relativePath: 'docs/video-matting/index.mdx',+ compId: 'articles-docs-video-matting-index',+ crumb: 'Separate a video into layers',+ noAi: false,+ slug: 'video-matting/index',+ },+ {+ id: 'video-matting/is-video-matting-model-cached',+ title: 'isVideoMattingModelCached()',+ relativePath: 'docs/video-matting/is-video-matting-model-cached.mdx',+ compId: 'articles-docs-video-matting-is-video-matting-model-cached',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/is-video-matting-model-cached',+ },+ {+ id: 'video-matting/load-video-matting-model',+ title: 'loadVideoMattingModel()',+ relativePath: 'docs/video-matting/load-video-matting-model.mdx',+ compId: 'articles-docs-video-matting-load-video-matting-model',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/load-video-matting-model',+ },+ {+ id: 'video-matting/separate-video-layers',+ title: 'separateVideoLayers()',+ relativePath: 'docs/video-matting/separate-video-layers.mdx',+ compId: 'articles-docs-video-matting-separate-video-layers',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/separate-video-layers',+ },+ {+ id: 'video-matting/video-matting-models',+ title: 'VIDEO_MATTING_MODELS',+ relativePath: 'docs/video-matting/video-matting-models.mdx',+ compId: 'articles-docs-video-matting-video-matting-models',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/video-matting-models',+ },{id: 'video-tags',title: 'Comparison of video tags',diff --git a/packages/docs/static/generated/articles-docs-video-matting-can-use-video-matting.png b/packages/docs/static/generated/articles-docs-video-matting-can-use-video-matting.pngnew file mode 100644index 0000000000..efef1c8e67Binary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-can-use-video-matting.png differdiff --git a/packages/docs/static/generated/articles-docs-video-matting-clear-stale-video-matting-models.png b/packages/docs/static/generated/articles-docs-video-matting-clear-stale-video-matting-models.pngnew file mode 100644index 0000000000..76b3cffc5dBinary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-clear-stale-video-matting-models.png differdiff --git a/packages/docs/static/generated/articles-docs-video-matting-dispose-video-matting-model.png b/packages/docs/static/generated/articles-docs-video-matting-dispose-video-matting-model.pngnew file mode 100644index 0000000000..8c6a360d1bBinary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-dispose-video-matting-model.png differdiff --git a/packages/docs/static/generated/articles-docs-video-matting-get-available-video-matting-models.png b/packages/docs/static/generated/articles-docs-video-matting-get-available-video-matting-models.pngnew file mode 100644index 0000000000..a4cd0b2899Binary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-get-available-video-matting-models.png differdiff --git a/packages/docs/static/generated/articles-docs-video-matting-index.png b/packages/docs/static/generated/articles-docs-video-matting-index.pngnew file mode 100644index 0000000000..c3fdd65ebeBinary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-index.png differdiff --git a/packages/docs/static/generated/articles-docs-video-matting-is-video-matting-model-cached.png b/packages/docs/static/generated/articles-docs-video-matting-is-video-matting-model-cached.pngnew file mode 100644index 0000000000..7329e5e45fBinary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-is-video-matting-model-cached.png differdiff --git a/packages/docs/static/generated/articles-docs-video-matting-load-video-matting-model.png b/packages/docs/static/generated/articles-docs-video-matting-load-video-matting-model.pngnew file mode 100644index 0000000000..0c42573580Binary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-load-video-matting-model.png differdiff --git a/packages/docs/static/generated/articles-docs-video-matting-separate-video-layers.png b/packages/docs/static/generated/articles-docs-video-matting-separate-video-layers.pngnew file mode 100644index 0000000000..e027c05d60Binary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-separate-video-layers.png differdiff --git a/packages/docs/static/generated/articles-docs-video-matting-video-matting-models.png b/packages/docs/static/generated/articles-docs-video-matting-video-matting-models.pngnew file mode 100644index 0000000000..a25e76352fBinary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-video-matting-models.png differdiff --git a/packages/example/package.json b/packages/example/package.jsonindex 67687eda49..358d15d715 100644--- a/packages/example/package.json+++ b/packages/example/package.json@@ -82,6 +82,7 @@"@remotion/tailwind-v4": "workspace:*","@remotion/three": "workspace:*","@remotion/transitions": "workspace:*",+ "@remotion/video-matting": "workspace:*","@remotion/webcodecs": "workspace:*","@remotion/whisper-web": "workspace:*","@remotion/whisper-webgpu": "workspace:*",diff --git a/packages/example/src/Root.tsx b/packages/example/src/Root.tsxindex 1820487223..e6dab93d63 100644--- a/packages/example/src/Root.tsx+++ b/packages/example/src/Root.tsx@@ -304,6 +304,7 @@ import {} from './TransitionSeriesOverlay';import {TriangleComp} from './Triangle';import {NewVideoBufferStateComp} from './VideoInterruptedByForeignBuffer';+import {VideoMatting} from './VideoMatting';import {VideoTestingPlayback} from './VideoTesting/playback';import {VideoTestingTrim} from './VideoTesting/trim';import {RemotionMediaVideoTexture} from './VideoTexture';@@ -2710,6 +2711,12 @@ export const Index: React.FC = () => {width={900}height={900}/>+ <Still+ id="video-matting"+ component={VideoMatting}+ width={1200}+ height={1000}+ /><Compositionid="empty"component={Empty}diff --git a/packages/example/src/VideoMatting/index.tsx b/packages/example/src/VideoMatting/index.tsxnew file mode 100644index 0000000000..d74815e4be--- /dev/null+++ b/packages/example/src/VideoMatting/index.tsx@@ -0,0 +1,216 @@+import {+ canUseVideoMatting,+ getAvailableVideoMattingModels,+ separateVideoLayers,+ type VideoMattingModel,+} from '@remotion/video-matting';+import {useCallback, useEffect, useMemo, useState} from 'react';++type OutputUrls = {+ base: string;+ foreground: string;+};++const formatMegabytes = (bytes: number) => {+ return `${Math.round(bytes / 1_000_000)} MB`;+};++export const VideoMatting = () => {+ const models = useMemo(() => getAvailableVideoMattingModels(), []);+ const [file, setFile] = useState<File | null>(null);+ const [isProcessing, setIsProcessing] = useState(false);+ const [selectedModel, setSelectedModel] =+ useState<VideoMattingModel>('modnet');+ const [status, setStatus] = useState('Choose a video to separate.');+ const [progress, setProgress] = useState<number | null>(null);+ const [outputUrls, setOutputUrls] = useState<OutputUrls | null>(null);+ const selectedModelInfo = models.find(+ (model) => model.name === selectedModel,+ );++ useEffect(() => {+ return () => {+ if (outputUrls) {+ URL.revokeObjectURL(outputUrls.base);+ URL.revokeObjectURL(outputUrls.foreground);+ }+ };+ }, [outputUrls]);++ const onSeparate = useCallback(async () => {+ if (file === null) {+ return;+ }++ setIsProcessing(true);+ setOutputUrls(null);+ setProgress(0);++ try {+ const support = await canUseVideoMatting({model: selectedModel});+ if (!support.supported) {+ throw new Error(support.detailedReason);+ }++ setStatus(`Loading ${selectedModel}…`);+ const result = await separateVideoLayers({+ src: file,+ model: selectedModel,+ onModelLoadProgress: (modelProgress) => {+ if (modelProgress.progress !== null) {+ setStatus(+ `Loading ${selectedModel}… ${Math.round(modelProgress.progress * 100)}%`,+ );+ }+ },+ onProgress: (videoProgress) => {+ if (videoProgress.progress === null) {+ setProgress(null);+ setStatus(`Finalizing ${videoProgress.processedFrames} frames…`);+ return;+ }++ setProgress(videoProgress.progress);+ setStatus(+ `Separating frame ${videoProgress.processedFrames}… ${Math.round(videoProgress.progress * 100)}%`,+ );+ },+ });++ setStatus('Preparing previews…');+ try {+ const [baseBlob, foregroundBlob] = await Promise.all([+ result.base.getBlob(),+ result.foreground.getBlob(),+ ]);+ setOutputUrls({+ base: URL.createObjectURL(baseBlob),+ foreground: URL.createObjectURL(foregroundBlob),+ });+ } finally {+ await Promise.allSettled([+ result.base.dispose(),+ result.foreground.dispose(),+ ]);+ }+ setProgress(1);+ setStatus(+ `Done: ${result.processedFrames} frames, ${result.width}×${result.height}, ${result.durationInSeconds.toFixed(2)}s.`,+ );+ } catch (error) {+ setProgress(null);+ setStatus(error instanceof Error ? error.message : String(error));+ } finally {+ setIsProcessing(false);+ }+ }, [file, selectedModel]);++ const downloadPrefix = file?.name.replace(/\.[^.]+$/, '') ?? 'video';++ return (+ <div className="min-h-screen bg-slate-950 p-8 text-slate-900">+ <div className="mx-auto w-full max-w-5xl space-y-5 rounded-xl bg-white p-7 shadow-2xl">+ <div>+ <h1 className="text-2xl font-semibold">Video matting</h1>+ <p className="mt-1 text-sm text-slate-600">+ Separate an opaque base video from a transparent foreground in the+ browser.+ </p>+ </div>++ <div className="grid gap-4 sm:grid-cols-2">+ <label className="block text-sm font-medium" htmlFor="matting-file">+ Video+ <input+ accept="video/*"+ className="mt-2 block w-full rounded-md border border-slate-300 p-2"+ disabled={isProcessing}+ id="matting-file"+ onChange={(event) => {+ setFile(event.target.files?.[0] ?? null);+ setOutputUrls(null);+ setProgress(null);+ setStatus('Ready to separate.');+ }}+ type="file"+ />+ </label>++ <label className="block text-sm font-medium" htmlFor="matting-model">+ Model+ <select+ className="mt-2 block w-full rounded-md border border-slate-300 p-2"+ disabled={isProcessing}+ id="matting-model"+ onChange={(event) => {+ setSelectedModel(event.target.value as VideoMattingModel);+ }}+ value={selectedModel}+ >+ {models.map((model) => (+ <option key={model.name} value={model.name}>+ {model.name} · {model.purpose} ·{' '}+ {formatMegabytes(model.webGpuDownloadSize)}+ </option>+ ))}+ </select>+ </label>+ </div>++ {selectedModelInfo?.experimental ? (+ <p className="rounded-md bg-amber-50 p-3 text-sm text-amber-900">+ This model is experimental and requires WebGPU shader-f16 support.+ </p>+ ) : null}++ <button+ className="w-full rounded-md bg-indigo-600 px-4 py-3 font-semibold text-white disabled:opacity-50"+ disabled={file === null || isProcessing}+ onClick={onSeparate}+ type="button"+ >+ {isProcessing ? 'Separating…' : 'Separate video'}+ </button>++ <div className="rounded-md bg-blue-50 p-3 text-sm text-blue-900">+ {status}+ {progress !== null ? (+ <progress className="mt-2 block w-full" max={1} value={progress} />+ ) : null}+ </div>++ {outputUrls ? (+ <div className="grid gap-5 md:grid-cols-2">+ {(+ [+ ['Base', outputUrls.base, `${downloadPrefix}-base.webm`],+ [+ 'Foreground',+ outputUrls.foreground,+ `${downloadPrefix}-foreground.webm`,+ ],+ ] as const+ ).map(([label, url, download]) => (+ <div className="space-y-2" key={label}>+ <h2 className="font-semibold">{label}</h2>+ {/* eslint-disable-next-line @remotion/warn-native-media-tag */}+ <video+ className="aspect-video w-full rounded-md bg-[repeating-conic-gradient(#e2e8f0_0_25%,#fff_0_50%)] bg-[length:24px_24px]"+ controls+ src={url}+ />+ <a+ className="inline-block text-sm font-medium text-indigo-700 underline"+ download={download}+ href={url}+ >+ Download {label.toLowerCase()}+ </a>+ </div>+ ))}+ </div>+ ) : null}+ </div>+ </div>+ );+};diff --git a/packages/example/tsconfig.json b/packages/example/tsconfig.jsonindex 23ced969fa..196c52955b 100644--- a/packages/example/tsconfig.json+++ b/packages/example/tsconfig.json@@ -43,6 +43,7 @@{"path": "../webcodecs"},{"path": "../whisper-web"},{"path": "../whisper-webgpu"},+ {"path": "../video-matting"},{"path": "../studio-protocol"},{"path": "../light-leaks"},{"path": "../mac-cursors"},diff --git a/packages/studio-shared/src/package-info.ts b/packages/studio-shared/src/package-info.tsindex af53d3b799..adb92f67b0 100644--- a/packages/studio-shared/src/package-info.ts+++ b/packages/studio-shared/src/package-info.ts@@ -95,6 +95,7 @@ const allPackages = ['example-videos','whisper-web','whisper-webgpu',+ 'video-matting','media','remotion-media','web-renderer',@@ -178,6 +179,8 @@ export const descriptions: {[key in Pkgs]: string | null} = {'install-whisper-cpp': 'Helpers for installing and using Whisper.cpp','whisper-web': 'Helpers for using Whisper.cpp in browser using WASM','whisper-webgpu': 'Transcribe audio in the browser using Whisper and WebGPU',+ 'video-matting':+ 'Separate a video into a base layer and transparent foreground in the browser','google-fonts': 'Use Google Fonts in Remotion',gsap: 'Use GSAP timelines in Remotion',mcp: "Remotion's Model Context Protocol",@@ -317,6 +320,7 @@ export const installableMap: {[key in Pkgs]: boolean} = {'install-whisper-cpp': true,'whisper-web': true,'whisper-webgpu': true,+ 'video-matting': true,'it-tests': false,'react18-tests': false,'lambda-go-example': false,@@ -417,6 +421,7 @@ export const apiDocs: {[key in Pkgs]: string | null} = {'install-whisper-cpp': 'https://www.remotion.dev/docs/install-whisper-cpp','whisper-web': 'https://www.remotion.dev/docs/whisper-web','whisper-webgpu': 'https://www.remotion.dev/docs/whisper-webgpu',+ 'video-matting': 'https://www.remotion.dev/docs/video-matting','google-fonts': 'https://www.remotion.dev/docs/google-fonts',gsap: 'https://www.remotion.dev/docs/gsap','media-utils': 'https://www.remotion.dev/docs/media-utils',diff --git a/packages/video-matting/.npmignore b/packages/video-matting/.npmignorenew file mode 100644index 0000000000..eb76a5ae81--- /dev/null+++ b/packages/video-matting/.npmignore@@ -0,0 +1,10 @@+dist/test/**+src+eslint.config.mjs+tsconfig.tsbuildinfo+*.tgz+.turbo+tsconfig.json+.prettierrc.js+bundle.ts+dist/cjs/testdiff --git a/packages/video-matting/README.md b/packages/video-matting/README.mdnew file mode 100644index 0000000000..312b6d5931--- /dev/null+++ b/packages/video-matting/README.md@@ -0,0 +1,18 @@+# @remotion/video-matting++Separate a video into a base layer and transparent foreground in the browser++[](https://npmcharts.com/compare/@remotion/video-matting?minimal=true)++## Installation++```bash+npm install @remotion/video-matting --save-exact+```++When installing a Remotion package, make sure to align the version of all `remotion` and `@remotion/*` packages to the same version.+Remove the `^` character from the version number to use the exact version.++## Usage++See the [documentation](https://www.remotion.dev/docs/video-matting) for more information.diff --git a/packages/video-matting/bundle.ts b/packages/video-matting/bundle.tsnew file mode 100644index 0000000000..af6870b9bd--- /dev/null+++ b/packages/video-matting/bundle.ts@@ -0,0 +1,36 @@+import path from 'path';+import {build} from 'bun';++if (process.env.NODE_ENV !== 'production') {+ throw new Error('This script must be run using NODE_ENV=production');+}++console.time('Generated.');+const output = await build({+ entrypoints: ['src/index.ts'],+ naming: '[name].mjs',+ external: [+ '@huggingface/transformers',+ 'mediabunny',+ 'remotion',+ 'remotion/no-react',+ 'react',+ 'react/jsx-runtime',+ 'react/jsx-dev-runtime',+ 'react-dom',+ ],+});++if (!output.success) {+ console.log(output.logs.join('\n'));+ process.exit(1);+}++for (const file of output.outputs) {+ const str = await file.text();+ const out = path.join('dist', 'esm', file.path);++ await Bun.write(out, str);+}++console.timeEnd('Generated.');diff --git a/packages/video-matting/eslint.config.mjs b/packages/video-matting/eslint.config.mjsnew file mode 100644index 0000000000..ce2fc683dd--- /dev/null+++ b/packages/video-matting/eslint.config.mjs@@ -0,0 +1,5 @@+import {remotionFlatConfig} from '@remotion/eslint-config-internal';++const config = remotionFlatConfig({react: false});++export default [config];diff --git a/packages/video-matting/package.json b/packages/video-matting/package.jsonnew file mode 100644index 0000000000..bc42a84e62--- /dev/null+++ b/packages/video-matting/package.json@@ -0,0 +1,53 @@+{+ "name": "@remotion/video-matting",+ "version": "4.0.521",+ "description": "Separate a video into a base layer and transparent foreground in the browser",+ "main": "dist/index.js",+ "types": "dist/index.d.ts",+ "module": "dist/esm/index.mjs",+ "repository": {+ "url": "https://github.com/remotion-dev/remotion/tree/main/packages/video-matting"+ },+ "type": "module",+ "scripts": {+ "test": "bun test src/test",+ "formatting": "oxfmt src --check",+ "format": "oxfmt src",+ "lint": "eslint src",+ "watch": "tsgo -w",+ "make": "tsgo && bun --env-file=../.env.bundle bundle.ts"+ },+ "author": "Jonny Burger <[email protected]>",+ "contributors": [],+ "license": "MIT",+ "bugs": {+ "url": "https://github.com/remotion-dev/remotion/issues"+ },+ "dependencies": {+ "@huggingface/transformers": "catalog:",+ "mediabunny": "catalog:"+ },+ "exports": {+ ".": {+ "types": "./dist/index.d.ts",+ "module": "./dist/esm/index.mjs",+ "import": "./dist/esm/index.mjs"+ },+ "./package.json": "./package.json"+ },+ "devDependencies": {+ "@remotion/eslint-config-internal": "workspace:*",+ "@typescript/native-preview": "catalog:",+ "eslint": "catalog:"+ },+ "keywords": [+ "remotion",+ "video-matting",+ "background-removal",+ "webgpu"+ ],+ "publishConfig": {+ "access": "public"+ },+ "homepage": "https://www.remotion.dev/docs/video-matting"+}diff --git a/packages/video-matting/src/can-use-video-matting.ts b/packages/video-matting/src/can-use-video-matting.tsnew file mode 100644index 0000000000..1f8f3a8020--- /dev/null+++ b/packages/video-matting/src/can-use-video-matting.ts@@ -0,0 +1,95 @@+import {getVideoMattingModelInfo, type VideoMattingModel} from './models';++export enum VideoMattingUnsupportedReason {+ WindowUndefined = 'window-undefined',+ WebGpuUnavailable = 'webgpu-unavailable',+ WebGpuRequiresSecureContext = 'webgpu-requires-secure-context',+ ShaderF16Unavailable = 'shader-f16-unavailable',+}++export type CanUseVideoMattingResult =+ | {+ supported: true;+ }+ | {+ supported: false;+ reason: VideoMattingUnsupportedReason;+ detailedReason: string;+ };++export type CanUseVideoMattingOptions = {+ model?: VideoMattingModel;+};++type WebGpuAdapter = {+ features?: {+ has: (feature: string) => boolean;+ };+};++export const canUseVideoMatting = async ({+ model = 'modnet',+}: CanUseVideoMattingOptions = {}): Promise<CanUseVideoMattingResult> => {+ const modelInfo = getVideoMattingModelInfo(model);++ if (typeof window === 'undefined' && typeof OffscreenCanvas === 'undefined') {+ return {+ supported: false,+ reason: VideoMattingUnsupportedReason.WindowUndefined,+ detailedReason:+ 'No browser window or worker canvas environment is available. @remotion/video-matting is intended for browser environments.',+ };+ }++ const isSecureContext =+ typeof window === 'undefined'+ ? (globalThis as {isSecureContext?: boolean}).isSecureContext+ : window.isSecureContext;+ if (isSecureContext === false) {+ return {+ supported: false,+ reason: VideoMattingUnsupportedReason.WebGpuRequiresSecureContext,+ detailedReason:+ 'WebGPU requires HTTPS in production or localhost during development.',+ };+ }++ if (typeof navigator === 'undefined' || !('gpu' in navigator)) {+ return {+ supported: false,+ reason: VideoMattingUnsupportedReason.WebGpuUnavailable,+ detailedReason: 'WebGPU is not available in this browser.',+ };+ }++ let adapter: WebGpuAdapter | null;+ try {+ const {gpu} = navigator as unknown as {+ gpu: {requestAdapter: () => Promise<WebGpuAdapter | null>};+ };+ adapter = await gpu.requestAdapter();+ } catch {+ adapter = null;+ }++ if (!adapter) {+ return {+ supported: false,+ reason: VideoMattingUnsupportedReason.WebGpuUnavailable,+ detailedReason: 'No usable WebGPU adapter is available in this browser.',+ };+ }++ if (+ modelInfo.requiresShaderF16 &&+ adapter.features?.has('shader-f16') !== true+ ) {+ return {+ supported: false,+ reason: VideoMattingUnsupportedReason.ShaderF16Unavailable,+ detailedReason: `The video matting model "${model}" requires the WebGPU shader-f16 feature, which is unavailable on this adapter.`,+ };+ }++ return {supported: true};+};diff --git a/packages/video-matting/src/clear-stale-video-matting-models.ts b/packages/video-matting/src/clear-stale-video-matting-models.tsnew file mode 100644index 0000000000..c46817affc--- /dev/null+++ b/packages/video-matting/src/clear-stale-video-matting-models.ts@@ -0,0 +1,4 @@+// Transformers.js owns the persistent model cache. Model revisions are pinned,+// so there is currently no package-owned cache entry that can become stale.+export const clearStaleVideoMattingModels = (): Promise<void> =>+ Promise.resolve();diff --git a/packages/video-matting/src/create-video-layer-output.ts b/packages/video-matting/src/create-video-layer-output.tsnew file mode 100644index 0000000000..e67dce1f2b--- /dev/null+++ b/packages/video-matting/src/create-video-layer-output.ts@@ -0,0 +1,288 @@+import {+ BufferTarget,+ Output,+ StreamTarget,+ type OutputFormat,+ type StreamTargetChunk,+} from 'mediabunny';+import type {VideoLayerOutput, VideoLayerOutputOptions} from './output-target';+import {+ canUseWebFsWriter,+ createWebFsVideoLayerTarget,+ type WebFsVideoLayerTarget,+} from './web-fs-target';++type VideoLayerMediabunnyTarget = BufferTarget | StreamTarget;++export type CreatedVideoLayerOutput<F extends OutputFormat> = {+ output: Output<F, VideoLayerMediabunnyTarget>;+ finalize: () => Promise<VideoLayerOutput>;+ cancel: () => Promise<void>;+ discard: () => Promise<void>;+};++export const createVideoLayerOutput = async <F extends OutputFormat>({+ format,+ options,+}: {+ format: F;+ options: VideoLayerOutputOptions | undefined;+}): Promise<CreatedVideoLayerOutput<F>> => {+ if (+ options?.outputTarget !== undefined &&+ options.outputWritable !== undefined+ ) {+ throw new Error(+ 'outputTarget and outputWritable cannot both be specified for a video layer',+ );+ }++ let webFsTarget: WebFsVideoLayerTarget | null = null;+ let target: VideoLayerMediabunnyTarget;+ let outputMode: 'arraybuffer' | 'web-fs' | 'writable';+ const outputWritable = options?.outputWritable;+ let outputWritableWriter: WritableStreamDefaultWriter<StreamTargetChunk> | null =+ null;+ let outputWritableClosePromise: Promise<void> | null = null;+ let outputWritableAbortPromise: Promise<void> | null = null;+ let outputWritableWasClosed = false;+ const getOutputWritableWriter = () => {+ if (outputWritable === undefined) {+ throw new Error('Expected a caller-provided output WritableStream');+ }++ outputWritableWriter ??= outputWritable.getWriter();+ return outputWritableWriter;+ };++ const releaseOutputWritableWriter = () => {+ if (outputWritableWriter === null) {+ return;+ }++ outputWritableWriter.releaseLock();+ outputWritableWriter = null;+ };++ const closeOutputWritable = () => {+ if (outputWritable === undefined || outputWritableWasClosed) {+ return Promise.resolve();+ }++ if (outputWritableAbortPromise !== null) {+ return outputWritableAbortPromise;+ }++ if (outputWritableClosePromise === null) {+ const writer = getOutputWritableWriter();+ outputWritableClosePromise = (async () => {+ try {+ await writer.close();+ outputWritableWasClosed = true;+ } finally {+ releaseOutputWritableWriter();+ }+ })();+ }++ return outputWritableClosePromise;+ };++ const abortOutputWritable = (reason: unknown) => {+ if (outputWritable === undefined || outputWritableWasClosed) {+ return Promise.resolve();+ }++ if (outputWritableAbortPromise === null) {+ const writer = getOutputWritableWriter();+ outputWritableAbortPromise = (async () => {+ try {+ await writer.abort(reason);+ } finally {+ releaseOutputWritableWriter();+ }+ })();+ }++ return outputWritableAbortPromise;+ };++ if (outputWritable !== undefined) {+ target = new StreamTarget(+ new WritableStream<StreamTargetChunk>({+ write: (chunk) => getOutputWritableWriter().write(chunk),+ // Mediabunny closes its target for both cancellation and finalization.+ // The caller's stream is settled once we know which case occurred.+ close: () => Promise.resolve(),+ abort: (reason) => abortOutputWritable(reason),+ }),+ );+ outputMode = 'writable';+ } else {+ const outputTarget =+ options?.outputTarget ??+ ((await canUseWebFsWriter()) ? 'web-fs' : 'arraybuffer');++ if (outputTarget === 'web-fs') {+ webFsTarget = await createWebFsVideoLayerTarget();+ target = new StreamTarget(webFsTarget.stream);+ outputMode = 'web-fs';+ } else {+ target = new BufferTarget();+ outputMode = 'arraybuffer';+ }+ }++ const output = new Output({format, target});+ let finalizationPromise: Promise<VideoLayerOutput> | null = null;+ let cancellationPromise: Promise<void> | null = null;+ let finalized = false;+ let cancellationRequested = false;+ const discardWithReason = (reason: Error) => {+ if (webFsTarget !== null) {+ return webFsTarget.remove();+ }++ return abortOutputWritable(reason);+ };++ const discard = () =>+ discardWithReason(new Error('Video layer output was discarded'));++ const cancelMediabunnyOutput = async () => {+ if (output.state === 'canceled' || output.state === 'finalized') {+ return;+ }++ await output.cancel();+ };++ const cancel = (): Promise<void> => {+ if (finalized) {+ return Promise.resolve();+ }++ if (cancellationPromise !== null) {+ return cancellationPromise;+ }++ cancellationRequested = true;+ cancellationPromise = (async () => {+ const discardPromise = discardWithReason(+ new Error('Video layer output was canceled'),+ );+ let cancellationError: unknown = null;+ try {+ await cancelMediabunnyOutput();+ } catch (error) {+ cancellationError = error;+ }++ if (finalizationPromise !== null) {+ try {+ await finalizationPromise;+ } catch {+ // The finalization promise is expected to reject after cancellation.+ }+ }++ try {+ await discardPromise;+ } catch (error) {+ cancellationError ??= error;+ }++ if (cancellationError !== null) {+ throw cancellationError;+ }+ })();++ return cancellationPromise;+ };++ const finalize = (): Promise<VideoLayerOutput> => {+ if (cancellationRequested) {+ return Promise.reject(new Error('Video layer output was canceled'));+ }++ if (finalizationPromise !== null) {+ return finalizationPromise;+ }++ finalizationPromise = (async () => {+ try {+ await output.finalize();+ if (cancellationRequested) {+ throw new Error('Video layer output was canceled');+ }++ const mimeType = await output.getMimeType();+ if (cancellationRequested) {+ throw new Error('Video layer output was canceled');+ }++ await closeOutputWritable();+ if (cancellationRequested) {+ throw new Error('Video layer output was canceled');+ }++ finalized = true;++ if (outputMode === 'writable') {+ return {+ dispose: () => Promise.resolve(),+ getBlob: () =>+ Promise.reject(+ new Error(+ 'getBlob() is unavailable when outputWritable is used',+ ),+ ),+ };+ }++ if (outputMode === 'web-fs') {+ if (webFsTarget === null) {+ throw new Error('Expected an OPFS-backed output target');+ }++ let blobPromise: Promise<Blob> | null = null;++ return {+ dispose: discard,+ getBlob: () => {+ blobPromise ??= (async () => {+ const file = await webFsTarget.getBlob();+ const buffer = await file.arrayBuffer();+ return new Blob([buffer], {type: mimeType});+ })();++ return blobPromise;+ },+ };+ }++ if (!(target instanceof BufferTarget)) {+ throw new Error('Expected an in-memory output target');+ }++ return {+ dispose: () => Promise.resolve(),+ getBlob: () => {+ if (target.buffer === null) {+ return Promise.reject(new Error('The resulting buffer is empty'));+ }++ return Promise.resolve(new Blob([target.buffer], {type: mimeType}));+ },+ };+ } catch (error) {+ await Promise.allSettled([cancelMediabunnyOutput(), discard()]);+ throw error;+ }+ })();++ return finalizationPromise;+ };++ return {output, finalize, cancel, discard};+};diff --git a/packages/video-matting/src/index.ts b/packages/video-matting/src/index.tsnew file mode 100644index 0000000000..fef200cd09--- /dev/null+++ b/packages/video-matting/src/index.ts@@ -0,0 +1,40 @@+export {+ canUseVideoMatting,+ VideoMattingUnsupportedReason,+} from './can-use-video-matting';+export type {+ CanUseVideoMattingOptions,+ CanUseVideoMattingResult,+} from './can-use-video-matting';+export {clearStaleVideoMattingModels} from './clear-stale-video-matting-models';+export {+ disposeVideoMattingModel,+ loadVideoMattingModel,+} from './load-video-matting-model';+export type {+ DisposeVideoMattingModelOptions,+ LoadVideoMattingModelOptions,+ LoadVideoMattingModelResult,+ OnVideoMattingModelLoadProgress,+ VideoMattingModelLoadProgress,+} from './load-video-matting-model';+export {isVideoMattingModelCached} from './is-video-matting-model-cached';+export type {IsVideoMattingModelCachedOptions} from './is-video-matting-model-cached';+export {getAvailableVideoMattingModels, VIDEO_MATTING_MODELS} from './models';+export type {VideoMattingModel, VideoMattingModelInfo} from './models';+export type {+ VideoLayerOutput,+ VideoLayerOutputOptions,+ VideoLayerOutputTarget,+} from './output-target';+export {separateVideoLayers} from './separate-video-layers';+export type {+ SeparateVideoLayersOptions,+ SeparateVideoLayersProgress,+ SeparateVideoLayersResult,+ VideoLayerAudio,+} from './separate-video-layers';+export type {+ VideoMattingBitrate,+ VideoMattingQuality,+} from './video-matting-quality';diff --git a/packages/video-matting/src/is-video-matting-model-cached.ts b/packages/video-matting/src/is-video-matting-model-cached.tsnew file mode 100644index 0000000000..4fef5a28bf--- /dev/null+++ b/packages/video-matting/src/is-video-matting-model-cached.ts@@ -0,0 +1,22 @@+import {getVideoMattingModelInfo, type VideoMattingModel} from './models';++export type IsVideoMattingModelCachedOptions = {+ model: VideoMattingModel;+};++export const isVideoMattingModelCached = async ({+ model,+}: IsVideoMattingModelCachedOptions): Promise<boolean> => {+ const {ModelRegistry} = await import('@huggingface/transformers');+ const modelInfo = getVideoMattingModelInfo(model);++ return ModelRegistry.is_pipeline_cached(+ 'background-removal',+ modelInfo.modelId,+ {+ device: 'webgpu',+ dtype: modelInfo.dtype,+ revision: modelInfo.revision,+ },+ );+};diff --git a/packages/video-matting/src/load-video-matting-model.ts b/packages/video-matting/src/load-video-matting-model.tsnew file mode 100644index 0000000000..a7087f7f39--- /dev/null+++ b/packages/video-matting/src/load-video-matting-model.ts@@ -0,0 +1,410 @@+import {getVideoMattingModelInfo, type VideoMattingModel} from './models';++export type VideoMattingModelLoadProgress = {+ status: string;+ file: string | null;+ progress: number | null;+ loadedBytes: number | null;+ totalBytes: number | null;+};++export type OnVideoMattingModelLoadProgress = (+ progress: VideoMattingModelLoadProgress,+) => void;++export type VideoMattingImageSource = HTMLCanvasElement | OffscreenCanvas;++export type VideoMattingPipelineResult = {+ data: Uint8ClampedArray<ArrayBuffer>;+ width: number;+ height: number;+ channels: 4;+};++export type LoadedVideoMattingPipeline = (+ image: VideoMattingImageSource,+) => Promise<VideoMattingPipelineResult>;++type TransformerPipelineResult = {+ data: Uint8Array | Uint8ClampedArray;+ width: number;+ height: number;+ channels: number;+};++type TransformerPipeline = {+ (image: VideoMattingImageSource): Promise<TransformerPipelineResult>;+ dispose: () => Promise<void>;+};++type LoadedPipeline = {+ run: LoadedVideoMattingPipeline;+ dispose: () => Promise<void>;+};++type LoadedVideoMattingPipelineState = {+ loading: Promise<LoadedPipeline>;+ pendingUses: number;+ activeUses: number;+ onIdle: Array<() => void>;+ progressListeners: Set<OnVideoMattingModelLoadProgress>;+ lastProgress: VideoMattingModelLoadProgress | null;+};++const pipelines = new Map<VideoMattingModel, LoadedVideoMattingPipelineState>();+const disposals = new Map<VideoMattingModel, Promise<void>>();++export type LoadVideoMattingModelOptions = {+ model: VideoMattingModel;+ onProgress?: OnVideoMattingModelLoadProgress;+};++export type LoadVideoMattingModelResult = {+ alreadyLoaded: boolean;+};++const notifyProgress = (+ state: LoadedVideoMattingPipelineState,+ progress: VideoMattingModelLoadProgress,+) => {+ state.lastProgress = progress;+ for (const listener of state.progressListeners) {+ try {+ listener(progress);+ } catch {+ // Progress callbacks are observers and must not affect shared loading.+ }+ }+};++const notifyIfIdle = (state: LoadedVideoMattingPipelineState) => {+ if (state.pendingUses === 0 && state.activeUses === 0) {+ for (const resolve of state.onIdle.splice(0)) {+ resolve();+ }+ }+};++const waitForLoadingOrAbort = <Value>({+ loading,+ signal,+}: {+ loading: Promise<Value>;+ signal: AbortSignal | null;+}): Promise<Value> => {+ if (signal === null) {+ return loading;+ }++ if (signal.aborted) {+ return Promise.reject(signal.reason);+ }++ return new Promise<Value>((resolve, reject) => {+ const onAbort = () => {+ signal.removeEventListener('abort', onAbort);+ reject(signal.reason);+ };++ signal.addEventListener('abort', onAbort, {once: true});++ loading.then(+ (value) => {+ signal.removeEventListener('abort', onAbort);+ resolve(value);+ },+ (error) => {+ signal.removeEventListener('abort', onAbort);+ reject(error);+ },+ );+ });+};++const getOrCreateVideoMattingPipeline = ({+ model,+}: {+ model: VideoMattingModel;+}): {+ state: LoadedVideoMattingPipelineState;+ alreadyLoaded: boolean;+} => {+ const modelInfo = getVideoMattingModelInfo(model);+ const existing = pipelines.get(model);+ if (existing) {+ return {state: existing, alreadyLoaded: true};+ }++ const state: LoadedVideoMattingPipelineState = {+ loading: Promise.resolve(null as never),+ pendingUses: 0,+ activeUses: 0,+ onIdle: [],+ progressListeners: new Set(),+ lastProgress: null,+ };+ const pendingDisposal = disposals.get(model) ?? null;++ state.loading = Promise.resolve().then(async () => {+ if (pendingDisposal !== null) {+ await pendingDisposal;+ }++ const {+ AutoModelForImageSegmentation,+ AutoProcessor,+ BackgroundRemovalPipeline,+ } = await import('@huggingface/transformers');+ const totalBytes = modelInfo.webGpuDownloadSize;+ const loadedByFile = new Map<string, number>();+ let lastProgress = 0;+ let lastLoadedBytes = 0;++ notifyProgress(state, {+ status: 'loading',+ file: null,+ progress: 0,+ loadedBytes: 0,+ totalBytes,+ });++ const pretrainedOptions = {+ device: 'webgpu' as const,+ dtype: modelInfo.dtype,+ revision: modelInfo.revision,+ progress_callback: (event: unknown) => {+ const record = event as Record<string, unknown>;+ if (+ record.status === 'progress' &&+ typeof record.file === 'string' &&+ typeof record.loaded === 'number' &&+ Number.isFinite(record.loaded)+ ) {+ loadedByFile.set(+ record.file,+ Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),+ );+ const loadedBytes = [...loadedByFile.values()].reduce(+ (sum, loaded) => sum + loaded,+ 0,+ );+ lastProgress = Math.max(+ lastProgress,+ Math.min(loadedBytes / totalBytes, 0.99),+ );+ lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);+ notifyProgress(state, {+ status: 'loading',+ file: null,+ progress: lastProgress,+ loadedBytes: Math.min(lastLoadedBytes, totalBytes),+ totalBytes,+ });+ }+ },+ };+ const transformerProcessor = await AutoProcessor.from_pretrained(+ modelInfo.modelId,+ pretrainedOptions,+ );+ const transformerModel =+ await AutoModelForImageSegmentation.from_pretrained(+ modelInfo.modelId,+ pretrainedOptions,+ );+ let transformerPipeline: TransformerPipeline | null = null;+ let returnedPipeline = false;+ try {+ transformerPipeline = new BackgroundRemovalPipeline({+ task: 'background-removal',+ model: transformerModel,+ processor: transformerProcessor,+ }) as unknown as TransformerPipeline;+ const loadedTransformerPipeline = transformerPipeline;++ notifyProgress(state, {+ status: 'ready',+ file: null,+ progress: 1,+ loadedBytes: totalBytes,+ totalBytes,+ });++ const loadedPipeline: LoadedPipeline = {+ run: async (image) => {+ const result = await loadedTransformerPipeline(image);+ if (result.channels !== 4) {+ throw new Error(+ `The video matting model "${model}" returned ${result.channels} channels instead of RGBA.`,+ );+ }++ const data =+ result.data instanceof Uint8ClampedArray &&+ result.data.buffer instanceof ArrayBuffer+ ? (result.data as Uint8ClampedArray<ArrayBuffer>)+ : new Uint8ClampedArray(result.data);++ return {+ data,+ width: result.width,+ height: result.height,+ channels: 4,+ };+ },+ dispose: () => loadedTransformerPipeline.dispose(),+ };+ returnedPipeline = true;+ return loadedPipeline;+ } finally {+ if (!returnedPipeline) {+ if (transformerPipeline === null) {+ await transformerModel.dispose();+ } else {+ await transformerPipeline.dispose();+ }+ }+ }+ });++ pipelines.set(model, state);+ state.loading.catch(() => {+ if (pipelines.get(model) === state) {+ pipelines.delete(model);+ }+ });++ return {state, alreadyLoaded: false};+};++const subscribeToProgress = (+ state: LoadedVideoMattingPipelineState,+ onProgress: OnVideoMattingModelLoadProgress | undefined,+): (() => void) => {+ if (!onProgress) {+ return () => undefined;+ }++ const listener: OnVideoMattingModelLoadProgress = (progress) => {+ onProgress(progress);+ };++ state.progressListeners.add(listener);+ if (state.lastProgress) {+ try {+ listener(state.lastProgress);+ } catch {+ // Progress callbacks are observers and must not affect shared loading.+ }+ }++ return () => {+ state.progressListeners.delete(listener);+ };+};++export const loadVideoMattingModel = async ({+ model,+ onProgress,+}: LoadVideoMattingModelOptions): Promise<LoadVideoMattingModelResult> => {+ const {state, alreadyLoaded} = getOrCreateVideoMattingPipeline({model});+ const unsubscribe = subscribeToProgress(state, onProgress);+ try {+ await state.loading;+ } finally {+ unsubscribe();+ }++ return {alreadyLoaded};+};++export const withLoadedVideoMattingPipeline = async <ReturnValue>({+ model,+ onProgress,+ signal,+ run,+}: {+ model: VideoMattingModel;+ onProgress?: OnVideoMattingModelLoadProgress;+ signal: AbortSignal | null;+ run: (pipeline: LoadedVideoMattingPipeline) => Promise<ReturnValue>;+}): Promise<ReturnValue> => {+ const {state} = getOrCreateVideoMattingPipeline({model});+ const unsubscribe = subscribeToProgress(state, onProgress);+ state.pendingUses++;+ let isActive = false;+ try {+ const loaded = await waitForLoadingOrAbort({+ loading: state.loading,+ signal,+ });+ state.pendingUses--;+ state.activeUses++;+ isActive = true;+ return await run(loaded.run);+ } finally {+ unsubscribe();+ if (isActive) {+ state.activeUses--;+ } else {+ state.pendingUses--;+ }++ notifyIfIdle(state);+ }+};++export type DisposeVideoMattingModelOptions = {+ model?: VideoMattingModel;+};++export const disposeVideoMattingModel = async ({+ model,+}: DisposeVideoMattingModelOptions = {}): Promise<void> => {+ if (model !== undefined) {+ getVideoMattingModelInfo(model);+ }++ const matching = [...pipelines.entries()].filter(([loadedModel]) => {+ return model === undefined || loadedModel === model;+ });+ for (const [key, state] of matching) {+ if (pipelines.get(key) === state) {+ pipelines.delete(key);+ }+ }++ const pending = new Map<VideoMattingModel, Promise<void>>();+ for (const [key, disposal] of disposals) {+ if (model === undefined || model === key) {+ pending.set(key, disposal);+ }+ }++ for (const [key, state] of matching) {+ const previousDisposal = disposals.get(key) ?? Promise.resolve();+ const disposal = previousDisposal.then(async () => {+ const loadedPipeline = await state.loading;+ if (state.pendingUses > 0 || state.activeUses > 0) {+ await new Promise<void>((resolve) => {+ state.onIdle.push(resolve);+ });+ }++ await loadedPipeline.dispose();+ });+ const disposalBarrier = disposal.then(+ () => undefined,+ () => undefined,+ );+ disposals.set(key, disposalBarrier);+ disposalBarrier.then(() => {+ if (disposals.get(key) === disposalBarrier) {+ disposals.delete(key);+ }+ });+ pending.set(key, disposal);+ }++ await Promise.all(pending.values());+};diff --git a/packages/video-matting/src/models.ts b/packages/video-matting/src/models.tsnew file mode 100644index 0000000000..d01fa610f0--- /dev/null+++ b/packages/video-matting/src/models.ts@@ -0,0 +1,64 @@+export const VIDEO_MATTING_MODELS = ['modnet', 'ben2-base'] as const;++export type VideoMattingModel = (typeof VIDEO_MATTING_MODELS)[number];++export type VideoMattingModelInfo = {+ name: VideoMattingModel;+ modelId: string;+ purpose: 'person' | 'general';+ experimental: boolean;+ webGpuDownloadSize: number;+};++type InternalVideoMattingModelInfo = VideoMattingModelInfo & {+ revision: string;+ dtype: 'fp32' | 'fp16';+ requiresShaderF16: boolean;+};++const MODEL_INFO: Record<VideoMattingModel, InternalVideoMattingModelInfo> = {+ modnet: {+ name: 'modnet',+ modelId: 'Xenova/modnet',+ purpose: 'person',+ experimental: false,+ webGpuDownloadSize: 25_888_640,+ revision: 'fa2fa546052fba4c08921230a26cc69a333fca12',+ dtype: 'fp32',+ requiresShaderF16: false,+ },+ 'ben2-base': {+ name: 'ben2-base',+ modelId: 'onnx-community/BEN2-ONNX',+ purpose: 'general',+ experimental: true,+ webGpuDownloadSize: 219_121_675,+ revision: 'c552aa82688edce09f0ac9d2e31ad53d9d629010',+ dtype: 'fp16',+ requiresShaderF16: true,+ },+};++export const getAvailableVideoMattingModels = (): VideoMattingModelInfo[] => {+ return VIDEO_MATTING_MODELS.map((model) => {+ const {+ revision: _revision,+ dtype: _dtype,+ requiresShaderF16: _requiresShaderF16,+ ...info+ } = MODEL_INFO[model];+ return {...info};+ });+};++export const getVideoMattingModelInfo = (+ model: VideoMattingModel,+): InternalVideoMattingModelInfo => {+ if (!Object.hasOwn(MODEL_INFO, model)) {+ throw new Error(+ `Unsupported video matting model "${model}". Available models: ${VIDEO_MATTING_MODELS.join(', ')}.`,+ );+ }++ return MODEL_INFO[model];+};diff --git a/packages/video-matting/src/output-target.ts b/packages/video-matting/src/output-target.tsnew file mode 100644index 0000000000..ab0a2f995c--- /dev/null+++ b/packages/video-matting/src/output-target.ts@@ -0,0 +1,18 @@+import type {StreamTargetChunk} from 'mediabunny';++export type VideoLayerOutputTarget = 'arraybuffer' | 'web-fs';++export type VideoLayerOutputOptions =+ | {+ outputTarget?: VideoLayerOutputTarget;+ outputWritable?: never;+ }+ | {+ outputTarget?: never;+ outputWritable: WritableStream<StreamTargetChunk>;+ };++export type VideoLayerOutput = {+ getBlob: () => Promise<Blob>;+ dispose: () => Promise<void>;+};diff --git a/packages/video-matting/src/prepare-audio.ts b/packages/video-matting/src/prepare-audio.tsnew file mode 100644index 0000000000..bf0b108d06--- /dev/null+++ b/packages/video-matting/src/prepare-audio.ts@@ -0,0 +1,542 @@+import {+ AudioSampleSink,+ AudioSampleSource,+ canEncodeAudio,+ EncodedAudioPacketSource,+ EncodedPacketSink,+ type Input,+ type Output,+ Quality,+ type Target,+ type WebMOutputFormat,+} from 'mediabunny';++export type VideoMattingAudioDestination =+ | 'base'+ | 'foreground'+ | 'both'+ | 'none';++export type PreparedVideoMattingAudio = {+ /**+ * Queues the first audio packet or sample, even when it starts after the+ * first video frame. Call this after the outputs have started.+ */+ prime: () => Promise<void>;+ /**+ * Writes audio whose rebased start timestamp is at or before the given+ * output timestamp. Calls must use monotonically increasing timestamps.+ */+ writeAudioUntil: (outputTimestamp: number) => Promise<void>;+ /** Writes the remaining audio up to the end of the video and closes it. */+ finishAudio: () => Promise<void>;+ /** Releases decoded samples that have not been written yet. */+ cancel: () => Promise<void>;+};++export const prepareAudio = async <+ BaseTarget extends Target,+ ForegroundTarget extends Target,+>({+ input,+ baseOutput,+ foregroundOutput,+ destination,+ videoStartTimestamp,+ videoEndTimestamp,+ audioQuality,+ forceTranscode,+}: {+ input: Input;+ baseOutput: Output<WebMOutputFormat, BaseTarget>;+ foregroundOutput: Output<WebMOutputFormat, ForegroundTarget>;+ destination: VideoMattingAudioDestination;+ videoStartTimestamp: number;+ videoEndTimestamp: number;+ audioQuality: Quality | null;+ forceTranscode: boolean;+}): Promise<PreparedVideoMattingAudio> => {+ if (!Number.isFinite(videoStartTimestamp)) {+ throw new TypeError('videoStartTimestamp must be a finite number.');+ }++ if (!Number.isFinite(videoEndTimestamp)) {+ throw new TypeError('videoEndTimestamp must be a finite number.');+ }++ if (videoEndTimestamp < videoStartTimestamp) {+ throw new RangeError(+ 'videoEndTimestamp must be greater than or equal to videoStartTimestamp.',+ );+ }++ if (typeof forceTranscode !== 'boolean') {+ throw new TypeError('forceTranscode must be a boolean.');+ }++ if (destination === 'none') {+ return {+ prime: () => Promise.resolve(),+ writeAudioUntil: () => Promise.resolve(),+ finishAudio: () => Promise.resolve(),+ cancel: () => Promise.resolve(),+ };+ }++ const audioTrack = await input.getPrimaryAudioTrack();+ if (audioTrack === null) {+ return {+ prime: () => Promise.resolve(),+ writeAudioUntil: () => Promise.resolve(),+ finishAudio: () => Promise.resolve(),+ cancel: () => Promise.resolve(),+ };+ }++ const [firstAudioTimestamp, audioEndTimestamp] = await Promise.all([+ audioTrack.getFirstTimestamp(),+ audioTrack.computeDuration(),+ ]);+ if (+ firstAudioTimestamp >= videoEndTimestamp ||+ audioEndTimestamp <= videoStartTimestamp+ ) {+ return {+ prime: () => Promise.resolve(),+ writeAudioUntil: () => Promise.resolve(),+ finishAudio: () => Promise.resolve(),+ cancel: () => Promise.resolve(),+ };+ }++ const outputs =+ destination === 'both'+ ? [baseOutput, foregroundOutput]+ : destination === 'base'+ ? [baseOutput]+ : [foregroundOutput];+ const sourceCodec = await audioTrack.getCodec();+ const canCopyPackets =+ !forceTranscode &&+ sourceCodec === 'opus' &&+ firstAudioTimestamp >= videoStartTimestamp;++ let lastWriteTimestamp = -Infinity;+ let primed = false;+ let finished = false;+ let canceled = false;++ if (canCopyPackets) {+ const decoderConfig = await audioTrack.getDecoderConfig();+ const packetSources = outputs.map((output) => {+ const source = new EncodedAudioPacketSource('opus');+ output.addAudioTrack(source, {+ decoderConfig: decoderConfig ?? undefined,+ });+ return source;+ });+ const packetIterator = new EncodedPacketSink(audioTrack).packets();+ let pendingPacket: Awaited<ReturnType<typeof packetIterator.next>>['value'];+ let packetIteratorDone = false;+ let packetSourcesClosed = false;++ const closePacketSources = () => {+ if (+ packetSourcesClosed ||+ outputs.some((output) => output.state !== 'started')+ ) {+ return;+ }++ packetSourcesClosed = true;+ for (const source of packetSources) {+ source.close();+ }+ };++ const stopPacketIterator = async () => {+ if (packetIteratorDone) {+ return;+ }++ packetIteratorDone = true;+ try {+ await packetIterator.return(undefined);+ } catch {+ // Iterator teardown is best-effort cleanup.+ }+ };++ const writePacketsUntil = async ({+ outputTimestamp,+ writeAtLeastOne,+ }: {+ outputTimestamp: number;+ writeAtLeastOne: boolean;+ }) => {+ let packetsWritten = 0;+ while (!packetIteratorDone) {+ if (pendingPacket === undefined) {+ const next = await packetIterator.next();+ if (next.done) {+ packetIteratorDone = true;+ closePacketSources();+ return;+ }++ pendingPacket = next.value;+ }++ if (pendingPacket.timestamp >= videoEndTimestamp) {+ pendingPacket = undefined;+ await stopPacketIterator();+ closePacketSources();+ return;+ }++ const rebasedTimestamp = pendingPacket.timestamp - videoStartTimestamp;+ if (+ rebasedTimestamp > outputTimestamp &&+ (!writeAtLeastOne || packetsWritten > 0)+ ) {+ return;+ }++ const packet = pendingPacket.clone({+ timestamp: rebasedTimestamp,+ });+ pendingPacket = undefined;+ const metadata: EncodedAudioChunkMetadata = {+ decoderConfig: decoderConfig ?? undefined,+ };+ await Promise.all(+ packetSources.map((source) => source.add(packet, metadata)),+ );+ packetsWritten++;+ if (writeAtLeastOne) {+ return;+ }+ }+ };++ return {+ prime: async () => {+ if (primed || finished || canceled) {+ return;+ }++ primed = true;+ await writePacketsUntil({+ outputTimestamp: 0,+ writeAtLeastOne: true,+ });+ },+ writeAudioUntil: async (outputTimestamp) => {+ if (!Number.isFinite(outputTimestamp) || outputTimestamp < 0) {+ throw new TypeError(+ 'outputTimestamp must be a finite, non-negative number.',+ );+ }++ if (outputTimestamp < lastWriteTimestamp) {+ throw new RangeError(+ 'writeAudioUntil() timestamps must be monotonically increasing.',+ );+ }++ if (finished || canceled) {+ return;+ }++ lastWriteTimestamp = outputTimestamp;+ await writePacketsUntil({+ outputTimestamp: Math.min(+ outputTimestamp,+ videoEndTimestamp - videoStartTimestamp,+ ),+ writeAtLeastOne: false,+ });+ },+ finishAudio: async () => {+ if (finished || canceled) {+ return;+ }++ finished = true;+ try {+ await writePacketsUntil({+ outputTimestamp: videoEndTimestamp - videoStartTimestamp,+ writeAtLeastOne: false,+ });+ } finally {+ await stopPacketIterator();+ closePacketSources();+ }+ },+ cancel: async () => {+ if (finished || canceled) {+ return;+ }++ canceled = true;+ pendingPacket = undefined;+ await stopPacketIterator();+ },+ };+ }++ if (!(await audioTrack.canDecode())) {+ throw new Error(+ `The primary audio track uses ${sourceCodec ?? 'an unknown codec'}, which ` ++ 'cannot be decoded in this browser. The audio cannot be converted to Opus.',+ );+ }++ const originalNumberOfChannels = await audioTrack.getNumberOfChannels();+ const originalSampleRate = await audioTrack.getSampleRate();+ const quality = audioQuality ?? new Quality('medium');+ let outputNumberOfChannels = originalNumberOfChannels;+ let outputSampleRate = originalSampleRate;++ if (+ !(await canEncodeAudio('opus', {+ numberOfChannels: outputNumberOfChannels,+ sampleRate: outputSampleRate,+ quality,+ }))+ ) {+ outputNumberOfChannels = Math.min(originalNumberOfChannels, 2);+ outputSampleRate = 48_000;++ if (+ !(await canEncodeAudio('opus', {+ numberOfChannels: outputNumberOfChannels,+ sampleRate: outputSampleRate,+ quality,+ }))+ ) {+ throw new Error(+ 'The primary audio track must be transcoded to Opus, but this browser ' ++ 'does not support a compatible Opus encoder.',+ );+ }+ }++ const sampleSources = outputs.map((output) => {+ const source = new AudioSampleSource({+ codec: 'opus',+ quality,+ transform: {+ numberOfChannels:+ outputNumberOfChannels === originalNumberOfChannels+ ? undefined+ : outputNumberOfChannels,+ sampleRate:+ outputSampleRate === originalSampleRate+ ? undefined+ : outputSampleRate,+ },+ });+ output.addAudioTrack(source);+ return source;+ });+ const sampleIterator = new AudioSampleSink(audioTrack).samples(+ videoStartTimestamp,+ videoEndTimestamp,+ );+ let pendingSample: Awaited<ReturnType<typeof sampleIterator.next>>['value'];+ let iteratorDone = false;+ let sourcesClosed = false;++ const closeSources = () => {+ if (sourcesClosed || outputs.some((output) => output.state !== 'started')) {+ return;+ }++ sourcesClosed = true;+ for (const source of sampleSources) {+ source.close();+ }+ };++ const stopSampleIterator = async () => {+ if (iteratorDone) {+ return;+ }++ iteratorDone = true;+ try {+ await sampleIterator.return(undefined);+ } catch {+ // Iterator teardown is best-effort cleanup.+ }+ };++ const writeUntil = async ({+ outputTimestamp,+ writeAtLeastOne,+ }: {+ outputTimestamp: number;+ writeAtLeastOne: boolean;+ }) => {+ let samplesWritten = 0;+ while (!iteratorDone) {+ if (pendingSample === undefined) {+ const next = await sampleIterator.next();+ if (next.done) {+ iteratorDone = true;+ closeSources();+ return;+ }++ pendingSample = next.value;+ }++ const effectiveTimestamp =+ Math.max(pendingSample.timestamp, videoStartTimestamp) -+ videoStartTimestamp;+ if (+ effectiveTimestamp > outputTimestamp &&+ (!writeAtLeastOne || samplesWritten > 0)+ ) {+ return;+ }++ const originalSample = pendingSample;+ pendingSample = undefined;+ let startFrame = 0;+ let endFrame = originalSample.numberOfFrames;+ if (originalSample.timestamp < videoStartTimestamp) {+ startFrame = Math.max(+ 0,+ Math.min(+ originalSample.numberOfFrames,+ Math.round(+ (videoStartTimestamp - originalSample.timestamp) *+ originalSample.sampleRate,+ ),+ ),+ );+ }++ if (+ originalSample.timestamp + originalSample.duration >+ videoEndTimestamp+ ) {+ endFrame = Math.max(+ 0,+ Math.min(+ originalSample.numberOfFrames,+ Math.round(+ (videoEndTimestamp - originalSample.timestamp) *+ originalSample.sampleRate,+ ),+ ),+ );+ }++ if (endFrame <= startFrame) {+ originalSample.close();+ continue;+ }++ const sample =+ startFrame === 0 && endFrame === originalSample.numberOfFrames+ ? originalSample+ : originalSample.trim(startFrame, endFrame);+ if (sample !== originalSample) {+ originalSample.close();+ }++ sample.setTimestamp(Math.max(0, sample.timestamp - videoStartTimestamp));++ const samples = sampleSources.map((_, index) =>+ index === sampleSources.length - 1 ? sample : sample.clone(),+ );+ try {+ await Promise.all(+ sampleSources.map((source, index) => source.add(samples[index]!)),+ );+ } finally {+ for (const sampleToClose of samples) {+ sampleToClose.close();+ }+ }++ samplesWritten++;+ if (writeAtLeastOne) {+ return;+ }+ }+ };++ return {+ prime: async () => {+ if (primed || finished || canceled) {+ return;+ }++ primed = true;+ await writeUntil({outputTimestamp: 0, writeAtLeastOne: true});+ },+ writeAudioUntil: async (outputTimestamp) => {+ if (!Number.isFinite(outputTimestamp) || outputTimestamp < 0) {+ throw new TypeError(+ 'outputTimestamp must be a finite, non-negative number.',+ );+ }++ if (outputTimestamp < lastWriteTimestamp) {+ throw new RangeError(+ 'writeAudioUntil() timestamps must be monotonically increasing.',+ );+ }++ if (finished || canceled) {+ return;+ }++ lastWriteTimestamp = outputTimestamp;+ await writeUntil({+ outputTimestamp: Math.min(+ outputTimestamp,+ videoEndTimestamp - videoStartTimestamp,+ ),+ writeAtLeastOne: false,+ });+ },+ finishAudio: async () => {+ if (finished || canceled) {+ return;+ }++ finished = true;+ try {+ await writeUntil({+ outputTimestamp: videoEndTimestamp - videoStartTimestamp,+ writeAtLeastOne: false,+ });+ } finally {+ pendingSample?.close();+ pendingSample = undefined;+ await stopSampleIterator();+ closeSources();+ }+ },+ cancel: async () => {+ if (finished || canceled) {+ return;+ }++ canceled = true;+ try {+ pendingSample?.close();+ } catch {+ // Cancellation cleanup must not replace the original error.+ }++ pendingSample = undefined;+ await stopSampleIterator();+ },+ };+};diff --git a/packages/video-matting/src/separate-video-layers.ts b/packages/video-matting/src/separate-video-layers.tsnew file mode 100644index 0000000000..20ef50d686--- /dev/null+++ b/packages/video-matting/src/separate-video-layers.ts@@ -0,0 +1,583 @@+import {+ ALL_FORMATS,+ BlobSource,+ CanvasSink,+ CanvasSource,+ Input,+ type InputVideoTrack,+ type Quality,+ UrlSource,+ WebMOutputFormat,+ canEncodeVideo,+} from 'mediabunny';+import {createVideoLayerOutput} from './create-video-layer-output';+import {+ type OnVideoMattingModelLoadProgress,+ withLoadedVideoMattingPipeline,+} from './load-video-matting-model';+import {getVideoMattingModelInfo, type VideoMattingModel} from './models';+import type {VideoLayerOutput, VideoLayerOutputOptions} from './output-target';+import {prepareAudio, type PreparedVideoMattingAudio} from './prepare-audio';+import {+ createVideoMattingCanvas,+ drawForegroundFrame,+ drawOpaqueBaseFrame,+ getVideoMattingCanvasContext,+} from './video-matting-canvas';+import {+ resolveVideoMattingQuality,+ type VideoMattingBitrate,+} from './video-matting-quality';+import {+ getClippedVideoFrameTiming,+ getVideoProcessingProgress,+} from './video-timing';++export type VideoLayerAudio = 'base' | 'foreground' | 'both' | 'none';++type VideoLayerProgressFields = {+ processedFrames: number;+ processedDurationInSeconds: number;+ durationInSeconds: number;+};++export type SeparateVideoLayersProgress =+ | (VideoLayerProgressFields & {+ stage: 'processing';+ progress: number;+ })+ | (VideoLayerProgressFields & {+ stage: 'finalizing';+ progress: null;+ });++export type SeparateVideoLayersOptions = {+ src: string | URL | Blob;+ model?: VideoMattingModel;+ audio?: VideoLayerAudio;+ outputs?: {+ base?: VideoLayerOutputOptions;+ foreground?: VideoLayerOutputOptions;+ };+ videoBitrate?: VideoMattingBitrate;+ audioBitrate?: VideoMattingBitrate;+ keyframeIntervalInSeconds?: number;+ signal?: AbortSignal;+ onModelLoadProgress?: OnVideoMattingModelLoadProgress;+ onProgress?: (progress: SeparateVideoLayersProgress) => void;+};++export type SeparateVideoLayersResult = {+ base: VideoLayerOutput;+ foreground: VideoLayerOutput;+ model: VideoMattingModel;+ width: number;+ height: number;+ durationInSeconds: number;+ processedFrames: number;+};++const AUDIO_DESTINATIONS: VideoLayerAudio[] = [+ 'base',+ 'foreground',+ 'both',+ 'none',+];++const createAbortError = (signal: AbortSignal): unknown => {+ if (signal.reason !== undefined) {+ return signal.reason;+ }++ const error = new Error('Video layer separation was aborted.');+ error.name = 'AbortError';+ return error;+};++const throwIfAborted = (signal: AbortSignal | undefined) => {+ if (signal?.aborted) {+ throw createAbortError(signal);+ }+};++const validateLayerOutputOptions = ({+ layer,+ output,+}: {+ layer: 'base' | 'foreground';+ output: VideoLayerOutputOptions | undefined;+}) => {+ if (output === undefined) {+ return;+ }++ if (!output || typeof output !== 'object' || Array.isArray(output)) {+ throw new TypeError(`outputs.${layer} must be an object.`);+ }++ if (+ output.outputTarget !== undefined &&+ output.outputTarget !== 'arraybuffer' &&+ output.outputTarget !== 'web-fs'+ ) {+ throw new TypeError(+ `outputs.${layer}.outputTarget must be arraybuffer or web-fs.`,+ );+ }++ if (+ output.outputTarget !== undefined &&+ output.outputWritable !== undefined+ ) {+ throw new TypeError(+ `outputs.${layer} cannot specify both outputTarget and outputWritable.`,+ );+ }++ if (output.outputWritable !== undefined) {+ if (+ typeof WritableStream === 'undefined' ||+ !(output.outputWritable instanceof WritableStream)+ ) {+ throw new TypeError(+ `outputs.${layer}.outputWritable must be a WritableStream.`,+ );+ }++ if (output.outputWritable.locked) {+ throw new TypeError(+ `outputs.${layer}.outputWritable must not already be locked.`,+ );+ }+ }+};++const validateOptions = (options: SeparateVideoLayersOptions) => {+ if (!options || typeof options !== 'object') {+ throw new TypeError('separateVideoLayers() expects an options object.');+ }++ const isBlob = typeof Blob !== 'undefined' && options.src instanceof Blob;+ const isUrl = options.src instanceof URL;+ if (typeof options.src !== 'string' && !isUrl && !isBlob) {+ throw new TypeError('src must be a string, URL, or Blob.');+ }++ if (typeof options.src === 'string' && options.src.length === 0) {+ throw new TypeError('src must not be an empty string.');+ }++ if (+ options.outputs !== undefined &&+ (!options.outputs ||+ typeof options.outputs !== 'object' ||+ Array.isArray(options.outputs))+ ) {+ throw new TypeError('outputs must be an object.');+ }++ validateLayerOutputOptions({+ layer: 'base',+ output: options.outputs?.base,+ });+ validateLayerOutputOptions({+ layer: 'foreground',+ output: options.outputs?.foreground,+ });+ if (+ options.outputs?.base?.outputWritable !== undefined &&+ options.outputs.base.outputWritable ===+ options.outputs.foreground?.outputWritable+ ) {+ throw new TypeError(+ 'outputs.base and outputs.foreground must not use the same outputWritable.',+ );+ }++ getVideoMattingModelInfo(options.model ?? 'modnet');++ if (+ options.audio !== undefined &&+ !AUDIO_DESTINATIONS.includes(options.audio)+ ) {+ throw new TypeError(+ 'audio must be one of base, foreground, both, or none.',+ );+ }++ resolveVideoMattingQuality(options.videoBitrate ?? 'very-high');+ resolveVideoMattingQuality(options.audioBitrate ?? 'medium');++ if (+ options.keyframeIntervalInSeconds !== undefined &&+ (!Number.isFinite(options.keyframeIntervalInSeconds) ||+ options.keyframeIntervalInSeconds <= 0)+ ) {+ throw new TypeError(+ 'keyframeIntervalInSeconds must be a positive finite number.',+ );+ }++ if (+ options.onProgress !== undefined &&+ typeof options.onProgress !== 'function'+ ) {+ throw new TypeError('onProgress must be a function.');+ }++ if (+ options.onModelLoadProgress !== undefined &&+ typeof options.onModelLoadProgress !== 'function'+ ) {+ throw new TypeError('onModelLoadProgress must be a function.');+ }+};++const makeInput = (src: string | URL | Blob): Input => {+ const source =+ typeof src === 'string' || src instanceof URL+ ? new UrlSource(src)+ : new BlobSource(src);++ return new Input({formats: ALL_FORMATS, source});+};++const probeVideoInput = async ({+ input,+ videoQuality,+}: {+ input: Input;+ videoQuality: Quality;+}): Promise<{videoTrack: InputVideoTrack; width: number; height: number}> => {+ if (!(await input.canRead())) {+ throw new Error('The input is not a supported media file.');+ }++ const videoTrack = await input.getPrimaryVideoTrack();+ if (videoTrack === null) {+ throw new Error('The input does not contain a video track.');+ }++ if (!(await videoTrack.canDecode())) {+ throw new Error('The primary video track cannot be decoded.');+ }++ const [width, height] = await Promise.all([+ videoTrack.getDisplayWidth(),+ videoTrack.getDisplayHeight(),+ ]);+ if (+ !Number.isInteger(width) ||+ width <= 0 ||+ !Number.isInteger(height) ||+ height <= 0+ ) {+ throw new Error('The input video has invalid dimensions.');+ }++ const [canEncodeBase, canEncodeForeground] = await Promise.all([+ canEncodeVideo('vp9', {+ width,+ height,+ quality: videoQuality,+ alpha: 'discard',+ }),+ canEncodeVideo('vp9', {+ width,+ height,+ quality: videoQuality,+ alpha: 'keep',+ }),+ ]);+ if (!canEncodeBase || !canEncodeForeground) {+ throw new Error(+ 'This browser cannot encode the VP9 video streams required for video layer separation.',+ );+ }++ return {videoTrack, width, height};+};++export const separateVideoLayers = async (+ options: SeparateVideoLayersOptions,+): Promise<SeparateVideoLayersResult> => {+ validateOptions(options);+ throwIfAborted(options.signal);++ const model = options.model ?? 'modnet';+ const audio = options.audio ?? 'base';+ const videoQuality = resolveVideoMattingQuality(+ options.videoBitrate ?? 'very-high',+ );+ const audioQuality = resolveVideoMattingQuality(+ options.audioBitrate ?? 'medium',+ );+ const keyframeIntervalInSeconds = options.keyframeIntervalInSeconds ?? 1;+ const input = makeInput(options.src);+ const onInputAbort = () => input.dispose();+ options.signal?.addEventListener('abort', onInputAbort, {once: true});++ try {+ const {videoTrack, width, height} = await probeVideoInput({+ input,+ videoQuality,+ });+ const [inputFirstVideoTimestamp, videoEndTimestamp] = await Promise.all([+ videoTrack.getFirstTimestamp(),+ videoTrack.computeDuration(),+ ]);+ const videoStartTimestamp = Math.max(inputFirstVideoTimestamp, 0);+ if (videoEndTimestamp <= videoStartTimestamp) {+ throw new Error(+ 'The primary video track contains no presentable video duration.',+ );+ }++ const durationInSeconds = videoEndTimestamp - videoStartTimestamp;+ throwIfAborted(options.signal);++ const result = await withLoadedVideoMattingPipeline({+ model,+ onProgress: options.onModelLoadProgress,+ signal: options.signal ?? null,+ run: async (pipeline) => {+ throwIfAborted(options.signal);+ let iterator: AsyncGenerator<+ {+ canvas: HTMLCanvasElement | OffscreenCanvas;+ timestamp: number;+ duration: number;+ },+ void,+ unknown+ > | null = null;+ let baseOutput: Awaited<+ ReturnType<typeof createVideoLayerOutput<WebMOutputFormat>>+ > | null = null;+ let foregroundOutput: Awaited<+ ReturnType<typeof createVideoLayerOutput<WebMOutputFormat>>+ > | null = null;+ let baseVideoSource: CanvasSource | null = null;+ let foregroundVideoSource: CanvasSource | null = null;+ let audioWriter: PreparedVideoMattingAudio | null = null;+ let completed = false;+ let abortCleanupPromise: Promise<void> | null = null;++ const cancelPendingMedia = async () => {+ await Promise.allSettled([+ baseOutput?.cancel(),+ foregroundOutput?.cancel(),+ ]);+ await Promise.allSettled([audioWriter?.cancel()]);+ await Promise.allSettled([+ baseOutput?.discard(),+ foregroundOutput?.discard(),+ ]);+ };++ const onAbort = () => {+ abortCleanupPromise = cancelPendingMedia();+ };++ options.signal?.addEventListener('abort', onAbort, {once: true});++ try {+ const canvasSink = new CanvasSink(videoTrack, {+ alpha: true,+ width,+ height,+ fit: 'fill',+ poolSize: 1,+ });+ iterator = canvasSink.canvases(+ videoStartTimestamp,+ videoEndTimestamp,+ );+ let nextFrame = await iterator.next();+ if (nextFrame.done) {+ throw new Error(+ 'The primary video track contains no presentable decodable frames.',+ );+ }++ throwIfAborted(options.signal);++ const baseCanvas = createVideoMattingCanvas({width, height});+ const foregroundCanvas = createVideoMattingCanvas({width, height});+ const baseContext = getVideoMattingCanvasContext(baseCanvas);+ const foregroundContext =+ getVideoMattingCanvasContext(foregroundCanvas);++ baseOutput = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: options.outputs?.base,+ });+ throwIfAborted(options.signal);+ foregroundOutput = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: options.outputs?.foreground,+ });+ throwIfAborted(options.signal);++ baseVideoSource = new CanvasSource(baseCanvas, {+ codec: 'vp9',+ quality: videoQuality,+ keyFrameInterval: keyframeIntervalInSeconds,+ alpha: 'discard',+ });+ foregroundVideoSource = new CanvasSource(foregroundCanvas, {+ codec: 'vp9',+ quality: videoQuality,+ keyFrameInterval: keyframeIntervalInSeconds,+ alpha: 'keep',+ });+ baseOutput.output.addVideoTrack(baseVideoSource);+ foregroundOutput.output.addVideoTrack(foregroundVideoSource);++ audioWriter = await prepareAudio({+ input,+ baseOutput: baseOutput.output,+ foregroundOutput: foregroundOutput.output,+ destination: audio,+ videoStartTimestamp,+ videoEndTimestamp,+ audioQuality,+ forceTranscode: options.audioBitrate !== undefined,+ });+ throwIfAborted(options.signal);++ await Promise.all([+ baseOutput.output.start(),+ foregroundOutput.output.start(),+ ]);+ throwIfAborted(options.signal);+ await audioWriter.prime();+ throwIfAborted(options.signal);++ let processedFrames = 0;+ let processedDurationInSeconds = 0;+ options.onProgress?.({+ stage: 'processing',+ progress: 0,+ processedFrames,+ processedDurationInSeconds,+ durationInSeconds,+ });++ while (!nextFrame.done) {+ throwIfAborted(options.signal);+ const frame = nextFrame.value;+ const timing = getClippedVideoFrameTiming({+ timestamp: frame.timestamp,+ duration: frame.duration,+ videoStartTimestamp,+ videoEndTimestamp,+ });+ if (timing === null) {+ nextFrame = await iterator.next();+ continue;+ }++ const foregroundFrame = await pipeline(frame.canvas);+ throwIfAborted(options.signal);++ drawOpaqueBaseFrame({+ context: baseContext,+ source: frame.canvas,+ width,+ height,+ });+ drawForegroundFrame({+ context: foregroundContext,+ result: foregroundFrame,+ source: frame.canvas,+ targetWidth: width,+ targetHeight: height,+ });++ await Promise.all([+ baseVideoSource.add(timing.timestamp, timing.duration),+ foregroundVideoSource.add(timing.timestamp, timing.duration),+ audioWriter.writeAudioUntil(timing.timestamp + timing.duration),+ ]);++ processedFrames++;+ const processingProgress = getVideoProcessingProgress({+ timestamp: videoStartTimestamp + timing.timestamp,+ duration: timing.duration,+ firstVideoTimestamp: videoStartTimestamp,+ durationInSeconds,+ });+ processedDurationInSeconds =+ processingProgress.processedDurationInSeconds;+ options.onProgress?.({+ stage: 'processing',+ progress: processingProgress.progress,+ processedFrames,+ processedDurationInSeconds,+ durationInSeconds,+ });++ nextFrame = await iterator.next();+ }++ options.onProgress?.({+ stage: 'finalizing',+ progress: null,+ processedFrames,+ processedDurationInSeconds,+ durationInSeconds,+ });+ await audioWriter.finishAudio();+ baseVideoSource.close();+ foregroundVideoSource.close();+ const [base, foreground] = await Promise.all([+ baseOutput.finalize(),+ foregroundOutput.finalize(),+ ]);+ throwIfAborted(options.signal);++ completed = true;+ return {+ base,+ foreground,+ model,+ width,+ height,+ durationInSeconds,+ processedFrames,+ };+ } catch (error) {+ await cancelPendingMedia();++ if (options.signal?.aborted) {+ throw createAbortError(options.signal);+ }++ throw error;+ } finally {+ options.signal?.removeEventListener('abort', onAbort);+ await abortCleanupPromise;+ if (!completed) {+ try {+ await iterator?.return();+ } catch {+ // Cleanup must not replace the operation's original error.+ }+ }+ }+ },+ });++ return result;+ } catch (error) {+ if (options.signal?.aborted) {+ throw createAbortError(options.signal);+ }++ throw error;+ } finally {+ options.signal?.removeEventListener('abort', onInputAbort);+ input.dispose();+ }+};diff --git a/packages/video-matting/src/test/model-runtime.test.ts b/packages/video-matting/src/test/model-runtime.test.tsnew file mode 100644index 0000000000..ead9334813--- /dev/null+++ b/packages/video-matting/src/test/model-runtime.test.ts@@ -0,0 +1,363 @@+import {beforeEach, expect, mock, test} from 'bun:test';++let initializationCount = 0;+let initializationStarted: ((count: number) => void) | null = null;+let initializationGate: Promise<void> | null = null;+let initializationOptions: Record<string, unknown> | null = null;+let initializedModelId: string | null = null;+let pipelineConstructionCount = 0;+let pipelineRunCount = 0;+let livePipelineCount = 0;+let maxLivePipelineCount = 0;+let disposeCalls = 0;+let disposalStarted: ((pipeline: number) => void) | null = null;+let disposalGate: Promise<void> | null = null;+let cacheCheck:+ | {+ task: string;+ modelId: string;+ options: Record<string, unknown>;+ }+ | undefined;++function FakeBackgroundRemovalPipeline() {+ const pipeline = ++pipelineConstructionCount;+ livePipelineCount++;+ maxLivePipelineCount = Math.max(maxLivePipelineCount, livePipelineCount);++ return Object.assign(+ () => {+ pipelineRunCount++;+ return Promise.resolve({+ data: new Uint8ClampedArray([10, 20, 30, 128]),+ width: 1,+ height: 1,+ channels: 4,+ });+ },+ {+ dispose: async () => {+ disposeCalls++;+ disposalStarted?.(pipeline);+ await disposalGate;+ livePipelineCount--;+ },+ },+ );+}++mock.module('@huggingface/transformers', () => ({+ AutoModelForImageSegmentation: {+ from_pretrained: async (+ modelId: string,+ options: Record<string, unknown>,+ ) => {+ initializationCount++;+ initializedModelId = modelId;+ initializationOptions = options;+ initializationStarted?.(initializationCount);+ await initializationGate;++ const onProgress = options.progress_callback as+ | ((event: Record<string, unknown>) => void)+ | undefined;+ onProgress?.({+ status: 'progress',+ file: 'onnx/model.onnx',+ loaded: 25_888_640,+ });+ return {+ dispose: () => Promise.resolve(),+ };+ },+ },+ AutoProcessor: {+ from_pretrained: (_modelId: string, options: Record<string, unknown>) => {+ const onProgress = options.progress_callback as+ | ((event: Record<string, unknown>) => void)+ | undefined;+ onProgress?.({+ status: 'progress',+ file: 'preprocessor_config.json',+ loaded: 365,+ });+ return Promise.resolve({});+ },+ },+ BackgroundRemovalPipeline: FakeBackgroundRemovalPipeline,+ ModelRegistry: {+ is_pipeline_cached: (+ task: string,+ modelId: string,+ options: Record<string, unknown>,+ ) => {+ cacheCheck = {task, modelId, options};+ return Promise.resolve(true);+ },+ },+}));++const getRuntime = () => import('../load-video-matting-model');++beforeEach(async () => {+ disposalGate = null;+ initializationGate = null;+ const {disposeVideoMattingModel} = await getRuntime();+ await disposeVideoMattingModel();++ initializationCount = 0;+ initializationStarted = null;+ initializationGate = null;+ initializationOptions = null;+ initializedModelId = null;+ pipelineConstructionCount = 0;+ pipelineRunCount = 0;+ livePipelineCount = 0;+ maxLivePipelineCount = 0;+ disposeCalls = 0;+ disposalStarted = null;+ disposalGate = null;+ cacheCheck = undefined;+});++test('coalesces initialization and defers disposal until active work is done', async () => {+ const {+ disposeVideoMattingModel,+ loadVideoMattingModel,+ withLoadedVideoMattingPipeline,+ } = await getRuntime();+ const {isVideoMattingModelCached} =+ await import('../is-video-matting-model-cached');++ let releaseInitialization: () => void = () => undefined;+ initializationGate = new Promise<void>((resolve) => {+ releaseInitialization = resolve;+ });+ const started = new Promise<void>((resolve) => {+ initializationStarted = () => resolve();+ });+ const firstProgress: number[] = [];+ const firstLoadedBytes: number[] = [];+ const secondProgress: number[] = [];+ const first = loadVideoMattingModel({+ model: 'modnet',+ onProgress: ({loadedBytes, progress}) => {+ if (progress !== null) {+ firstProgress.push(progress);+ }++ if (loadedBytes !== null) {+ firstLoadedBytes.push(loadedBytes);+ }+ },+ });+ const second = loadVideoMattingModel({+ model: 'modnet',+ onProgress: ({progress}) => {+ if (progress !== null) {+ secondProgress.push(progress);+ }+ },+ });++ await started;+ expect(initializationCount).toBe(1);+ releaseInitialization();+ expect(await Promise.all([first, second])).toEqual([+ {alreadyLoaded: false},+ {alreadyLoaded: true},+ ]);+ expect(initializedModelId).toBe('Xenova/modnet');+ expect(initializationOptions).toMatchObject({+ device: 'webgpu',+ dtype: 'fp32',+ revision: 'fa2fa546052fba4c08921230a26cc69a333fca12',+ });+ expect(firstProgress.at(-1)).toBe(1);+ expect(secondProgress.at(-1)).toBe(1);+ expect(Math.max(...firstLoadedBytes)).toBe(25_888_640);++ expect(await isVideoMattingModelCached({model: 'modnet'})).toBe(true);+ expect(cacheCheck).toEqual({+ task: 'background-removal',+ modelId: 'Xenova/modnet',+ options: {+ device: 'webgpu',+ dtype: 'fp32',+ revision: 'fa2fa546052fba4c08921230a26cc69a333fca12',+ },+ });++ let releaseUse: () => void = () => undefined;+ const useGate = new Promise<void>((resolve) => {+ releaseUse = resolve;+ });+ let markUseStarted: () => void = () => undefined;+ const useStarted = new Promise<void>((resolve) => {+ markUseStarted = resolve;+ });+ const use = withLoadedVideoMattingPipeline({+ model: 'modnet',+ signal: null,+ run: async (pipeline) => {+ const result = await pipeline({} as OffscreenCanvas);+ markUseStarted();+ await useGate;+ return result;+ },+ });+ await useStarted;+ const disposal = disposeVideoMattingModel({model: 'modnet'});+ await Promise.resolve();+ expect(disposeCalls).toBe(0);++ releaseUse();+ expect(await use).toEqual({+ data: new Uint8ClampedArray([10, 20, 30, 128]),+ width: 1,+ height: 1,+ channels: 4,+ });+ await disposal;+ expect(disposeCalls).toBe(1);+ expect(livePipelineCount).toBe(0);+});++test('isolates throwing progress observers from shared loading', async () => {+ const {disposeVideoMattingModel, loadVideoMattingModel} = await getRuntime();++ let throwingObserverCalls = 0;+ const successfulProgress: number[] = [];+ const throwingLoad = loadVideoMattingModel({+ model: 'modnet',+ onProgress: () => {+ throwingObserverCalls++;+ throw new Error('observer failed');+ },+ });+ const successfulLoad = loadVideoMattingModel({+ model: 'modnet',+ onProgress: ({progress}) => {+ if (progress !== null) {+ successfulProgress.push(progress);+ }+ },+ });++ expect(await Promise.all([throwingLoad, successfulLoad])).toEqual([+ {alreadyLoaded: false},+ {alreadyLoaded: true},+ ]);+ expect(throwingObserverCalls).toBeGreaterThan(0);+ expect(successfulProgress.at(-1)).toBe(1);+ expect(initializationCount).toBe(1);+ expect(disposeCalls).toBe(0);++ let replayCalls = 0;+ expect(+ await loadVideoMattingModel({+ model: 'modnet',+ onProgress: () => {+ replayCalls++;+ throw new Error('replayed observer failed');+ },+ }),+ ).toEqual({alreadyLoaded: true});+ expect(replayCalls).toBe(1);++ await disposeVideoMattingModel({model: 'modnet'});+ expect(disposeCalls).toBe(1);+ expect(livePipelineCount).toBe(0);+});++test('aborts a caller waiting for shared initialization without canceling it', async () => {+ const {+ disposeVideoMattingModel,+ loadVideoMattingModel,+ withLoadedVideoMattingPipeline,+ } = await getRuntime();++ let releaseInitialization: () => void = () => undefined;+ initializationGate = new Promise<void>((resolve) => {+ releaseInitialization = resolve;+ });+ const started = new Promise<void>((resolve) => {+ initializationStarted = () => resolve();+ });+ const controller = new AbortController();+ const abortReason = new Error('stop waiting');+ let runCalls = 0;+ let progressCalls = 0;+ const use = withLoadedVideoMattingPipeline({+ model: 'modnet',+ onProgress: () => {+ progressCalls++;+ },+ signal: controller.signal,+ run: () => {+ runCalls++;+ return Promise.resolve();+ },+ });++ await started;+ const progressCallsBeforeAbort = progressCalls;+ controller.abort(abortReason);+ await expect(use).rejects.toBe(abortReason);+ expect(runCalls).toBe(0);++ const sharedLoad = loadVideoMattingModel({model: 'modnet'});+ releaseInitialization();+ expect(await sharedLoad).toEqual({alreadyLoaded: true});+ expect(initializationCount).toBe(1);+ expect(progressCalls).toBe(progressCallsBeforeAbort);++ const result = await withLoadedVideoMattingPipeline({+ model: 'modnet',+ signal: null,+ run: (pipeline) => pipeline({} as OffscreenCanvas),+ });+ expect(result.data).toEqual(new Uint8ClampedArray([10, 20, 30, 128]));+ expect(pipelineRunCount).toBe(1);++ await disposeVideoMattingModel({model: 'modnet'});+ expect(disposeCalls).toBe(1);+ expect(livePipelineCount).toBe(0);+});++test('waits for disposal before constructing a replacement pipeline', async () => {+ const {disposeVideoMattingModel, loadVideoMattingModel} = await getRuntime();++ await loadVideoMattingModel({model: 'modnet'});+ expect(initializationCount).toBe(1);+ expect(livePipelineCount).toBe(1);++ let releaseDisposal: () => void = () => undefined;+ disposalGate = new Promise<void>((resolve) => {+ releaseDisposal = resolve;+ });+ const disposalHasStarted = new Promise<void>((resolve) => {+ disposalStarted = () => resolve();+ });+ const disposal = disposeVideoMattingModel({model: 'modnet'});+ await disposalHasStarted;++ const replacement = loadVideoMattingModel({model: 'modnet'});+ await new Promise<void>((resolve) => {+ setTimeout(resolve, 0);+ });+ expect(initializationCount).toBe(1);+ expect(pipelineConstructionCount).toBe(1);++ releaseDisposal();+ await Promise.all([disposal, replacement]);+ expect(initializationCount).toBe(2);+ expect(pipelineConstructionCount).toBe(2);+ expect(maxLivePipelineCount).toBe(1);+ expect(livePipelineCount).toBe(1);++ disposalGate = null;+ await disposeVideoMattingModel({model: 'modnet'});+ expect(disposeCalls).toBe(2);+ expect(livePipelineCount).toBe(0);+});diff --git a/packages/video-matting/src/test/models.test.ts b/packages/video-matting/src/test/models.test.tsnew file mode 100644index 0000000000..d0b179a4b1--- /dev/null+++ b/packages/video-matting/src/test/models.test.ts@@ -0,0 +1,159 @@+import {expect, test} from 'bun:test';+import {+ canUseVideoMatting,+ VideoMattingUnsupportedReason,+} from '../can-use-video-matting';+import {+ getAvailableVideoMattingModels,+ getVideoMattingModelInfo,+} from '../models';++test('returns public metadata without mutable internal model settings', () => {+ const models = getAvailableVideoMattingModels();++ expect(models).toEqual([+ {+ name: 'modnet',+ modelId: 'Xenova/modnet',+ purpose: 'person',+ experimental: false,+ webGpuDownloadSize: 25_888_640,+ },+ {+ name: 'ben2-base',+ modelId: 'onnx-community/BEN2-ONNX',+ purpose: 'general',+ experimental: true,+ webGpuDownloadSize: 219_121_675,+ },+ ]);++ models[0].experimental = true;+ expect(getAvailableVideoMattingModels()[0].experimental).toBe(false);+});++test('rejects unsupported runtime model values', () => {+ expect(() => getVideoMattingModelInfo('not-a-model' as never)).toThrow(+ 'Unsupported video matting model "not-a-model". Available models: modnet, ben2-base.',+ );+ expect(() => getVideoMattingModelInfo('toString' as never)).toThrow(+ 'Unsupported video matting model "toString". Available models: modnet, ben2-base.',+ );+});++test('requires shader-f16 only for the fp16 BEN2 model', async () => {+ const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');+ const originalNavigator = Object.getOwnPropertyDescriptor(+ globalThis,+ 'navigator',+ );+ const features = new Set<string>();+ Object.defineProperty(globalThis, 'window', {+ configurable: true,+ value: {isSecureContext: true},+ });+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ value: {+ gpu: {+ requestAdapter: () => Promise.resolve({features}),+ },+ },+ });++ try {+ expect(await canUseVideoMatting({model: 'modnet'})).toEqual({+ supported: true,+ });+ expect(await canUseVideoMatting({model: 'ben2-base'})).toEqual({+ supported: false,+ reason: VideoMattingUnsupportedReason.ShaderF16Unavailable,+ detailedReason:+ 'The video matting model "ben2-base" requires the WebGPU shader-f16 feature, which is unavailable on this adapter.',+ });++ features.add('shader-f16');+ expect(await canUseVideoMatting({model: 'ben2-base'})).toEqual({+ supported: true,+ });+ } finally {+ if (originalWindow) {+ Object.defineProperty(globalThis, 'window', originalWindow);+ } else {+ delete (globalThis as {window?: unknown}).window;+ }++ if (originalNavigator) {+ Object.defineProperty(globalThis, 'navigator', originalNavigator);+ } else {+ delete (globalThis as {navigator?: unknown}).navigator;+ }+ }+});++test('supports a secure worker with OffscreenCanvas and WebGPU', async () => {+ const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');+ const originalNavigator = Object.getOwnPropertyDescriptor(+ globalThis,+ 'navigator',+ );+ const originalOffscreenCanvas = Object.getOwnPropertyDescriptor(+ globalThis,+ 'OffscreenCanvas',+ );+ const originalSecureContext = Object.getOwnPropertyDescriptor(+ globalThis,+ 'isSecureContext',+ );+ delete (globalThis as {window?: unknown}).window;+ Object.defineProperty(globalThis, 'OffscreenCanvas', {+ configurable: true,+ value: class FakeOffscreenCanvas {},+ });+ Object.defineProperty(globalThis, 'isSecureContext', {+ configurable: true,+ value: true,+ });+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ value: {+ gpu: {+ requestAdapter: () => Promise.resolve({features: new Set<string>()}),+ },+ },+ });++ try {+ expect(await canUseVideoMatting()).toEqual({supported: true});+ } finally {+ if (originalWindow) {+ Object.defineProperty(globalThis, 'window', originalWindow);+ }++ if (originalNavigator) {+ Object.defineProperty(globalThis, 'navigator', originalNavigator);+ } else {+ delete (globalThis as {navigator?: unknown}).navigator;+ }++ if (originalOffscreenCanvas) {+ Object.defineProperty(+ globalThis,+ 'OffscreenCanvas',+ originalOffscreenCanvas,+ );+ } else {+ delete (globalThis as {OffscreenCanvas?: unknown}).OffscreenCanvas;+ }++ if (originalSecureContext) {+ Object.defineProperty(+ globalThis,+ 'isSecureContext',+ originalSecureContext,+ );+ } else {+ delete (globalThis as {isSecureContext?: unknown}).isSecureContext;+ }+ }+});diff --git a/packages/video-matting/src/test/output-target.test.ts b/packages/video-matting/src/test/output-target.test.tsnew file mode 100644index 0000000000..5728095fd5--- /dev/null+++ b/packages/video-matting/src/test/output-target.test.ts@@ -0,0 +1,532 @@+import {afterEach, describe, expect, test} from 'bun:test';+import {Output, WebMOutputFormat, type StreamTargetChunk} from 'mediabunny';+import {createVideoLayerOutput} from '../create-video-layer-output';+import type {VideoLayerOutputOptions} from '../output-target';++const originalNavigator = globalThis.navigator;++type FakeFile = {+ bytes: Uint8Array<ArrayBuffer>;+ position: number;+ closed: boolean;+ aborted: boolean;+ seekPositions: number[];+};++const makeFakeFile = (): FakeFile => ({+ bytes: new Uint8Array(),+ position: 0,+ closed: false,+ aborted: false,+ seekPositions: [],+});++const makeFakeOpfs = ({+ createWritableError,+ closeError,+ abortError,+}: {+ createWritableError?: Error;+ closeError?: Error;+ abortError?: Error;+} = {}) => {+ const files = new Map<string, FakeFile>();+ const removedNames: string[] = [];++ const getFileHandle = (+ name: string,+ options?: FileSystemGetFileOptions,+ ): Promise<FileSystemFileHandle> => {+ if (!files.has(name)) {+ if (!options?.create) {+ throw new DOMException('File not found', 'NotFoundError');+ }++ files.set(name, makeFakeFile());+ }++ const file = files.get(name)!;+ return Promise.resolve({+ kind: 'file',+ name,+ isSameEntry: () => Promise.resolve(false),+ createWritable: () => {+ if (createWritableError !== undefined) {+ return Promise.reject(createWritableError);+ }++ return Promise.resolve({+ seek: (position: number) => {+ file.position = position;+ file.seekPositions.push(position);+ return Promise.resolve();+ },+ write: (chunk: StreamTargetChunk) => {+ const end = file.position + chunk.data.byteLength;+ if (end > file.bytes.byteLength) {+ const resized = new Uint8Array(end);+ resized.set(file.bytes);+ file.bytes = resized;+ }++ file.bytes.set(chunk.data, file.position);+ file.position = end;+ return Promise.resolve();+ },+ close: () => {+ file.closed = true;+ return closeError === undefined+ ? Promise.resolve()+ : Promise.reject(closeError);+ },+ abort: () => {+ file.aborted = true;+ return abortError === undefined+ ? Promise.resolve()+ : Promise.reject(abortError);+ },+ } as unknown as FileSystemWritableFileStream);+ },+ getFile: () => Promise.resolve(new File([file.bytes], name)),+ } as FileSystemFileHandle);+ };++ const directory = {+ kind: 'directory',+ name: '',+ isSameEntry: () => Promise.resolve(false),+ getFileHandle,+ removeEntry: (name: string) => {+ removedNames.push(name);+ if (!files.delete(name)) {+ return Promise.reject(+ new DOMException('File not found', 'NotFoundError'),+ );+ }++ return Promise.resolve();+ },+ async *entries() {+ for (const name of files.keys()) {+ yield [name, await getFileHandle(name)] as const;+ }+ },+ } as unknown as FileSystemDirectoryHandle;++ const seedFile = (name: string) => files.set(name, makeFakeFile());++ return {directory, files, removedNames, seedFile};+};++afterEach(() => {+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ enumerable: true,+ value: originalNavigator,+ writable: true,+ });+});++describe('video layer output targets', () => {+ test('creates an in-memory Blob result', async () => {+ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputTarget: 'arraybuffer'},+ });++ await created.output.start();+ const result = await created.finalize();+ const blob = await result.getBlob();+ await result.dispose();+ await result.dispose();++ expect(blob.type).toBe('application/webm');+ expect(blob.size).toBeGreaterThan(0);+ });++ test('writes to and closes an outputWritable', async () => {+ const chunks: StreamTargetChunk[] = [];+ let closed = false;+ const outputWritable = new WritableStream<StreamTargetChunk>({+ write: (chunk) => {+ chunks.push(chunk);+ },+ close: () => {+ closed = true;+ },+ });+ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputWritable},+ });++ await created.output.start();+ const result = await created.finalize();++ expect(chunks.length).toBeGreaterThan(0);+ expect(closed).toBe(true);+ await expect(result.getBlob()).rejects.toThrow(+ 'getBlob() is unavailable when outputWritable is used',+ );+ await result.dispose();+ await result.dispose();+ });++ test('rejects finalization when cancellation is requested during stream close', async () => {+ let cancelOutput: (() => Promise<void>) | null = null;+ let cancellationPromise: Promise<void> | null = null;+ const outputWritable = new WritableStream<StreamTargetChunk>({+ close: () => {+ if (cancelOutput === null) {+ throw new Error('Expected output cancellation to be initialized');+ }++ cancellationPromise = cancelOutput();+ },+ });+ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputWritable},+ });+ cancelOutput = created.cancel;++ await created.output.start();+ await expect(created.finalize()).rejects.toThrow(+ 'Video layer output was canceled',+ );+ await Promise.allSettled([cancellationPromise]);+ });++ test('aborts an outputWritable when canceled before starting', async () => {+ let closed = false;+ let abortReason: unknown = null;+ const outputWritable = new WritableStream<StreamTargetChunk>({+ close: () => {+ closed = true;+ },+ abort: (reason) => {+ abortReason = reason;+ },+ });+ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputWritable},+ });++ await created.cancel();+ expect(closed).toBe(false);+ expect(abortReason).toBeInstanceOf(Error);+ expect((abortReason as Error).message).toBe(+ 'Video layer output was canceled',+ );+ });++ test('aborts an outputWritable when canceled after starting', async () => {+ let closed = false;+ let abortReason: unknown = null;+ const outputWritable = new WritableStream<StreamTargetChunk>({+ close: () => {+ closed = true;+ },+ abort: (reason) => {+ abortReason = reason;+ },+ });+ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputWritable},+ });+ await created.output.start();++ await created.cancel();+ expect(closed).toBe(false);+ expect(abortReason).toBeInstanceOf(Error);+ expect((abortReason as Error).message).toBe(+ 'Video layer output was canceled',+ );+ });++ test('aborts an outputWritable when discarded after starting', async () => {+ let closed = false;+ let abortReason: unknown = null;+ const outputWritable = new WritableStream<StreamTargetChunk>({+ close: () => {+ closed = true;+ },+ abort: (reason) => {+ abortReason = reason;+ },+ });+ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputWritable},+ });+ await created.output.start();++ await created.discard();+ expect(closed).toBe(false);+ expect(abortReason).toBeInstanceOf(Error);+ expect((abortReason as Error).message).toBe(+ 'Video layer output was discarded',+ );+ });++ test('uses OPFS automatically and honors positional writes', async () => {+ const {directory, files, removedNames} = makeFakeOpfs();+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ enumerable: true,+ value: {storage: {getDirectory: () => Promise.resolve(directory)}},+ writable: true,+ });++ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: undefined,+ });+ await created.output.start();+ const result = await created.finalize();+ const blob = await result.getBlob();+ const [filename, file] = [...files.entries()][0];++ expect(files.size).toBe(1);+ expect(file.closed).toBe(true);+ expect(file.seekPositions.length).toBeGreaterThan(0);+ expect(blob.size).toBeGreaterThan(0);+ expect(blob.type).toBe('application/webm');+ const bytesBeforeDisposal = new Uint8Array(await blob.arrayBuffer());++ await Promise.all([result.dispose(), result.dispose()]);+ expect(files.size).toBe(0);+ expect(removedNames.filter((name) => name === filename)).toHaveLength(1);+ const blobAfterDisposal = await result.getBlob();+ expect(blobAfterDisposal).toBe(blob);+ expect(new Uint8Array(await blobAfterDisposal.arrayBuffer())).toEqual(+ bytesBeforeDisposal,+ );+ });++ test('does not remove an OPFS output owned by another session', async () => {+ const {directory, files, seedFile} = makeFakeOpfs();+ const otherSessionFilename =+ '__remotion_video_matting:another-session:existing-output';+ seedFile(otherSessionFilename);+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ enumerable: true,+ value: {storage: {getDirectory: () => Promise.resolve(directory)}},+ writable: true,+ });++ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputTarget: 'web-fs'},+ });++ expect(files.has(otherSessionFilename)).toBe(true);+ expect(files.size).toBe(2);+ await created.cancel();+ expect(files.has(otherSessionFilename)).toBe(true);+ expect(files.size).toBe(1);+ });++ test('can discard a finalized OPFS output internally', async () => {+ const {directory, files, removedNames} = makeFakeOpfs();+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ enumerable: true,+ value: {storage: {getDirectory: () => Promise.resolve(directory)}},+ writable: true,+ });++ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputTarget: 'web-fs'},+ });+ await created.output.start();+ await created.finalize();+ const [filename] = files.keys();++ await created.discard();+ await created.discard();+ expect(files.size).toBe(0);+ expect(removedNames.filter((name) => name === filename)).toHaveLength(1);+ });++ test('can retry OPFS disposal after a transient removal failure', async () => {+ const {directory, files} = makeFakeOpfs();+ const removeEntry = directory.removeEntry.bind(directory);+ let removalAttempts = 0;+ Object.defineProperty(directory, 'removeEntry', {+ value: (name: string) => {+ removalAttempts++;+ if (removalAttempts === 1) {+ return Promise.reject(new Error('temporary removal failure'));+ }++ return removeEntry(name);+ },+ });+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ enumerable: true,+ value: {storage: {getDirectory: () => Promise.resolve(directory)}},+ writable: true,+ });++ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputTarget: 'web-fs'},+ });+ await created.output.start();+ const result = await created.finalize();++ await expect(result.dispose()).rejects.toThrow('temporary removal failure');+ expect(files.size).toBe(1);+ await expect(result.dispose()).resolves.toBeUndefined();+ expect(files.size).toBe(0);+ expect(removalAttempts).toBe(2);+ });++ test('removes a partial OPFS output when canceled', async () => {+ const {directory, files} = makeFakeOpfs();+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ enumerable: true,+ value: {storage: {getDirectory: () => Promise.resolve(directory)}},+ writable: true,+ });++ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputTarget: 'web-fs'},+ });+ await created.output.start();+ expect(files.size).toBe(1);++ await created.cancel();+ expect(files.size).toBe(0);+ });++ test('removes an OPFS output even when Mediabunny cancellation fails', async () => {+ const {directory, files} = makeFakeOpfs();+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ enumerable: true,+ value: {storage: {getDirectory: () => Promise.resolve(directory)}},+ writable: true,+ });++ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputTarget: 'web-fs'},+ });+ Object.defineProperty(created.output, 'cancel', {+ value: () => Promise.reject(new Error('cancel failed')),+ });++ await expect(created.cancel()).rejects.toThrow('cancel failed');+ expect(files.size).toBe(0);+ });++ test('removes a partial OPFS output when finalization fails', async () => {+ const {directory, files} = makeFakeOpfs();+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ enumerable: true,+ value: {storage: {getDirectory: () => Promise.resolve(directory)}},+ writable: true,+ });++ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputTarget: 'web-fs'},+ });+ expect(files.size).toBe(1);++ await expect(created.finalize()).rejects.toThrow(+ 'Cannot finalize before starting',+ );+ expect(files.size).toBe(0);+ });++ test('removes an OPFS output when close and cancellation both fail', async () => {+ const {directory, files} = makeFakeOpfs({+ closeError: new Error('close failed'),+ });+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ enumerable: true,+ value: {storage: {getDirectory: () => Promise.resolve(directory)}},+ writable: true,+ });++ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputTarget: 'web-fs'},+ });+ await created.output.start();+ Object.defineProperty(created.output, 'cancel', {+ value: () => Promise.reject(new Error('cancel failed')),+ });++ await expect(created.finalize()).rejects.toThrow('close failed');+ expect(files.size).toBe(0);+ });++ test('removes the created OPFS entry if createWritable fails', async () => {+ const {directory, files} = makeFakeOpfs({+ createWritableError: new Error('createWritable failed'),+ });+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ enumerable: true,+ value: {storage: {getDirectory: () => Promise.resolve(directory)}},+ writable: true,+ });++ await expect(+ createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputTarget: 'web-fs'},+ }),+ ).rejects.toThrow('createWritable failed');+ expect(files.size).toBe(0);+ });++ test('rejects conflicting destinations at runtime', async () => {+ await expect(+ createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {+ outputTarget: 'arraybuffer',+ outputWritable: new WritableStream<StreamTargetChunk>(),+ } as unknown as VideoLayerOutputOptions,+ }),+ ).rejects.toThrow(+ 'outputTarget and outputWritable cannot both be specified for a video layer',+ );+ });+});++const acceptOutputOptions = (_options: VideoLayerOutputOptions) => undefined;++acceptOutputOptions({outputTarget: 'arraybuffer'});+acceptOutputOptions({+ outputWritable: new WritableStream<StreamTargetChunk>(),+});+// @ts-expect-error outputTarget and outputWritable are mutually exclusive.+acceptOutputOptions({+ outputTarget: 'web-fs',+ outputWritable: new WritableStream<StreamTargetChunk>(),+});++test('the created output exposes the Mediabunny Output primitive', async () => {+ const created = await createVideoLayerOutput({+ format: new WebMOutputFormat(),+ options: {outputTarget: 'arraybuffer'},+ });++ expect(created.output).toBeInstanceOf(Output);+ await created.cancel();+});diff --git a/packages/video-matting/src/test/prepare-audio.test.ts b/packages/video-matting/src/test/prepare-audio.test.tsnew file mode 100644index 0000000000..a6a6c52cd1--- /dev/null+++ b/packages/video-matting/src/test/prepare-audio.test.ts@@ -0,0 +1,255 @@+import {expect, test} from 'bun:test';+import {+ ALL_FORMATS,+ BlobSource,+ BufferSource,+ BufferTarget,+ EncodedPacket,+ EncodedVideoPacketSource,+ Input,+ Output,+ StreamTarget,+ type StreamTargetChunk,+ WebMOutputFormat,+} from 'mediabunny';+import {prepareAudio} from '../prepare-audio';++test('packet-copies Opus audio to both outputs and rebases its timestamps', async () => {+ const fixture = Bun.file(+ new URL('../../../example/public/opus.webm', import.meta.url),+ );+ const input = new Input({+ formats: ALL_FORMATS,+ source: new BlobSource(fixture),+ });+ const inputAudioTrack = await input.getPrimaryAudioTrack();+ expect(inputAudioTrack).not.toBeNull();+ const audioStartTimestamp = await inputAudioTrack!.getFirstTimestamp();++ const baseTarget = new BufferTarget();+ const foregroundTarget = new BufferTarget();+ const baseOutput = new Output({+ format: new WebMOutputFormat(),+ target: baseTarget,+ });+ const foregroundOutput = new Output({+ format: new WebMOutputFormat(),+ target: foregroundTarget,+ });+ const audio = await prepareAudio({+ input,+ baseOutput,+ foregroundOutput,+ destination: 'both',+ videoStartTimestamp: audioStartTimestamp,+ videoEndTimestamp: audioStartTimestamp + 0.25,+ audioQuality: null,+ forceTranscode: false,+ });++ await Promise.all([baseOutput.start(), foregroundOutput.start()]);+ await audio.prime();+ await audio.writeAudioUntil(0);+ await audio.finishAudio();+ await Promise.all([baseOutput.finalize(), foregroundOutput.finalize()]);++ expect(baseTarget.buffer).not.toBeNull();+ expect(foregroundTarget.buffer).not.toBeNull();+ for (const buffer of [baseTarget.buffer!, foregroundTarget.buffer!]) {+ const outputInput = new Input({+ formats: ALL_FORMATS,+ source: new BufferSource(buffer),+ });+ const outputAudioTrack = await outputInput.getPrimaryAudioTrack();+ expect(await outputAudioTrack?.getCodec()).toBe('opus');+ expect(await outputAudioTrack?.getFirstTimestamp()).toBe(0);+ outputInput.dispose();+ }++ input.dispose();+});++test('primes delayed audio so queued video data can be muxed', async () => {+ const fixture = Bun.file(+ new URL('../../../example/public/opus.webm', import.meta.url),+ );+ const input = new Input({+ formats: ALL_FORMATS,+ source: new BlobSource(fixture),+ });+ const inputAudioTrack = await input.getPrimaryAudioTrack();+ expect(inputAudioTrack).not.toBeNull();+ const audioStartTimestamp = await inputAudioTrack!.getFirstTimestamp();++ let bytesWritten = 0;+ const baseOutput = new Output({+ format: new WebMOutputFormat(),+ target: new StreamTarget(+ new WritableStream<StreamTargetChunk>({+ write: (chunk) => {+ bytesWritten += chunk.data.byteLength;+ },+ }),+ ),+ });+ const foregroundOutput = new Output({+ format: new WebMOutputFormat(),+ target: new BufferTarget(),+ });+ const videoSource = new EncodedVideoPacketSource('vp9');+ const videoDecoderConfig: VideoDecoderConfig = {+ codec: 'vp09.00.10.08',+ codedWidth: 2,+ codedHeight: 2,+ };+ baseOutput.addVideoTrack(videoSource, {+ decoderConfig: videoDecoderConfig,+ });+ const audio = await prepareAudio({+ input,+ baseOutput,+ foregroundOutput,+ destination: 'base',+ videoStartTimestamp: audioStartTimestamp - 1,+ videoEndTimestamp: audioStartTimestamp + 0.25,+ audioQuality: null,+ forceTranscode: false,+ });++ await baseOutput.start();+ await videoSource.add(+ new EncodedPacket(new Uint8Array([0]), 'key', 0, 0.04),+ {decoderConfig: videoDecoderConfig},+ );+ await Promise.resolve();+ const bytesWrittenBeforePrime = bytesWritten;++ await audio.prime();+ await Promise.resolve();++ expect(bytesWritten).toBeGreaterThan(bytesWrittenBeforePrime);+ await audio.cancel();+ await Promise.all([baseOutput.cancel(), foregroundOutput.cancel()]);+ input.dispose();+});++test('cancel is idempotent before the outputs start', async () => {+ const fixture = Bun.file(+ new URL('../../../example/public/opus.webm', import.meta.url),+ );+ const input = new Input({+ formats: ALL_FORMATS,+ source: new BlobSource(fixture),+ });+ const inputAudioTrack = await input.getPrimaryAudioTrack();+ expect(inputAudioTrack).not.toBeNull();+ const audioStartTimestamp = await inputAudioTrack!.getFirstTimestamp();+ const baseOutput = new Output({+ format: new WebMOutputFormat(),+ target: new BufferTarget(),+ });+ const foregroundOutput = new Output({+ format: new WebMOutputFormat(),+ target: new BufferTarget(),+ });+ const audio = await prepareAudio({+ input,+ baseOutput,+ foregroundOutput,+ destination: 'both',+ videoStartTimestamp: audioStartTimestamp,+ videoEndTimestamp: audioStartTimestamp + 0.25,+ audioQuality: null,+ forceTranscode: false,+ });++ await audio.cancel();+ await audio.cancel();+ await Promise.all([baseOutput.cancel(), foregroundOutput.cancel()]);++ expect(baseOutput.state).toBe('canceled');+ expect(foregroundOutput.state).toBe('canceled');+ input.dispose();+});++test('does not attach audio when it does not overlap the video', async () => {+ const fixture = Bun.file(+ new URL('../../../example/public/opus.webm', import.meta.url),+ );+ const input = new Input({+ formats: ALL_FORMATS,+ source: new BlobSource(fixture),+ });+ const inputAudioTrack = await input.getPrimaryAudioTrack();+ expect(inputAudioTrack).not.toBeNull();+ const audioEndTimestamp = await inputAudioTrack!.computeDuration();+ const baseOutput = new Output({+ format: new WebMOutputFormat(),+ target: new BufferTarget(),+ });+ const foregroundOutput = new Output({+ format: new WebMOutputFormat(),+ target: new BufferTarget(),+ });+ const audio = await prepareAudio({+ input,+ baseOutput,+ foregroundOutput,+ destination: 'both',+ videoStartTimestamp: audioEndTimestamp + 1,+ videoEndTimestamp: audioEndTimestamp + 2,+ audioQuality: null,+ forceTranscode: false,+ });++ expect(baseOutput.tracks).toHaveLength(0);+ expect(foregroundOutput.tracks).toHaveLength(0);+ await audio.prime();+ await audio.finishAudio();+ await Promise.all([baseOutput.cancel(), foregroundOutput.cancel()]);+ input.dispose();+});++test('explicit bitrate requests force transcoding instead of Opus packet copy', async () => {+ const fixture = Bun.file(+ new URL('../../../example/public/opus.webm', import.meta.url),+ );+ const input = new Input({+ formats: ALL_FORMATS,+ source: new BlobSource(fixture),+ });+ const inputAudioTrack = await input.getPrimaryAudioTrack();+ expect(inputAudioTrack).not.toBeNull();+ const audioStartTimestamp = await inputAudioTrack!.getFirstTimestamp();+ let decodeChecks = 0;+ Object.defineProperty(inputAudioTrack, 'canDecode', {+ value: () => {+ decodeChecks++;+ return Promise.resolve(false);+ },+ });+ const baseOutput = new Output({+ format: new WebMOutputFormat(),+ target: new BufferTarget(),+ });+ const foregroundOutput = new Output({+ format: new WebMOutputFormat(),+ target: new BufferTarget(),+ });++ await expect(+ prepareAudio({+ input,+ baseOutput,+ foregroundOutput,+ destination: 'base',+ videoStartTimestamp: audioStartTimestamp,+ videoEndTimestamp: audioStartTimestamp + 0.25,+ audioQuality: null,+ forceTranscode: true,+ }),+ ).rejects.toThrow('cannot be decoded');+ expect(decodeChecks).toBe(1);+ await Promise.all([baseOutput.cancel(), foregroundOutput.cancel()]);+ input.dispose();+});diff --git a/packages/video-matting/src/test/separate-video-layers-options.test.ts b/packages/video-matting/src/test/separate-video-layers-options.test.tsnew file mode 100644index 0000000000..d6ae689607--- /dev/null+++ b/packages/video-matting/src/test/separate-video-layers-options.test.ts@@ -0,0 +1,98 @@+import {describe, expect, test} from 'bun:test';+import {separateVideoLayers} from '../separate-video-layers';++describe('separateVideoLayers() option validation', () => {+ test('rejects an invalid audio destination before loading a model', async () => {+ await expect(+ separateVideoLayers({+ src: 'video.mp4',+ audio: 'background' as never,+ }),+ ).rejects.toThrow('audio must be one of base, foreground, both, or none.');+ });++ test('rejects invalid encoding options before loading a model', async () => {+ await expect(+ separateVideoLayers({+ src: 'video.mp4',+ videoBitrate: 0,+ }),+ ).rejects.toThrow('A numeric bitrate must be a positive integer.');++ await expect(+ separateVideoLayers({+ src: 'video.mp4',+ keyframeIntervalInSeconds: 0,+ }),+ ).rejects.toThrow(+ 'keyframeIntervalInSeconds must be a positive finite number.',+ );+ });++ test('rejects conflicting layer destinations before loading a model', async () => {+ await expect(+ separateVideoLayers({+ src: 'video.mp4',+ outputs: {+ base: {+ outputTarget: 'arraybuffer',+ outputWritable: new WritableStream(),+ } as never,+ },+ }),+ ).rejects.toThrow(+ 'outputs.base cannot specify both outputTarget and outputWritable.',+ );+ });++ test('rejects invalid, locked, and shared output streams before loading a model', async () => {+ await expect(+ separateVideoLayers({+ src: 'video.mp4',+ outputs: {+ base: {outputWritable: {} as WritableStream},+ },+ }),+ ).rejects.toThrow('outputs.base.outputWritable must be a WritableStream.');++ const locked = new WritableStream();+ const writer = locked.getWriter();+ try {+ await expect(+ separateVideoLayers({+ src: 'video.mp4',+ outputs: {base: {outputWritable: locked}},+ }),+ ).rejects.toThrow(+ 'outputs.base.outputWritable must not already be locked.',+ );+ } finally {+ writer.releaseLock();+ }++ const shared = new WritableStream();+ await expect(+ separateVideoLayers({+ src: 'video.mp4',+ outputs: {+ base: {outputWritable: shared},+ foreground: {outputWritable: shared},+ },+ }),+ ).rejects.toThrow(+ 'outputs.base and outputs.foreground must not use the same outputWritable.',+ );+ });++ test('honors an already-aborted signal before loading a model', async () => {+ const controller = new AbortController();+ controller.abort(new Error('stop now'));++ await expect(+ separateVideoLayers({+ src: 'video.mp4',+ signal: controller.signal,+ }),+ ).rejects.toThrow('stop now');+ });+});diff --git a/packages/video-matting/src/test/video-matting-quality.test.ts b/packages/video-matting/src/test/video-matting-quality.test.tsnew file mode 100644index 0000000000..dd12995107--- /dev/null+++ b/packages/video-matting/src/test/video-matting-quality.test.ts@@ -0,0 +1,18 @@+import {describe, expect, test} from 'bun:test';+import {resolveVideoMattingQuality} from '../video-matting-quality';++describe('video matting quality', () => {+ test('accepts named qualities and bitrates', () => {+ expect(resolveVideoMattingQuality('very-high')).toBeDefined();+ expect(resolveVideoMattingQuality(2_000_000)).toBeDefined();+ });++ test('rejects invalid numeric bitrates', () => {+ expect(() => resolveVideoMattingQuality(0)).toThrow(+ 'A numeric bitrate must be a positive integer.',+ );+ expect(() => resolveVideoMattingQuality(1.5)).toThrow(+ 'A numeric bitrate must be a positive integer.',+ );+ });+});diff --git a/packages/video-matting/src/test/video-timing.test.ts b/packages/video-matting/src/test/video-timing.test.tsnew file mode 100644index 0000000000..4a363fbf67--- /dev/null+++ b/packages/video-matting/src/test/video-timing.test.ts@@ -0,0 +1,81 @@+import {describe, expect, test} from 'bun:test';+import {+ getClippedVideoFrameTiming,+ getVideoProcessingProgress,+ rebaseVideoTimestamp,+} from '../video-timing';++describe('video timing', () => {+ test('rebases a non-zero input timeline without rounding', () => {+ expect(+ rebaseVideoTimestamp({+ timestamp: 4.979,+ firstVideoTimestamp: 4.045,+ }),+ ).toBeCloseTo(0.934, 12);+ });++ test('uses frame timestamps and durations for VFR progress', () => {+ expect(+ getVideoProcessingProgress({+ timestamp: 10.125,+ duration: 0.125,+ firstVideoTimestamp: 10,+ durationInSeconds: 1,+ }),+ ).toEqual({processedDurationInSeconds: 0.25, progress: 0.25});+ });++ test('clamps the final frame to the video duration', () => {+ expect(+ getVideoProcessingProgress({+ timestamp: 2.9,+ duration: 0.2,+ firstVideoTimestamp: 2,+ durationInSeconds: 1,+ }),+ ).toEqual({processedDurationInSeconds: 1, progress: 1});+ });++ test('discards frames outside the presentation range', () => {+ expect(+ getClippedVideoFrameTiming({+ timestamp: -0.5,+ duration: 0.25,+ videoStartTimestamp: 0,+ videoEndTimestamp: 1,+ }),+ ).toBeNull();+ });++ test('clips frames crossing the start and end boundaries', () => {+ expect(+ getClippedVideoFrameTiming({+ timestamp: -0.25,+ duration: 0.5,+ videoStartTimestamp: 0,+ videoEndTimestamp: 1,+ }),+ ).toEqual({timestamp: 0, duration: 0.25});++ expect(+ getClippedVideoFrameTiming({+ timestamp: 0.875,+ duration: 0.25,+ videoStartTimestamp: 0,+ videoEndTimestamp: 1,+ }),+ ).toEqual({timestamp: 0.875, duration: 0.125});+ });++ test('preserves a frame inside a positive VFR presentation range', () => {+ expect(+ getClippedVideoFrameTiming({+ timestamp: 4.25,+ duration: 0.125,+ videoStartTimestamp: 4,+ videoEndTimestamp: 5,+ }),+ ).toEqual({timestamp: 0.25, duration: 0.125});+ });+});diff --git a/packages/video-matting/src/video-matting-canvas.ts b/packages/video-matting/src/video-matting-canvas.tsnew file mode 100644index 0000000000..f33535c176--- /dev/null+++ b/packages/video-matting/src/video-matting-canvas.ts@@ -0,0 +1,112 @@+import type {VideoMattingPipelineResult} from './load-video-matting-model';++export type VideoMattingCanvas = HTMLCanvasElement | OffscreenCanvas;++type VideoMattingCanvasContext =+ | CanvasRenderingContext2D+ | OffscreenCanvasRenderingContext2D;++export const createVideoMattingCanvas = ({+ width,+ height,+}: {+ width: number;+ height: number;+}): VideoMattingCanvas => {+ if (typeof OffscreenCanvas !== 'undefined') {+ return new OffscreenCanvas(width, height);+ }++ if (typeof document !== 'undefined') {+ const canvas = document.createElement('canvas');+ canvas.width = width;+ canvas.height = height;+ return canvas;+ }++ throw new Error(+ 'Could not create a canvas. This API must run in a browser with OffscreenCanvas or the DOM available.',+ );+};++export const getVideoMattingCanvasContext = (+ canvas: VideoMattingCanvas,+): VideoMattingCanvasContext => {+ const context = canvas.getContext('2d', {willReadFrequently: true});+ if (!context) {+ throw new Error('Could not create a 2D canvas context.');+ }++ return context as VideoMattingCanvasContext;+};++export const drawOpaqueBaseFrame = ({+ context,+ source,+ width,+ height,+}: {+ context: VideoMattingCanvasContext;+ source: VideoMattingCanvas;+ width: number;+ height: number;+}) => {+ context.save();+ context.globalCompositeOperation = 'copy';+ context.fillStyle = '#000000';+ context.fillRect(0, 0, width, height);+ context.globalCompositeOperation = 'source-over';+ context.drawImage(source, 0, 0, width, height);+ context.restore();+};++export const drawForegroundFrame = ({+ context,+ result,+ source,+ targetWidth,+ targetHeight,+}: {+ context: VideoMattingCanvasContext;+ result: VideoMattingPipelineResult;+ source: VideoMattingCanvas;+ targetWidth: number;+ targetHeight: number;+}) => {+ if (+ !Number.isInteger(result.width) ||+ result.width <= 0 ||+ !Number.isInteger(result.height) ||+ result.height <= 0+ ) {+ throw new Error('The video matting model returned invalid dimensions.');+ }++ const expectedLength = result.width * result.height * result.channels;+ if (result.data.length !== expectedLength) {+ throw new Error(+ `The video matting model returned ${result.data.length} bytes, but ${expectedLength} RGBA bytes were expected.`,+ );+ }++ const imageData = new ImageData(result.data, result.width, result.height);++ context.clearRect(0, 0, targetWidth, targetHeight);+ if (result.width === targetWidth && result.height === targetHeight) {+ context.putImageData(imageData, 0, 0);+ } else {+ const intermediateCanvas = createVideoMattingCanvas({+ width: result.width,+ height: result.height,+ });+ const intermediateContext =+ getVideoMattingCanvasContext(intermediateCanvas);+ intermediateContext.putImageData(imageData, 0, 0);+ context.drawImage(intermediateCanvas, 0, 0, targetWidth, targetHeight);+ }++ context.save();+ context.globalCompositeOperation = 'destination-in';+ context.drawImage(source, 0, 0, targetWidth, targetHeight);+ context.restore();+};diff --git a/packages/video-matting/src/video-matting-quality.ts b/packages/video-matting/src/video-matting-quality.tsnew file mode 100644index 0000000000..c6f564c214--- /dev/null+++ b/packages/video-matting/src/video-matting-quality.ts@@ -0,0 +1,33 @@+import {Quality} from 'mediabunny';++export const VIDEO_MATTING_QUALITIES = [+ 'very-low',+ 'low',+ 'medium',+ 'high',+ 'very-high',+] as const;++export type VideoMattingQuality = (typeof VIDEO_MATTING_QUALITIES)[number];++export type VideoMattingBitrate = number | VideoMattingQuality;++export const resolveVideoMattingQuality = (+ value: VideoMattingBitrate,+): Quality => {+ if (typeof value === 'number') {+ if (!Number.isInteger(value) || value <= 0) {+ throw new TypeError('A numeric bitrate must be a positive integer.');+ }++ return new Quality({bitrate: value});+ }++ if (!VIDEO_MATTING_QUALITIES.includes(value)) {+ throw new TypeError(+ `The bitrate quality must be one of ${VIDEO_MATTING_QUALITIES.join(', ')}.`,+ );+ }++ return new Quality({quality: value, preferBitrate: true});+};diff --git a/packages/video-matting/src/video-timing.ts b/packages/video-matting/src/video-timing.tsnew file mode 100644index 0000000000..c66b568670--- /dev/null+++ b/packages/video-matting/src/video-timing.ts@@ -0,0 +1,94 @@+export const rebaseVideoTimestamp = ({+ timestamp,+ firstVideoTimestamp,+}: {+ timestamp: number;+ firstVideoTimestamp: number;+}): number => {+ if (!Number.isFinite(timestamp) || !Number.isFinite(firstVideoTimestamp)) {+ throw new TypeError('Video timestamps must be finite numbers.');+ }++ const rebased = timestamp - firstVideoTimestamp;+ if (rebased < -Number.EPSILON * 16) {+ throw new Error('Video frames must be in presentation order.');+ }++ return Math.max(0, rebased);+};++export const getClippedVideoFrameTiming = ({+ timestamp,+ duration,+ videoStartTimestamp,+ videoEndTimestamp,+}: {+ timestamp: number;+ duration: number;+ videoStartTimestamp: number;+ videoEndTimestamp: number;+}): {timestamp: number; duration: number} | null => {+ if (!Number.isFinite(timestamp)) {+ throw new TypeError('The video frame timestamp must be finite.');+ }++ if (!Number.isFinite(duration) || duration < 0) {+ throw new TypeError('The video frame duration must be non-negative.');+ }++ if (+ !Number.isFinite(videoStartTimestamp) ||+ !Number.isFinite(videoEndTimestamp) ||+ videoEndTimestamp < videoStartTimestamp+ ) {+ throw new TypeError('The video presentation range is invalid.');+ }++ const visibleStart = Math.max(timestamp, videoStartTimestamp);+ const visibleEnd = Math.min(timestamp + duration, videoEndTimestamp);+ if (visibleEnd <= visibleStart) {+ return null;+ }++ return {+ timestamp: visibleStart - videoStartTimestamp,+ duration: visibleEnd - visibleStart,+ };+};++export const getVideoProcessingProgress = ({+ timestamp,+ duration,+ firstVideoTimestamp,+ durationInSeconds,+}: {+ timestamp: number;+ duration: number;+ firstVideoTimestamp: number;+ durationInSeconds: number;+}): {processedDurationInSeconds: number; progress: number} => {+ if (!Number.isFinite(duration) || duration < 0) {+ throw new TypeError('Video frame durations must be non-negative.');+ }++ if (!Number.isFinite(durationInSeconds) || durationInSeconds < 0) {+ throw new TypeError('The video duration must be non-negative.');+ }++ const rebasedTimestamp = rebaseVideoTimestamp({+ timestamp,+ firstVideoTimestamp,+ });+ const processedDurationInSeconds = Math.min(+ durationInSeconds,+ Math.max(0, rebasedTimestamp + duration),+ );++ return {+ processedDurationInSeconds,+ progress:+ durationInSeconds === 0+ ? 1+ : Math.min(1, processedDurationInSeconds / durationInSeconds),+ };+};diff --git a/packages/video-matting/src/web-fs-target.ts b/packages/video-matting/src/web-fs-target.tsnew file mode 100644index 0000000000..f45163dbb4--- /dev/null+++ b/packages/video-matting/src/web-fs-target.ts@@ -0,0 +1,144 @@+import type {StreamTargetChunk} from 'mediabunny';++const filePrefix = '__remotion_video_matting:';+let sessionId: string | null = null;++const getSessionPrefix = () => {+ if (sessionId === null) {+ sessionId = crypto.randomUUID();+ }++ return `${filePrefix}${sessionId}:`;+};++export const canUseWebFsWriter = async (): Promise<boolean> => {+ if (typeof navigator === 'undefined' || !('storage' in navigator)) {+ return false;+ }++ if (!('getDirectory' in navigator.storage)) {+ return false;+ }++ const probeName = `${filePrefix}probe:${crypto.randomUUID()}`;+ try {+ const directoryHandle = await navigator.storage.getDirectory();+ const fileHandle = await directoryHandle.getFileHandle(probeName, {+ create: true,+ });+ const writable = await fileHandle.createWritable();+ await writable.close();+ await directoryHandle.removeEntry(probeName);+ return true;+ } catch {+ try {+ const directoryHandle = await navigator.storage.getDirectory();+ await directoryHandle.removeEntry(probeName);+ } catch {+ // The probe may not have created a file.+ }++ return false;+ }+};++export type WebFsVideoLayerTarget = {+ stream: WritableStream<StreamTargetChunk>;+ getBlob: () => Promise<Blob>;+ remove: () => Promise<void>;+};++export const createWebFsVideoLayerTarget =+ async (): Promise<WebFsVideoLayerTarget> => {+ const directoryHandle = await navigator.storage.getDirectory();+ const filename = `${getSessionPrefix()}${crypto.randomUUID()}`;+ const fileHandle = await directoryHandle.getFileHandle(filename, {+ create: true,+ });+ let writable: FileSystemWritableFileStream;+ try {+ writable = await fileHandle.createWritable();+ } catch (error) {+ try {+ await directoryHandle.removeEntry(filename);+ } catch {+ // Preserve the error that prevented the writable from being created.+ }++ throw error;+ }++ let closed = false;+ let removed = false;+ let removalPromise: Promise<void> | null = null;++ const close = async () => {+ if (closed) {+ return;+ }++ closed = true;+ await writable.close();+ };++ const abort = async (reason?: unknown) => {+ if (closed) {+ return;+ }++ closed = true;+ await writable.abort(reason);+ };++ const stream = new WritableStream<StreamTargetChunk>({+ async write(chunk) {+ await writable.seek(chunk.position);+ await writable.write(chunk);+ },+ close,+ abort,+ });++ const getBlob = async () => {+ const currentFileHandle = await directoryHandle.getFileHandle(filename);+ return currentFileHandle.getFile();+ };++ const remove = () => {+ if (removed) {+ return Promise.resolve();+ }++ if (removalPromise === null) {+ removalPromise = (async () => {+ try {+ await abort(new Error('Video layer output was discarded'));+ } catch {+ // Removing the file must still be attempted if aborting it fails.+ }++ try {+ await directoryHandle.removeEntry(filename);+ } catch (error) {+ if (+ typeof error !== 'object' ||+ error === null ||+ !('name' in error) ||+ error.name !== 'NotFoundError'+ ) {+ throw error;+ }+ }++ removed = true;+ })().catch((error: unknown) => {+ removalPromise = null;+ throw error;+ });+ }++ return removalPromise;+ };++ return {stream, getBlob, remove};+ };diff --git a/packages/video-matting/tsconfig.json b/packages/video-matting/tsconfig.jsonnew file mode 100644index 0000000000..e5c5f5163c--- /dev/null+++ b/packages/video-matting/tsconfig.json@@ -0,0 +1,14 @@+{+ "extends": "../tsconfig.settings.json",+ "compilerOptions": {+ "rootDir": "src",+ "outDir": "dist",+ "skipLibCheck": true,+ "target": "ES2022",+ "module": "es2020",+ "moduleResolution": "bundler",+ "declaration": true,+ "emitDeclarationOnly": true+ },+ "include": ["src"]+}diff --git a/tsconfig.json b/tsconfig.jsonindex bcf5580d63..151c1280dd 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -282,6 +282,9 @@{"path": "./packages/whisper-webgpu"},+ {+ "path": "./packages/video-matting"+ },{"path": "./packages/maptiler"},