🎥 Make videos programmatically with React
Host video matting models on R2
JonnyBurgerb357137edc73
diff --git a/bun.lock b/bun.lockindex 3aeb53def1..14df9d199d 100644--- a/bun.lock+++ b/bun.lock@@ -780,6 +780,7 @@"name": "@remotion/example","version": "4.0.521","dependencies": {+ "@huggingface/transformers": "catalog:","@mdx-js/loader": "2.3.0","@mdx-js/react": "2.3.0","@mediabunny/ac3": "catalog:",@@ -2772,14 +2773,17 @@"name": "@remotion/video-matting","version": "4.0.521","dependencies": {- "@huggingface/transformers": "catalog:","mediabunny": "catalog:",},"devDependencies": {+ "@huggingface/transformers": "catalog:","@remotion/eslint-config-internal": "workspace:*","@typescript/native-preview": "catalog:","eslint": "catalog:",},+ "peerDependencies": {+ "@huggingface/transformers": "4.2.0",+ },},"packages/web-renderer": {"name": "@remotion/web-renderer",diff --git a/packages/docs/docs/video-matting/can-use-video-matting.mdx b/packages/docs/docs/video-matting/can-use-video-matting.mdxindex 91f52b3472..742633af9c 100644--- a/packages/docs/docs/video-matting/can-use-video-matting.mdx+++ b/packages/docs/docs/video-matting/can-use-video-matting.mdx@@ -4,7 +4,7 @@ title: canUseVideoMatting()crumb: '@remotion/video-matting'----# canUseVideoMatting()<AvailableFrom v="4.0.522" />+# canUseVideoMatting()<AvailableFrom v="4.0.523" />Checks whether a video matting model can run in the current browser.diff --git a/packages/docs/docs/video-matting/clear-stale-video-matting-models.mdx b/packages/docs/docs/video-matting/clear-stale-video-matting-models.mdxindex a071f516a8..69282d6a61 100644--- a/packages/docs/docs/video-matting/clear-stale-video-matting-models.mdx+++ b/packages/docs/docs/video-matting/clear-stale-video-matting-models.mdx@@ -4,14 +4,16 @@ title: clearStaleVideoMattingModels()crumb: '@remotion/video-matting'----# clearStaleVideoMattingModels()<AvailableFrom v="4.0.522" />+# clearStaleVideoMattingModels()<AvailableFrom v="4.0.523" />Removes cached model files that are no longer returned by [`getAvailableVideoMattingModels()`](/docs/video-matting/get-available-video-matting-models).-There are no stale models in the current version, so the function resolves without changing the cache.+It also removes legacy entries for the pinned Hugging Face revisions that were used before models moved to `remotion.media`.Call this function after upgrading `@remotion/video-matting` to remove files for models discontinued by a newer version.+Only known legacy video-matting entries are removed. Current files from `remotion.media` and cache entries used by unrelated Transformers.js workflows are preserved.+```ts twoslash title="clear-stale-models.ts"import {clearStaleVideoMattingModels} from '@remotion/video-matting';diff --git a/packages/docs/docs/video-matting/dispose-video-matting-model.mdx b/packages/docs/docs/video-matting/dispose-video-matting-model.mdxindex 54e611a682..00a46e2bae 100644--- a/packages/docs/docs/video-matting/dispose-video-matting-model.mdx+++ b/packages/docs/docs/video-matting/dispose-video-matting-model.mdx@@ -4,7 +4,7 @@ title: disposeVideoMattingModel()crumb: '@remotion/video-matting'----# disposeVideoMattingModel()<AvailableFrom v="4.0.522" />+# disposeVideoMattingModel()<AvailableFrom v="4.0.523" />Releases the memory used by initialized video matting models. Persistent browser-cached model files are not deleted.diff --git a/packages/docs/docs/video-matting/get-available-video-matting-models.mdx b/packages/docs/docs/video-matting/get-available-video-matting-models.mdxindex 48c0edb18a..21519e1d79 100644--- a/packages/docs/docs/video-matting/get-available-video-matting-models.mdx+++ b/packages/docs/docs/video-matting/get-available-video-matting-models.mdx@@ -4,7 +4,7 @@ title: getAvailableVideoMattingModels()crumb: '@remotion/video-matting'----# getAvailableVideoMattingModels()<AvailableFrom v="4.0.522" />+# getAvailableVideoMattingModels()<AvailableFrom v="4.0.523" />Returns metadata for the models supported by [`separateVideoLayers()`](/docs/video-matting/separate-video-layers).@@ -22,6 +22,8 @@ console.log(models);The [`modnet` model](https://huggingface.co/Xenova/modnet) is Apache-2.0 licensed. The [`ben2-base` model](https://huggingface.co/onnx-community/BEN2-ONNX) is the MIT-licensed BEN2 Base export.+The `modelId` values identify these upstream Hugging Face models for provenance. Runtime files are downloaded from immutable paths on `remotion.media` and are byte-identical to files from the pinned upstream revisions.+Each model contains these properties:### name@@ -30,7 +32,7 @@ The model name accepted by APIs in this package.### modelId-The Hugging Face model identifier.+The upstream Hugging Face model identifier. This is provenance metadata, not the runtime download location.### purpose@@ -42,7 +44,7 @@ Whether the model is experimental.### webGpuDownloadSize-The estimated WebGPU model download size in bytes. Browser caches and supporting model files may add a small amount.+The total size in bytes of all runtime files downloaded for WebGPU.Models process each frame independently and may show temporal flicker. The set of available models may change between package versions.diff --git a/packages/docs/docs/video-matting/index.mdx b/packages/docs/docs/video-matting/index.mdxindex 058d9766b2..9810998db4 100644--- a/packages/docs/docs/video-matting/index.mdx+++ b/packages/docs/docs/video-matting/index.mdx@@ -4,7 +4,7 @@ title: '@remotion/video-matting'crumb: 'Separate a video into layers'----# @remotion/video-matting<AvailableFrom v="4.0.522" />+# @remotion/video-matting<AvailableFrom v="4.0.523" />Separate a video into an opaque base layer and a foreground layer with alpha, locally in the browser using WebGPU.@@ -12,7 +12,9 @@ import {TableOfContents} from './table-of-contents';## Installation-<Installation pkg="@remotion/video-matting" />+<Installation pkg="@remotion/video-matting @huggingface/[email protected]" />++`@huggingface/transformers` is a peer dependency. Install exactly version `4.2.0`.## Example@@ -35,6 +37,10 @@ Call `dispose()` after retrieving the blobs to release package-owned temporary sThe model is downloaded on the first call and cached by the browser. Call [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model) first to display download progress.+## Model hosting++Model files are downloaded from immutable paths on `remotion.media`. They are byte-identical mirrors of files from pinned upstream Hugging Face revisions. The public `modelId` metadata identifies the upstream source for provenance; it is not the download location.+## Layer orderThe base is an opaque copy of the input video. The foreground contains the detected subject and transparent pixels everywhere else.@@ -57,6 +63,8 @@ Audio is included in the base by default and omitted from the foreground, so staWebGPU requires HTTPS in production or `localhost` during development. Use [`canUseVideoMatting()`](/docs/video-matting/can-use-video-matting) to check whether the selected model is supported.+If the page uses a strict Content Security Policy, add `https://remotion.media` to its `connect-src` directive, for example `connect-src 'self' https://remotion.media`.+The browser must also be able to decode the input video and encode VP9 with alpha through WebCodecs. [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) checks these media capabilities for the selected input.## Licensediff --git a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdxindex 80523d45e7..59614d4e5c 100644--- a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx+++ b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx@@ -4,9 +4,9 @@ title: isVideoMattingModelCached()crumb: '@remotion/video-matting'----# isVideoMattingModelCached()<AvailableFrom v="4.0.522" />+# isVideoMattingModelCached()<AvailableFrom v="4.0.523" />-Checks whether all files needed for a video matting model are present in the browser cache. The check does not download or initialize the model.+Checks whether all files hosted on `remotion.media` for a video matting model are present in the browser cache. The check does not download or initialize the model.```ts twoslash title="model-cache.ts"import {isVideoMattingModelCached} from '@remotion/video-matting';@@ -26,6 +26,8 @@ One of the names returned by [`getAvailableVideoMattingModels()`](/docs/video-maA `Promise<boolean>` resolving to `true` if every required model file is cached.+Legacy model files cached from Hugging Face do not satisfy this check. Loading the model downloads and caches the current byte-identical files from their immutable `remotion.media` paths.+## Compatibility<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />diff --git a/packages/docs/docs/video-matting/load-video-matting-model.mdx b/packages/docs/docs/video-matting/load-video-matting-model.mdxindex d66d3af9f9..0d6df92dab 100644--- a/packages/docs/docs/video-matting/load-video-matting-model.mdx+++ b/packages/docs/docs/video-matting/load-video-matting-model.mdx@@ -4,9 +4,9 @@ title: loadVideoMattingModel()crumb: '@remotion/video-matting'----# loadVideoMattingModel()<AvailableFrom v="4.0.522" />+# loadVideoMattingModel()<AvailableFrom v="4.0.523" />-Downloads and initializes a video matting model. Calling it before [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) lets you show model-loading progress.+Downloads a video matting model from an immutable path on `remotion.media` and initializes it. Calling it before [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) lets you show model-loading progress.```ts twoslash title="load-model.ts"import {loadVideoMattingModel} from '@remotion/video-matting';@@ -63,6 +63,8 @@ Whether the same model was already initialized in memory.The browser manages the persistent [Transformers.js](https://huggingface.co/docs/transformers.js) cache. This function also retains the initialized model in memory until [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model) is called or the page closes.+The hosted files are byte-identical to files from the pinned upstream Hugging Face revision. Legacy Hugging Face downloads use different cache entries and are downloaded once more from `remotion.media`.+## Compatibility<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />diff --git a/packages/docs/docs/video-matting/separate-video-layers.mdx b/packages/docs/docs/video-matting/separate-video-layers.mdxindex a2806d6fa9..b72c47bf31 100644--- a/packages/docs/docs/video-matting/separate-video-layers.mdx+++ b/packages/docs/docs/video-matting/separate-video-layers.mdx@@ -4,7 +4,7 @@ title: separateVideoLayers()crumb: '@remotion/video-matting'----# separateVideoLayers()<AvailableFrom v="4.0.522" />+# separateVideoLayers()<AvailableFrom v="4.0.523" />Separates a video into an opaque base WebM and a foreground WebM with alpha.diff --git a/packages/docs/docs/video-matting/video-matting-models.mdx b/packages/docs/docs/video-matting/video-matting-models.mdxindex 36f3cae8d7..3804ce5e55 100644--- a/packages/docs/docs/video-matting/video-matting-models.mdx+++ b/packages/docs/docs/video-matting/video-matting-models.mdx@@ -4,7 +4,7 @@ title: VIDEO_MATTING_MODELScrumb: '@remotion/video-matting'----# VIDEO_MATTING_MODELS<AvailableFrom v="4.0.522" />+# VIDEO_MATTING_MODELS<AvailableFrom v="4.0.523" />A readonly array containing the model names supported by `@remotion/video-matting`.diff --git a/packages/example/package.json b/packages/example/package.jsonindex 358d15d715..d72446a5e0 100644--- a/packages/example/package.json+++ b/packages/example/package.json@@ -31,6 +31,7 @@"render-transparent": "remotion render --image-format=png --pixel-format=yuva420p --codec=vp8 react-svg out/video.webm"},"dependencies": {+ "@huggingface/transformers": "catalog:","@remotion/studio-protocol": "workspace:*","@mdx-js/loader": "2.3.0","@mdx-js/react": "2.3.0",diff --git a/packages/video-matting/package.json b/packages/video-matting/package.jsonindex bc42a84e62..1545f44c28 100644--- a/packages/video-matting/package.json+++ b/packages/video-matting/package.json@@ -24,9 +24,11 @@"url": "https://github.com/remotion-dev/remotion/issues"},"dependencies": {- "@huggingface/transformers": "catalog:","mediabunny": "catalog:"},+ "peerDependencies": {+ "@huggingface/transformers": "4.2.0"+ },"exports": {".": {"types": "./dist/index.d.ts",@@ -36,6 +38,7 @@"./package.json": "./package.json"},"devDependencies": {+ "@huggingface/transformers": "catalog:","@remotion/eslint-config-internal": "workspace:*","@typescript/native-preview": "catalog:","eslint": "catalog:"diff --git a/packages/video-matting/src/clear-stale-video-matting-models.ts b/packages/video-matting/src/clear-stale-video-matting-models.tsindex c46817affc..22ad2cc327 100644--- a/packages/video-matting/src/clear-stale-video-matting-models.ts+++ b/packages/video-matting/src/clear-stale-video-matting-models.ts@@ -1,4 +1,34 @@-// Transformers.js owns the persistent model cache. Model revisions are pinned,-// so there is currently no package-owned cache entry that can become stale.-export const clearStaleVideoMattingModels = (): Promise<void> =>- Promise.resolve();+import {getVideoMattingModelInfo, VIDEO_MATTING_MODELS} from './models';++const legacyModelUrlPrefixes = VIDEO_MATTING_MODELS.map((model) => {+ const {modelId, revision} = getVideoMattingModelInfo(model);+ return `https://huggingface.co/${modelId}/resolve/${revision}/`;+});++export const clearStaleVideoMattingModels = async (): Promise<void> => {+ if (typeof caches === 'undefined') {+ return;+ }++ const {env} = await import('@huggingface/transformers');+ let cache: Cache;+ let requests: readonly Request[];+ try {+ cache = await caches.open(env.cacheKey);+ requests = await cache.keys();+ } catch {+ return;+ }++ await Promise.all(+ requests.map((request) => {+ if (+ legacyModelUrlPrefixes.some((prefix) => request.url.startsWith(prefix))+ ) {+ return cache.delete(request);+ }++ return Promise.resolve(false);+ }),+ );+};diff --git a/packages/video-matting/src/is-video-matting-model-cached.ts b/packages/video-matting/src/is-video-matting-model-cached.tsindex 4fef5a28bf..a8e87dd6f8 100644--- a/packages/video-matting/src/is-video-matting-model-cached.ts+++ b/packages/video-matting/src/is-video-matting-model-cached.ts@@ -1,22 +1,27 @@-import {getVideoMattingModelInfo, type VideoMattingModel} from './models';+import {+ getHostedVideoMattingModelId,+ getVideoMattingModelInfo,+ type VideoMattingModel,+} from './models';+import {withRemotionModelHost} from './with-remotion-model-host';export type IsVideoMattingModelCachedOptions = {model: VideoMattingModel;};-export const isVideoMattingModelCached = async ({+export const isVideoMattingModelCached = ({model,}: IsVideoMattingModelCachedOptions): Promise<boolean> => {- const {ModelRegistry} = await import('@huggingface/transformers');const modelInfo = getVideoMattingModelInfo(model);- return ModelRegistry.is_pipeline_cached(- 'background-removal',- modelInfo.modelId,- {- device: 'webgpu',- dtype: modelInfo.dtype,- revision: modelInfo.revision,- },- );+ return withRemotionModelHost(({ModelRegistry}) => {+ return ModelRegistry.is_pipeline_cached(+ 'background-removal',+ getHostedVideoMattingModelId(model),+ {+ device: 'webgpu',+ dtype: modelInfo.dtype,+ },+ );+ });};diff --git a/packages/video-matting/src/load-video-matting-model.ts b/packages/video-matting/src/load-video-matting-model.tsindex a7087f7f39..181dadb62c 100644--- a/packages/video-matting/src/load-video-matting-model.ts+++ b/packages/video-matting/src/load-video-matting-model.ts@@ -1,4 +1,9 @@-import {getVideoMattingModelInfo, type VideoMattingModel} from './models';+import {+ getHostedVideoMattingModelId,+ getVideoMattingModelInfo,+ type VideoMattingModel,+} from './models';+import {withRemotionModelHost} from './with-remotion-model-host';export type VideoMattingModelLoadProgress = {status: string;@@ -150,121 +155,124 @@ const getOrCreateVideoMattingPipeline = ({await pendingDisposal;}- const {- AutoModelForImageSegmentation,- AutoProcessor,- BackgroundRemovalPipeline,- } = await import('@huggingface/transformers');- const totalBytes = modelInfo.webGpuDownloadSize;- const loadedByFile = new Map<string, number>();- let lastProgress = 0;- let lastLoadedBytes = 0;-- notifyProgress(state, {- status: 'loading',- file: null,- progress: 0,- loadedBytes: 0,- totalBytes,- });+ return withRemotionModelHost(+ async ({+ AutoModelForImageSegmentation,+ AutoProcessor,+ BackgroundRemovalPipeline,+ }) => {+ const hostedModelId = getHostedVideoMattingModelId(model);+ const totalBytes = modelInfo.webGpuDownloadSize;+ const loadedByFile = new Map<string, number>();+ let lastProgress = 0;+ let lastLoadedBytes = 0;++ notifyProgress(state, {+ status: 'loading',+ file: null,+ progress: 0,+ loadedBytes: 0,+ totalBytes,+ });- const pretrainedOptions = {- device: 'webgpu' as const,- dtype: modelInfo.dtype,- revision: modelInfo.revision,- progress_callback: (event: unknown) => {- const record = event as Record<string, unknown>;- if (- record.status === 'progress' &&- typeof record.file === 'string' &&- typeof record.loaded === 'number' &&- Number.isFinite(record.loaded)- ) {- loadedByFile.set(- record.file,- Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),- );- const loadedBytes = [...loadedByFile.values()].reduce(- (sum, loaded) => sum + loaded,- 0,- );- lastProgress = Math.max(- lastProgress,- Math.min(loadedBytes / totalBytes, 0.99),+ const pretrainedOptions = {+ device: 'webgpu' as const,+ dtype: modelInfo.dtype,+ progress_callback: (event: unknown) => {+ const record = event as Record<string, unknown>;+ if (+ record.status === 'progress' &&+ typeof record.file === 'string' &&+ typeof record.loaded === 'number' &&+ Number.isFinite(record.loaded)+ ) {+ loadedByFile.set(+ record.file,+ Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),+ );+ const loadedBytes = [...loadedByFile.values()].reduce(+ (sum, loaded) => sum + loaded,+ 0,+ );+ lastProgress = Math.max(+ lastProgress,+ Math.min(loadedBytes / totalBytes, 0.99),+ );+ lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);+ notifyProgress(state, {+ status: 'loading',+ file: null,+ progress: lastProgress,+ loadedBytes: Math.min(lastLoadedBytes, totalBytes),+ totalBytes,+ });+ }+ },+ };+ const transformerProcessor = await AutoProcessor.from_pretrained(+ hostedModelId,+ pretrainedOptions,+ );+ const transformerModel =+ await AutoModelForImageSegmentation.from_pretrained(+ hostedModelId,+ pretrainedOptions,);- lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);+ let transformerPipeline: TransformerPipeline | null = null;+ let returnedPipeline = false;+ try {+ transformerPipeline = new BackgroundRemovalPipeline({+ task: 'background-removal',+ model: transformerModel,+ processor: transformerProcessor,+ }) as unknown as TransformerPipeline;+ const loadedTransformerPipeline = transformerPipeline;+notifyProgress(state, {- status: 'loading',+ status: 'ready',file: null,- progress: lastProgress,- loadedBytes: Math.min(lastLoadedBytes, totalBytes),+ progress: 1,+ loadedBytes: totalBytes,totalBytes,});- }- },- };- const transformerProcessor = await AutoProcessor.from_pretrained(- modelInfo.modelId,- pretrainedOptions,- );- const transformerModel =- await AutoModelForImageSegmentation.from_pretrained(- modelInfo.modelId,- pretrainedOptions,- );- let transformerPipeline: TransformerPipeline | null = null;- let returnedPipeline = false;- try {- transformerPipeline = new BackgroundRemovalPipeline({- task: 'background-removal',- model: transformerModel,- processor: transformerProcessor,- }) as unknown as TransformerPipeline;- const loadedTransformerPipeline = transformerPipeline;-- notifyProgress(state, {- status: 'ready',- file: null,- progress: 1,- loadedBytes: totalBytes,- totalBytes,- });-- const loadedPipeline: LoadedPipeline = {- run: async (image) => {- const result = await loadedTransformerPipeline(image);- if (result.channels !== 4) {- throw new Error(- `The video matting model "${model}" returned ${result.channels} channels instead of RGBA.`,- );- }- const data =- result.data instanceof Uint8ClampedArray &&- result.data.buffer instanceof ArrayBuffer- ? (result.data as Uint8ClampedArray<ArrayBuffer>)- : new Uint8ClampedArray(result.data);-- return {- data,- width: result.width,- height: result.height,- channels: 4,+ const loadedPipeline: LoadedPipeline = {+ run: async (image) => {+ const result = await loadedTransformerPipeline(image);+ if (result.channels !== 4) {+ throw new Error(+ `The video matting model "${model}" returned ${result.channels} channels instead of RGBA.`,+ );+ }++ const data =+ result.data instanceof Uint8ClampedArray &&+ result.data.buffer instanceof ArrayBuffer+ ? (result.data as Uint8ClampedArray<ArrayBuffer>)+ : new Uint8ClampedArray(result.data);++ return {+ data,+ width: result.width,+ height: result.height,+ channels: 4,+ };+ },+ dispose: () => loadedTransformerPipeline.dispose(),};- },- dispose: () => loadedTransformerPipeline.dispose(),- };- returnedPipeline = true;- return loadedPipeline;- } finally {- if (!returnedPipeline) {- if (transformerPipeline === null) {- await transformerModel.dispose();- } else {- await transformerPipeline.dispose();+ returnedPipeline = true;+ return loadedPipeline;+ } finally {+ if (!returnedPipeline) {+ if (transformerPipeline === null) {+ await transformerModel.dispose();+ } else {+ await transformerPipeline.dispose();+ }+ }}- }- }+ },+ );});pipelines.set(model, state);diff --git a/packages/video-matting/src/models.ts b/packages/video-matting/src/models.tsindex d01fa610f0..bd1e83e670 100644--- a/packages/video-matting/src/models.ts+++ b/packages/video-matting/src/models.ts@@ -22,7 +22,7 @@ const MODEL_INFO: Record<VideoMattingModel, InternalVideoMattingModelInfo> = {modelId: 'Xenova/modnet',purpose: 'person',experimental: false,- webGpuDownloadSize: 25_888_640,+ webGpuDownloadSize: 25_889_088,revision: 'fa2fa546052fba4c08921230a26cc69a333fca12',dtype: 'fp32',requiresShaderF16: false,@@ -32,13 +32,18 @@ const MODEL_INFO: Record<VideoMattingModel, InternalVideoMattingModelInfo> = {modelId: 'onnx-community/BEN2-ONNX',purpose: 'general',experimental: true,- webGpuDownloadSize: 219_121_675,+ webGpuDownloadSize: 219_122_146,revision: 'c552aa82688edce09f0ac9d2e31ad53d9d629010',dtype: 'fp16',requiresShaderF16: true,},};+const HOSTED_MODEL_IDS: Record<VideoMattingModel, string> = {+ modnet: 'modnet-v1',+ 'ben2-base': 'ben2-base-v1',+};+export const getAvailableVideoMattingModels = (): VideoMattingModelInfo[] => {return VIDEO_MATTING_MODELS.map((model) => {const {@@ -62,3 +67,9 @@ export const getVideoMattingModelInfo = (return MODEL_INFO[model];};++export const getHostedVideoMattingModelId = (+ model: VideoMattingModel,+): string => {+ return HOSTED_MODEL_IDS[model];+};diff --git a/packages/video-matting/src/test/model-runtime.test.ts b/packages/video-matting/src/test/model-runtime.test.tsindex ead9334813..0d715f530a 100644--- a/packages/video-matting/src/test/model-runtime.test.ts+++ b/packages/video-matting/src/test/model-runtime.test.ts@@ -4,7 +4,21 @@ let initializationCount = 0;let initializationStarted: ((count: number) => void) | null = null;let initializationGate: Promise<void> | null = null;let initializationOptions: Record<string, unknown> | null = null;+let initializationEnvironment:+ | {+ remoteHost: string;+ remotePathTemplate: string;+ }+ | undefined;+let initializationEnvironmentAfterLoading:+ | {+ remoteHost: string;+ remotePathTemplate: string;+ }+ | undefined;let initializedModelId: string | null = null;+const initializedModelIds: string[] = [];+const initializedProcessorModelIds: string[] = [];let pipelineConstructionCount = 0;let pipelineRunCount = 0;let livePipelineCount = 0;@@ -19,6 +33,30 @@ let cacheCheck:options: Record<string, unknown>;}| undefined;+let cacheCheckEnvironment:+ | {+ remoteHost: string;+ remotePathTemplate: string;+ }+ | undefined;+let cacheCheckError: Error | null = null;++const originalTransformersEnvironment = {+ remoteHost: 'https://huggingface.co/',+ remotePathTemplate: '{model}/resolve/{revision}/',+};+const transformersEnvironment = {...originalTransformersEnvironment};+const preexistingWhisperModelHostState = {+ activeOperations: 0,+ previousRemoteConfiguration: null as+ | typeof originalTransformersEnvironment+ | null,+};+Object.defineProperty(+ transformersEnvironment,+ Symbol.for('@remotion/whisper-webgpu/model-host-state'),+ {value: preexistingWhisperModelHostState},+);function FakeBackgroundRemovalPipeline() {const pipeline = ++pipelineConstructionCount;@@ -47,6 +85,7 @@ function FakeBackgroundRemovalPipeline() {}mock.module('@huggingface/transformers', () => ({+ env: transformersEnvironment,AutoModelForImageSegmentation: {from_pretrained: async (modelId: string,@@ -54,13 +93,21 @@ mock.module('@huggingface/transformers', () => ({) => {initializationCount++;initializedModelId = modelId;+ initializedModelIds.push(modelId);initializationOptions = options;+ initializationEnvironment = {...transformersEnvironment};initializationStarted?.(initializationCount);await initializationGate;+ initializationEnvironmentAfterLoading = {...transformersEnvironment};const onProgress = options.progress_callback as| ((event: Record<string, unknown>) => void)| undefined;+ onProgress?.({+ status: 'progress',+ file: 'config.json',+ loaded: 83,+ });onProgress?.({status: 'progress',file: 'onnx/model.onnx',@@ -72,7 +119,8 @@ mock.module('@huggingface/transformers', () => ({},},AutoProcessor: {- from_pretrained: (_modelId: string, options: Record<string, unknown>) => {+ from_pretrained: (modelId: string, options: Record<string, unknown>) => {+ initializedProcessorModelIds.push(modelId);const onProgress = options.progress_callback as| ((event: Record<string, unknown>) => void)| undefined;@@ -92,6 +140,11 @@ mock.module('@huggingface/transformers', () => ({options: Record<string, unknown>,) => {cacheCheck = {task, modelId, options};+ cacheCheckEnvironment = {...transformersEnvironment};+ if (cacheCheckError) {+ return Promise.reject(cacheCheckError);+ }+return Promise.resolve(true);},},@@ -109,7 +162,11 @@ beforeEach(async () => {initializationStarted = null;initializationGate = null;initializationOptions = null;+ initializationEnvironment = undefined;+ initializationEnvironmentAfterLoading = undefined;initializedModelId = null;+ initializedModelIds.length = 0;+ initializedProcessorModelIds.length = 0;pipelineConstructionCount = 0;pipelineRunCount = 0;livePipelineCount = 0;@@ -118,6 +175,71 @@ beforeEach(async () => {disposalStarted = null;disposalGate = null;cacheCheck = undefined;+ cacheCheckEnvironment = undefined;+ cacheCheckError = null;+ preexistingWhisperModelHostState.activeOperations = 0;+ preexistingWhisperModelHostState.previousRemoteConfiguration = null;+ Object.assign(transformersEnvironment, originalTransformersEnvironment);+});++test('removes only legacy pinned Hugging Face models from the shared browser cache', async () => {+ const originalCaches = Object.getOwnPropertyDescriptor(globalThis, 'caches');+ const cacheEnvironment =+ transformersEnvironment as typeof transformersEnvironment & {+ cacheKey: string;+ };+ cacheEnvironment.cacheKey = 'custom-transformers-cache';+ const legacyModnet = new Request(+ 'https://huggingface.co/Xenova/modnet/resolve/fa2fa546052fba4c08921230a26cc69a333fca12/config.json',+ );+ const legacyBen2 = new Request(+ 'https://huggingface.co/onnx-community/BEN2-ONNX/resolve/c552aa82688edce09f0ac9d2e31ad53d9d629010/onnx/model_fp16.onnx',+ );+ const sameModelDifferentRevision = new Request(+ 'https://huggingface.co/Xenova/modnet/resolve/main/config.json',+ );+ const currentHostedModel = new Request(+ 'https://remotion.media/models/modnet-v1/config.json',+ );+ const unrelatedModel = new Request(+ 'https://huggingface.co/onnx-community/background-removal/resolve/main/config.json',+ );+ const deleted: string[] = [];+ Object.defineProperty(globalThis, 'caches', {+ configurable: true,+ value: {+ open: (cacheKey: string) => {+ expect(cacheKey).toBe('custom-transformers-cache');+ return Promise.resolve({+ delete: (request: Request) => {+ deleted.push(request.url);+ return Promise.resolve(true);+ },+ keys: () =>+ Promise.resolve([+ legacyModnet,+ legacyBen2,+ sameModelDifferentRevision,+ currentHostedModel,+ unrelatedModel,+ ]),+ });+ },+ } as unknown as CacheStorage,+ });++ try {+ const {clearStaleVideoMattingModels} = await import('../index');+ await clearStaleVideoMattingModels();+ expect(deleted).toEqual([legacyModnet.url, legacyBen2.url]);+ } finally {+ Reflect.deleteProperty(cacheEnvironment, 'cacheKey');+ if (originalCaches) {+ Object.defineProperty(globalThis, 'caches', originalCaches);+ } else {+ Reflect.deleteProperty(globalThis, 'caches');+ }+ }});test('coalesces initialization and defers disposal until active work is done', async () => {@@ -138,10 +260,11 @@ test('coalesces initialization and defers disposal until active work is done', a});const firstProgress: number[] = [];const firstLoadedBytes: number[] = [];+ const firstTotalBytes: number[] = [];const secondProgress: number[] = [];const first = loadVideoMattingModel({model: 'modnet',- onProgress: ({loadedBytes, progress}) => {+ onProgress: ({loadedBytes, progress, totalBytes}) => {if (progress !== null) {firstProgress.push(progress);}@@ -149,6 +272,10 @@ test('coalesces initialization and defers disposal until active work is done', aif (loadedBytes !== null) {firstLoadedBytes.push(loadedBytes);}++ if (totalBytes !== null) {+ firstTotalBytes.push(totalBytes);+ }},});const second = loadVideoMattingModel({@@ -162,31 +289,60 @@ test('coalesces initialization and defers disposal until active work is done', aawait started;expect(initializationCount).toBe(1);+ expect(initializedModelId).toBe('modnet-v1');+ expect(initializationEnvironment).toEqual({+ remoteHost: 'https://remotion.media/',+ remotePathTemplate: 'models/{model}/',+ });+ expect(transformersEnvironment).toEqual({+ remoteHost: 'https://remotion.media/',+ remotePathTemplate: 'models/{model}/',+ });+ expect(+ (+ transformersEnvironment as typeof transformersEnvironment &+ Record<symbol, unknown>+ )[Symbol.for('@remotion/whisper-webgpu/model-host-state')],+ ).toBe(preexistingWhisperModelHostState);+ expect(preexistingWhisperModelHostState).toMatchObject({+ activeOperations: 1,+ previousRemoteConfiguration: originalTransformersEnvironment,+ });+ expect(await isVideoMattingModelCached({model: 'modnet'})).toBe(true);+ expect(cacheCheck).toEqual({+ task: 'background-removal',+ modelId: 'modnet-v1',+ options: {+ device: 'webgpu',+ dtype: 'fp32',+ },+ });+ expect(cacheCheckEnvironment).toEqual({+ remoteHost: 'https://remotion.media/',+ remotePathTemplate: 'models/{model}/',+ });+ expect(transformersEnvironment).toEqual({+ remoteHost: 'https://remotion.media/',+ remotePathTemplate: 'models/{model}/',+ });releaseInitialization();expect(await Promise.all([first, second])).toEqual([{alreadyLoaded: false},{alreadyLoaded: true},]);- expect(initializedModelId).toBe('Xenova/modnet');+ expect(initializationEnvironmentAfterLoading).toEqual(+ initializationEnvironment,+ );expect(initializationOptions).toMatchObject({device: 'webgpu',dtype: 'fp32',- revision: 'fa2fa546052fba4c08921230a26cc69a333fca12',});+ expect(initializationOptions).not.toHaveProperty('revision');expect(firstProgress.at(-1)).toBe(1);expect(secondProgress.at(-1)).toBe(1);- expect(Math.max(...firstLoadedBytes)).toBe(25_888_640);-- expect(await isVideoMattingModelCached({model: 'modnet'})).toBe(true);- expect(cacheCheck).toEqual({- task: 'background-removal',- modelId: 'Xenova/modnet',- options: {- device: 'webgpu',- dtype: 'fp32',- revision: 'fa2fa546052fba4c08921230a26cc69a333fca12',- },- });+ expect(Math.max(...firstLoadedBytes)).toBe(25_889_088);+ expect(new Set(firstTotalBytes)).toEqual(new Set([25_889_088]));+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);let releaseUse: () => void = () => undefined;const useGate = new Promise<void>((resolve) => {@@ -223,6 +379,129 @@ test('coalesces initialization and defers disposal until active work is done', aexpect(livePipelineCount).toBe(0);});+test('loads every public model from its immutable hosted model ID', async () => {+ const {+ disposeVideoMattingModel,+ getAvailableVideoMattingModels,+ loadVideoMattingModel,+ } = await import('../index');+ const models = getAvailableVideoMattingModels();++ for (const model of models) {+ await loadVideoMattingModel({model: model.name});+ await disposeVideoMattingModel({model: model.name});+ }++ expect(initializedProcessorModelIds).toEqual(['modnet-v1', 'ben2-base-v1']);+ expect(initializedModelIds).toEqual(['modnet-v1', 'ben2-base-v1']);+ expect(models.map(({modelId}) => modelId)).toEqual([+ 'Xenova/modnet',+ 'onnx-community/BEN2-ONNX',+ ]);+ expect(models.map(({webGpuDownloadSize}) => webGpuDownloadSize)).toEqual([+ 25_889_088, 219_122_146,+ ]);+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);+});++test('keeps the shared hosted environment active for concurrent model loads', async () => {+ const {disposeVideoMattingModel, loadVideoMattingModel} = await getRuntime();+ let releaseInitialization: () => void = () => undefined;+ initializationGate = new Promise<void>((resolve) => {+ releaseInitialization = resolve;+ });+ const bothStarted = new Promise<void>((resolve) => {+ initializationStarted = (count) => {+ if (count === 2) {+ resolve();+ }+ };+ });+ const modnet = loadVideoMattingModel({model: 'modnet'});+ const ben2 = loadVideoMattingModel({model: 'ben2-base'});++ await bothStarted;+ const environmentWithState =+ transformersEnvironment as typeof transformersEnvironment &+ Record<symbol, unknown>;+ const sharedState =+ environmentWithState[+ Symbol.for('@remotion/whisper-webgpu/model-host-state')+ ];+ expect(sharedState).toMatchObject({+ activeOperations: 2,+ previousRemoteConfiguration: originalTransformersEnvironment,+ });+ expect(+ environmentWithState[Symbol.for('@remotion/transformers/model-host-state')],+ ).toBe(sharedState);+ expect(transformersEnvironment).toEqual({+ remoteHost: 'https://remotion.media/',+ remotePathTemplate: 'models/{model}/',+ });++ releaseInitialization();+ expect(await Promise.all([modnet, ben2])).toEqual([+ {alreadyLoaded: false},+ {alreadyLoaded: false},+ ]);+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);++ await disposeVideoMattingModel();+});++test('restores the Transformers environment after a hosted operation fails', async () => {+ const {isVideoMattingModelCached} =+ await import('../is-video-matting-model-cached');+ cacheCheckError = new Error('Could not read the model cache');++ try {+ await expect(+ isVideoMattingModelCached({model: 'ben2-base'}),+ ).rejects.toThrow('Could not read the model cache');+ expect(cacheCheck).toMatchObject({+ modelId: 'ben2-base-v1',+ options: {device: 'webgpu', dtype: 'fp16'},+ });+ expect(cacheCheckEnvironment).toEqual({+ remoteHost: 'https://remotion.media/',+ remotePathTemplate: 'models/{model}/',+ });+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);+ } finally {+ cacheCheckError = null;+ }+});++test('only restores Transformers environment fields left unchanged by the consumer', async () => {+ const {disposeVideoMattingModel, loadVideoMattingModel} = await getRuntime();+ let releaseInitialization: () => void = () => undefined;+ initializationGate = new Promise<void>((resolve) => {+ releaseInitialization = resolve;+ });+ const started = new Promise<void>((resolve) => {+ initializationStarted = () => resolve();+ });+ const loading = loadVideoMattingModel({model: 'modnet'});++ try {+ await started;+ transformersEnvironment.remoteHost = 'https://models.example.com/';+ releaseInitialization();+ await loading;+ expect(transformersEnvironment).toEqual({+ remoteHost: 'https://models.example.com/',+ remotePathTemplate: '{model}/resolve/{revision}/',+ });+ await disposeVideoMattingModel({model: 'modnet'});+ } finally {+ releaseInitialization();+ initializationGate = null;+ initializationStarted = null;+ Object.assign(transformersEnvironment, originalTransformersEnvironment);+ }+});+test('isolates throwing progress observers from shared loading', async () => {const {disposeVideoMattingModel, loadVideoMattingModel} = await getRuntime();diff --git a/packages/video-matting/src/test/models.test.ts b/packages/video-matting/src/test/models.test.tsindex d0b179a4b1..e438e17f6c 100644--- a/packages/video-matting/src/test/models.test.ts+++ b/packages/video-matting/src/test/models.test.ts@@ -17,14 +17,14 @@ test('returns public metadata without mutable internal model settings', () => {modelId: 'Xenova/modnet',purpose: 'person',experimental: false,- webGpuDownloadSize: 25_888_640,+ webGpuDownloadSize: 25_889_088,},{name: 'ben2-base',modelId: 'onnx-community/BEN2-ONNX',purpose: 'general',experimental: true,- webGpuDownloadSize: 219_121_675,+ webGpuDownloadSize: 219_122_146,},]);diff --git a/packages/video-matting/src/with-remotion-model-host.ts b/packages/video-matting/src/with-remotion-model-host.tsnew file mode 100644index 0000000000..2d92551193--- /dev/null+++ b/packages/video-matting/src/with-remotion-model-host.ts@@ -0,0 +1,94 @@+import type {+ AutoModelForImageSegmentation,+ AutoProcessor,+ BackgroundRemovalPipeline,+ env,+ ModelRegistry,+} from '@huggingface/transformers';++const REMOTION_MODEL_HOST = 'https://remotion.media/';+const REMOTION_MODEL_PATH_TEMPLATE = 'models/{model}/';+const REMOTION_MODEL_HOST_STATE = Symbol.for(+ '@remotion/transformers/model-host-state',+);+const WHISPER_MODEL_HOST_STATE = Symbol.for(+ '@remotion/whisper-webgpu/model-host-state',+);++type TransformersModule = {+ readonly AutoModelForImageSegmentation: typeof AutoModelForImageSegmentation;+ readonly AutoProcessor: typeof AutoProcessor;+ readonly BackgroundRemovalPipeline: typeof BackgroundRemovalPipeline;+ readonly env: typeof env;+ readonly ModelRegistry: typeof ModelRegistry;+};++type ModelHostState = {+ activeOperations: number;+ previousRemoteConfiguration: {+ remoteHost: string;+ remotePathTemplate: string;+ } | null;+};++export const withRemotionModelHost = async <ReturnValue>(+ operation: (transformers: TransformersModule) => Promise<ReturnValue>,+): Promise<ReturnValue> => {+ // Transformers.js is an external peer so AI workflows in one application can+ // share a runtime. The alias coordinates with @remotion/whisper-webgpu while+ // both packages transition to the generic state key.+ const transformers = await import('@huggingface/transformers');+ const {env} = transformers;+ const environmentWithState = env as typeof env &+ Record<symbol, ModelHostState | undefined>;+ let state =+ environmentWithState[REMOTION_MODEL_HOST_STATE] ??+ environmentWithState[WHISPER_MODEL_HOST_STATE];+ if (!state) {+ state = {+ activeOperations: 0,+ previousRemoteConfiguration: null,+ };+ }++ if (!environmentWithState[REMOTION_MODEL_HOST_STATE]) {+ Object.defineProperty(environmentWithState, REMOTION_MODEL_HOST_STATE, {+ value: state,+ });+ }++ if (!environmentWithState[WHISPER_MODEL_HOST_STATE]) {+ Object.defineProperty(environmentWithState, WHISPER_MODEL_HOST_STATE, {+ value: state,+ });+ }++ if (state.activeOperations === 0) {+ state.previousRemoteConfiguration = {+ remoteHost: env.remoteHost,+ remotePathTemplate: env.remotePathTemplate,+ };+ env.remoteHost = REMOTION_MODEL_HOST;+ env.remotePathTemplate = REMOTION_MODEL_PATH_TEMPLATE;+ }++ state.activeOperations++;+ try {+ return await operation(transformers);+ } finally {+ state.activeOperations--;+ if (state.activeOperations === 0) {+ const configurationToRestore = state.previousRemoteConfiguration;+ state.previousRemoteConfiguration = null;+ if (configurationToRestore) {+ if (env.remoteHost === REMOTION_MODEL_HOST) {+ env.remoteHost = configurationToRestore.remoteHost;+ }++ if (env.remotePathTemplate === REMOTION_MODEL_PATH_TEMPLATE) {+ env.remotePathTemplate = configurationToRestore.remotePathTemplate;+ }+ }+ }+ }+};