🎥 Make videos programmatically with React
Add async disposal to video matting resources
JonnyBurger80ad8225b5dc
diff --git a/packages/docs/docs/video-matting/index.mdx b/packages/docs/docs/video-matting/index.mdxindex 143a0390f2..4a3cf26c40 100644--- a/packages/docs/docs/video-matting/index.mdx+++ b/packages/docs/docs/video-matting/index.mdx@@ -21,17 +21,16 @@ import {separateVideoLayers} from '@remotion/video-matting';const file = new File([], 'input.mp4');-const {base, foreground} = await separateVideoLayers({+await using result = await separateVideoLayers({src: file,});+const {base, foreground} = result;const baseBlob = await base.getBlob();const foregroundBlob = await foreground.getBlob();--await Promise.all([base.dispose(), foreground.dispose()]);```-Call `dispose()` after retrieving the blobs to release package-owned temporary storage.+`await using` releases package-owned temporary storage after the blobs have been retrieved. It does not unload the model.The model is downloaded on the first call and cached by the browser. Call [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model) first to display download progress.diff --git a/packages/docs/docs/video-matting/load-video-matting-model.mdx b/packages/docs/docs/video-matting/load-video-matting-model.mdxindex 5cc0d7b1a3..cfe180a2b4 100644--- a/packages/docs/docs/video-matting/load-video-matting-model.mdx+++ b/packages/docs/docs/video-matting/load-video-matting-model.mdx@@ -11,7 +11,7 @@ Downloads a video matting model from an immutable path on `remotion.media` and i```ts twoslash title="load-model.ts"import {loadVideoMattingModel} from '@remotion/video-matting';-const result = await loadVideoMattingModel({+await using result = await loadVideoMattingModel({model: 'modnet',onProgress: ({progress}) => {console.log(progress);@@ -61,7 +61,13 @@ Returns an object with this property:Whether the same model was already initialized in memory.-The browser manages the persistent [Transformers.js](https://huggingface.co/docs/transformers.js) cache. This function also retains the initialized model in memory until [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model) is called or the page closes.+### `[Symbol.asyncDispose]()`<AvailableFrom v="4.0.527" />++Releases this call's hold on the initialized model. The pipeline is disposed after every load handle is released and active separations finish.++Use `await using` to call this method automatically when leaving the current scope. Call [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model) to dispose the pipeline explicitly regardless of retained load handles.++The browser manages the persistent [Transformers.js](https://huggingface.co/docs/transformers.js) cache. Releasing the initialized pipeline does not delete the downloaded model files.The hosted files are byte-identical to files from the pinned upstream Hugging Face revision.diff --git a/packages/docs/docs/video-matting/separate-video-layers.mdx b/packages/docs/docs/video-matting/separate-video-layers.mdxindex 8820e180f7..e34f1a2e87 100644--- a/packages/docs/docs/video-matting/separate-video-layers.mdx+++ b/packages/docs/docs/video-matting/separate-video-layers.mdx@@ -13,7 +13,7 @@ import {separateVideoLayers} from '@remotion/video-matting';const file = new File([], 'input.mp4');-const result = await separateVideoLayers({+await using result = await separateVideoLayers({src: file,model: 'modnet',audio: 'base',@@ -24,8 +24,6 @@ const result = await separateVideoLayers({const baseBlob = await result.base.getBlob();const foregroundBlob = await result.foreground.getBlob();--await Promise.all([result.base.dispose(), result.foreground.dispose()]);```Place inserted content between the base and foreground. The base remains opaque and includes the original subject; the foreground repeats the subject with alpha so it appears in front of the inserted content.@@ -171,6 +169,10 @@ Returns a promise resolving to a `Blob`, unless `outputs.base.outputWritable` waReleases temporary storage owned by the base output. Call it after `base.getBlob()` has resolved. The method is idempotent and has no effect for `arraybuffer` or custom writable outputs.+#### `base[Symbol.asyncDispose]()`<AvailableFrom v="4.0.527" />++Calls `base.dispose()` when an `await using` scope ends.+### foregroundThe foreground output with alpha.@@ -183,6 +185,10 @@ Returns a promise resolving to a `Blob`, unless `outputs.foreground.outputWritabReleases temporary storage owned by the foreground output. Call it after `foreground.getBlob()` has resolved. The method is idempotent and has no effect for `arraybuffer` or custom writable outputs.+#### `foreground[Symbol.asyncDispose]()`<AvailableFrom v="4.0.527" />++Calls `foreground.dispose()` when an `await using` scope ends.+### modelThe model used for separation.@@ -203,6 +209,10 @@ The duration of the processed video in seconds.The number of video frames processed.+### `[Symbol.asyncDispose]()`<AvailableFrom v="4.0.527" />++Disposes both outputs when an `await using` scope ends. Retrieve any blobs before leaving the scope; blobs returned by `getBlob()` remain usable afterward. The loaded model remains in memory for reuse.+## Output formatBoth layers are WebM files encoded with VP9. The foreground preserves alpha. Audio is encoded as Opus when included.diff --git a/packages/video-matting/src/create-video-layer-output.ts b/packages/video-matting/src/create-video-layer-output.tsindex 1a85ba892c..9168e8de8a 100644--- a/packages/video-matting/src/create-video-layer-output.ts+++ b/packages/video-matting/src/create-video-layer-output.ts@@ -1,3 +1,4 @@+import './symbol-async-dispose';import {BufferTarget,Output,@@ -15,6 +16,16 @@ import {type VideoLayerMediabunnyTarget = BufferTarget | StreamTarget;+const withAsyncDispose = (+ result: Pick<VideoLayerOutput, 'getBlob' | 'dispose'>,+): VideoLayerOutput => {+ Object.defineProperty(result, Symbol.asyncDispose, {+ enumerable: false,+ value: result.dispose,+ });+ return result as VideoLayerOutput;+};+export type CreatedVideoLayerOutput<F extends OutputFormat> = {output: Output<F, VideoLayerMediabunnyTarget>;finalize: () => Promise<VideoLayerOutput>;@@ -233,7 +244,7 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({finalized = true;if (outputMode === 'writable') {- return {+ return withAsyncDispose({dispose: () => Promise.resolve(),getBlob: () =>Promise.reject(@@ -241,7 +252,7 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({'getBlob() is unavailable when outputWritable is used',),),- };+ });}if (outputMode === 'web-fs') {@@ -251,7 +262,7 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({let blobPromise: Promise<Blob> | null = null;- return {+ return withAsyncDispose({dispose: discard,getBlob: () => {blobPromise ??= (async () => {@@ -262,14 +273,14 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({return blobPromise;},- };+ });}if (!(target instanceof BufferTarget)) {throw new Error('Expected an in-memory output target');}- return {+ return withAsyncDispose({dispose: () => Promise.resolve(),getBlob: () => {if (target.buffer === null) {@@ -278,7 +289,7 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({return Promise.resolve(new Blob([target.buffer], {type: mimeType}));},- };+ });} catch (error) {await Promise.allSettled([cancelMediabunnyOutput(), discard()]);throw error;diff --git a/packages/video-matting/src/load-video-matting-model.ts b/packages/video-matting/src/load-video-matting-model.tsindex 181dadb62c..a20faab7d5 100644--- a/packages/video-matting/src/load-video-matting-model.ts+++ b/packages/video-matting/src/load-video-matting-model.ts@@ -1,3 +1,4 @@+import './symbol-async-dispose';import {getHostedVideoMattingModelId,getVideoMattingModelInfo,@@ -54,6 +55,7 @@ type LoadedVideoMattingPipelineState = {onIdle: Array<() => void>;progressListeners: Set<OnVideoMattingModelLoadProgress>;lastProgress: VideoMattingModelLoadProgress | null;+ retainedLoadHandles: number;};const pipelines = new Map<VideoMattingModel, LoadedVideoMattingPipelineState>();@@ -64,7 +66,7 @@ export type LoadVideoMattingModelOptions = {onProgress?: OnVideoMattingModelLoadProgress;};-export type LoadVideoMattingModelResult = {+export type LoadVideoMattingModelResult = AsyncDisposable & {alreadyLoaded: boolean;};@@ -147,6 +149,7 @@ const getOrCreateVideoMattingPipeline = ({onIdle: [],progressListeners: new Set(),lastProgress: null,+ retainedLoadHandles: 0,};const pendingDisposal = disposals.get(model) ?? null;@@ -311,57 +314,6 @@ const subscribeToProgress = (};};-export const loadVideoMattingModel = async ({- model,- onProgress,-}: LoadVideoMattingModelOptions): Promise<LoadVideoMattingModelResult> => {- const {state, alreadyLoaded} = getOrCreateVideoMattingPipeline({model});- const unsubscribe = subscribeToProgress(state, onProgress);- try {- await state.loading;- } finally {- unsubscribe();- }-- return {alreadyLoaded};-};--export const withLoadedVideoMattingPipeline = async <ReturnValue>({- model,- onProgress,- signal,- run,-}: {- model: VideoMattingModel;- onProgress?: OnVideoMattingModelLoadProgress;- signal: AbortSignal | null;- run: (pipeline: LoadedVideoMattingPipeline) => Promise<ReturnValue>;-}): Promise<ReturnValue> => {- const {state} = getOrCreateVideoMattingPipeline({model});- const unsubscribe = subscribeToProgress(state, onProgress);- state.pendingUses++;- let isActive = false;- try {- const loaded = await waitForLoadingOrAbort({- loading: state.loading,- signal,- });- state.pendingUses--;- state.activeUses++;- isActive = true;- return await run(loaded.run);- } finally {- unsubscribe();- if (isActive) {- state.activeUses--;- } else {- state.pendingUses--;- }-- notifyIfIdle(state);- }-};-export type DisposeVideoMattingModelOptions = {model?: VideoMattingModel;};@@ -416,3 +368,71 @@ export const disposeVideoMattingModel = async ({await Promise.all(pending.values());};++export const loadVideoMattingModel = async ({+ model,+ onProgress,+}: LoadVideoMattingModelOptions): Promise<LoadVideoMattingModelResult> => {+ const {state, alreadyLoaded} = getOrCreateVideoMattingPipeline({model});+ const unsubscribe = subscribeToProgress(state, onProgress);+ try {+ await state.loading;+ } finally {+ unsubscribe();+ }++ state.retainedLoadHandles++;+ let released = false;+ const result = {alreadyLoaded} as LoadVideoMattingModelResult;+ Object.defineProperty(result, Symbol.asyncDispose, {+ enumerable: false,+ value: async () => {+ if (released) {+ return;+ }++ released = true;+ state.retainedLoadHandles--;+ if (state.retainedLoadHandles === 0 && pipelines.get(model) === state) {+ await disposeVideoMattingModel({model});+ }+ },+ });+ return result;+};++export const withLoadedVideoMattingPipeline = async <ReturnValue>({+ model,+ onProgress,+ signal,+ run,+}: {+ model: VideoMattingModel;+ onProgress?: OnVideoMattingModelLoadProgress;+ signal: AbortSignal | null;+ run: (pipeline: LoadedVideoMattingPipeline) => Promise<ReturnValue>;+}): Promise<ReturnValue> => {+ const {state} = getOrCreateVideoMattingPipeline({model});+ const unsubscribe = subscribeToProgress(state, onProgress);+ state.pendingUses++;+ let isActive = false;+ try {+ const loaded = await waitForLoadingOrAbort({+ loading: state.loading,+ signal,+ });+ state.pendingUses--;+ state.activeUses++;+ isActive = true;+ return await run(loaded.run);+ } finally {+ unsubscribe();+ if (isActive) {+ state.activeUses--;+ } else {+ state.pendingUses--;+ }++ notifyIfIdle(state);+ }+};diff --git a/packages/video-matting/src/output-target.ts b/packages/video-matting/src/output-target.tsindex ab0a2f995c..a207fd8dc9 100644--- a/packages/video-matting/src/output-target.ts+++ b/packages/video-matting/src/output-target.ts@@ -12,7 +12,7 @@ export type VideoLayerOutputOptions =outputWritable: WritableStream<StreamTargetChunk>;};-export type VideoLayerOutput = {+export type VideoLayerOutput = AsyncDisposable & {getBlob: () => Promise<Blob>;dispose: () => Promise<void>;};diff --git a/packages/video-matting/src/separate-video-layers.ts b/packages/video-matting/src/separate-video-layers.tsindex 20ef50d686..1b7063746e 100644--- a/packages/video-matting/src/separate-video-layers.ts+++ b/packages/video-matting/src/separate-video-layers.ts@@ -67,7 +67,7 @@ export type SeparateVideoLayersOptions = {onProgress?: (progress: SeparateVideoLayersProgress) => void;};-export type SeparateVideoLayersResult = {+export type SeparateVideoLayersResult = AsyncDisposable & {base: VideoLayerOutput;foreground: VideoLayerOutput;model: VideoMattingModel;@@ -538,15 +538,21 @@ export const separateVideoLayers = async (throwIfAborted(options.signal);completed = true;- return {+ const separated = {base,foreground,model,width,height,durationInSeconds,- processedFrames,- };+ } as SeparateVideoLayersResult;+ Object.defineProperty(separated, Symbol.asyncDispose, {+ enumerable: false,+ value: async () => {+ await Promise.all([base.dispose(), foreground.dispose()]);+ },+ });+ return separated;} catch (error) {await cancelPendingMedia();diff --git a/packages/video-matting/src/symbol-async-dispose.ts b/packages/video-matting/src/symbol-async-dispose.tsnew file mode 100644index 0000000000..1cde9540f8--- /dev/null+++ b/packages/video-matting/src/symbol-async-dispose.ts@@ -0,0 +1,5 @@+if (typeof Symbol.asyncDispose !== 'symbol') {+ Object.defineProperty(Symbol, 'asyncDispose', {+ value: Symbol.for('asyncDispose'),+ });+}diff --git a/packages/video-matting/src/test/model-runtime.test.ts b/packages/video-matting/src/test/model-runtime.test.tsindex da4b1c7105..532e6bbf7d 100644--- a/packages/video-matting/src/test/model-runtime.test.ts+++ b/packages/video-matting/src/test/model-runtime.test.ts@@ -176,10 +176,11 @@ test('shares loading, reports progress, and defers disposal while in use', async});expect(cacheCheckEnvironment).toEqual(initializationEnvironment);releaseInitialization();- expect(await Promise.all([first, second])).toEqual([- {alreadyLoaded: false},- {alreadyLoaded: true},- ]);+ expect(+ (await Promise.all([first, second])).map(+ ({alreadyLoaded}) => alreadyLoaded,+ ),+ ).toEqual([false, true]);expect(initializationOptions).toMatchObject({device: 'webgpu',dtype: 'fp32',@@ -218,6 +219,50 @@ test('shares loading, reports progress, and defers disposal while in use', asyncexpect(livePipelineCount).toBe(0);});+test('await using releases a model after its last load handle', async () => {+ const {loadVideoMattingModel} = await import('../index');+ const {withLoadedVideoMattingPipeline} = await getRuntime();++ {+ await using first = await loadVideoMattingModel({model: 'modnet'});+ expect(first.alreadyLoaded).toBe(false);+ {+ await using second = await loadVideoMattingModel({model: 'modnet'});+ expect(second.alreadyLoaded).toBe(true);+ await second[Symbol.asyncDispose]();+ }++ expect(disposeCalls).toBe(0);+ const result = await withLoadedVideoMattingPipeline({+ model: 'modnet',+ signal: null,+ run: (pipeline) => pipeline({} as OffscreenCanvas),+ });+ expect(result.data).toEqual(new Uint8ClampedArray([10, 20, 30, 128]));+ }++ expect(initializationCount).toBe(1);+ expect(pipelineRunCount).toBe(1);+ expect(disposeCalls).toBe(1);+ expect(livePipelineCount).toBe(0);+});++test('an old load handle cannot dispose a replacement model', async () => {+ const {disposeVideoMattingModel, loadVideoMattingModel} =+ await import('../index');+ const oldHandle = await loadVideoMattingModel({model: 'modnet'});+ await disposeVideoMattingModel({model: 'modnet'});+ const newHandle = await loadVideoMattingModel({model: 'modnet'});++ await oldHandle[Symbol.asyncDispose]();+ expect(disposeCalls).toBe(1);+ expect(livePipelineCount).toBe(1);++ await newHandle[Symbol.asyncDispose]();+ expect(disposeCalls).toBe(2);+ expect(livePipelineCount).toBe(0);+});+test('loads every public model from its immutable hosted model ID', async () => {const {disposeVideoMattingModel, getAvailableModels, loadVideoMattingModel} =await import('../index');@@ -282,10 +327,9 @@ test('coordinates the shared Transformers environment across model loads', asyncenvironmentWithState[Symbol.for('@remotion/transformers/model-host-state')],).toBe(sharedState);releaseInitialization();- expect(await Promise.all(loads)).toEqual([- {alreadyLoaded: false},- {alreadyLoaded: false},- ]);+ expect(+ (await Promise.all(loads)).map(({alreadyLoaded}) => alreadyLoaded),+ ).toEqual([false, false]);expect(transformersEnvironment).toEqual(originalTransformersEnvironment);await disposeVideoMattingModel();});diff --git a/packages/video-matting/src/test/output-target.test.ts b/packages/video-matting/src/test/output-target.test.tsindex 264adfa559..08415d1dab 100644--- a/packages/video-matting/src/test/output-target.test.ts+++ b/packages/video-matting/src/test/output-target.test.ts@@ -141,14 +141,18 @@ describe('video layer output targets', () => {});await created.output.start();- const result = await created.finalize();- const blob = await result.getBlob();- const [filename, file] = [...files.entries()][0];+ let filename: string;+ {+ await using result = await created.finalize();+ const blob = await result.getBlob();+ const [name, file] = [...files.entries()][0];+ filename = name;++ expect(file.closed).toBe(true);+ expect(file.seekPositions.length).toBeGreaterThan(0);+ expect(blob.size).toBeGreaterThan(0);+ }- expect(file.closed).toBe(true);- expect(file.seekPositions.length).toBeGreaterThan(0);- expect(blob.size).toBeGreaterThan(0);- await result.dispose();expect(files.size).toBe(0);expect(removedNames.at(-1)).toBe(filename);});