Skip to content

🎥 Make videos programmatically with React

Add async disposal to video matting resources

JonnyBurger80ad8225b5dc
diff --git a/packages/docs/docs/video-matting/index.mdx b/packages/docs/docs/video-matting/index.mdx
index 143a0390f2..4a3cf26c40 100644
--- a/packages/docs/docs/video-matting/index.mdx
+++ b/packages/docs/docs/video-matting/index.mdx
@@ -21,17 +21,16 @@ import {separateVideoLayers} from '@remotion/video-matting';
const file = new File([], 'input.mp4');
-const {base, foreground} = await separateVideoLayers({
+await using result = await separateVideoLayers({
src: file,
});
+const {base, foreground} = result;
const baseBlob = await base.getBlob();
const foregroundBlob = await foreground.getBlob();
-
-await Promise.all([base.dispose(), foreground.dispose()]);
```
-Call `dispose()` after retrieving the blobs to release package-owned temporary storage.
+`await using` releases package-owned temporary storage after the blobs have been retrieved. It does not unload the model.
The model is downloaded on the first call and cached by the browser. Call [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model) first to display download progress.
diff --git a/packages/docs/docs/video-matting/load-video-matting-model.mdx b/packages/docs/docs/video-matting/load-video-matting-model.mdx
index 5cc0d7b1a3..cfe180a2b4 100644
--- a/packages/docs/docs/video-matting/load-video-matting-model.mdx
+++ b/packages/docs/docs/video-matting/load-video-matting-model.mdx
@@ -11,7 +11,7 @@ Downloads a video matting model from an immutable path on `remotion.media` and i
```ts twoslash title="load-model.ts"
import {loadVideoMattingModel} from '@remotion/video-matting';
-const result = await loadVideoMattingModel({
+await using result = await loadVideoMattingModel({
model: 'modnet',
onProgress: ({progress}) => {
console.log(progress);
@@ -61,7 +61,13 @@ Returns an object with this property:
Whether the same model was already initialized in memory.
-The browser manages the persistent [Transformers.js](https://huggingface.co/docs/transformers.js) cache. This function also retains the initialized model in memory until [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model) is called or the page closes.
+### `[Symbol.asyncDispose]()`<AvailableFrom v="4.0.527" />
+
+Releases this call's hold on the initialized model. The pipeline is disposed after every load handle is released and active separations finish.
+
+Use `await using` to call this method automatically when leaving the current scope. Call [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model) to dispose the pipeline explicitly regardless of retained load handles.
+
+The browser manages the persistent [Transformers.js](https://huggingface.co/docs/transformers.js) cache. Releasing the initialized pipeline does not delete the downloaded model files.
The hosted files are byte-identical to files from the pinned upstream Hugging Face revision.
diff --git a/packages/docs/docs/video-matting/separate-video-layers.mdx b/packages/docs/docs/video-matting/separate-video-layers.mdx
index 8820e180f7..e34f1a2e87 100644
--- a/packages/docs/docs/video-matting/separate-video-layers.mdx
+++ b/packages/docs/docs/video-matting/separate-video-layers.mdx
@@ -13,7 +13,7 @@ import {separateVideoLayers} from '@remotion/video-matting';
const file = new File([], 'input.mp4');
-const result = await separateVideoLayers({
+await using result = await separateVideoLayers({
src: file,
model: 'modnet',
audio: 'base',
@@ -24,8 +24,6 @@ const result = await separateVideoLayers({
const baseBlob = await result.base.getBlob();
const foregroundBlob = await result.foreground.getBlob();
-
-await Promise.all([result.base.dispose(), result.foreground.dispose()]);
```
Place inserted content between the base and foreground. The base remains opaque and includes the original subject; the foreground repeats the subject with alpha so it appears in front of the inserted content.
@@ -171,6 +169,10 @@ Returns a promise resolving to a `Blob`, unless `outputs.base.outputWritable` wa
Releases temporary storage owned by the base output. Call it after `base.getBlob()` has resolved. The method is idempotent and has no effect for `arraybuffer` or custom writable outputs.
+#### `base[Symbol.asyncDispose]()`<AvailableFrom v="4.0.527" />
+
+Calls `base.dispose()` when an `await using` scope ends.
+
### foreground
The foreground output with alpha.
@@ -183,6 +185,10 @@ Returns a promise resolving to a `Blob`, unless `outputs.foreground.outputWritab
Releases temporary storage owned by the foreground output. Call it after `foreground.getBlob()` has resolved. The method is idempotent and has no effect for `arraybuffer` or custom writable outputs.
+#### `foreground[Symbol.asyncDispose]()`<AvailableFrom v="4.0.527" />
+
+Calls `foreground.dispose()` when an `await using` scope ends.
+
### model
The model used for separation.
@@ -203,6 +209,10 @@ The duration of the processed video in seconds.
The number of video frames processed.
+### `[Symbol.asyncDispose]()`<AvailableFrom v="4.0.527" />
+
+Disposes both outputs when an `await using` scope ends. Retrieve any blobs before leaving the scope; blobs returned by `getBlob()` remain usable afterward. The loaded model remains in memory for reuse.
+
## Output format
Both layers are WebM files encoded with VP9. The foreground preserves alpha. Audio is encoded as Opus when included.
diff --git a/packages/video-matting/src/create-video-layer-output.ts b/packages/video-matting/src/create-video-layer-output.ts
index 1a85ba892c..9168e8de8a 100644
--- a/packages/video-matting/src/create-video-layer-output.ts
+++ b/packages/video-matting/src/create-video-layer-output.ts
@@ -1,3 +1,4 @@
+import './symbol-async-dispose';
import {
BufferTarget,
Output,
@@ -15,6 +16,16 @@ import {
type VideoLayerMediabunnyTarget = BufferTarget | StreamTarget;
+const withAsyncDispose = (
+ result: Pick<VideoLayerOutput, 'getBlob' | 'dispose'>,
+): VideoLayerOutput => {
+ Object.defineProperty(result, Symbol.asyncDispose, {
+ enumerable: false,
+ value: result.dispose,
+ });
+ return result as VideoLayerOutput;
+};
+
export type CreatedVideoLayerOutput<F extends OutputFormat> = {
output: Output<F, VideoLayerMediabunnyTarget>;
finalize: () => Promise<VideoLayerOutput>;
@@ -233,7 +244,7 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({
finalized = true;
if (outputMode === 'writable') {
- return {
+ return withAsyncDispose({
dispose: () => Promise.resolve(),
getBlob: () =>
Promise.reject(
@@ -241,7 +252,7 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({
'getBlob() is unavailable when outputWritable is used',
),
),
- };
+ });
}
if (outputMode === 'web-fs') {
@@ -251,7 +262,7 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({
let blobPromise: Promise<Blob> | null = null;
- return {
+ return withAsyncDispose({
dispose: discard,
getBlob: () => {
blobPromise ??= (async () => {
@@ -262,14 +273,14 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({
return blobPromise;
},
- };
+ });
}
if (!(target instanceof BufferTarget)) {
throw new Error('Expected an in-memory output target');
}
- return {
+ return withAsyncDispose({
dispose: () => Promise.resolve(),
getBlob: () => {
if (target.buffer === null) {
@@ -278,7 +289,7 @@ export const createVideoLayerOutput = async <F extends OutputFormat>({
return Promise.resolve(new Blob([target.buffer], {type: mimeType}));
},
- };
+ });
} catch (error) {
await Promise.allSettled([cancelMediabunnyOutput(), discard()]);
throw error;
diff --git a/packages/video-matting/src/load-video-matting-model.ts b/packages/video-matting/src/load-video-matting-model.ts
index 181dadb62c..a20faab7d5 100644
--- a/packages/video-matting/src/load-video-matting-model.ts
+++ b/packages/video-matting/src/load-video-matting-model.ts
@@ -1,3 +1,4 @@
+import './symbol-async-dispose';
import {
getHostedVideoMattingModelId,
getVideoMattingModelInfo,
@@ -54,6 +55,7 @@ type LoadedVideoMattingPipelineState = {
onIdle: Array<() => void>;
progressListeners: Set<OnVideoMattingModelLoadProgress>;
lastProgress: VideoMattingModelLoadProgress | null;
+ retainedLoadHandles: number;
};
const pipelines = new Map<VideoMattingModel, LoadedVideoMattingPipelineState>();
@@ -64,7 +66,7 @@ export type LoadVideoMattingModelOptions = {
onProgress?: OnVideoMattingModelLoadProgress;
};
-export type LoadVideoMattingModelResult = {
+export type LoadVideoMattingModelResult = AsyncDisposable & {
alreadyLoaded: boolean;
};
@@ -147,6 +149,7 @@ const getOrCreateVideoMattingPipeline = ({
onIdle: [],
progressListeners: new Set(),
lastProgress: null,
+ retainedLoadHandles: 0,
};
const pendingDisposal = disposals.get(model) ?? null;
@@ -311,57 +314,6 @@ const subscribeToProgress = (
};
};
-export const loadVideoMattingModel = async ({
- model,
- onProgress,
-}: LoadVideoMattingModelOptions): Promise<LoadVideoMattingModelResult> => {
- const {state, alreadyLoaded} = getOrCreateVideoMattingPipeline({model});
- const unsubscribe = subscribeToProgress(state, onProgress);
- try {
- await state.loading;
- } finally {
- unsubscribe();
- }
-
- return {alreadyLoaded};
-};
-
-export const withLoadedVideoMattingPipeline = async <ReturnValue>({
- model,
- onProgress,
- signal,
- run,
-}: {
- model: VideoMattingModel;
- onProgress?: OnVideoMattingModelLoadProgress;
- signal: AbortSignal | null;
- run: (pipeline: LoadedVideoMattingPipeline) => Promise<ReturnValue>;
-}): Promise<ReturnValue> => {
- const {state} = getOrCreateVideoMattingPipeline({model});
- const unsubscribe = subscribeToProgress(state, onProgress);
- state.pendingUses++;
- let isActive = false;
- try {
- const loaded = await waitForLoadingOrAbort({
- loading: state.loading,
- signal,
- });
- state.pendingUses--;
- state.activeUses++;
- isActive = true;
- return await run(loaded.run);
- } finally {
- unsubscribe();
- if (isActive) {
- state.activeUses--;
- } else {
- state.pendingUses--;
- }
-
- notifyIfIdle(state);
- }
-};
-
export type DisposeVideoMattingModelOptions = {
model?: VideoMattingModel;
};
@@ -416,3 +368,71 @@ export const disposeVideoMattingModel = async ({
await Promise.all(pending.values());
};
+
+export const loadVideoMattingModel = async ({
+ model,
+ onProgress,
+}: LoadVideoMattingModelOptions): Promise<LoadVideoMattingModelResult> => {
+ const {state, alreadyLoaded} = getOrCreateVideoMattingPipeline({model});
+ const unsubscribe = subscribeToProgress(state, onProgress);
+ try {
+ await state.loading;
+ } finally {
+ unsubscribe();
+ }
+
+ state.retainedLoadHandles++;
+ let released = false;
+ const result = {alreadyLoaded} as LoadVideoMattingModelResult;
+ Object.defineProperty(result, Symbol.asyncDispose, {
+ enumerable: false,
+ value: async () => {
+ if (released) {
+ return;
+ }
+
+ released = true;
+ state.retainedLoadHandles--;
+ if (state.retainedLoadHandles === 0 && pipelines.get(model) === state) {
+ await disposeVideoMattingModel({model});
+ }
+ },
+ });
+ return result;
+};
+
+export const withLoadedVideoMattingPipeline = async <ReturnValue>({
+ model,
+ onProgress,
+ signal,
+ run,
+}: {
+ model: VideoMattingModel;
+ onProgress?: OnVideoMattingModelLoadProgress;
+ signal: AbortSignal | null;
+ run: (pipeline: LoadedVideoMattingPipeline) => Promise<ReturnValue>;
+}): Promise<ReturnValue> => {
+ const {state} = getOrCreateVideoMattingPipeline({model});
+ const unsubscribe = subscribeToProgress(state, onProgress);
+ state.pendingUses++;
+ let isActive = false;
+ try {
+ const loaded = await waitForLoadingOrAbort({
+ loading: state.loading,
+ signal,
+ });
+ state.pendingUses--;
+ state.activeUses++;
+ isActive = true;
+ return await run(loaded.run);
+ } finally {
+ unsubscribe();
+ if (isActive) {
+ state.activeUses--;
+ } else {
+ state.pendingUses--;
+ }
+
+ notifyIfIdle(state);
+ }
+};
diff --git a/packages/video-matting/src/output-target.ts b/packages/video-matting/src/output-target.ts
index ab0a2f995c..a207fd8dc9 100644
--- a/packages/video-matting/src/output-target.ts
+++ b/packages/video-matting/src/output-target.ts
@@ -12,7 +12,7 @@ export type VideoLayerOutputOptions =
outputWritable: WritableStream<StreamTargetChunk>;
};
-export type VideoLayerOutput = {
+export type VideoLayerOutput = AsyncDisposable & {
getBlob: () => Promise<Blob>;
dispose: () => Promise<void>;
};
diff --git a/packages/video-matting/src/separate-video-layers.ts b/packages/video-matting/src/separate-video-layers.ts
index 20ef50d686..1b7063746e 100644
--- a/packages/video-matting/src/separate-video-layers.ts
+++ b/packages/video-matting/src/separate-video-layers.ts
@@ -67,7 +67,7 @@ export type SeparateVideoLayersOptions = {
onProgress?: (progress: SeparateVideoLayersProgress) => void;
};
-export type SeparateVideoLayersResult = {
+export type SeparateVideoLayersResult = AsyncDisposable & {
base: VideoLayerOutput;
foreground: VideoLayerOutput;
model: VideoMattingModel;
@@ -538,15 +538,21 @@ export const separateVideoLayers = async (
throwIfAborted(options.signal);
completed = true;
- return {
+ const separated = {
base,
foreground,
model,
width,
height,
durationInSeconds,
- processedFrames,
- };
+ } as SeparateVideoLayersResult;
+ Object.defineProperty(separated, Symbol.asyncDispose, {
+ enumerable: false,
+ value: async () => {
+ await Promise.all([base.dispose(), foreground.dispose()]);
+ },
+ });
+ return separated;
} catch (error) {
await cancelPendingMedia();
diff --git a/packages/video-matting/src/symbol-async-dispose.ts b/packages/video-matting/src/symbol-async-dispose.ts
new file mode 100644
index 0000000000..1cde9540f8
--- /dev/null
+++ b/packages/video-matting/src/symbol-async-dispose.ts
@@ -0,0 +1,5 @@
+if (typeof Symbol.asyncDispose !== 'symbol') {
+ Object.defineProperty(Symbol, 'asyncDispose', {
+ value: Symbol.for('asyncDispose'),
+ });
+}
diff --git a/packages/video-matting/src/test/model-runtime.test.ts b/packages/video-matting/src/test/model-runtime.test.ts
index da4b1c7105..532e6bbf7d 100644
--- a/packages/video-matting/src/test/model-runtime.test.ts
+++ b/packages/video-matting/src/test/model-runtime.test.ts
@@ -176,10 +176,11 @@ test('shares loading, reports progress, and defers disposal while in use', async
});
expect(cacheCheckEnvironment).toEqual(initializationEnvironment);
releaseInitialization();
- expect(await Promise.all([first, second])).toEqual([
- {alreadyLoaded: false},
- {alreadyLoaded: true},
- ]);
+ expect(
+ (await Promise.all([first, second])).map(
+ ({alreadyLoaded}) => alreadyLoaded,
+ ),
+ ).toEqual([false, true]);
expect(initializationOptions).toMatchObject({
device: 'webgpu',
dtype: 'fp32',
@@ -218,6 +219,50 @@ test('shares loading, reports progress, and defers disposal while in use', async
expect(livePipelineCount).toBe(0);
});
+test('await using releases a model after its last load handle', async () => {
+ const {loadVideoMattingModel} = await import('../index');
+ const {withLoadedVideoMattingPipeline} = await getRuntime();
+
+ {
+ await using first = await loadVideoMattingModel({model: 'modnet'});
+ expect(first.alreadyLoaded).toBe(false);
+ {
+ await using second = await loadVideoMattingModel({model: 'modnet'});
+ expect(second.alreadyLoaded).toBe(true);
+ await second[Symbol.asyncDispose]();
+ }
+
+ expect(disposeCalls).toBe(0);
+ const result = await withLoadedVideoMattingPipeline({
+ model: 'modnet',
+ signal: null,
+ run: (pipeline) => pipeline({} as OffscreenCanvas),
+ });
+ expect(result.data).toEqual(new Uint8ClampedArray([10, 20, 30, 128]));
+ }
+
+ expect(initializationCount).toBe(1);
+ expect(pipelineRunCount).toBe(1);
+ expect(disposeCalls).toBe(1);
+ expect(livePipelineCount).toBe(0);
+});
+
+test('an old load handle cannot dispose a replacement model', async () => {
+ const {disposeVideoMattingModel, loadVideoMattingModel} =
+ await import('../index');
+ const oldHandle = await loadVideoMattingModel({model: 'modnet'});
+ await disposeVideoMattingModel({model: 'modnet'});
+ const newHandle = await loadVideoMattingModel({model: 'modnet'});
+
+ await oldHandle[Symbol.asyncDispose]();
+ expect(disposeCalls).toBe(1);
+ expect(livePipelineCount).toBe(1);
+
+ await newHandle[Symbol.asyncDispose]();
+ expect(disposeCalls).toBe(2);
+ expect(livePipelineCount).toBe(0);
+});
+
test('loads every public model from its immutable hosted model ID', async () => {
const {disposeVideoMattingModel, getAvailableModels, loadVideoMattingModel} =
await import('../index');
@@ -282,10 +327,9 @@ test('coordinates the shared Transformers environment across model loads', async
environmentWithState[Symbol.for('@remotion/transformers/model-host-state')],
).toBe(sharedState);
releaseInitialization();
- expect(await Promise.all(loads)).toEqual([
- {alreadyLoaded: false},
- {alreadyLoaded: false},
- ]);
+ expect(
+ (await Promise.all(loads)).map(({alreadyLoaded}) => alreadyLoaded),
+ ).toEqual([false, false]);
expect(transformersEnvironment).toEqual(originalTransformersEnvironment);
await disposeVideoMattingModel();
});
diff --git a/packages/video-matting/src/test/output-target.test.ts b/packages/video-matting/src/test/output-target.test.ts
index 264adfa559..08415d1dab 100644
--- a/packages/video-matting/src/test/output-target.test.ts
+++ b/packages/video-matting/src/test/output-target.test.ts
@@ -141,14 +141,18 @@ describe('video layer output targets', () => {
});
await created.output.start();
- const result = await created.finalize();
- const blob = await result.getBlob();
- const [filename, file] = [...files.entries()][0];
+ let filename: string;
+ {
+ await using result = await created.finalize();
+ const blob = await result.getBlob();
+ const [name, file] = [...files.entries()][0];
+ filename = name;
+
+ expect(file.closed).toBe(true);
+ expect(file.seekPositions.length).toBeGreaterThan(0);
+ expect(blob.size).toBeGreaterThan(0);
+ }
- expect(file.closed).toBe(true);
- expect(file.seekPositions.length).toBeGreaterThan(0);
- expect(blob.size).toBeGreaterThan(0);
- await result.dispose();
expect(files.size).toBe(0);
expect(removedNames.at(-1)).toBe(filename);
});