Skip to content

🎥 Make videos programmatically with React

Separate AI model downloads from WebGPU initialization

JonnyBurgera2bb86ed1854
diff --git a/packages/convert/app/components/transcribe/state.ts b/packages/convert/app/components/transcribe/state.ts
index 4f378bc754..c6a67eb871 100644
--- a/packages/convert/app/components/transcribe/state.ts
+++ b/packages/convert/app/components/transcribe/state.ts
@@ -1,6 +1,6 @@
import type {Caption} from '@remotion/captions';
import type {
- WhisperWebGpuModelLoadProgress,
+ WhisperModelDownloadProgress,
WhisperWebGpuTranscription,
} from '@remotion/whisper-webgpu';
@@ -13,7 +13,7 @@ export type TranscriptionState =
}
| {
type: 'downloading-model';
- progress: WhisperWebGpuModelLoadProgress;
+ progress: WhisperModelDownloadProgress;
}
| {
type: 'transcribing';
diff --git a/packages/convert/app/components/transcribe/transcribeAudio.tsx b/packages/convert/app/components/transcribe/transcribeAudio.tsx
index 4cdf96f267..8ae4fe8e12 100644
--- a/packages/convert/app/components/transcribe/transcribeAudio.tsx
+++ b/packages/convert/app/components/transcribe/transcribeAudio.tsx
@@ -1,4 +1,5 @@
import {
+ downloadWhisperModel,
loadWhisperModel,
resampleTo16Khz,
toCaptions,
@@ -46,7 +47,7 @@ export default function TranscribeAudio({
file: await sourceToBlob(source),
});
- await loadWhisperModel({
+ await downloadWhisperModel({
model: selectedModel,
onProgress: (progress) =>
setState(() => ({
@@ -54,6 +55,7 @@ export default function TranscribeAudio({
progress,
})),
});
+ await loadWhisperModel({model: selectedModel});
setState(() => ({
type: 'transcribing',
@@ -87,7 +89,7 @@ export default function TranscribeAudio({
<div
className="w-[50%] h-5 bg-brand"
style={{
- width: `${(state.progress.progress ?? 0) * 100}%`,
+ width: `${state.progress.progress * 100}%`,
}}
/>
</div>
@@ -100,12 +102,8 @@ export default function TranscribeAudio({
</div>
<div className="tabular-nums text-muted-foreground font-brand text-sm">
<span>
- {state.progress.progress === null
- ? state.progress.status
- : `${Math.round(state.progress.progress * 100)}%`}{' '}
- {state.progress.loadedBytes === null
- ? null
- : formatBytes(state.progress.loadedBytes)}
+ {Math.round(state.progress.progress * 100)}%{' '}
+ {formatBytes(state.progress.loadedBytes)}
</span>
</div>
</div>
diff --git a/packages/docs/docs/video-matting/download-video-matting-model.mdx b/packages/docs/docs/video-matting/download-video-matting-model.mdx
new file mode 100644
index 0000000000..39515d2c52
--- /dev/null
+++ b/packages/docs/docs/video-matting/download-video-matting-model.mdx
@@ -0,0 +1,64 @@
+---
+image: /generated/articles-docs-video-matting-download-video-matting-model.png
+title: downloadVideoMattingModel()
+crumb: '@remotion/video-matting'
+---
+
+# downloadVideoMattingModel()<AvailableFrom v="4.0.528" />
+
+Downloads a video matting model from `remotion.media` into the browser cache without initializing WebGPU. Call [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model) or [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) afterwards.
+
+The browser Cache API or a Transformers.js custom cache must be enabled.
+
+```ts twoslash title="download-model.ts"
+import {downloadVideoMattingModel} from '@remotion/video-matting';
+
+const {alreadyDownloaded} = await downloadVideoMattingModel({
+ model: 'modnet',
+ onProgress: ({progress}) => console.log(progress),
+});
+
+console.log(alreadyDownloaded);
+```
+
+## Options
+
+### model
+
+One of the names returned by [`getAvailableModels()`](/docs/video-matting/get-available-models).
+
+### onProgress?
+
+Called as files download. Default: no callback. The callback receives:
+
+#### file
+
+The file being downloaded, or `null` for aggregate progress.
+
+#### progress
+
+Aggregate progress between `0` and `1`.
+
+#### loadedBytes
+
+Bytes downloaded in this call.
+
+#### totalBytes
+
+Estimated total model size in bytes.
+
+## Return value
+
+### alreadyDownloaded
+
+`true` if all files were already cached before this call.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+
+## See also
+
+- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)
+- [`isVideoMattingModelCached()`](/docs/video-matting/is-video-matting-model-cached)
+- [`removeVideoMattingModel()`](/docs/video-matting/remove-video-matting-model)
diff --git a/packages/docs/docs/video-matting/get-available-models.mdx b/packages/docs/docs/video-matting/get-available-models.mdx
index d685248131..3ab99262ff 100644
--- a/packages/docs/docs/video-matting/get-available-models.mdx
+++ b/packages/docs/docs/video-matting/get-available-models.mdx
@@ -51,5 +51,6 @@ The total size in bytes of all runtime files downloaded for WebGPU.
## See also
- [`isVideoMattingModelCached()`](/docs/video-matting/is-video-matting-model-cached)
+- [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model)
- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)
- [`removeVideoMattingModel()`](/docs/video-matting/remove-video-matting-model)
diff --git a/packages/docs/docs/video-matting/index.mdx b/packages/docs/docs/video-matting/index.mdx
index 1814cfa61f..843e9e4c95 100644
--- a/packages/docs/docs/video-matting/index.mdx
+++ b/packages/docs/docs/video-matting/index.mdx
@@ -18,15 +18,19 @@ import {TableOfContents} from './table-of-contents';
```ts twoslash title="separate-video.ts"
import {
+ downloadVideoMattingModel,
loadVideoMattingModel,
separateVideoLayers,
} from '@remotion/video-matting';
export const matteVideo = async (file: File) => {
const model = 'modnet';
+ await downloadVideoMattingModel({
+ model,
+ onProgress: ({progress}) => console.log('Downloading model:', progress),
+ });
await using modelHandle = await loadVideoMattingModel({
model,
- onProgress: ({progress}) => console.log('Loading model:', progress),
});
console.log('Model already in memory:', modelHandle.alreadyLoaded);
diff --git a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx
index 20d886cd7b..279ad7da9c 100644
--- a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx
+++ b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx
@@ -33,5 +33,6 @@ A `Promise<boolean>` resolving to `true` if every required model file is cached.
## See also
- [`getAvailableModels()`](/docs/video-matting/get-available-models)
+- [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model)
- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)
- [`removeVideoMattingModel()`](/docs/video-matting/remove-video-matting-model)
diff --git a/packages/docs/docs/video-matting/load-video-matting-model.mdx b/packages/docs/docs/video-matting/load-video-matting-model.mdx
index be9348af2f..fe5350f7f4 100644
--- a/packages/docs/docs/video-matting/load-video-matting-model.mdx
+++ b/packages/docs/docs/video-matting/load-video-matting-model.mdx
@@ -6,18 +6,16 @@ crumb: '@remotion/video-matting'
# loadVideoMattingModel()<AvailableFrom v="4.0.523" />
-Downloads a video matting model from our `remotion.media` CDN.
+Initializes a previously downloaded video matting model on WebGPU. Call [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model) first.
Call it before [`separateVideoLayers()`](/docs/video-matting/separate-video-layers).
```ts twoslash title="load-model.ts"
-import {loadVideoMattingModel} from '@remotion/video-matting';
+import {downloadVideoMattingModel, loadVideoMattingModel} from '@remotion/video-matting';
+await downloadVideoMattingModel({model: 'modnet'});
await using result = await loadVideoMattingModel({
model: 'modnet',
- onProgress: ({progress}) => {
- console.log(progress);
- },
});
console.log(result.alreadyLoaded);
@@ -31,7 +29,7 @@ One of the names returned by [`getAvailableModels()`](/docs/video-matting/get-av
### onProgress?
-Called while model files are resolved and downloaded. Default: no callback.
+Called while the cached model is initialized. Use [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model) for download progress. Default: no callback.
The callback receives these properties:
@@ -45,15 +43,15 @@ The model file associated with the update, or `null` for aggregate progress.
#### progress
-Download progress between `0` and `1`, or `null` when it cannot be calculated.
+`null` during initialization and `1` when ready.
#### loadedBytes
-The number of bytes downloaded, or `null` when it is unknown.
+Always `null`; downloads happen in [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model).
#### totalBytes
-The total number of bytes, or `null` when it is unknown.
+Always `null`.
## Return value
@@ -65,9 +63,11 @@ Whether the same model was already initialized in memory.
### `[Symbol.asyncDispose]()`<AvailableFrom v="4.0.528" />
-Releases this call's hold on the initialized model. The pipeline is disposed after every load handle is released and active separations finish.
+Releases resources allocated by this function.
-Use `await using` to call this method automatically when leaving the current scope. Call [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model) to dispose the pipeline explicitly regardless of retained load handles.
+Use `await using` to call this method automatically when leaving the current scope.
+
+Call [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model) to dispose the pipeline explicitly regardless of retained load handles.
## Compatibility
@@ -75,6 +75,7 @@ Use `await using` to call this method automatically when leaving the current sco
## See also
+- [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model)
- [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model)
- [`isVideoMattingModelCached()`](/docs/video-matting/is-video-matting-model-cached)
- [`removeVideoMattingModel()`](/docs/video-matting/remove-video-matting-model)
diff --git a/packages/docs/docs/video-matting/separate-video-layers.mdx b/packages/docs/docs/video-matting/separate-video-layers.mdx
index 645f891b80..bbec704dc6 100644
--- a/packages/docs/docs/video-matting/separate-video-layers.mdx
+++ b/packages/docs/docs/video-matting/separate-video-layers.mdx
@@ -9,8 +9,9 @@ crumb: '@remotion/video-matting'
Separates a video into an opaque base WebM and a foreground WebM with alpha.
```ts twoslash title="separate-video.ts"
-import {separateVideoLayers} from '@remotion/video-matting';
+import {downloadVideoMattingModel, separateVideoLayers} from '@remotion/video-matting';
+await downloadVideoMattingModel({model: 'modnet'});
const file = new File([], 'input.mp4');
await using result = await separateVideoLayers({
@@ -121,7 +122,7 @@ Aborting rejects the promise and cancels both outputs. Model inference may finis
### onModelLoadProgress?
-Called while model files are resolved and downloaded. Default: no callback.
+Called while a previously downloaded model is initialized. Default: no callback. Call [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model) first to download the files and receive download progress.
To load the model separately, use [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model).
diff --git a/packages/docs/docs/video-matting/table-of-contents.tsx b/packages/docs/docs/video-matting/table-of-contents.tsx
index 45896f1fc5..3fd9b28a21 100644
--- a/packages/docs/docs/video-matting/table-of-contents.tsx
+++ b/packages/docs/docs/video-matting/table-of-contents.tsx
@@ -17,9 +17,13 @@ export const TableOfContents: React.FC = () => {
<strong>isVideoMattingModelCached()</strong>
<div>Check whether a model is downloaded</div>
</TOCItem>
+ <TOCItem link="/docs/video-matting/download-video-matting-model">
+ <strong>downloadVideoMattingModel()</strong>
+ <div>Download a model without initializing WebGPU</div>
+ </TOCItem>
<TOCItem link="/docs/video-matting/load-video-matting-model">
<strong>loadVideoMattingModel()</strong>
- <div>Download and initialize a model</div>
+ <div>Initialize a downloaded model</div>
</TOCItem>
<TOCItem link="/docs/video-matting/remove-video-matting-model">
<strong>removeVideoMattingModel()</strong>
diff --git a/packages/docs/docs/whisper-webgpu/download-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/download-whisper-model.mdx
new file mode 100644
index 0000000000..40f96e55fd
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/download-whisper-model.mdx
@@ -0,0 +1,64 @@
+---
+image: /generated/articles-docs-whisper-webgpu-download-whisper-model.png
+title: downloadWhisperModel()
+crumb: '@remotion/whisper-webgpu'
+---
+
+# downloadWhisperModel()<AvailableFrom v="4.0.528" />
+
+Downloads a Whisper model from `remotion.media` into the Transformers.js cache without initializing WebGPU. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) or [`transcribe()`](/docs/whisper-webgpu/transcribe) afterwards.
+
+The browser uses its Cache API; Node.js uses the Transformers.js filesystem cache. A cache must be enabled.
+
+```ts twoslash title="download-model.ts"
+import {downloadWhisperModel} from '@remotion/whisper-webgpu';
+
+const {alreadyDownloaded} = await downloadWhisperModel({
+ model: 'small.en',
+ onProgress: ({progress}) => console.log(progress),
+});
+
+console.log(alreadyDownloaded);
+```
+
+## Options
+
+### model
+
+One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models).
+
+### onProgress?
+
+Called as files download. Default: no callback. The callback receives:
+
+#### file
+
+The file being downloaded, or `null` for aggregate progress.
+
+#### progress
+
+Aggregate progress between `0` and `1`.
+
+#### loadedBytes
+
+Bytes downloaded in this call.
+
+#### totalBytes
+
+Estimated total model size in bytes.
+
+## Return value
+
+### alreadyDownloaded
+
+`true` if all files were already cached before this call.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
+
+## See also
+
+- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)
+- [`isWhisperModelCached()`](/docs/whisper-webgpu/is-whisper-model-cached)
+- [`removeWhisperModel()`](/docs/whisper-webgpu/remove-whisper-model)
diff --git a/packages/docs/docs/whisper-webgpu/get-available-models.mdx b/packages/docs/docs/whisper-webgpu/get-available-models.mdx
index cf18b13add..d443b1a5d1 100644
--- a/packages/docs/docs/whisper-webgpu/get-available-models.mdx
+++ b/packages/docs/docs/whisper-webgpu/get-available-models.mdx
@@ -62,4 +62,5 @@ We noticed that hosting the models on R2 leads to faster downloading than throug
- [`transcribe()`](/docs/whisper-webgpu/transcribe)
- [`clearStaleModels()`](/docs/whisper-webgpu/clear-stale-models)
- [`isWhisperModelCached()`](/docs/whisper-webgpu/is-whisper-model-cached)
+- [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model)
- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)
diff --git a/packages/docs/docs/whisper-webgpu/index.mdx b/packages/docs/docs/whisper-webgpu/index.mdx
index f6518a4954..de2105a629 100644
--- a/packages/docs/docs/whisper-webgpu/index.mdx
+++ b/packages/docs/docs/whisper-webgpu/index.mdx
@@ -17,9 +17,10 @@ import {TableOfContents} from './table-of-contents';
## Example
```ts twoslash title="transcribe.ts"
-import {clearStaleModels, resampleTo16Khz, toCaptions, transcribe} from '@remotion/whisper-webgpu';
+import {clearStaleModels, downloadWhisperModel, resampleTo16Khz, toCaptions, transcribe} from '@remotion/whisper-webgpu';
await clearStaleModels();
+await downloadWhisperModel({model: 'small.en'});
const file = new File([], 'audio.wav');
const channelWaveform = await resampleTo16Khz({file});
@@ -34,7 +35,7 @@ const {captions} = toCaptions({whisperWebGpuOutput: transcription});
console.log(captions);
```
-The model is downloaded on the first call and cached by the runtime. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) first to display model download progress.
+[`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model) stores model files in the persistent cache without initializing WebGPU. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) when you want to initialize the model explicitly.
See [Using `@remotion/whisper-webgpu` in Node.js](/docs/whisper-webgpu/node) for a complete Node.js example.
diff --git a/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx
index a66c0e3b62..d1dec8ef4b 100644
--- a/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx
+++ b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx
@@ -26,7 +26,7 @@ One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-a
A `Promise<boolean>` resolving to `true` if every required model file is cached.
-Models cached from Hugging Face by versions before 4.0.522 do not satisfy this check. Loading the model downloads and caches a new copy from `remotion.media`.
+Models cached from Hugging Face by versions before 4.0.522 do not satisfy this check. Use [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model) to download and cache a copy from `remotion.media`.
## Compatibility
@@ -35,5 +35,6 @@ Models cached from Hugging Face by versions before 4.0.522 do not satisfy this c
## See also
- [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models)
+- [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model)
- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)
- [`removeWhisperModel()`](/docs/whisper-webgpu/remove-whisper-model)
diff --git a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
index 4177571a60..a92afec360 100644
--- a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
+++ b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
@@ -6,16 +6,14 @@ crumb: '@remotion/whisper-webgpu'
# loadWhisperModel()<AvailableFrom v="4.0.518" />
-Downloads a timestamped Whisper model from `remotion.media` and initializes it. Calling it before [`transcribe()`](/docs/whisper-webgpu/transcribe) lets you show model-loading progress.
+Initializes a previously downloaded Whisper model on WebGPU. Call [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model) first.
```ts twoslash title="load-model.ts"
-import {loadWhisperModel} from '@remotion/whisper-webgpu';
+import {downloadWhisperModel, loadWhisperModel} from '@remotion/whisper-webgpu';
+await downloadWhisperModel({model: 'small.en'});
await using result = await loadWhisperModel({
model: 'small.en',
- onProgress: ({progress}) => {
- console.log(progress);
- },
});
console.log(result.alreadyLoaded);
@@ -29,7 +27,7 @@ One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-a
### onProgress?
-Called while model files are resolved and downloaded. `progress` is the aggregate download progress between `0` and `1` and never decreases.
+Called while the cached model is initialized. `progress` is `null` during initialization and `1` when ready. Use [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model) for download progress.
## Return value
@@ -51,6 +49,7 @@ Use `await using` to call this method automatically when leaving the current sco
## See also
+- [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model)
- [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model)
- [`isWhisperModelCached()`](/docs/whisper-webgpu/is-whisper-model-cached)
- [`removeWhisperModel()`](/docs/whisper-webgpu/remove-whisper-model)
diff --git a/packages/docs/docs/whisper-webgpu/node.mdx b/packages/docs/docs/whisper-webgpu/node.mdx
index b97d4ebefd..7fe0e583c6 100644
--- a/packages/docs/docs/whisper-webgpu/node.mdx
+++ b/packages/docs/docs/whisper-webgpu/node.mdx
@@ -20,6 +20,7 @@ The following example uses [`@mediabunny/server`](https://mediabunny.dev/guide/e
import {registerMediabunnyServer} from '@mediabunny/server';
import {
WHISPER_WEBGPU_SAMPLE_RATE,
+ downloadWhisperModel,
loadWhisperModel,
toCaptions,
transcribe,
@@ -118,6 +119,7 @@ for (const chunk of chunks) {
}
const model = 'small.en';
+await downloadWhisperModel({model});
await using modelHandle = await loadWhisperModel({model});
const transcription = await transcribe({channelWaveform, model});
const {captions} = toCaptions({whisperWebGpuOutput: transcription});
diff --git a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
index 5537c3ede0..4859bc19c7 100644
--- a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
+++ b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
@@ -25,9 +25,13 @@ export const TableOfContents: React.FC = () => {
<strong>isWhisperModelCached()</strong>
<div>Check whether a model is downloaded</div>
</TOCItem>
+ <TOCItem link="/docs/whisper-webgpu/download-whisper-model">
+ <strong>downloadWhisperModel()</strong>
+ <div>Download a model without initializing WebGPU</div>
+ </TOCItem>
<TOCItem link="/docs/whisper-webgpu/load-whisper-model">
<strong>loadWhisperModel()</strong>
- <div>Download and initialize a model</div>
+ <div>Initialize a downloaded model</div>
</TOCItem>
<TOCItem link="/docs/whisper-webgpu/remove-whisper-model">
<strong>removeWhisperModel()</strong>
diff --git a/packages/docs/docs/whisper-webgpu/transcribe.mdx b/packages/docs/docs/whisper-webgpu/transcribe.mdx
index a04cb7707a..a48a7000a5 100644
--- a/packages/docs/docs/whisper-webgpu/transcribe.mdx
+++ b/packages/docs/docs/whisper-webgpu/transcribe.mdx
@@ -9,8 +9,9 @@ crumb: '@remotion/whisper-webgpu'
Transcribes a 16kHz mono waveform and returns words with start and end timestamps.
```ts twoslash title="transcribe.ts"
-import {resampleTo16Khz, transcribe} from '@remotion/whisper-webgpu';
+import {downloadWhisperModel, resampleTo16Khz, transcribe} from '@remotion/whisper-webgpu';
+await downloadWhisperModel({model: 'small.en'});
const file = new File([], 'audio.wav');
const channelWaveform = await resampleTo16Khz({file});
@@ -76,7 +77,7 @@ Prevents a generated token sequence of this length from occurring more than once
### onModelLoadProgress?
-Called while the model is loaded. To load independently, use [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model).
+Called while a previously downloaded model is initialized. Call [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model) first to download the files and receive download progress.
## Return value
diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.ts
index a4bce17cae..82266fa34b 100644
--- a/packages/docs/sidebars.ts
+++ b/packages/docs/sidebars.ts
@@ -963,6 +963,7 @@ const sidebars: SidebarsConfig = {
'video-matting/can-use-video-matting',
'video-matting/get-available-models',
'video-matting/is-video-matting-model-cached',
+ 'video-matting/download-video-matting-model',
'video-matting/load-video-matting-model',
'video-matting/remove-video-matting-model',
'video-matting/separate-video-layers',
@@ -1024,6 +1025,7 @@ const sidebars: SidebarsConfig = {
'whisper-webgpu/get-available-models',
'whisper-webgpu/clear-stale-models',
'whisper-webgpu/is-whisper-model-cached',
+ 'whisper-webgpu/download-whisper-model',
'whisper-webgpu/load-whisper-model',
'whisper-webgpu/remove-whisper-model',
'whisper-webgpu/transcribe',
@@ -1591,8 +1593,8 @@ const sidebars: SidebarsConfig = {
'miscellaneous/pexels',
'lovable-for-motion-graphics',
],
- },
- {
+ },
+ {
type: 'category',
label: 'Terminology',
link: {
diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.ts
index 9ad27f4e06..fbc485717d 100644
--- a/packages/docs/src/data/articles.ts
+++ b/packages/docs/src/data/articles.ts
@@ -8661,6 +8661,15 @@ export const articles = [
noAi: false,
slug: 'video-matting/dispose-video-matting-model',
},
+ {
+ id: 'video-matting/download-video-matting-model',
+ title: 'downloadVideoMattingModel()',
+ relativePath: 'docs/video-matting/download-video-matting-model.mdx',
+ compId: 'articles-docs-video-matting-download-video-matting-model',
+ crumb: '@remotion/video-matting',
+ noAi: false,
+ slug: 'video-matting/download-video-matting-model',
+ },
{
id: 'video-matting/get-available-models',
title: 'getAvailableModels()',
@@ -9318,6 +9327,15 @@ export const articles = [
noAi: false,
slug: 'whisper-webgpu/dispose-whisper-model',
},
+ {
+ id: 'whisper-webgpu/download-whisper-model',
+ title: 'downloadWhisperModel()',
+ relativePath: 'docs/whisper-webgpu/download-whisper-model.mdx',
+ compId: 'articles-docs-whisper-webgpu-download-whisper-model',
+ crumb: '@remotion/whisper-webgpu',
+ noAi: false,
+ slug: 'whisper-webgpu/download-whisper-model',
+ },
{
id: 'whisper-webgpu/get-available-models',
title: 'getAvailableModels()',
diff --git a/packages/docs/static/generated/articles-docs-video-matting-download-video-matting-model.png b/packages/docs/static/generated/articles-docs-video-matting-download-video-matting-model.png
new file mode 100644
index 0000000000..fac8e3ce02
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-download-video-matting-model.png differ
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-download-whisper-model.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-download-whisper-model.png
new file mode 100644
index 0000000000..fbf24e77cb
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-download-whisper-model.png differ
diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx
index bcca6abd45..ed5fb09095 100644
--- a/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx
+++ b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx
@@ -2,6 +2,7 @@ import {
canUseWhisperWebGpu,
clearStaleModels,
disposeWhisperModel,
+ downloadWhisperModel,
isWhisperModelCached,
loadWhisperModel,
toCaptions,
@@ -49,11 +50,13 @@ export const CaptionQueueProcessor: React.FC = () => {
progressStart: 0.03,
progressSpan: 0.27,
isModelCached: (model) => isWhisperModelCached({model}),
- loadModel: (model, onProgress) =>
- loadWhisperModel({
+ loadModel: async (model, onProgress) => {
+ await downloadWhisperModel({
model,
onProgress: (progress) => onProgress(progress.progress),
- }),
+ });
+ await loadWhisperModel({model});
+ },
updateProgress: (progress) =>
updateCaptionJobProgress(job.id, progress),
});
diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx
index 8063683a57..61186c9b1d 100644
--- a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx
+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx
@@ -1,6 +1,7 @@
import {
canUseVideoMatting,
disposeVideoMattingModel,
+ downloadVideoMattingModel,
isVideoMattingModelCached,
loadVideoMattingModel,
separateVideoLayers,
@@ -40,11 +41,13 @@ export const VideoMattingQueueProcessor: React.FC = () => {
progressStart: 0,
progressSpan: 0.2,
isModelCached: (model) => isVideoMattingModelCached({model}),
- loadModel: (model, onProgress) =>
- loadVideoMattingModel({
+ loadModel: async (model, onProgress) => {
+ await downloadVideoMattingModel({
model,
onProgress: (progress) => onProgress(progress.progress),
- }),
+ });
+ await loadVideoMattingModel({model});
+ },
updateProgress: (progress) =>
updateVideoMattingJobProgress(job.id, {
...progress,
diff --git a/packages/studio/src/components/Transcription/Models.tsx b/packages/studio/src/components/Transcription/Models.tsx
index 7493a3aeb7..793cb44a36 100644
--- a/packages/studio/src/components/Transcription/Models.tsx
+++ b/packages/studio/src/components/Transcription/Models.tsx
@@ -1,8 +1,8 @@
import {
clearStaleModels,
+ downloadWhisperModel,
getAvailableModels,
isWhisperModelCached,
- loadWhisperModel,
removeWhisperModel,
type WhisperWebGpuModel,
} from '@remotion/whisper-webgpu';
@@ -25,7 +25,7 @@ export const Models: React.FC<{
model: WhisperWebGpuModel,
onProgress: (progress: number | null) => void,
) =>
- loadWhisperModel({
+ downloadWhisperModel({
model,
onProgress: (progress) => onProgress(progress.progress),
}),
diff --git a/packages/studio/src/components/VideoMatting/Models.tsx b/packages/studio/src/components/VideoMatting/Models.tsx
index b2f3e5fca1..29c61566a8 100644
--- a/packages/studio/src/components/VideoMatting/Models.tsx
+++ b/packages/studio/src/components/VideoMatting/Models.tsx
@@ -1,7 +1,7 @@
import {
+ downloadVideoMattingModel,
getAvailableModels,
isVideoMattingModelCached,
- loadVideoMattingModel,
removeVideoMattingModel,
type VideoMattingModel,
} from '@remotion/video-matting';
@@ -21,7 +21,7 @@ export const Models: React.FC<{
);
const loadModel = useCallback(
(model: VideoMattingModel, onProgress: (progress: number | null) => void) =>
- loadVideoMattingModel({
+ downloadVideoMattingModel({
model,
onProgress: (progress) => onProgress(progress.progress),
}),
diff --git a/packages/studio/src/test/caption-queue-processor.test.tsx b/packages/studio/src/test/caption-queue-processor.test.tsx
index e102045789..ea05526ac3 100644
--- a/packages/studio/src/test/caption-queue-processor.test.tsx
+++ b/packages/studio/src/test/caption-queue-processor.test.tsx
@@ -24,11 +24,10 @@ const getModelLoadCall = () => modelLoadCall;
let modelIsCached = false;
let modelLoadProgressCallback:
| ((progress: {
- status: string;
file: string | null;
- progress: number | null;
- loadedBytes: number | null;
- totalBytes: number | null;
+ progress: number;
+ loadedBytes: number;
+ totalBytes: number;
}) => void)
| undefined;
let resolveModelLoading: (() => void) | null = null;
@@ -96,7 +95,7 @@ mock.module('@remotion/whisper-webgpu', () => ({
cacheCheck = {model};
return Promise.resolve(modelIsCached);
},
- loadWhisperModel: async ({
+ downloadWhisperModel: async ({
model,
onProgress,
}: {
@@ -109,14 +108,14 @@ mock.module('@remotion/whisper-webgpu', () => ({
resolveModelLoading = resolve;
});
modelLoadProgressCallback?.({
- status: 'ready',
file: null,
progress: 1,
loadedBytes: 119_699_015,
totalBytes: 119_699_015,
});
- return {alreadyLoaded: false};
+ return {alreadyDownloaded: false};
},
+ loadWhisperModel: () => Promise.resolve({alreadyLoaded: false}),
removeWhisperModel: () => Promise.resolve(),
toCaptions: () => ({
captions: [
@@ -267,7 +266,6 @@ test('downloads a model in the queued job before transcribing', async () => {
act(() => {
modelLoadProgressCallback?.({
- status: 'loading',
file: null,
progress: 0.5,
loadedBytes: 119_699_015 / 2,
diff --git a/packages/studio/src/test/video-matting-queue-processor.test.tsx b/packages/studio/src/test/video-matting-queue-processor.test.tsx
index aa1cf3c2d2..7ac94b70de 100644
--- a/packages/studio/src/test/video-matting-queue-processor.test.tsx
+++ b/packages/studio/src/test/video-matting-queue-processor.test.tsx
@@ -14,13 +14,17 @@ let processJob: ((job: VideoMattingJob) => Promise<void>) | null = null;
mock.module('@remotion/video-matting', () => ({
canUseVideoMatting: () => Promise.resolve({supported: true}),
isVideoMattingModelCached: () => Promise.resolve(false),
- loadVideoMattingModel: ({
+ downloadVideoMattingModel: ({
onProgress,
}: {
onProgress: (progress: {progress: number}) => void;
}) => {
- calls.push('load-model');
+ calls.push('download-model');
onProgress({progress: 1});
+ return Promise.resolve({alreadyDownloaded: false});
+ },
+ loadVideoMattingModel: () => {
+ calls.push('load-model');
return Promise.resolve({alreadyLoaded: false});
},
separateVideoLayers: ({
@@ -120,6 +124,7 @@ test('loads the model, separates the layers and writes both outputs', async () =
expect(failed).toBeNull();
expect(done).toBe(true);
expect(calls).toEqual([
+ 'download-model',
'load-model',
'separate',
'write:input-base.webm',
diff --git a/packages/video-matting/src/download-video-matting-model.ts b/packages/video-matting/src/download-video-matting-model.ts
new file mode 100644
index 0000000000..b5de024201
--- /dev/null
+++ b/packages/video-matting/src/download-video-matting-model.ts
@@ -0,0 +1,128 @@
+import {
+ getHostedVideoMattingModelId,
+ getVideoMattingModelInfo,
+ type VideoMattingModel,
+} from './models';
+import {withRemotionModelHost} from './with-remotion-model-host';
+
+export type VideoMattingModelDownloadProgress = {
+ file: string | null;
+ progress: number;
+ loadedBytes: number;
+ totalBytes: number;
+};
+
+export type DownloadVideoMattingModelOptions = {
+ model: VideoMattingModel;
+ onProgress?: (progress: VideoMattingModelDownloadProgress) => void;
+};
+
+export type DownloadVideoMattingModelResult = {
+ alreadyDownloaded: boolean;
+};
+
+export const downloadVideoMattingModel = ({
+ model,
+ onProgress,
+}: DownloadVideoMattingModelOptions): Promise<DownloadVideoMattingModelResult> => {
+ const modelInfo = getVideoMattingModelInfo(model);
+ const totalBytes = modelInfo.webGpuDownloadSize;
+ const hostedModelId = getHostedVideoMattingModelId(model);
+ return withRemotionModelHost(async ({env, ModelRegistry}) => {
+ const cache = env.useCustomCache
+ ? env.customCache
+ : env.useBrowserCache && typeof caches !== 'undefined'
+ ? await caches.open(env.cacheKey)
+ : null;
+ if (cache === null) {
+ throw new Error(
+ 'A Transformers.js browser or custom cache is required to download a video matting model.',
+ );
+ }
+
+ const registryOptions = {
+ device: 'webgpu' as const,
+ dtype: modelInfo.dtype,
+ };
+ const cacheStatus = await ModelRegistry.is_pipeline_cached_files(
+ 'background-removal',
+ hostedModelId,
+ registryOptions,
+ );
+ if (cacheStatus.allCached) {
+ onProgress?.({
+ file: null,
+ progress: 1,
+ loadedBytes: totalBytes,
+ totalBytes,
+ });
+ return {alreadyDownloaded: true};
+ }
+
+ let loadedBytes = 0;
+ const emitProgress = (file: string | null) => {
+ onProgress?.({
+ file,
+ progress: Math.min(loadedBytes / totalBytes, 0.99),
+ loadedBytes,
+ totalBytes,
+ });
+ };
+
+ emitProgress(null);
+
+ for (const {file, cached} of cacheStatus.files) {
+ if (cached) {
+ continue;
+ }
+
+ const url = `${env.remoteHost}${env.remotePathTemplate.replace('{model}', hostedModelId)}${file}`;
+ // Transformers.js 4.2.0 uses the remote URL as the browser cache key.
+ const response = await env.fetch(url);
+ if (!response.ok || response.body === null) {
+ throw new Error(`Could not download ${url}: HTTP ${response.status}.`);
+ }
+
+ const reader = response.body.getReader();
+ const stream = new ReadableStream<Uint8Array>({
+ async pull(controller) {
+ try {
+ const {done, value} = await reader.read();
+ if (done) {
+ controller.close();
+ return;
+ }
+
+ loadedBytes += value.byteLength;
+ emitProgress(file);
+ controller.enqueue(value);
+ } catch (error) {
+ controller.error(error);
+ }
+ },
+ cancel: () => reader.cancel(),
+ });
+ await cache.put(url, new Response(stream, {headers: response.headers}));
+ }
+
+ if (
+ !(await ModelRegistry.is_pipeline_cached(
+ 'background-removal',
+ hostedModelId,
+ registryOptions,
+ ))
+ ) {
+ throw new Error(
+ `The video matting model "${model}" was not fully cached.`,
+ );
+ }
+
+ onProgress?.({
+ file: null,
+ progress: 1,
+ loadedBytes: totalBytes,
+ totalBytes,
+ });
+ return {alreadyDownloaded: false};
+ });
+};
diff --git a/packages/video-matting/src/index.ts b/packages/video-matting/src/index.ts
index 8aea58a0d1..57acd6497f 100644
--- a/packages/video-matting/src/index.ts
+++ b/packages/video-matting/src/index.ts
@@ -10,6 +10,12 @@ export {
disposeVideoMattingModel,
loadVideoMattingModel,
} from './load-video-matting-model';
+export {downloadVideoMattingModel} from './download-video-matting-model';
+export type {
+ DownloadVideoMattingModelOptions,
+ DownloadVideoMattingModelResult,
+ VideoMattingModelDownloadProgress,
+} from './download-video-matting-model';
export type {
DisposeVideoMattingModelOptions,
LoadVideoMattingModelOptions,
diff --git a/packages/video-matting/src/load-video-matting-model.ts b/packages/video-matting/src/load-video-matting-model.ts
index a20faab7d5..0ad40d953a 100644
--- a/packages/video-matting/src/load-video-matting-model.ts
+++ b/packages/video-matting/src/load-video-matting-model.ts
@@ -163,54 +163,32 @@ const getOrCreateVideoMattingPipeline = ({
AutoModelForImageSegmentation,
AutoProcessor,
BackgroundRemovalPipeline,
+ ModelRegistry,
}) => {
const hostedModelId = getHostedVideoMattingModelId(model);
- const totalBytes = modelInfo.webGpuDownloadSize;
- const loadedByFile = new Map<string, number>();
- let lastProgress = 0;
- let lastLoadedBytes = 0;
+ if (
+ !(await ModelRegistry.is_pipeline_cached(
+ 'background-removal',
+ hostedModelId,
+ {device: 'webgpu', dtype: modelInfo.dtype},
+ ))
+ ) {
+ throw new Error(
+ `The video matting model "${model}" is not downloaded. Call downloadVideoMattingModel() first.`,
+ );
+ }
notifyProgress(state, {
- status: 'loading',
+ status: 'initializing',
file: null,
- progress: 0,
- loadedBytes: 0,
- totalBytes,
+ progress: null,
+ loadedBytes: null,
+ totalBytes: null,
});
const pretrainedOptions = {
device: 'webgpu' as const,
dtype: modelInfo.dtype,
- progress_callback: (event: unknown) => {
- const record = event as Record<string, unknown>;
- if (
- record.status === 'progress' &&
- typeof record.file === 'string' &&
- typeof record.loaded === 'number' &&
- Number.isFinite(record.loaded)
- ) {
- loadedByFile.set(
- record.file,
- Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),
- );
- const loadedBytes = [...loadedByFile.values()].reduce(
- (sum, loaded) => sum + loaded,
- 0,
- );
- lastProgress = Math.max(
- lastProgress,
- Math.min(loadedBytes / totalBytes, 0.99),
- );
- lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);
- notifyProgress(state, {
- status: 'loading',
- file: null,
- progress: lastProgress,
- loadedBytes: Math.min(lastLoadedBytes, totalBytes),
- totalBytes,
- });
- }
- },
};
const transformerProcessor = await AutoProcessor.from_pretrained(
hostedModelId,
@@ -235,8 +213,8 @@ const getOrCreateVideoMattingPipeline = ({
status: 'ready',
file: null,
progress: 1,
- loadedBytes: totalBytes,
- totalBytes,
+ loadedBytes: null,
+ totalBytes: null,
});
const loadedPipeline: LoadedPipeline = {
diff --git a/packages/video-matting/src/test/model-runtime.test.ts b/packages/video-matting/src/test/model-runtime.test.ts
index 532e6bbf7d..100277435b 100644
--- a/packages/video-matting/src/test/model-runtime.test.ts
+++ b/packages/video-matting/src/test/model-runtime.test.ts
@@ -21,10 +21,27 @@ let cacheClear:
let cacheCheckEnvironment:
| {remoteHost: string; remotePathTemplate: string}
| undefined;
+const downloadedFiles = new Map<string, Response>();
+const requestedFiles: string[] = [];
+let checkDownloadedFiles = false;
+const customCache = {
+ match: (url: string) => Promise.resolve(downloadedFiles.get(url)?.clone()),
+ put: async (url: string, response: Response) => {
+ downloadedFiles.set(url, new Response(await response.arrayBuffer()));
+ },
+};
const originalTransformersEnvironment = {
remoteHost: 'https://huggingface.co/',
remotePathTemplate: '{model}/resolve/{revision}/',
+ useCustomCache: true,
+ customCache,
+ useBrowserCache: false,
+ cacheKey: 'transformers-cache',
+ fetch: (url: string) => {
+ requestedFiles.push(url);
+ return Promise.resolve(new Response(url));
+ },
};
const transformersEnvironment = {...originalTransformersEnvironment};
const preexistingWhisperModelHostState = {
@@ -71,7 +88,10 @@ mock.module('@huggingface/transformers', () => ({
initializationCount++;
initializedModelIds.push(modelId);
initializationOptions = options;
- initializationEnvironment = {...transformersEnvironment};
+ initializationEnvironment = {
+ remoteHost: transformersEnvironment.remoteHost,
+ remotePathTemplate: transformersEnvironment.remotePathTemplate,
+ };
initializationStarted?.(initializationCount);
await initializationGate;
const onProgress = options.progress_callback as
@@ -93,6 +113,18 @@ mock.module('@huggingface/transformers', () => ({
},
BackgroundRemovalPipeline,
ModelRegistry: {
+ is_pipeline_cached_files: () => {
+ const files = ['config.json', 'onnx/model.onnx'].map((file) => ({
+ file,
+ cached: downloadedFiles.has(
+ `https://remotion.media/models/modnet-v1/${file}`,
+ ),
+ }));
+ return Promise.resolve({
+ files,
+ allCached: files.every(({cached}) => cached),
+ });
+ },
clear_pipeline_cache: (
task: string,
modelId: string,
@@ -107,8 +139,13 @@ mock.module('@huggingface/transformers', () => ({
options: Record<string, unknown>,
) => {
cacheCheck = {task, modelId, options};
- cacheCheckEnvironment = {...transformersEnvironment};
- return Promise.resolve(true);
+ cacheCheckEnvironment = {
+ remoteHost: transformersEnvironment.remoteHost,
+ remotePathTemplate: transformersEnvironment.remotePathTemplate,
+ };
+ return Promise.resolve(
+ checkDownloadedFiles ? downloadedFiles.size === 2 : true,
+ );
},
},
}));
@@ -131,11 +168,66 @@ beforeEach(async () => {
cacheCheck = undefined;
cacheClear = undefined;
cacheCheckEnvironment = undefined;
+ downloadedFiles.clear();
+ requestedFiles.length = 0;
+ checkDownloadedFiles = false;
preexistingWhisperModelHostState.activeOperations = 0;
preexistingWhisperModelHostState.previousRemoteConfiguration = null;
Object.assign(transformersEnvironment, originalTransformersEnvironment);
});
+test('downloads without initializing WebGPU, then initializes from the cached files', async () => {
+ checkDownloadedFiles = true;
+ transformersEnvironment.useCustomCache = false;
+ transformersEnvironment.useBrowserCache = true;
+ const originalCaches = Object.getOwnPropertyDescriptor(globalThis, 'caches');
+ Object.defineProperty(globalThis, 'caches', {
+ configurable: true,
+ value: {open: () => Promise.resolve(customCache)},
+ });
+ try {
+ const {downloadVideoMattingModel, loadVideoMattingModel} =
+ await import('../index');
+ await expect(loadVideoMattingModel({model: 'modnet'})).rejects.toThrow(
+ 'downloadVideoMattingModel() first',
+ );
+ const progress: number[] = [];
+ const download = await downloadVideoMattingModel({
+ model: 'modnet',
+ onProgress: ({progress: value}) => progress.push(value),
+ });
+
+ expect(download.alreadyDownloaded).toBe(false);
+ expect(initializationCount).toBe(0);
+ expect(requestedFiles).toEqual([
+ 'https://remotion.media/models/modnet-v1/config.json',
+ 'https://remotion.media/models/modnet-v1/onnx/model.onnx',
+ ]);
+ expect(progress.at(-1)).toBe(1);
+ downloadedFiles.delete(
+ 'https://remotion.media/models/modnet-v1/onnx/model.onnx',
+ );
+ expect(
+ (await downloadVideoMattingModel({model: 'modnet'})).alreadyDownloaded,
+ ).toBe(false);
+ expect(requestedFiles.slice(2)).toEqual([requestedFiles[1]]);
+
+ await using loaded = await loadVideoMattingModel({model: 'modnet'});
+ expect(loaded.alreadyLoaded).toBe(false);
+ expect(initializationCount).toBe(1);
+ expect(
+ (await downloadVideoMattingModel({model: 'modnet'})).alreadyDownloaded,
+ ).toBe(true);
+ expect(requestedFiles).toHaveLength(3);
+ } finally {
+ if (originalCaches) {
+ Object.defineProperty(globalThis, 'caches', originalCaches);
+ } else {
+ Reflect.deleteProperty(globalThis, 'caches');
+ }
+ }
+});
+
test('shares loading, reports progress, and defers disposal while in use', async () => {
const {
disposeVideoMattingModel,
diff --git a/packages/whisper-webgpu/src/download-whisper-model.ts b/packages/whisper-webgpu/src/download-whisper-model.ts
new file mode 100644
index 0000000000..413edff57a
--- /dev/null
+++ b/packages/whisper-webgpu/src/download-whisper-model.ts
@@ -0,0 +1,171 @@
+import {
+ getHostedModelId,
+ getModelInfo,
+ getWhisperWebGpuDtype,
+ type WhisperWebGpuModel,
+} from './models';
+import {withRemotionModelHost} from './with-remotion-model-host';
+
+export type WhisperModelDownloadProgress = {
+ file: string | null;
+ progress: number;
+ loadedBytes: number;
+ totalBytes: number;
+};
+
+export type DownloadWhisperModelOptions = {
+ model: WhisperWebGpuModel;
+ onProgress?: (progress: WhisperModelDownloadProgress) => void;
+};
+
+export type DownloadWhisperModelResult = {
+ alreadyDownloaded: boolean;
+};
+
+export const downloadWhisperModel = ({
+ model,
+ onProgress,
+}: DownloadWhisperModelOptions): Promise<DownloadWhisperModelResult> => {
+ const totalBytes = getModelInfo(model).webGpuDownloadSize;
+ const hostedModelId = getHostedModelId(model);
+ return withRemotionModelHost(async ({env, ModelRegistry}) => {
+ const cache = env.useCustomCache
+ ? env.customCache
+ : env.useBrowserCache && typeof caches !== 'undefined'
+ ? await caches.open(env.cacheKey)
+ : null;
+ const useFileCache = cache === null && env.useFSCache;
+ if (cache === null && !useFileCache) {
+ throw new Error(
+ 'A Transformers.js model cache is required to download a Whisper model.',
+ );
+ }
+
+ const registryOptions = {
+ device: 'webgpu' as const,
+ dtype: getWhisperWebGpuDtype(model),
+ };
+ const cacheStatus = await ModelRegistry.is_pipeline_cached_files(
+ 'automatic-speech-recognition',
+ hostedModelId,
+ registryOptions,
+ );
+ if (cacheStatus.allCached) {
+ onProgress?.({
+ file: null,
+ progress: 1,
+ loadedBytes: totalBytes,
+ totalBytes,
+ });
+ return {alreadyDownloaded: true};
+ }
+
+ let loadedBytes = 0;
+ const emitProgress = (file: string | null) => {
+ onProgress?.({
+ file,
+ progress: Math.min(loadedBytes / totalBytes, 0.99),
+ loadedBytes,
+ totalBytes,
+ });
+ };
+
+ emitProgress(null);
+
+ for (const {file, cached} of cacheStatus.files) {
+ if (cached) {
+ continue;
+ }
+
+ const url = `${env.remoteHost}${env.remotePathTemplate.replace('{model}', hostedModelId)}${file}`;
+ // Transformers.js 4.2.0 keys browser caches by URL and its FileCache by model/file.
+ const cacheKey = cache === null ? `${hostedModelId}/${file}` : url;
+ const response = await env.fetch(url);
+ if (!response.ok || response.body === null) {
+ throw new Error(`Could not download ${url}: HTTP ${response.status}.`);
+ }
+
+ const reader = response.body.getReader();
+ if (cache !== null) {
+ const stream = new ReadableStream<Uint8Array>({
+ async pull(controller) {
+ try {
+ const {done, value} = await reader.read();
+ if (done) {
+ controller.close();
+ return;
+ }
+
+ loadedBytes += value.byteLength;
+ emitProgress(file);
+ controller.enqueue(value);
+ } catch (error) {
+ controller.error(error);
+ }
+ },
+ cancel: () => reader.cancel(),
+ });
+ await cache.put(
+ cacheKey,
+ new Response(stream, {headers: response.headers}),
+ );
+ } else {
+ // Match Transformers.js FileCache's model/file layout without loading ONNX.
+ const fsName: string = 'node:fs/promises';
+ const pathName: string = 'node:path';
+ const cryptoName: string = 'node:crypto';
+ const [{mkdir, open, rename, unlink}, {join, dirname}, {randomUUID}] =
+ await Promise.all([
+ import(/* @vite-ignore */ fsName),
+ import(/* @vite-ignore */ pathName),
+ import(/* @vite-ignore */ cryptoName),
+ ]);
+ if (env.cacheDir === null) {
+ throw new Error('Transformers.js env.cacheDir must be set.');
+ }
+
+ const destination = join(env.cacheDir, cacheKey);
+ const temporary = `${destination}.tmp.${randomUUID()}`;
+ await mkdir(dirname(destination), {recursive: true});
+ const handle = await open(temporary, 'wx');
+ try {
+ while (true) {
+ const {done, value} = await reader.read();
+ if (done) {
+ break;
+ }
+
+ await handle.writeFile(value);
+ loadedBytes += value.byteLength;
+ emitProgress(file);
+ }
+ } catch (error) {
+ await handle.close();
+ await unlink(temporary);
+ throw error;
+ }
+
+ await handle.close();
+ await rename(temporary, destination);
+ }
+ }
+
+ if (
+ !(await ModelRegistry.is_pipeline_cached(
+ 'automatic-speech-recognition',
+ hostedModelId,
+ registryOptions,
+ ))
+ ) {
+ throw new Error(`The Whisper model "${model}" was not fully cached.`);
+ }
+
+ onProgress?.({
+ file: null,
+ progress: 1,
+ loadedBytes: totalBytes,
+ totalBytes,
+ });
+ return {alreadyDownloaded: false};
+ });
+};
diff --git a/packages/whisper-webgpu/src/index.ts b/packages/whisper-webgpu/src/index.ts
index 3fdc37b3ea..5ad37fa9a5 100644
--- a/packages/whisper-webgpu/src/index.ts
+++ b/packages/whisper-webgpu/src/index.ts
@@ -4,6 +4,12 @@ export {
} from './can-use-whisper-webgpu';
export type {CanUseWhisperWebGpuResult} from './can-use-whisper-webgpu';
export {clearStaleModels} from './clear-stale-models';
+export {downloadWhisperModel} from './download-whisper-model';
+export type {
+ DownloadWhisperModelOptions,
+ DownloadWhisperModelResult,
+ WhisperModelDownloadProgress,
+} from './download-whisper-model';
export {isWhisperModelCached} from './is-whisper-model-cached';
export type {IsWhisperModelCachedOptions} from './is-whisper-model-cached';
export {disposeWhisperModel, loadWhisperModel} from './load-whisper-model';
diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.ts
index 915f04330b..77da1fce69 100644
--- a/packages/whisper-webgpu/src/load-whisper-model.ts
+++ b/packages/whisper-webgpu/src/load-whisper-model.ts
@@ -52,8 +52,10 @@ const getOrCreateWhisperPipeline = ({
alreadyLoaded: boolean;
unsubscribe: () => void;
} => {
- const modelInfo = getModelInfo(model);
- const totalBytes = modelInfo.webGpuDownloadSize;
+ if (!getModelInfo(model)) {
+ throw new Error(`Unsupported Whisper model "${model}".`);
+ }
+
const existing = pipelines.get(model);
if (existing) {
if (onProgress) {
@@ -95,59 +97,40 @@ const getOrCreateWhisperPipeline = ({
};
const loading = Promise.resolve().then(() => {
- return withRemotionModelHost(({pipeline}) => {
+ return withRemotionModelHost(async ({ModelRegistry, pipeline}) => {
+ const hostedModelId = getHostedModelId(model);
+ if (
+ !(await ModelRegistry.is_pipeline_cached(
+ 'automatic-speech-recognition',
+ hostedModelId,
+ {device: 'webgpu', dtype: getWhisperWebGpuDtype(model)},
+ ))
+ ) {
+ throw new Error(
+ `The Whisper model "${model}" is not downloaded. Call downloadWhisperModel() first.`,
+ );
+ }
+
emitProgress({
- status: 'loading',
+ status: 'initializing',
file: null,
- progress: 0,
- loadedBytes: 0,
- totalBytes,
+ progress: null,
+ loadedBytes: null,
+ totalBytes: null,
});
- const hostedModelId = getHostedModelId(model);
- const loadedByFile = new Map<string, number>();
- let lastProgress = 0;
- let lastLoadedBytes = 0;
return pipeline('automatic-speech-recognition', hostedModelId, {
device: 'webgpu',
dtype: getWhisperWebGpuDtype(model),
progress_callback: (event) => {
const record = event as Record<string, unknown>;
- if (
- record.status === 'progress' &&
- typeof record.file === 'string' &&
- typeof record.loaded === 'number' &&
- Number.isFinite(record.loaded)
- ) {
- loadedByFile.set(
- record.file,
- Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),
- );
- const loadedBytes = [...loadedByFile.values()].reduce(
- (sum, loaded) => sum + loaded,
- 0,
- );
- lastProgress = Math.max(
- lastProgress,
- Math.min(loadedBytes / totalBytes, 0.99),
- );
- lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);
- emitProgress({
- status: 'loading',
- file: null,
- progress: lastProgress,
- loadedBytes: lastLoadedBytes,
- totalBytes,
- });
- }
-
if (record.status === 'ready') {
emitProgress({
status: 'ready',
file: null,
progress: 1,
- loadedBytes: Math.max(lastLoadedBytes, totalBytes),
- totalBytes,
+ loadedBytes: null,
+ totalBytes: null,
});
}
},
diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.ts
index bd44c48d30..6f8177d159 100644
--- a/packages/whisper-webgpu/src/test/transcribe.test.ts
+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts
@@ -1,4 +1,8 @@
import {expect, mock, test} from 'bun:test';
+import {existsSync} from 'node:fs';
+import {mkdtemp, readFile, rm, unlink} from 'node:fs/promises';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
let pipelineInitialization:
| {
@@ -67,6 +71,38 @@ const originalTransformersEnvironment = {
remotePathTemplate: '{model}/resolve/{revision}/',
};
const transformersEnvironment = {...originalTransformersEnvironment};
+const downloadFiles = ['config.json', 'onnx/encoder_model.onnx'];
+const downloadRequests: string[] = [];
+let downloadCacheDir = '';
+let checkDownloadedFiles = false;
+for (const [key, value] of Object.entries({
+ useCustomCache: false,
+ customCache: null,
+ useBrowserCache: false,
+ useFSCache: true,
+ cacheDir: downloadCacheDir,
+ fetch: (url: string) => {
+ downloadRequests.push(url);
+ const middle = Math.floor(url.length / 2);
+ return Promise.resolve(
+ new Response(
+ new ReadableStream({
+ start(controller) {
+ controller.enqueue(new TextEncoder().encode(url.slice(0, middle)));
+ controller.enqueue(new TextEncoder().encode(url.slice(middle)));
+ controller.close();
+ },
+ }),
+ ),
+ );
+ },
+})) {
+ Object.defineProperty(transformersEnvironment, key, {
+ configurable: true,
+ writable: true,
+ value,
+ });
+}
const fakePipeline = Object.assign(
async (audio: Float32Array, options: Record<string, unknown>) => {
@@ -95,6 +131,16 @@ const fakePipeline = Object.assign(
mock.module('@huggingface/transformers', () => ({
env: transformersEnvironment,
ModelRegistry: {
+ is_pipeline_cached_files: (_task: string, modelId: string) => {
+ const files = downloadFiles.map((file) => ({
+ file,
+ cached: existsSync(join(downloadCacheDir, modelId, file)),
+ }));
+ return Promise.resolve({
+ files,
+ allCached: files.every(({cached}) => cached),
+ });
+ },
clear_pipeline_cache: (
task: string,
modelId: string,
@@ -114,7 +160,13 @@ mock.module('@huggingface/transformers', () => ({
return Promise.reject(cacheCheckError);
}
- return Promise.resolve(true);
+ return Promise.resolve(
+ checkDownloadedFiles
+ ? downloadFiles.every((file) =>
+ existsSync(join(downloadCacheDir, modelId, file)),
+ )
+ : true,
+ );
},
},
pipeline: async (
@@ -190,6 +242,67 @@ mock.module('onnxruntime-node', () => ({
},
}));
+test('downloads files without WebGPU and initializes only after download', async () => {
+ downloadCacheDir = await mkdtemp(
+ join(tmpdir(), 'remotion-whisper-download-'),
+ );
+ (
+ transformersEnvironment as typeof transformersEnvironment & {
+ cacheDir: string;
+ }
+ ).cacheDir = downloadCacheDir;
+ checkDownloadedFiles = true;
+ downloadRequests.length = 0;
+ try {
+ const {downloadWhisperModel, loadWhisperModel} = await import('../index');
+ await expect(loadWhisperModel({model: 'tiny.en'})).rejects.toThrow(
+ 'downloadWhisperModel() first',
+ );
+
+ const initializationsBeforeDownload = pipelineInitializationCount;
+ const progress: number[] = [];
+ const result = await downloadWhisperModel({
+ model: 'tiny.en',
+ onProgress: ({progress: value}) => progress.push(value),
+ });
+ expect(result.alreadyDownloaded).toBe(false);
+ expect(pipelineInitializationCount).toBe(initializationsBeforeDownload);
+ expect(downloadRequests).toEqual([
+ 'https://remotion.media/models/whisper-tiny.en_timestamped-v1/config.json',
+ 'https://remotion.media/models/whisper-tiny.en_timestamped-v1/onnx/encoder_model.onnx',
+ ]);
+ expect(progress.at(-1)).toBe(1);
+ expect(
+ await readFile(
+ join(downloadCacheDir, 'whisper-tiny.en_timestamped-v1', 'config.json'),
+ 'utf8',
+ ),
+ ).toBe(downloadRequests[0]);
+ await unlink(
+ join(
+ downloadCacheDir,
+ 'whisper-tiny.en_timestamped-v1',
+ 'onnx/encoder_model.onnx',
+ ),
+ );
+ expect(
+ (await downloadWhisperModel({model: 'tiny.en'})).alreadyDownloaded,
+ ).toBe(false);
+ expect(downloadRequests.slice(2)).toEqual([downloadRequests[1]]);
+
+ await using loaded = await loadWhisperModel({model: 'tiny.en'});
+ expect(loaded.alreadyLoaded).toBe(false);
+ expect(pipelineInitializationCount).toBe(initializationsBeforeDownload + 1);
+ expect(
+ (await downloadWhisperModel({model: 'tiny.en'})).alreadyDownloaded,
+ ).toBe(true);
+ expect(downloadRequests).toHaveLength(3);
+ } finally {
+ checkDownloadedFiles = false;
+ await rm(downloadCacheDir, {recursive: true, force: true});
+ }
+});
+
test('removes legacy Hugging Face Whisper entries from the shared browser cache', async () => {
const originalCaches = Object.getOwnPropertyDescriptor(globalThis, 'caches');
const cacheEnvironment =
@@ -497,7 +610,7 @@ test('restores the Transformers environment if a hosted model operation fails',
}
});
-test('reports model progress without reaching 100% before every file loads', async () => {
+test('reports initialization separately from download progress', async () => {
const {disposeWhisperModel, loadWhisperModel} = await import('../index');
const progressValues: Array<{
progress: number | null;
@@ -516,27 +629,16 @@ test('reports model progress without reaching 100% before every file loads', asy
});
expect(progressValues[0]).toEqual({
- progress: 0,
- loadedBytes: 0,
- totalBytes: 1_698_504_047,
+ progress: null,
+ loadedBytes: null,
+ totalBytes: null,
});
expect(progressValues.at(-1)).toEqual({
progress: 1,
- loadedBytes: 1_698_504_047,
- totalBytes: 1_698_504_047,
+ loadedBytes: null,
+ totalBytes: null,
});
- const downloading = progressValues.slice(0, -1);
- expect(
- downloading.every(({progress}) => progress !== null && progress < 1),
- ).toBe(true);
- expect(downloading.map(({loadedBytes}) => loadedBytes)).toEqual([
- 0, 500_000_000, 600_000_000, 985_710_974,
- ]);
- expect(downloading.map(({progress}) => progress)).toEqual(
- [0, 500_000_000, 600_000_000, 985_710_974].map((loaded) =>
- Math.min(loaded / 1_698_504_047, 0.99),
- ),
- );
+ expect(progressValues).toHaveLength(2);
await disposeWhisperModel({model: 'medium.en'});
});