🎥 Make videos programmatically with React
Separate AI model downloads from WebGPU initialization
JonnyBurgera2bb86ed1854
diff --git a/packages/convert/app/components/transcribe/state.ts b/packages/convert/app/components/transcribe/state.tsindex 4f378bc754..c6a67eb871 100644--- a/packages/convert/app/components/transcribe/state.ts+++ b/packages/convert/app/components/transcribe/state.ts@@ -1,6 +1,6 @@import type {Caption} from '@remotion/captions';import type {- WhisperWebGpuModelLoadProgress,+ WhisperModelDownloadProgress,WhisperWebGpuTranscription,} from '@remotion/whisper-webgpu';@@ -13,7 +13,7 @@ export type TranscriptionState =}| {type: 'downloading-model';- progress: WhisperWebGpuModelLoadProgress;+ progress: WhisperModelDownloadProgress;}| {type: 'transcribing';diff --git a/packages/convert/app/components/transcribe/transcribeAudio.tsx b/packages/convert/app/components/transcribe/transcribeAudio.tsxindex 4cdf96f267..8ae4fe8e12 100644--- a/packages/convert/app/components/transcribe/transcribeAudio.tsx+++ b/packages/convert/app/components/transcribe/transcribeAudio.tsx@@ -1,4 +1,5 @@import {+ downloadWhisperModel,loadWhisperModel,resampleTo16Khz,toCaptions,@@ -46,7 +47,7 @@ export default function TranscribeAudio({file: await sourceToBlob(source),});- await loadWhisperModel({+ await downloadWhisperModel({model: selectedModel,onProgress: (progress) =>setState(() => ({@@ -54,6 +55,7 @@ export default function TranscribeAudio({progress,})),});+ await loadWhisperModel({model: selectedModel});setState(() => ({type: 'transcribing',@@ -87,7 +89,7 @@ export default function TranscribeAudio({<divclassName="w-[50%] h-5 bg-brand"style={{- width: `${(state.progress.progress ?? 0) * 100}%`,+ width: `${state.progress.progress * 100}%`,}}/></div>@@ -100,12 +102,8 @@ export default function TranscribeAudio({</div><div className="tabular-nums text-muted-foreground font-brand text-sm"><span>- {state.progress.progress === null- ? state.progress.status- : `${Math.round(state.progress.progress * 100)}%`}{' '}- {state.progress.loadedBytes === null- ? null- : formatBytes(state.progress.loadedBytes)}+ {Math.round(state.progress.progress * 100)}%{' '}+ {formatBytes(state.progress.loadedBytes)}</span></div></div>diff --git a/packages/docs/docs/video-matting/download-video-matting-model.mdx b/packages/docs/docs/video-matting/download-video-matting-model.mdxnew file mode 100644index 0000000000..39515d2c52--- /dev/null+++ b/packages/docs/docs/video-matting/download-video-matting-model.mdx@@ -0,0 +1,64 @@+---+image: /generated/articles-docs-video-matting-download-video-matting-model.png+title: downloadVideoMattingModel()+crumb: '@remotion/video-matting'+---++# downloadVideoMattingModel()<AvailableFrom v="4.0.528" />++Downloads a video matting model from `remotion.media` into the browser cache without initializing WebGPU. Call [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model) or [`separateVideoLayers()`](/docs/video-matting/separate-video-layers) afterwards.++The browser Cache API or a Transformers.js custom cache must be enabled.++```ts twoslash title="download-model.ts"+import {downloadVideoMattingModel} from '@remotion/video-matting';++const {alreadyDownloaded} = await downloadVideoMattingModel({+ model: 'modnet',+ onProgress: ({progress}) => console.log(progress),+});++console.log(alreadyDownloaded);+```++## Options++### model++One of the names returned by [`getAvailableModels()`](/docs/video-matting/get-available-models).++### onProgress?++Called as files download. Default: no callback. The callback receives:++#### file++The file being downloaded, or `null` for aggregate progress.++#### progress++Aggregate progress between `0` and `1`.++#### loadedBytes++Bytes downloaded in this call.++#### totalBytes++Estimated total model size in bytes.++## Return value++### alreadyDownloaded++`true` if all files were already cached before this call.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)+- [`isVideoMattingModelCached()`](/docs/video-matting/is-video-matting-model-cached)+- [`removeVideoMattingModel()`](/docs/video-matting/remove-video-matting-model)diff --git a/packages/docs/docs/video-matting/get-available-models.mdx b/packages/docs/docs/video-matting/get-available-models.mdxindex d685248131..3ab99262ff 100644--- a/packages/docs/docs/video-matting/get-available-models.mdx+++ b/packages/docs/docs/video-matting/get-available-models.mdx@@ -51,5 +51,6 @@ The total size in bytes of all runtime files downloaded for WebGPU.## See also- [`isVideoMattingModelCached()`](/docs/video-matting/is-video-matting-model-cached)+- [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model)- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)- [`removeVideoMattingModel()`](/docs/video-matting/remove-video-matting-model)diff --git a/packages/docs/docs/video-matting/index.mdx b/packages/docs/docs/video-matting/index.mdxindex 1814cfa61f..843e9e4c95 100644--- a/packages/docs/docs/video-matting/index.mdx+++ b/packages/docs/docs/video-matting/index.mdx@@ -18,15 +18,19 @@ import {TableOfContents} from './table-of-contents';```ts twoslash title="separate-video.ts"import {+ downloadVideoMattingModel,loadVideoMattingModel,separateVideoLayers,} from '@remotion/video-matting';export const matteVideo = async (file: File) => {const model = 'modnet';+ await downloadVideoMattingModel({+ model,+ onProgress: ({progress}) => console.log('Downloading model:', progress),+ });await using modelHandle = await loadVideoMattingModel({model,- onProgress: ({progress}) => console.log('Loading model:', progress),});console.log('Model already in memory:', modelHandle.alreadyLoaded);diff --git a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdxindex 20d886cd7b..279ad7da9c 100644--- a/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx+++ b/packages/docs/docs/video-matting/is-video-matting-model-cached.mdx@@ -33,5 +33,6 @@ A `Promise<boolean>` resolving to `true` if every required model file is cached.## See also- [`getAvailableModels()`](/docs/video-matting/get-available-models)+- [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model)- [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model)- [`removeVideoMattingModel()`](/docs/video-matting/remove-video-matting-model)diff --git a/packages/docs/docs/video-matting/load-video-matting-model.mdx b/packages/docs/docs/video-matting/load-video-matting-model.mdxindex be9348af2f..fe5350f7f4 100644--- a/packages/docs/docs/video-matting/load-video-matting-model.mdx+++ b/packages/docs/docs/video-matting/load-video-matting-model.mdx@@ -6,18 +6,16 @@ crumb: '@remotion/video-matting'# loadVideoMattingModel()<AvailableFrom v="4.0.523" />-Downloads a video matting model from our `remotion.media` CDN.+Initializes a previously downloaded video matting model on WebGPU. Call [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model) first.Call it before [`separateVideoLayers()`](/docs/video-matting/separate-video-layers).```ts twoslash title="load-model.ts"-import {loadVideoMattingModel} from '@remotion/video-matting';+import {downloadVideoMattingModel, loadVideoMattingModel} from '@remotion/video-matting';+await downloadVideoMattingModel({model: 'modnet'});await using result = await loadVideoMattingModel({model: 'modnet',- onProgress: ({progress}) => {- console.log(progress);- },});console.log(result.alreadyLoaded);@@ -31,7 +29,7 @@ One of the names returned by [`getAvailableModels()`](/docs/video-matting/get-av### onProgress?-Called while model files are resolved and downloaded. Default: no callback.+Called while the cached model is initialized. Use [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model) for download progress. Default: no callback.The callback receives these properties:@@ -45,15 +43,15 @@ The model file associated with the update, or `null` for aggregate progress.#### progress-Download progress between `0` and `1`, or `null` when it cannot be calculated.+`null` during initialization and `1` when ready.#### loadedBytes-The number of bytes downloaded, or `null` when it is unknown.+Always `null`; downloads happen in [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model).#### totalBytes-The total number of bytes, or `null` when it is unknown.+Always `null`.## Return value@@ -65,9 +63,11 @@ Whether the same model was already initialized in memory.### `[Symbol.asyncDispose]()`<AvailableFrom v="4.0.528" />-Releases this call's hold on the initialized model. The pipeline is disposed after every load handle is released and active separations finish.+Releases resources allocated by this function.-Use `await using` to call this method automatically when leaving the current scope. Call [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model) to dispose the pipeline explicitly regardless of retained load handles.+Use `await using` to call this method automatically when leaving the current scope.++Call [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model) to dispose the pipeline explicitly regardless of retained load handles.## Compatibility@@ -75,6 +75,7 @@ Use `await using` to call this method automatically when leaving the current sco## See also+- [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model)- [`disposeVideoMattingModel()`](/docs/video-matting/dispose-video-matting-model)- [`isVideoMattingModelCached()`](/docs/video-matting/is-video-matting-model-cached)- [`removeVideoMattingModel()`](/docs/video-matting/remove-video-matting-model)diff --git a/packages/docs/docs/video-matting/separate-video-layers.mdx b/packages/docs/docs/video-matting/separate-video-layers.mdxindex 645f891b80..bbec704dc6 100644--- a/packages/docs/docs/video-matting/separate-video-layers.mdx+++ b/packages/docs/docs/video-matting/separate-video-layers.mdx@@ -9,8 +9,9 @@ crumb: '@remotion/video-matting'Separates a video into an opaque base WebM and a foreground WebM with alpha.```ts twoslash title="separate-video.ts"-import {separateVideoLayers} from '@remotion/video-matting';+import {downloadVideoMattingModel, separateVideoLayers} from '@remotion/video-matting';+await downloadVideoMattingModel({model: 'modnet'});const file = new File([], 'input.mp4');await using result = await separateVideoLayers({@@ -121,7 +122,7 @@ Aborting rejects the promise and cancels both outputs. Model inference may finis### onModelLoadProgress?-Called while model files are resolved and downloaded. Default: no callback.+Called while a previously downloaded model is initialized. Default: no callback. Call [`downloadVideoMattingModel()`](/docs/video-matting/download-video-matting-model) first to download the files and receive download progress.To load the model separately, use [`loadVideoMattingModel()`](/docs/video-matting/load-video-matting-model).diff --git a/packages/docs/docs/video-matting/table-of-contents.tsx b/packages/docs/docs/video-matting/table-of-contents.tsxindex 45896f1fc5..3fd9b28a21 100644--- a/packages/docs/docs/video-matting/table-of-contents.tsx+++ b/packages/docs/docs/video-matting/table-of-contents.tsx@@ -17,9 +17,13 @@ export const TableOfContents: React.FC = () => {<strong>isVideoMattingModelCached()</strong><div>Check whether a model is downloaded</div></TOCItem>+ <TOCItem link="/docs/video-matting/download-video-matting-model">+ <strong>downloadVideoMattingModel()</strong>+ <div>Download a model without initializing WebGPU</div>+ </TOCItem><TOCItem link="/docs/video-matting/load-video-matting-model"><strong>loadVideoMattingModel()</strong>- <div>Download and initialize a model</div>+ <div>Initialize a downloaded model</div></TOCItem><TOCItem link="/docs/video-matting/remove-video-matting-model"><strong>removeVideoMattingModel()</strong>diff --git a/packages/docs/docs/whisper-webgpu/download-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/download-whisper-model.mdxnew file mode 100644index 0000000000..40f96e55fd--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/download-whisper-model.mdx@@ -0,0 +1,64 @@+---+image: /generated/articles-docs-whisper-webgpu-download-whisper-model.png+title: downloadWhisperModel()+crumb: '@remotion/whisper-webgpu'+---++# downloadWhisperModel()<AvailableFrom v="4.0.528" />++Downloads a Whisper model from `remotion.media` into the Transformers.js cache without initializing WebGPU. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) or [`transcribe()`](/docs/whisper-webgpu/transcribe) afterwards.++The browser uses its Cache API; Node.js uses the Transformers.js filesystem cache. A cache must be enabled.++```ts twoslash title="download-model.ts"+import {downloadWhisperModel} from '@remotion/whisper-webgpu';++const {alreadyDownloaded} = await downloadWhisperModel({+ model: 'small.en',+ onProgress: ({progress}) => console.log(progress),+});++console.log(alreadyDownloaded);+```++## Options++### model++One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models).++### onProgress?++Called as files download. Default: no callback. The callback receives:++#### file++The file being downloaded, or `null` for aggregate progress.++#### progress++Aggregate progress between `0` and `1`.++#### loadedBytes++Bytes downloaded in this call.++#### totalBytes++Estimated total model size in bytes.++## Return value++### alreadyDownloaded++`true` if all files were already cached before this call.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />++## See also++- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)+- [`isWhisperModelCached()`](/docs/whisper-webgpu/is-whisper-model-cached)+- [`removeWhisperModel()`](/docs/whisper-webgpu/remove-whisper-model)diff --git a/packages/docs/docs/whisper-webgpu/get-available-models.mdx b/packages/docs/docs/whisper-webgpu/get-available-models.mdxindex cf18b13add..d443b1a5d1 100644--- a/packages/docs/docs/whisper-webgpu/get-available-models.mdx+++ b/packages/docs/docs/whisper-webgpu/get-available-models.mdx@@ -62,4 +62,5 @@ We noticed that hosting the models on R2 leads to faster downloading than throug- [`transcribe()`](/docs/whisper-webgpu/transcribe)- [`clearStaleModels()`](/docs/whisper-webgpu/clear-stale-models)- [`isWhisperModelCached()`](/docs/whisper-webgpu/is-whisper-model-cached)+- [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model)- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)diff --git a/packages/docs/docs/whisper-webgpu/index.mdx b/packages/docs/docs/whisper-webgpu/index.mdxindex f6518a4954..de2105a629 100644--- a/packages/docs/docs/whisper-webgpu/index.mdx+++ b/packages/docs/docs/whisper-webgpu/index.mdx@@ -17,9 +17,10 @@ import {TableOfContents} from './table-of-contents';## Example```ts twoslash title="transcribe.ts"-import {clearStaleModels, resampleTo16Khz, toCaptions, transcribe} from '@remotion/whisper-webgpu';+import {clearStaleModels, downloadWhisperModel, resampleTo16Khz, toCaptions, transcribe} from '@remotion/whisper-webgpu';await clearStaleModels();+await downloadWhisperModel({model: 'small.en'});const file = new File([], 'audio.wav');const channelWaveform = await resampleTo16Khz({file});@@ -34,7 +35,7 @@ const {captions} = toCaptions({whisperWebGpuOutput: transcription});console.log(captions);```-The model is downloaded on the first call and cached by the runtime. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) first to display model download progress.+[`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model) stores model files in the persistent cache without initializing WebGPU. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) when you want to initialize the model explicitly.See [Using `@remotion/whisper-webgpu` in Node.js](/docs/whisper-webgpu/node) for a complete Node.js example.diff --git a/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdxindex a66c0e3b62..d1dec8ef4b 100644--- a/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx+++ b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx@@ -26,7 +26,7 @@ One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-aA `Promise<boolean>` resolving to `true` if every required model file is cached.-Models cached from Hugging Face by versions before 4.0.522 do not satisfy this check. Loading the model downloads and caches a new copy from `remotion.media`.+Models cached from Hugging Face by versions before 4.0.522 do not satisfy this check. Use [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model) to download and cache a copy from `remotion.media`.## Compatibility@@ -35,5 +35,6 @@ Models cached from Hugging Face by versions before 4.0.522 do not satisfy this c## See also- [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models)+- [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model)- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)- [`removeWhisperModel()`](/docs/whisper-webgpu/remove-whisper-model)diff --git a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdxindex 4177571a60..a92afec360 100644--- a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx+++ b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx@@ -6,16 +6,14 @@ crumb: '@remotion/whisper-webgpu'# loadWhisperModel()<AvailableFrom v="4.0.518" />-Downloads a timestamped Whisper model from `remotion.media` and initializes it. Calling it before [`transcribe()`](/docs/whisper-webgpu/transcribe) lets you show model-loading progress.+Initializes a previously downloaded Whisper model on WebGPU. Call [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model) first.```ts twoslash title="load-model.ts"-import {loadWhisperModel} from '@remotion/whisper-webgpu';+import {downloadWhisperModel, loadWhisperModel} from '@remotion/whisper-webgpu';+await downloadWhisperModel({model: 'small.en'});await using result = await loadWhisperModel({model: 'small.en',- onProgress: ({progress}) => {- console.log(progress);- },});console.log(result.alreadyLoaded);@@ -29,7 +27,7 @@ One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-a### onProgress?-Called while model files are resolved and downloaded. `progress` is the aggregate download progress between `0` and `1` and never decreases.+Called while the cached model is initialized. `progress` is `null` during initialization and `1` when ready. Use [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model) for download progress.## Return value@@ -51,6 +49,7 @@ Use `await using` to call this method automatically when leaving the current sco## See also+- [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model)- [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model)- [`isWhisperModelCached()`](/docs/whisper-webgpu/is-whisper-model-cached)- [`removeWhisperModel()`](/docs/whisper-webgpu/remove-whisper-model)diff --git a/packages/docs/docs/whisper-webgpu/node.mdx b/packages/docs/docs/whisper-webgpu/node.mdxindex b97d4ebefd..7fe0e583c6 100644--- a/packages/docs/docs/whisper-webgpu/node.mdx+++ b/packages/docs/docs/whisper-webgpu/node.mdx@@ -20,6 +20,7 @@ The following example uses [`@mediabunny/server`](https://mediabunny.dev/guide/eimport {registerMediabunnyServer} from '@mediabunny/server';import {WHISPER_WEBGPU_SAMPLE_RATE,+ downloadWhisperModel,loadWhisperModel,toCaptions,transcribe,@@ -118,6 +119,7 @@ for (const chunk of chunks) {}const model = 'small.en';+await downloadWhisperModel({model});await using modelHandle = await loadWhisperModel({model});const transcription = await transcribe({channelWaveform, model});const {captions} = toCaptions({whisperWebGpuOutput: transcription});diff --git a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx b/packages/docs/docs/whisper-webgpu/table-of-contents.tsxindex 5537c3ede0..4859bc19c7 100644--- a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx+++ b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx@@ -25,9 +25,13 @@ export const TableOfContents: React.FC = () => {<strong>isWhisperModelCached()</strong><div>Check whether a model is downloaded</div></TOCItem>+ <TOCItem link="/docs/whisper-webgpu/download-whisper-model">+ <strong>downloadWhisperModel()</strong>+ <div>Download a model without initializing WebGPU</div>+ </TOCItem><TOCItem link="/docs/whisper-webgpu/load-whisper-model"><strong>loadWhisperModel()</strong>- <div>Download and initialize a model</div>+ <div>Initialize a downloaded model</div></TOCItem><TOCItem link="/docs/whisper-webgpu/remove-whisper-model"><strong>removeWhisperModel()</strong>diff --git a/packages/docs/docs/whisper-webgpu/transcribe.mdx b/packages/docs/docs/whisper-webgpu/transcribe.mdxindex a04cb7707a..a48a7000a5 100644--- a/packages/docs/docs/whisper-webgpu/transcribe.mdx+++ b/packages/docs/docs/whisper-webgpu/transcribe.mdx@@ -9,8 +9,9 @@ crumb: '@remotion/whisper-webgpu'Transcribes a 16kHz mono waveform and returns words with start and end timestamps.```ts twoslash title="transcribe.ts"-import {resampleTo16Khz, transcribe} from '@remotion/whisper-webgpu';+import {downloadWhisperModel, resampleTo16Khz, transcribe} from '@remotion/whisper-webgpu';+await downloadWhisperModel({model: 'small.en'});const file = new File([], 'audio.wav');const channelWaveform = await resampleTo16Khz({file});@@ -76,7 +77,7 @@ Prevents a generated token sequence of this length from occurring more than once### onModelLoadProgress?-Called while the model is loaded. To load independently, use [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model).+Called while a previously downloaded model is initialized. Call [`downloadWhisperModel()`](/docs/whisper-webgpu/download-whisper-model) first to download the files and receive download progress.## Return valuediff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.tsindex a4bce17cae..82266fa34b 100644--- a/packages/docs/sidebars.ts+++ b/packages/docs/sidebars.ts@@ -963,6 +963,7 @@ const sidebars: SidebarsConfig = {'video-matting/can-use-video-matting','video-matting/get-available-models','video-matting/is-video-matting-model-cached',+ 'video-matting/download-video-matting-model','video-matting/load-video-matting-model','video-matting/remove-video-matting-model','video-matting/separate-video-layers',@@ -1024,6 +1025,7 @@ const sidebars: SidebarsConfig = {'whisper-webgpu/get-available-models','whisper-webgpu/clear-stale-models','whisper-webgpu/is-whisper-model-cached',+ 'whisper-webgpu/download-whisper-model','whisper-webgpu/load-whisper-model','whisper-webgpu/remove-whisper-model','whisper-webgpu/transcribe',@@ -1591,8 +1593,8 @@ const sidebars: SidebarsConfig = {'miscellaneous/pexels','lovable-for-motion-graphics',],- },- {+ },+ {type: 'category',label: 'Terminology',link: {diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.tsindex 9ad27f4e06..fbc485717d 100644--- a/packages/docs/src/data/articles.ts+++ b/packages/docs/src/data/articles.ts@@ -8661,6 +8661,15 @@ export const articles = [noAi: false,slug: 'video-matting/dispose-video-matting-model',},+ {+ id: 'video-matting/download-video-matting-model',+ title: 'downloadVideoMattingModel()',+ relativePath: 'docs/video-matting/download-video-matting-model.mdx',+ compId: 'articles-docs-video-matting-download-video-matting-model',+ crumb: '@remotion/video-matting',+ noAi: false,+ slug: 'video-matting/download-video-matting-model',+ },{id: 'video-matting/get-available-models',title: 'getAvailableModels()',@@ -9318,6 +9327,15 @@ export const articles = [noAi: false,slug: 'whisper-webgpu/dispose-whisper-model',},+ {+ id: 'whisper-webgpu/download-whisper-model',+ title: 'downloadWhisperModel()',+ relativePath: 'docs/whisper-webgpu/download-whisper-model.mdx',+ compId: 'articles-docs-whisper-webgpu-download-whisper-model',+ crumb: '@remotion/whisper-webgpu',+ noAi: false,+ slug: 'whisper-webgpu/download-whisper-model',+ },{id: 'whisper-webgpu/get-available-models',title: 'getAvailableModels()',diff --git a/packages/docs/static/generated/articles-docs-video-matting-download-video-matting-model.png b/packages/docs/static/generated/articles-docs-video-matting-download-video-matting-model.pngnew file mode 100644index 0000000000..fac8e3ce02Binary files /dev/null and b/packages/docs/static/generated/articles-docs-video-matting-download-video-matting-model.png differdiff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-download-whisper-model.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-download-whisper-model.pngnew file mode 100644index 0000000000..fbf24e77cbBinary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-download-whisper-model.png differdiff --git a/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsxindex bcca6abd45..ed5fb09095 100644--- a/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx+++ b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx@@ -2,6 +2,7 @@ import {canUseWhisperWebGpu,clearStaleModels,disposeWhisperModel,+ downloadWhisperModel,isWhisperModelCached,loadWhisperModel,toCaptions,@@ -49,11 +50,13 @@ export const CaptionQueueProcessor: React.FC = () => {progressStart: 0.03,progressSpan: 0.27,isModelCached: (model) => isWhisperModelCached({model}),- loadModel: (model, onProgress) =>- loadWhisperModel({+ loadModel: async (model, onProgress) => {+ await downloadWhisperModel({model,onProgress: (progress) => onProgress(progress.progress),- }),+ });+ await loadWhisperModel({model});+ },updateProgress: (progress) =>updateCaptionJobProgress(job.id, progress),});diff --git a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsxindex 8063683a57..61186c9b1d 100644--- a/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx+++ b/packages/studio/src/components/RenderQueue/VideoMattingQueueProcessor.tsx@@ -1,6 +1,7 @@import {canUseVideoMatting,disposeVideoMattingModel,+ downloadVideoMattingModel,isVideoMattingModelCached,loadVideoMattingModel,separateVideoLayers,@@ -40,11 +41,13 @@ export const VideoMattingQueueProcessor: React.FC = () => {progressStart: 0,progressSpan: 0.2,isModelCached: (model) => isVideoMattingModelCached({model}),- loadModel: (model, onProgress) =>- loadVideoMattingModel({+ loadModel: async (model, onProgress) => {+ await downloadVideoMattingModel({model,onProgress: (progress) => onProgress(progress.progress),- }),+ });+ await loadVideoMattingModel({model});+ },updateProgress: (progress) =>updateVideoMattingJobProgress(job.id, {...progress,diff --git a/packages/studio/src/components/Transcription/Models.tsx b/packages/studio/src/components/Transcription/Models.tsxindex 7493a3aeb7..793cb44a36 100644--- a/packages/studio/src/components/Transcription/Models.tsx+++ b/packages/studio/src/components/Transcription/Models.tsx@@ -1,8 +1,8 @@import {clearStaleModels,+ downloadWhisperModel,getAvailableModels,isWhisperModelCached,- loadWhisperModel,removeWhisperModel,type WhisperWebGpuModel,} from '@remotion/whisper-webgpu';@@ -25,7 +25,7 @@ export const Models: React.FC<{model: WhisperWebGpuModel,onProgress: (progress: number | null) => void,) =>- loadWhisperModel({+ downloadWhisperModel({model,onProgress: (progress) => onProgress(progress.progress),}),diff --git a/packages/studio/src/components/VideoMatting/Models.tsx b/packages/studio/src/components/VideoMatting/Models.tsxindex b2f3e5fca1..29c61566a8 100644--- a/packages/studio/src/components/VideoMatting/Models.tsx+++ b/packages/studio/src/components/VideoMatting/Models.tsx@@ -1,7 +1,7 @@import {+ downloadVideoMattingModel,getAvailableModels,isVideoMattingModelCached,- loadVideoMattingModel,removeVideoMattingModel,type VideoMattingModel,} from '@remotion/video-matting';@@ -21,7 +21,7 @@ export const Models: React.FC<{);const loadModel = useCallback((model: VideoMattingModel, onProgress: (progress: number | null) => void) =>- loadVideoMattingModel({+ downloadVideoMattingModel({model,onProgress: (progress) => onProgress(progress.progress),}),diff --git a/packages/studio/src/test/caption-queue-processor.test.tsx b/packages/studio/src/test/caption-queue-processor.test.tsxindex e102045789..ea05526ac3 100644--- a/packages/studio/src/test/caption-queue-processor.test.tsx+++ b/packages/studio/src/test/caption-queue-processor.test.tsx@@ -24,11 +24,10 @@ const getModelLoadCall = () => modelLoadCall;let modelIsCached = false;let modelLoadProgressCallback:| ((progress: {- status: string;file: string | null;- progress: number | null;- loadedBytes: number | null;- totalBytes: number | null;+ progress: number;+ loadedBytes: number;+ totalBytes: number;}) => void)| undefined;let resolveModelLoading: (() => void) | null = null;@@ -96,7 +95,7 @@ mock.module('@remotion/whisper-webgpu', () => ({cacheCheck = {model};return Promise.resolve(modelIsCached);},- loadWhisperModel: async ({+ downloadWhisperModel: async ({model,onProgress,}: {@@ -109,14 +108,14 @@ mock.module('@remotion/whisper-webgpu', () => ({resolveModelLoading = resolve;});modelLoadProgressCallback?.({- status: 'ready',file: null,progress: 1,loadedBytes: 119_699_015,totalBytes: 119_699_015,});- return {alreadyLoaded: false};+ return {alreadyDownloaded: false};},+ loadWhisperModel: () => Promise.resolve({alreadyLoaded: false}),removeWhisperModel: () => Promise.resolve(),toCaptions: () => ({captions: [@@ -267,7 +266,6 @@ test('downloads a model in the queued job before transcribing', async () => {act(() => {modelLoadProgressCallback?.({- status: 'loading',file: null,progress: 0.5,loadedBytes: 119_699_015 / 2,diff --git a/packages/studio/src/test/video-matting-queue-processor.test.tsx b/packages/studio/src/test/video-matting-queue-processor.test.tsxindex aa1cf3c2d2..7ac94b70de 100644--- a/packages/studio/src/test/video-matting-queue-processor.test.tsx+++ b/packages/studio/src/test/video-matting-queue-processor.test.tsx@@ -14,13 +14,17 @@ let processJob: ((job: VideoMattingJob) => Promise<void>) | null = null;mock.module('@remotion/video-matting', () => ({canUseVideoMatting: () => Promise.resolve({supported: true}),isVideoMattingModelCached: () => Promise.resolve(false),- loadVideoMattingModel: ({+ downloadVideoMattingModel: ({onProgress,}: {onProgress: (progress: {progress: number}) => void;}) => {- calls.push('load-model');+ calls.push('download-model');onProgress({progress: 1});+ return Promise.resolve({alreadyDownloaded: false});+ },+ loadVideoMattingModel: () => {+ calls.push('load-model');return Promise.resolve({alreadyLoaded: false});},separateVideoLayers: ({@@ -120,6 +124,7 @@ test('loads the model, separates the layers and writes both outputs', async () =expect(failed).toBeNull();expect(done).toBe(true);expect(calls).toEqual([+ 'download-model','load-model','separate','write:input-base.webm',diff --git a/packages/video-matting/src/download-video-matting-model.ts b/packages/video-matting/src/download-video-matting-model.tsnew file mode 100644index 0000000000..b5de024201--- /dev/null+++ b/packages/video-matting/src/download-video-matting-model.ts@@ -0,0 +1,128 @@+import {+ getHostedVideoMattingModelId,+ getVideoMattingModelInfo,+ type VideoMattingModel,+} from './models';+import {withRemotionModelHost} from './with-remotion-model-host';++export type VideoMattingModelDownloadProgress = {+ file: string | null;+ progress: number;+ loadedBytes: number;+ totalBytes: number;+};++export type DownloadVideoMattingModelOptions = {+ model: VideoMattingModel;+ onProgress?: (progress: VideoMattingModelDownloadProgress) => void;+};++export type DownloadVideoMattingModelResult = {+ alreadyDownloaded: boolean;+};++export const downloadVideoMattingModel = ({+ model,+ onProgress,+}: DownloadVideoMattingModelOptions): Promise<DownloadVideoMattingModelResult> => {+ const modelInfo = getVideoMattingModelInfo(model);+ const totalBytes = modelInfo.webGpuDownloadSize;+ const hostedModelId = getHostedVideoMattingModelId(model);+ return withRemotionModelHost(async ({env, ModelRegistry}) => {+ const cache = env.useCustomCache+ ? env.customCache+ : env.useBrowserCache && typeof caches !== 'undefined'+ ? await caches.open(env.cacheKey)+ : null;+ if (cache === null) {+ throw new Error(+ 'A Transformers.js browser or custom cache is required to download a video matting model.',+ );+ }++ const registryOptions = {+ device: 'webgpu' as const,+ dtype: modelInfo.dtype,+ };+ const cacheStatus = await ModelRegistry.is_pipeline_cached_files(+ 'background-removal',+ hostedModelId,+ registryOptions,+ );+ if (cacheStatus.allCached) {+ onProgress?.({+ file: null,+ progress: 1,+ loadedBytes: totalBytes,+ totalBytes,+ });+ return {alreadyDownloaded: true};+ }++ let loadedBytes = 0;+ const emitProgress = (file: string | null) => {+ onProgress?.({+ file,+ progress: Math.min(loadedBytes / totalBytes, 0.99),+ loadedBytes,+ totalBytes,+ });+ };++ emitProgress(null);++ for (const {file, cached} of cacheStatus.files) {+ if (cached) {+ continue;+ }++ const url = `${env.remoteHost}${env.remotePathTemplate.replace('{model}', hostedModelId)}${file}`;+ // Transformers.js 4.2.0 uses the remote URL as the browser cache key.+ const response = await env.fetch(url);+ if (!response.ok || response.body === null) {+ throw new Error(`Could not download ${url}: HTTP ${response.status}.`);+ }++ const reader = response.body.getReader();+ const stream = new ReadableStream<Uint8Array>({+ async pull(controller) {+ try {+ const {done, value} = await reader.read();+ if (done) {+ controller.close();+ return;+ }++ loadedBytes += value.byteLength;+ emitProgress(file);+ controller.enqueue(value);+ } catch (error) {+ controller.error(error);+ }+ },+ cancel: () => reader.cancel(),+ });+ await cache.put(url, new Response(stream, {headers: response.headers}));+ }++ if (+ !(await ModelRegistry.is_pipeline_cached(+ 'background-removal',+ hostedModelId,+ registryOptions,+ ))+ ) {+ throw new Error(+ `The video matting model "${model}" was not fully cached.`,+ );+ }++ onProgress?.({+ file: null,+ progress: 1,+ loadedBytes: totalBytes,+ totalBytes,+ });+ return {alreadyDownloaded: false};+ });+};diff --git a/packages/video-matting/src/index.ts b/packages/video-matting/src/index.tsindex 8aea58a0d1..57acd6497f 100644--- a/packages/video-matting/src/index.ts+++ b/packages/video-matting/src/index.ts@@ -10,6 +10,12 @@ export {disposeVideoMattingModel,loadVideoMattingModel,} from './load-video-matting-model';+export {downloadVideoMattingModel} from './download-video-matting-model';+export type {+ DownloadVideoMattingModelOptions,+ DownloadVideoMattingModelResult,+ VideoMattingModelDownloadProgress,+} from './download-video-matting-model';export type {DisposeVideoMattingModelOptions,LoadVideoMattingModelOptions,diff --git a/packages/video-matting/src/load-video-matting-model.ts b/packages/video-matting/src/load-video-matting-model.tsindex a20faab7d5..0ad40d953a 100644--- a/packages/video-matting/src/load-video-matting-model.ts+++ b/packages/video-matting/src/load-video-matting-model.ts@@ -163,54 +163,32 @@ const getOrCreateVideoMattingPipeline = ({AutoModelForImageSegmentation,AutoProcessor,BackgroundRemovalPipeline,+ ModelRegistry,}) => {const hostedModelId = getHostedVideoMattingModelId(model);- const totalBytes = modelInfo.webGpuDownloadSize;- const loadedByFile = new Map<string, number>();- let lastProgress = 0;- let lastLoadedBytes = 0;+ if (+ !(await ModelRegistry.is_pipeline_cached(+ 'background-removal',+ hostedModelId,+ {device: 'webgpu', dtype: modelInfo.dtype},+ ))+ ) {+ throw new Error(+ `The video matting model "${model}" is not downloaded. Call downloadVideoMattingModel() first.`,+ );+ }notifyProgress(state, {- status: 'loading',+ status: 'initializing',file: null,- progress: 0,- loadedBytes: 0,- totalBytes,+ progress: null,+ loadedBytes: null,+ totalBytes: null,});const pretrainedOptions = {device: 'webgpu' as const,dtype: modelInfo.dtype,- progress_callback: (event: unknown) => {- const record = event as Record<string, unknown>;- if (- record.status === 'progress' &&- typeof record.file === 'string' &&- typeof record.loaded === 'number' &&- Number.isFinite(record.loaded)- ) {- loadedByFile.set(- record.file,- Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),- );- const loadedBytes = [...loadedByFile.values()].reduce(- (sum, loaded) => sum + loaded,- 0,- );- lastProgress = Math.max(- lastProgress,- Math.min(loadedBytes / totalBytes, 0.99),- );- lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);- notifyProgress(state, {- status: 'loading',- file: null,- progress: lastProgress,- loadedBytes: Math.min(lastLoadedBytes, totalBytes),- totalBytes,- });- }- },};const transformerProcessor = await AutoProcessor.from_pretrained(hostedModelId,@@ -235,8 +213,8 @@ const getOrCreateVideoMattingPipeline = ({status: 'ready',file: null,progress: 1,- loadedBytes: totalBytes,- totalBytes,+ loadedBytes: null,+ totalBytes: null,});const loadedPipeline: LoadedPipeline = {diff --git a/packages/video-matting/src/test/model-runtime.test.ts b/packages/video-matting/src/test/model-runtime.test.tsindex 532e6bbf7d..100277435b 100644--- a/packages/video-matting/src/test/model-runtime.test.ts+++ b/packages/video-matting/src/test/model-runtime.test.ts@@ -21,10 +21,27 @@ let cacheClear:let cacheCheckEnvironment:| {remoteHost: string; remotePathTemplate: string}| undefined;+const downloadedFiles = new Map<string, Response>();+const requestedFiles: string[] = [];+let checkDownloadedFiles = false;+const customCache = {+ match: (url: string) => Promise.resolve(downloadedFiles.get(url)?.clone()),+ put: async (url: string, response: Response) => {+ downloadedFiles.set(url, new Response(await response.arrayBuffer()));+ },+};const originalTransformersEnvironment = {remoteHost: 'https://huggingface.co/',remotePathTemplate: '{model}/resolve/{revision}/',+ useCustomCache: true,+ customCache,+ useBrowserCache: false,+ cacheKey: 'transformers-cache',+ fetch: (url: string) => {+ requestedFiles.push(url);+ return Promise.resolve(new Response(url));+ },};const transformersEnvironment = {...originalTransformersEnvironment};const preexistingWhisperModelHostState = {@@ -71,7 +88,10 @@ mock.module('@huggingface/transformers', () => ({initializationCount++;initializedModelIds.push(modelId);initializationOptions = options;- initializationEnvironment = {...transformersEnvironment};+ initializationEnvironment = {+ remoteHost: transformersEnvironment.remoteHost,+ remotePathTemplate: transformersEnvironment.remotePathTemplate,+ };initializationStarted?.(initializationCount);await initializationGate;const onProgress = options.progress_callback as@@ -93,6 +113,18 @@ mock.module('@huggingface/transformers', () => ({},BackgroundRemovalPipeline,ModelRegistry: {+ is_pipeline_cached_files: () => {+ const files = ['config.json', 'onnx/model.onnx'].map((file) => ({+ file,+ cached: downloadedFiles.has(+ `https://remotion.media/models/modnet-v1/${file}`,+ ),+ }));+ return Promise.resolve({+ files,+ allCached: files.every(({cached}) => cached),+ });+ },clear_pipeline_cache: (task: string,modelId: string,@@ -107,8 +139,13 @@ mock.module('@huggingface/transformers', () => ({options: Record<string, unknown>,) => {cacheCheck = {task, modelId, options};- cacheCheckEnvironment = {...transformersEnvironment};- return Promise.resolve(true);+ cacheCheckEnvironment = {+ remoteHost: transformersEnvironment.remoteHost,+ remotePathTemplate: transformersEnvironment.remotePathTemplate,+ };+ return Promise.resolve(+ checkDownloadedFiles ? downloadedFiles.size === 2 : true,+ );},},}));@@ -131,11 +168,66 @@ beforeEach(async () => {cacheCheck = undefined;cacheClear = undefined;cacheCheckEnvironment = undefined;+ downloadedFiles.clear();+ requestedFiles.length = 0;+ checkDownloadedFiles = false;preexistingWhisperModelHostState.activeOperations = 0;preexistingWhisperModelHostState.previousRemoteConfiguration = null;Object.assign(transformersEnvironment, originalTransformersEnvironment);});+test('downloads without initializing WebGPU, then initializes from the cached files', async () => {+ checkDownloadedFiles = true;+ transformersEnvironment.useCustomCache = false;+ transformersEnvironment.useBrowserCache = true;+ const originalCaches = Object.getOwnPropertyDescriptor(globalThis, 'caches');+ Object.defineProperty(globalThis, 'caches', {+ configurable: true,+ value: {open: () => Promise.resolve(customCache)},+ });+ try {+ const {downloadVideoMattingModel, loadVideoMattingModel} =+ await import('../index');+ await expect(loadVideoMattingModel({model: 'modnet'})).rejects.toThrow(+ 'downloadVideoMattingModel() first',+ );+ const progress: number[] = [];+ const download = await downloadVideoMattingModel({+ model: 'modnet',+ onProgress: ({progress: value}) => progress.push(value),+ });++ expect(download.alreadyDownloaded).toBe(false);+ expect(initializationCount).toBe(0);+ expect(requestedFiles).toEqual([+ 'https://remotion.media/models/modnet-v1/config.json',+ 'https://remotion.media/models/modnet-v1/onnx/model.onnx',+ ]);+ expect(progress.at(-1)).toBe(1);+ downloadedFiles.delete(+ 'https://remotion.media/models/modnet-v1/onnx/model.onnx',+ );+ expect(+ (await downloadVideoMattingModel({model: 'modnet'})).alreadyDownloaded,+ ).toBe(false);+ expect(requestedFiles.slice(2)).toEqual([requestedFiles[1]]);++ await using loaded = await loadVideoMattingModel({model: 'modnet'});+ expect(loaded.alreadyLoaded).toBe(false);+ expect(initializationCount).toBe(1);+ expect(+ (await downloadVideoMattingModel({model: 'modnet'})).alreadyDownloaded,+ ).toBe(true);+ expect(requestedFiles).toHaveLength(3);+ } finally {+ if (originalCaches) {+ Object.defineProperty(globalThis, 'caches', originalCaches);+ } else {+ Reflect.deleteProperty(globalThis, 'caches');+ }+ }+});+test('shares loading, reports progress, and defers disposal while in use', async () => {const {disposeVideoMattingModel,diff --git a/packages/whisper-webgpu/src/download-whisper-model.ts b/packages/whisper-webgpu/src/download-whisper-model.tsnew file mode 100644index 0000000000..413edff57a--- /dev/null+++ b/packages/whisper-webgpu/src/download-whisper-model.ts@@ -0,0 +1,171 @@+import {+ getHostedModelId,+ getModelInfo,+ getWhisperWebGpuDtype,+ type WhisperWebGpuModel,+} from './models';+import {withRemotionModelHost} from './with-remotion-model-host';++export type WhisperModelDownloadProgress = {+ file: string | null;+ progress: number;+ loadedBytes: number;+ totalBytes: number;+};++export type DownloadWhisperModelOptions = {+ model: WhisperWebGpuModel;+ onProgress?: (progress: WhisperModelDownloadProgress) => void;+};++export type DownloadWhisperModelResult = {+ alreadyDownloaded: boolean;+};++export const downloadWhisperModel = ({+ model,+ onProgress,+}: DownloadWhisperModelOptions): Promise<DownloadWhisperModelResult> => {+ const totalBytes = getModelInfo(model).webGpuDownloadSize;+ const hostedModelId = getHostedModelId(model);+ return withRemotionModelHost(async ({env, ModelRegistry}) => {+ const cache = env.useCustomCache+ ? env.customCache+ : env.useBrowserCache && typeof caches !== 'undefined'+ ? await caches.open(env.cacheKey)+ : null;+ const useFileCache = cache === null && env.useFSCache;+ if (cache === null && !useFileCache) {+ throw new Error(+ 'A Transformers.js model cache is required to download a Whisper model.',+ );+ }++ const registryOptions = {+ device: 'webgpu' as const,+ dtype: getWhisperWebGpuDtype(model),+ };+ const cacheStatus = await ModelRegistry.is_pipeline_cached_files(+ 'automatic-speech-recognition',+ hostedModelId,+ registryOptions,+ );+ if (cacheStatus.allCached) {+ onProgress?.({+ file: null,+ progress: 1,+ loadedBytes: totalBytes,+ totalBytes,+ });+ return {alreadyDownloaded: true};+ }++ let loadedBytes = 0;+ const emitProgress = (file: string | null) => {+ onProgress?.({+ file,+ progress: Math.min(loadedBytes / totalBytes, 0.99),+ loadedBytes,+ totalBytes,+ });+ };++ emitProgress(null);++ for (const {file, cached} of cacheStatus.files) {+ if (cached) {+ continue;+ }++ const url = `${env.remoteHost}${env.remotePathTemplate.replace('{model}', hostedModelId)}${file}`;+ // Transformers.js 4.2.0 keys browser caches by URL and its FileCache by model/file.+ const cacheKey = cache === null ? `${hostedModelId}/${file}` : url;+ const response = await env.fetch(url);+ if (!response.ok || response.body === null) {+ throw new Error(`Could not download ${url}: HTTP ${response.status}.`);+ }++ const reader = response.body.getReader();+ if (cache !== null) {+ const stream = new ReadableStream<Uint8Array>({+ async pull(controller) {+ try {+ const {done, value} = await reader.read();+ if (done) {+ controller.close();+ return;+ }++ loadedBytes += value.byteLength;+ emitProgress(file);+ controller.enqueue(value);+ } catch (error) {+ controller.error(error);+ }+ },+ cancel: () => reader.cancel(),+ });+ await cache.put(+ cacheKey,+ new Response(stream, {headers: response.headers}),+ );+ } else {+ // Match Transformers.js FileCache's model/file layout without loading ONNX.+ const fsName: string = 'node:fs/promises';+ const pathName: string = 'node:path';+ const cryptoName: string = 'node:crypto';+ const [{mkdir, open, rename, unlink}, {join, dirname}, {randomUUID}] =+ await Promise.all([+ import(/* @vite-ignore */ fsName),+ import(/* @vite-ignore */ pathName),+ import(/* @vite-ignore */ cryptoName),+ ]);+ if (env.cacheDir === null) {+ throw new Error('Transformers.js env.cacheDir must be set.');+ }++ const destination = join(env.cacheDir, cacheKey);+ const temporary = `${destination}.tmp.${randomUUID()}`;+ await mkdir(dirname(destination), {recursive: true});+ const handle = await open(temporary, 'wx');+ try {+ while (true) {+ const {done, value} = await reader.read();+ if (done) {+ break;+ }++ await handle.writeFile(value);+ loadedBytes += value.byteLength;+ emitProgress(file);+ }+ } catch (error) {+ await handle.close();+ await unlink(temporary);+ throw error;+ }++ await handle.close();+ await rename(temporary, destination);+ }+ }++ if (+ !(await ModelRegistry.is_pipeline_cached(+ 'automatic-speech-recognition',+ hostedModelId,+ registryOptions,+ ))+ ) {+ throw new Error(`The Whisper model "${model}" was not fully cached.`);+ }++ onProgress?.({+ file: null,+ progress: 1,+ loadedBytes: totalBytes,+ totalBytes,+ });+ return {alreadyDownloaded: false};+ });+};diff --git a/packages/whisper-webgpu/src/index.ts b/packages/whisper-webgpu/src/index.tsindex 3fdc37b3ea..5ad37fa9a5 100644--- a/packages/whisper-webgpu/src/index.ts+++ b/packages/whisper-webgpu/src/index.ts@@ -4,6 +4,12 @@ export {} from './can-use-whisper-webgpu';export type {CanUseWhisperWebGpuResult} from './can-use-whisper-webgpu';export {clearStaleModels} from './clear-stale-models';+export {downloadWhisperModel} from './download-whisper-model';+export type {+ DownloadWhisperModelOptions,+ DownloadWhisperModelResult,+ WhisperModelDownloadProgress,+} from './download-whisper-model';export {isWhisperModelCached} from './is-whisper-model-cached';export type {IsWhisperModelCachedOptions} from './is-whisper-model-cached';export {disposeWhisperModel, loadWhisperModel} from './load-whisper-model';diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.tsindex 915f04330b..77da1fce69 100644--- a/packages/whisper-webgpu/src/load-whisper-model.ts+++ b/packages/whisper-webgpu/src/load-whisper-model.ts@@ -52,8 +52,10 @@ const getOrCreateWhisperPipeline = ({alreadyLoaded: boolean;unsubscribe: () => void;} => {- const modelInfo = getModelInfo(model);- const totalBytes = modelInfo.webGpuDownloadSize;+ if (!getModelInfo(model)) {+ throw new Error(`Unsupported Whisper model "${model}".`);+ }+const existing = pipelines.get(model);if (existing) {if (onProgress) {@@ -95,59 +97,40 @@ const getOrCreateWhisperPipeline = ({};const loading = Promise.resolve().then(() => {- return withRemotionModelHost(({pipeline}) => {+ return withRemotionModelHost(async ({ModelRegistry, pipeline}) => {+ const hostedModelId = getHostedModelId(model);+ if (+ !(await ModelRegistry.is_pipeline_cached(+ 'automatic-speech-recognition',+ hostedModelId,+ {device: 'webgpu', dtype: getWhisperWebGpuDtype(model)},+ ))+ ) {+ throw new Error(+ `The Whisper model "${model}" is not downloaded. Call downloadWhisperModel() first.`,+ );+ }+emitProgress({- status: 'loading',+ status: 'initializing',file: null,- progress: 0,- loadedBytes: 0,- totalBytes,+ progress: null,+ loadedBytes: null,+ totalBytes: null,});- const hostedModelId = getHostedModelId(model);- const loadedByFile = new Map<string, number>();- let lastProgress = 0;- let lastLoadedBytes = 0;return pipeline('automatic-speech-recognition', hostedModelId, {device: 'webgpu',dtype: getWhisperWebGpuDtype(model),progress_callback: (event) => {const record = event as Record<string, unknown>;- if (- record.status === 'progress' &&- typeof record.file === 'string' &&- typeof record.loaded === 'number' &&- Number.isFinite(record.loaded)- ) {- loadedByFile.set(- record.file,- Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),- );- const loadedBytes = [...loadedByFile.values()].reduce(- (sum, loaded) => sum + loaded,- 0,- );- lastProgress = Math.max(- lastProgress,- Math.min(loadedBytes / totalBytes, 0.99),- );- lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);- emitProgress({- status: 'loading',- file: null,- progress: lastProgress,- loadedBytes: lastLoadedBytes,- totalBytes,- });- }-if (record.status === 'ready') {emitProgress({status: 'ready',file: null,progress: 1,- loadedBytes: Math.max(lastLoadedBytes, totalBytes),- totalBytes,+ loadedBytes: null,+ totalBytes: null,});}},diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.tsindex bd44c48d30..6f8177d159 100644--- a/packages/whisper-webgpu/src/test/transcribe.test.ts+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts@@ -1,4 +1,8 @@import {expect, mock, test} from 'bun:test';+import {existsSync} from 'node:fs';+import {mkdtemp, readFile, rm, unlink} from 'node:fs/promises';+import {tmpdir} from 'node:os';+import {join} from 'node:path';let pipelineInitialization:| {@@ -67,6 +71,38 @@ const originalTransformersEnvironment = {remotePathTemplate: '{model}/resolve/{revision}/',};const transformersEnvironment = {...originalTransformersEnvironment};+const downloadFiles = ['config.json', 'onnx/encoder_model.onnx'];+const downloadRequests: string[] = [];+let downloadCacheDir = '';+let checkDownloadedFiles = false;+for (const [key, value] of Object.entries({+ useCustomCache: false,+ customCache: null,+ useBrowserCache: false,+ useFSCache: true,+ cacheDir: downloadCacheDir,+ fetch: (url: string) => {+ downloadRequests.push(url);+ const middle = Math.floor(url.length / 2);+ return Promise.resolve(+ new Response(+ new ReadableStream({+ start(controller) {+ controller.enqueue(new TextEncoder().encode(url.slice(0, middle)));+ controller.enqueue(new TextEncoder().encode(url.slice(middle)));+ controller.close();+ },+ }),+ ),+ );+ },+})) {+ Object.defineProperty(transformersEnvironment, key, {+ configurable: true,+ writable: true,+ value,+ });+}const fakePipeline = Object.assign(async (audio: Float32Array, options: Record<string, unknown>) => {@@ -95,6 +131,16 @@ const fakePipeline = Object.assign(mock.module('@huggingface/transformers', () => ({env: transformersEnvironment,ModelRegistry: {+ is_pipeline_cached_files: (_task: string, modelId: string) => {+ const files = downloadFiles.map((file) => ({+ file,+ cached: existsSync(join(downloadCacheDir, modelId, file)),+ }));+ return Promise.resolve({+ files,+ allCached: files.every(({cached}) => cached),+ });+ },clear_pipeline_cache: (task: string,modelId: string,@@ -114,7 +160,13 @@ mock.module('@huggingface/transformers', () => ({return Promise.reject(cacheCheckError);}- return Promise.resolve(true);+ return Promise.resolve(+ checkDownloadedFiles+ ? downloadFiles.every((file) =>+ existsSync(join(downloadCacheDir, modelId, file)),+ )+ : true,+ );},},pipeline: async (@@ -190,6 +242,67 @@ mock.module('onnxruntime-node', () => ({},}));+test('downloads files without WebGPU and initializes only after download', async () => {+ downloadCacheDir = await mkdtemp(+ join(tmpdir(), 'remotion-whisper-download-'),+ );+ (+ transformersEnvironment as typeof transformersEnvironment & {+ cacheDir: string;+ }+ ).cacheDir = downloadCacheDir;+ checkDownloadedFiles = true;+ downloadRequests.length = 0;+ try {+ const {downloadWhisperModel, loadWhisperModel} = await import('../index');+ await expect(loadWhisperModel({model: 'tiny.en'})).rejects.toThrow(+ 'downloadWhisperModel() first',+ );++ const initializationsBeforeDownload = pipelineInitializationCount;+ const progress: number[] = [];+ const result = await downloadWhisperModel({+ model: 'tiny.en',+ onProgress: ({progress: value}) => progress.push(value),+ });+ expect(result.alreadyDownloaded).toBe(false);+ expect(pipelineInitializationCount).toBe(initializationsBeforeDownload);+ expect(downloadRequests).toEqual([+ 'https://remotion.media/models/whisper-tiny.en_timestamped-v1/config.json',+ 'https://remotion.media/models/whisper-tiny.en_timestamped-v1/onnx/encoder_model.onnx',+ ]);+ expect(progress.at(-1)).toBe(1);+ expect(+ await readFile(+ join(downloadCacheDir, 'whisper-tiny.en_timestamped-v1', 'config.json'),+ 'utf8',+ ),+ ).toBe(downloadRequests[0]);+ await unlink(+ join(+ downloadCacheDir,+ 'whisper-tiny.en_timestamped-v1',+ 'onnx/encoder_model.onnx',+ ),+ );+ expect(+ (await downloadWhisperModel({model: 'tiny.en'})).alreadyDownloaded,+ ).toBe(false);+ expect(downloadRequests.slice(2)).toEqual([downloadRequests[1]]);++ await using loaded = await loadWhisperModel({model: 'tiny.en'});+ expect(loaded.alreadyLoaded).toBe(false);+ expect(pipelineInitializationCount).toBe(initializationsBeforeDownload + 1);+ expect(+ (await downloadWhisperModel({model: 'tiny.en'})).alreadyDownloaded,+ ).toBe(true);+ expect(downloadRequests).toHaveLength(3);+ } finally {+ checkDownloadedFiles = false;+ await rm(downloadCacheDir, {recursive: true, force: true});+ }+});+test('removes legacy Hugging Face Whisper entries from the shared browser cache', async () => {const originalCaches = Object.getOwnPropertyDescriptor(globalThis, 'caches');const cacheEnvironment =@@ -497,7 +610,7 @@ test('restores the Transformers environment if a hosted model operation fails',}});-test('reports model progress without reaching 100% before every file loads', async () => {+test('reports initialization separately from download progress', async () => {const {disposeWhisperModel, loadWhisperModel} = await import('../index');const progressValues: Array<{progress: number | null;@@ -516,27 +629,16 @@ test('reports model progress without reaching 100% before every file loads', asy});expect(progressValues[0]).toEqual({- progress: 0,- loadedBytes: 0,- totalBytes: 1_698_504_047,+ progress: null,+ loadedBytes: null,+ totalBytes: null,});expect(progressValues.at(-1)).toEqual({progress: 1,- loadedBytes: 1_698_504_047,- totalBytes: 1_698_504_047,+ loadedBytes: null,+ totalBytes: null,});- const downloading = progressValues.slice(0, -1);- expect(- downloading.every(({progress}) => progress !== null && progress < 1),- ).toBe(true);- expect(downloading.map(({loadedBytes}) => loadedBytes)).toEqual([- 0, 500_000_000, 600_000_000, 985_710_974,- ]);- expect(downloading.map(({progress}) => progress)).toEqual(- [0, 500_000_000, 600_000_000, 985_710_974].map((loaded) =>- Math.min(loaded / 1_698_504_047, 0.99),- ),- );+ expect(progressValues).toHaveLength(2);await disposeWhisperModel({model: 'medium.en'});});