Skip to content

🎥 Make videos programmatically with React

Show downloaded Whisper models

Jonny Burger9e4f2347dd9a
diff --git a/packages/convert/app/components/transcribe/App.tsx b/packages/convert/app/components/transcribe/App.tsx
index 7d8e0dd7c4..3d1f9eaf43 100644
--- a/packages/convert/app/components/transcribe/App.tsx
+++ b/packages/convert/app/components/transcribe/App.tsx
@@ -1,5 +1,9 @@
-import type {WhisperWebGpuModel} from '@remotion/whisper-webgpu';
-import React, {useState} from 'react';
+import {
+ getAvailableModels,
+ isWhisperModelCached,
+ type WhisperWebGpuModel,
+} from '@remotion/whisper-webgpu';
+import React, {useEffect, useState} from 'react';
import type {Source} from '~/lib/convert-state';
import Display from './display';
import ModelSelector from './modelSelector';
@@ -15,6 +19,36 @@ const Transcribe: React.FC<{
const [selectedModel, setSelectedModel] =
useState<WhisperWebGpuModel>('tiny.en');
+ const [cachedModels, setCachedModels] = useState<
+ WhisperWebGpuModel[] | null
+ >(null);
+
+ useEffect(() => {
+ if (state.type !== 'idle' && state.type !== 'error') {
+ return;
+ }
+
+ let cancelled = false;
+ Promise.all(
+ getAvailableModels().map(async ({name: modelName}) => {
+ const cached = await isWhisperModelCached({
+ model: modelName,
+ backend: 'webgpu',
+ }).catch(() => false);
+ return cached ? modelName : null;
+ }),
+ ).then((models) => {
+ if (!cancelled) {
+ setCachedModels(
+ models.filter((model): model is WhisperWebGpuModel => model !== null),
+ );
+ }
+ });
+
+ return () => {
+ cancelled = true;
+ };
+ }, [state.type]);
return (
<>
@@ -28,6 +62,7 @@ const Transcribe: React.FC<{
selectedModel={selectedModel}
setSelectedModel={setSelectedModel}
disabled={state.type === 'initializing'}
+ cachedModels={cachedModels}
/>
<div className="h-4" />
</>
diff --git a/packages/convert/app/components/transcribe/modelSelector.tsx b/packages/convert/app/components/transcribe/modelSelector.tsx
index e78c9bc601..db97d62c13 100644
--- a/packages/convert/app/components/transcribe/modelSelector.tsx
+++ b/packages/convert/app/components/transcribe/modelSelector.tsx
@@ -17,10 +17,12 @@ export default function ModelSelector({
selectedModel,
setSelectedModel,
disabled,
+ cachedModels,
}: {
readonly selectedModel: WhisperWebGpuModel;
readonly setSelectedModel: (model: WhisperWebGpuModel) => void;
readonly disabled: boolean;
+ readonly cachedModels: WhisperWebGpuModel[] | null;
}) {
return (
<div className="flex flex-col gap-5">
@@ -45,6 +47,9 @@ export default function ModelSelector({
<div className="font-medium">{model.name}</div>
<div className="text-xs text-muted-foreground">
{formatBytes(model.webGpuDownloadSize)} WebGPU
+ {cachedModels?.includes(model.name)
+ ? ' · Downloaded'
+ : null}
</div>
</div>
</SelectItem>
diff --git a/packages/convert/test/transcribe-webgpu.test.tsx b/packages/convert/test/transcribe-webgpu.test.tsx
index d784863059..09dcfc018e 100644
--- a/packages/convert/test/transcribe-webgpu.test.tsx
+++ b/packages/convert/test/transcribe-webgpu.test.tsx
@@ -7,6 +7,7 @@ afterEach(() => cleanup());
test('offers WebGPU model downloads without a WASM alternative', () => {
const rendered = render(
<ModelSelector
+ cachedModels={['tiny.en']}
selectedModel="tiny.en"
setSelectedModel={() => undefined}
disabled={false}
@@ -15,5 +16,6 @@ test('offers WebGPU model downloads without a WASM alternative', () => {
const model = rendered.getByRole('combobox', {name: 'Whisper model'});
expect(model.textContent).toContain('119.7 MB WebGPU');
+ expect(model.textContent).toContain('Downloaded');
expect(model.textContent).not.toContain('WASM');
});
diff --git a/packages/docs/docs/whisper-webgpu/get-available-models.mdx b/packages/docs/docs/whisper-webgpu/get-available-models.mdx
index c9f97935be..71aed2440a 100644
--- a/packages/docs/docs/whisper-webgpu/get-available-models.mdx
+++ b/packages/docs/docs/whisper-webgpu/get-available-models.mdx
@@ -28,4 +28,5 @@ The estimates cover model weights. Browser caches and supporting model files may
## See also
- [`transcribe()`](/docs/whisper-webgpu/transcribe)
+- [`isWhisperModelCached()`](/docs/whisper-webgpu/is-whisper-model-cached)
- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)
diff --git a/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx
new file mode 100644
index 0000000000..1c9a270883
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx
@@ -0,0 +1,41 @@
+---
+image: /generated/articles-docs-whisper-webgpu-is-whisper-model-cached.png
+title: isWhisperModelCached()
+crumb: '@remotion/whisper-webgpu'
+---
+
+# isWhisperModelCached()<AvailableFrom v="4.0.518" />
+
+Checks whether all files needed for a timestamped Whisper model are present in the browser cache. The check does not download or initialize the model.
+
+```ts twoslash title="model-cache.ts"
+import {isWhisperModelCached} from '@remotion/whisper-webgpu';
+
+const downloaded = await isWhisperModelCached({
+ model: 'small.en',
+ backend: 'webgpu',
+});
+```
+
+## Options
+
+### model
+
+One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models).
+
+### backend
+
+`'webgpu'` or `'wasm'`. The cache is checked for the model files used by that specific backend.
+
+## Return value
+
+A `Promise<boolean>` resolving to `true` if every required model file is cached.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+
+## See also
+
+- [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models)
+- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)
diff --git a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
index e14d8bd98b..3323dfe791 100644
--- a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
+++ b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
@@ -48,4 +48,5 @@ The browser manages its persistent Transformers.js cache. This function also ret
## See also
- [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model)
+- [`isWhisperModelCached()`](/docs/whisper-webgpu/is-whisper-model-cached)
- [`transcribe()`](/docs/whisper-webgpu/transcribe)
diff --git a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
index 52664f7b21..b09afa5fbc 100644
--- a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
+++ b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
@@ -13,6 +13,10 @@ export const TableOfContents: React.FC = () => {
<strong>getAvailableModels()</strong>
<div>List timestamped models and their download sizes</div>
</TOCItem>
+ <TOCItem link="/docs/whisper-webgpu/is-whisper-model-cached">
+ <strong>isWhisperModelCached()</strong>
+ <div>Check whether a model is downloaded</div>
+ </TOCItem>
<TOCItem link="/docs/whisper-webgpu/load-whisper-model">
<strong>loadWhisperModel()</strong>
<div>Download and initialize a model</div>
diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.ts
index 836afb476f..d50e1a0f1a 100644
--- a/packages/docs/sidebars.ts
+++ b/packages/docs/sidebars.ts
@@ -989,6 +989,7 @@ const sidebars: SidebarsConfig = {
items: [
'whisper-webgpu/can-use-whisper-webgpu',
'whisper-webgpu/get-available-models',
+ 'whisper-webgpu/is-whisper-model-cached',
'whisper-webgpu/load-whisper-model',
'whisper-webgpu/transcribe',
'whisper-webgpu/to-captions',
diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.ts
index 43ca5f8692..cbd6a70d11 100644
--- a/packages/docs/src/data/articles.ts
+++ b/packages/docs/src/data/articles.ts
@@ -9011,6 +9011,15 @@ export const articles = [
noAi: false,
slug: 'whisper-webgpu/index',
},
+ {
+ id: 'whisper-webgpu/is-whisper-model-cached',
+ title: 'isWhisperModelCached()',
+ relativePath: 'docs/whisper-webgpu/is-whisper-model-cached.mdx',
+ compId: 'articles-docs-whisper-webgpu-is-whisper-model-cached',
+ crumb: '@remotion/whisper-webgpu',
+ noAi: false,
+ slug: 'whisper-webgpu/is-whisper-model-cached',
+ },
{
id: 'whisper-webgpu/load-whisper-model',
title: 'loadWhisperModel()',
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-is-whisper-model-cached.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-is-whisper-model-cached.png
new file mode 100644
index 0000000000..5701e022c1
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-is-whisper-model-cached.png differ
diff --git a/packages/whisper-webgpu/src/index.ts b/packages/whisper-webgpu/src/index.ts
index c39445dcfc..8331233013 100644
--- a/packages/whisper-webgpu/src/index.ts
+++ b/packages/whisper-webgpu/src/index.ts
@@ -18,6 +18,8 @@ export type {
OnWhisperWebGpuModelLoadProgress,
WhisperWebGpuModelLoadProgress,
} from './load-whisper-model';
+export {isWhisperModelCached} from './is-whisper-model-cached';
+export type {IsWhisperModelCachedOptions} from './is-whisper-model-cached';
export {getAvailableModels, WHISPER_WEBGPU_MODELS} from './models';
export type {WhisperWebGpuModel, WhisperWebGpuModelInfo} from './models';
export {resampleTo16Khz, WHISPER_WEBGPU_SAMPLE_RATE} from './resample-to-16khz';
diff --git a/packages/whisper-webgpu/src/is-whisper-model-cached.ts b/packages/whisper-webgpu/src/is-whisper-model-cached.ts
new file mode 100644
index 0000000000..64750d2d65
--- /dev/null
+++ b/packages/whisper-webgpu/src/is-whisper-model-cached.ts
@@ -0,0 +1,28 @@
+import type {ResolvedWhisperWebGpuBackend} from './backend';
+import {
+ getModelInfo,
+ getWhisperModelDtype,
+ type WhisperWebGpuModel,
+} from './models';
+
+export type IsWhisperModelCachedOptions = {
+ model: WhisperWebGpuModel;
+ backend: ResolvedWhisperWebGpuBackend;
+};
+
+export const isWhisperModelCached = async ({
+ model,
+ backend,
+}: IsWhisperModelCachedOptions): Promise<boolean> => {
+ const {ModelRegistry} = await import('@huggingface/transformers');
+ const {modelId} = getModelInfo(model);
+
+ return ModelRegistry.is_pipeline_cached(
+ 'automatic-speech-recognition',
+ modelId,
+ {
+ device: backend,
+ dtype: getWhisperModelDtype(backend),
+ },
+ );
+};
diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.ts
index c003c27bbc..809cdc590c 100644
--- a/packages/whisper-webgpu/src/load-whisper-model.ts
+++ b/packages/whisper-webgpu/src/load-whisper-model.ts
@@ -3,7 +3,11 @@ import type {
WhisperWebGpuBackend,
} from './backend';
import {resolveBackend} from './backend';
-import {getModelInfo, type WhisperWebGpuModel} from './models';
+import {
+ getModelInfo,
+ getWhisperModelDtype,
+ type WhisperWebGpuModel,
+} from './models';
export type WhisperWebGpuModelLoadProgress = {
status: string;
@@ -84,10 +88,7 @@ export const loadWhisperModel = async ({
let lastLoadedBytes = 0;
const loading = pipeline('automatic-speech-recognition', modelId, {
device: resolvedBackend,
- dtype:
- resolvedBackend === 'webgpu'
- ? {encoder_model: 'fp32', decoder_model_merged: 'q4'}
- : 'q8',
+ dtype: getWhisperModelDtype(resolvedBackend),
progress_callback: onProgress
? (event) => {
const record = event as Record<string, unknown>;
diff --git a/packages/whisper-webgpu/src/models.ts b/packages/whisper-webgpu/src/models.ts
index d83efe778d..8a7e68c666 100644
--- a/packages/whisper-webgpu/src/models.ts
+++ b/packages/whisper-webgpu/src/models.ts
@@ -1,3 +1,5 @@
+import type {ResolvedWhisperWebGpuBackend} from './backend';
+
export const WHISPER_WEBGPU_MODELS = [
'tiny',
'tiny.en',
@@ -105,3 +107,9 @@ export const getModelInfo = (
): WhisperWebGpuModelInfo => {
return MODEL_INFO[model];
};
+
+export const getWhisperModelDtype = (backend: ResolvedWhisperWebGpuBackend) => {
+ return backend === 'webgpu'
+ ? ({encoder_model: 'fp32', decoder_model_merged: 'q4'} as const)
+ : ('q8' as const);
+};
diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.ts
index b855e6304f..e8beab3435 100644
--- a/packages/whisper-webgpu/src/test/transcribe.test.ts
+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts
@@ -13,6 +13,13 @@ let transcriptionCall:
}
| undefined;
let disposed = false;
+let cacheCheck:
+ | {
+ task: string;
+ modelId: string;
+ options: Record<string, unknown>;
+ }
+ | undefined;
const fakePipeline = Object.assign(
(audio: Float32Array, options: Record<string, unknown>) => {
@@ -37,6 +44,16 @@ const fakePipeline = Object.assign(
mock.module('@huggingface/transformers', () => ({
env: {backends: {onnx: {wasm: {numThreads: 0}}}},
+ ModelRegistry: {
+ is_pipeline_cached: (
+ task: string,
+ modelId: string,
+ options: Record<string, unknown>,
+ ) => {
+ cacheCheck = {task, modelId, options};
+ return Promise.resolve(true);
+ },
+ },
pipeline: (
_task: string,
modelId: string,
@@ -92,6 +109,7 @@ test('transcribes with timestamps and selects a usable backend', async () => {
canUseWhisperWebGpu,
disposeWhisperModel,
getAvailableModels,
+ isWhisperModelCached,
toCaptions,
transcribe,
WhisperWebGpuUnsupportedReason,
@@ -180,6 +198,17 @@ test('transcribes with timestamps and selects a usable backend', async () => {
multilingual: false,
webGpuDownloadSize: 586_209_938,
});
+ expect(
+ await isWhisperModelCached({backend: 'webgpu', model: 'small.en'}),
+ ).toBe(true);
+ expect(cacheCheck).toEqual({
+ task: 'automatic-speech-recognition',
+ modelId: 'onnx-community/whisper-small.en_timestamped',
+ options: {
+ device: 'webgpu',
+ dtype: {encoder_model: 'fp32', decoder_model_merged: 'q4'},
+ },
+ });
const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');
const originalNavigator = Object.getOwnPropertyDescriptor(