🎥 Make videos programmatically with React
Show downloaded Whisper models
Jonny Burger9e4f2347dd9a
diff --git a/packages/convert/app/components/transcribe/App.tsx b/packages/convert/app/components/transcribe/App.tsxindex 7d8e0dd7c4..3d1f9eaf43 100644--- a/packages/convert/app/components/transcribe/App.tsx+++ b/packages/convert/app/components/transcribe/App.tsx@@ -1,5 +1,9 @@-import type {WhisperWebGpuModel} from '@remotion/whisper-webgpu';-import React, {useState} from 'react';+import {+ getAvailableModels,+ isWhisperModelCached,+ type WhisperWebGpuModel,+} from '@remotion/whisper-webgpu';+import React, {useEffect, useState} from 'react';import type {Source} from '~/lib/convert-state';import Display from './display';import ModelSelector from './modelSelector';@@ -15,6 +19,36 @@ const Transcribe: React.FC<{const [selectedModel, setSelectedModel] =useState<WhisperWebGpuModel>('tiny.en');+ const [cachedModels, setCachedModels] = useState<+ WhisperWebGpuModel[] | null+ >(null);++ useEffect(() => {+ if (state.type !== 'idle' && state.type !== 'error') {+ return;+ }++ let cancelled = false;+ Promise.all(+ getAvailableModels().map(async ({name: modelName}) => {+ const cached = await isWhisperModelCached({+ model: modelName,+ backend: 'webgpu',+ }).catch(() => false);+ return cached ? modelName : null;+ }),+ ).then((models) => {+ if (!cancelled) {+ setCachedModels(+ models.filter((model): model is WhisperWebGpuModel => model !== null),+ );+ }+ });++ return () => {+ cancelled = true;+ };+ }, [state.type]);return (<>@@ -28,6 +62,7 @@ const Transcribe: React.FC<{selectedModel={selectedModel}setSelectedModel={setSelectedModel}disabled={state.type === 'initializing'}+ cachedModels={cachedModels}/><div className="h-4" /></>diff --git a/packages/convert/app/components/transcribe/modelSelector.tsx b/packages/convert/app/components/transcribe/modelSelector.tsxindex e78c9bc601..db97d62c13 100644--- a/packages/convert/app/components/transcribe/modelSelector.tsx+++ b/packages/convert/app/components/transcribe/modelSelector.tsx@@ -17,10 +17,12 @@ export default function ModelSelector({selectedModel,setSelectedModel,disabled,+ cachedModels,}: {readonly selectedModel: WhisperWebGpuModel;readonly setSelectedModel: (model: WhisperWebGpuModel) => void;readonly disabled: boolean;+ readonly cachedModels: WhisperWebGpuModel[] | null;}) {return (<div className="flex flex-col gap-5">@@ -45,6 +47,9 @@ export default function ModelSelector({<div className="font-medium">{model.name}</div><div className="text-xs text-muted-foreground">{formatBytes(model.webGpuDownloadSize)} WebGPU+ {cachedModels?.includes(model.name)+ ? ' · Downloaded'+ : null}</div></div></SelectItem>diff --git a/packages/convert/test/transcribe-webgpu.test.tsx b/packages/convert/test/transcribe-webgpu.test.tsxindex d784863059..09dcfc018e 100644--- a/packages/convert/test/transcribe-webgpu.test.tsx+++ b/packages/convert/test/transcribe-webgpu.test.tsx@@ -7,6 +7,7 @@ afterEach(() => cleanup());test('offers WebGPU model downloads without a WASM alternative', () => {const rendered = render(<ModelSelector+ cachedModels={['tiny.en']}selectedModel="tiny.en"setSelectedModel={() => undefined}disabled={false}@@ -15,5 +16,6 @@ test('offers WebGPU model downloads without a WASM alternative', () => {const model = rendered.getByRole('combobox', {name: 'Whisper model'});expect(model.textContent).toContain('119.7 MB WebGPU');+ expect(model.textContent).toContain('Downloaded');expect(model.textContent).not.toContain('WASM');});diff --git a/packages/docs/docs/whisper-webgpu/get-available-models.mdx b/packages/docs/docs/whisper-webgpu/get-available-models.mdxindex c9f97935be..71aed2440a 100644--- a/packages/docs/docs/whisper-webgpu/get-available-models.mdx+++ b/packages/docs/docs/whisper-webgpu/get-available-models.mdx@@ -28,4 +28,5 @@ The estimates cover model weights. Browser caches and supporting model files may## See also- [`transcribe()`](/docs/whisper-webgpu/transcribe)+- [`isWhisperModelCached()`](/docs/whisper-webgpu/is-whisper-model-cached)- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)diff --git a/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdxnew file mode 100644index 0000000000..1c9a270883--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx@@ -0,0 +1,41 @@+---+image: /generated/articles-docs-whisper-webgpu-is-whisper-model-cached.png+title: isWhisperModelCached()+crumb: '@remotion/whisper-webgpu'+---++# isWhisperModelCached()<AvailableFrom v="4.0.518" />++Checks whether all files needed for a timestamped Whisper model are present in the browser cache. The check does not download or initialize the model.++```ts twoslash title="model-cache.ts"+import {isWhisperModelCached} from '@remotion/whisper-webgpu';++const downloaded = await isWhisperModelCached({+ model: 'small.en',+ backend: 'webgpu',+});+```++## Options++### model++One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models).++### backend++`'webgpu'` or `'wasm'`. The cache is checked for the model files used by that specific backend.++## Return value++A `Promise<boolean>` resolving to `true` if every required model file is cached.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models)+- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)diff --git a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdxindex e14d8bd98b..3323dfe791 100644--- a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx+++ b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx@@ -48,4 +48,5 @@ The browser manages its persistent Transformers.js cache. This function also ret## See also- [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model)+- [`isWhisperModelCached()`](/docs/whisper-webgpu/is-whisper-model-cached)- [`transcribe()`](/docs/whisper-webgpu/transcribe)diff --git a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx b/packages/docs/docs/whisper-webgpu/table-of-contents.tsxindex 52664f7b21..b09afa5fbc 100644--- a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx+++ b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx@@ -13,6 +13,10 @@ export const TableOfContents: React.FC = () => {<strong>getAvailableModels()</strong><div>List timestamped models and their download sizes</div></TOCItem>+ <TOCItem link="/docs/whisper-webgpu/is-whisper-model-cached">+ <strong>isWhisperModelCached()</strong>+ <div>Check whether a model is downloaded</div>+ </TOCItem><TOCItem link="/docs/whisper-webgpu/load-whisper-model"><strong>loadWhisperModel()</strong><div>Download and initialize a model</div>diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.tsindex 836afb476f..d50e1a0f1a 100644--- a/packages/docs/sidebars.ts+++ b/packages/docs/sidebars.ts@@ -989,6 +989,7 @@ const sidebars: SidebarsConfig = {items: ['whisper-webgpu/can-use-whisper-webgpu','whisper-webgpu/get-available-models',+ 'whisper-webgpu/is-whisper-model-cached','whisper-webgpu/load-whisper-model','whisper-webgpu/transcribe','whisper-webgpu/to-captions',diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.tsindex 43ca5f8692..cbd6a70d11 100644--- a/packages/docs/src/data/articles.ts+++ b/packages/docs/src/data/articles.ts@@ -9011,6 +9011,15 @@ export const articles = [noAi: false,slug: 'whisper-webgpu/index',},+ {+ id: 'whisper-webgpu/is-whisper-model-cached',+ title: 'isWhisperModelCached()',+ relativePath: 'docs/whisper-webgpu/is-whisper-model-cached.mdx',+ compId: 'articles-docs-whisper-webgpu-is-whisper-model-cached',+ crumb: '@remotion/whisper-webgpu',+ noAi: false,+ slug: 'whisper-webgpu/is-whisper-model-cached',+ },{id: 'whisper-webgpu/load-whisper-model',title: 'loadWhisperModel()',diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-is-whisper-model-cached.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-is-whisper-model-cached.pngnew file mode 100644index 0000000000..5701e022c1Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-is-whisper-model-cached.png differdiff --git a/packages/whisper-webgpu/src/index.ts b/packages/whisper-webgpu/src/index.tsindex c39445dcfc..8331233013 100644--- a/packages/whisper-webgpu/src/index.ts+++ b/packages/whisper-webgpu/src/index.ts@@ -18,6 +18,8 @@ export type {OnWhisperWebGpuModelLoadProgress,WhisperWebGpuModelLoadProgress,} from './load-whisper-model';+export {isWhisperModelCached} from './is-whisper-model-cached';+export type {IsWhisperModelCachedOptions} from './is-whisper-model-cached';export {getAvailableModels, WHISPER_WEBGPU_MODELS} from './models';export type {WhisperWebGpuModel, WhisperWebGpuModelInfo} from './models';export {resampleTo16Khz, WHISPER_WEBGPU_SAMPLE_RATE} from './resample-to-16khz';diff --git a/packages/whisper-webgpu/src/is-whisper-model-cached.ts b/packages/whisper-webgpu/src/is-whisper-model-cached.tsnew file mode 100644index 0000000000..64750d2d65--- /dev/null+++ b/packages/whisper-webgpu/src/is-whisper-model-cached.ts@@ -0,0 +1,28 @@+import type {ResolvedWhisperWebGpuBackend} from './backend';+import {+ getModelInfo,+ getWhisperModelDtype,+ type WhisperWebGpuModel,+} from './models';++export type IsWhisperModelCachedOptions = {+ model: WhisperWebGpuModel;+ backend: ResolvedWhisperWebGpuBackend;+};++export const isWhisperModelCached = async ({+ model,+ backend,+}: IsWhisperModelCachedOptions): Promise<boolean> => {+ const {ModelRegistry} = await import('@huggingface/transformers');+ const {modelId} = getModelInfo(model);++ return ModelRegistry.is_pipeline_cached(+ 'automatic-speech-recognition',+ modelId,+ {+ device: backend,+ dtype: getWhisperModelDtype(backend),+ },+ );+};diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.tsindex c003c27bbc..809cdc590c 100644--- a/packages/whisper-webgpu/src/load-whisper-model.ts+++ b/packages/whisper-webgpu/src/load-whisper-model.ts@@ -3,7 +3,11 @@ import type {WhisperWebGpuBackend,} from './backend';import {resolveBackend} from './backend';-import {getModelInfo, type WhisperWebGpuModel} from './models';+import {+ getModelInfo,+ getWhisperModelDtype,+ type WhisperWebGpuModel,+} from './models';export type WhisperWebGpuModelLoadProgress = {status: string;@@ -84,10 +88,7 @@ export const loadWhisperModel = async ({let lastLoadedBytes = 0;const loading = pipeline('automatic-speech-recognition', modelId, {device: resolvedBackend,- dtype:- resolvedBackend === 'webgpu'- ? {encoder_model: 'fp32', decoder_model_merged: 'q4'}- : 'q8',+ dtype: getWhisperModelDtype(resolvedBackend),progress_callback: onProgress? (event) => {const record = event as Record<string, unknown>;diff --git a/packages/whisper-webgpu/src/models.ts b/packages/whisper-webgpu/src/models.tsindex d83efe778d..8a7e68c666 100644--- a/packages/whisper-webgpu/src/models.ts+++ b/packages/whisper-webgpu/src/models.ts@@ -1,3 +1,5 @@+import type {ResolvedWhisperWebGpuBackend} from './backend';+export const WHISPER_WEBGPU_MODELS = ['tiny','tiny.en',@@ -105,3 +107,9 @@ export const getModelInfo = (): WhisperWebGpuModelInfo => {return MODEL_INFO[model];};++export const getWhisperModelDtype = (backend: ResolvedWhisperWebGpuBackend) => {+ return backend === 'webgpu'+ ? ({encoder_model: 'fp32', decoder_model_merged: 'q4'} as const)+ : ('q8' as const);+};diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.tsindex b855e6304f..e8beab3435 100644--- a/packages/whisper-webgpu/src/test/transcribe.test.ts+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts@@ -13,6 +13,13 @@ let transcriptionCall:}| undefined;let disposed = false;+let cacheCheck:+ | {+ task: string;+ modelId: string;+ options: Record<string, unknown>;+ }+ | undefined;const fakePipeline = Object.assign((audio: Float32Array, options: Record<string, unknown>) => {@@ -37,6 +44,16 @@ const fakePipeline = Object.assign(mock.module('@huggingface/transformers', () => ({env: {backends: {onnx: {wasm: {numThreads: 0}}}},+ ModelRegistry: {+ is_pipeline_cached: (+ task: string,+ modelId: string,+ options: Record<string, unknown>,+ ) => {+ cacheCheck = {task, modelId, options};+ return Promise.resolve(true);+ },+ },pipeline: (_task: string,modelId: string,@@ -92,6 +109,7 @@ test('transcribes with timestamps and selects a usable backend', async () => {canUseWhisperWebGpu,disposeWhisperModel,getAvailableModels,+ isWhisperModelCached,toCaptions,transcribe,WhisperWebGpuUnsupportedReason,@@ -180,6 +198,17 @@ test('transcribes with timestamps and selects a usable backend', async () => {multilingual: false,webGpuDownloadSize: 586_209_938,});+ expect(+ await isWhisperModelCached({backend: 'webgpu', model: 'small.en'}),+ ).toBe(true);+ expect(cacheCheck).toEqual({+ task: 'automatic-speech-recognition',+ modelId: 'onnx-community/whisper-small.en_timestamped',+ options: {+ device: 'webgpu',+ dtype: {encoder_model: 'fp32', decoder_model_merged: 'q4'},+ },+ });const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');const originalNavigator = Object.getOwnPropertyDescriptor(