🎥 Make videos programmatically with React
Support large Whisper model in browsers
JonnyBurger38756ff78c91
diff --git a/packages/whisper-webgpu/src/is-whisper-model-cached.ts b/packages/whisper-webgpu/src/is-whisper-model-cached.tsindex 0b7f41252e..a1c04d7c8e 100644--- a/packages/whisper-webgpu/src/is-whisper-model-cached.ts+++ b/packages/whisper-webgpu/src/is-whisper-model-cached.ts@@ -1,6 +1,6 @@import {getHostedModelId,- WHISPER_WEBGPU_DTYPE,+ getWhisperWebGpuDtype,type WhisperWebGpuModel,} from './models';import {withRemotionModelHost} from './with-remotion-model-host';@@ -18,7 +18,7 @@ export const isWhisperModelCached = ({getHostedModelId(model),{device: 'webgpu',- dtype: WHISPER_WEBGPU_DTYPE,+ dtype: getWhisperWebGpuDtype(model),},);});diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.tsindex ce18e1d560..1eb39610d5 100644--- a/packages/whisper-webgpu/src/load-whisper-model.ts+++ b/packages/whisper-webgpu/src/load-whisper-model.ts@@ -1,7 +1,7 @@import {getHostedModelId,getModelInfo,- WHISPER_WEBGPU_DTYPE,+ getWhisperWebGpuDtype,type WhisperWebGpuModel,} from './models';import {withRemotionModelHost} from './with-remotion-model-host';@@ -107,7 +107,7 @@ const getOrCreateWhisperPipeline = ({let lastLoadedBytes = 0;return pipeline('automatic-speech-recognition', hostedModelId, {device: 'webgpu',- dtype: WHISPER_WEBGPU_DTYPE,+ dtype: getWhisperWebGpuDtype(model),progress_callback: (event) => {const record = event as Record<string, unknown>;if (diff --git a/packages/whisper-webgpu/src/models.ts b/packages/whisper-webgpu/src/models.tsindex 623f3ff2bd..c8e3615c78 100644--- a/packages/whisper-webgpu/src/models.ts+++ b/packages/whisper-webgpu/src/models.ts@@ -92,7 +92,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {parameters: 809_000_000,multilingual: true,supportsTranslation: false,- webGpuDownloadSize: 2_882_584_170,+ webGpuDownloadSize: 1_608_611_679,},};@@ -122,7 +122,20 @@ export const getHostedModelId = (model: WhisperWebGpuModel): string => {return HOSTED_MODEL_IDS[model];};-export const WHISPER_WEBGPU_DTYPE = {+const DEFAULT_WHISPER_WEBGPU_DTYPE = {encoder_model: 'fp32',decoder_model_merged: 'q4',} as const;++// The FP32 encoder is 2.55 GB and Transformers.js reads it into one ArrayBuffer,+// which exceeds the browser allocation limit.+const LARGE_V3_TURBO_WHISPER_WEBGPU_DTYPE = {+ encoder_model: 'fp16',+ decoder_model_merged: 'q4',+} as const;++export const getWhisperWebGpuDtype = (model: WhisperWebGpuModel) => {+ return model === 'large-v3-turbo'+ ? LARGE_V3_TURBO_WHISPER_WEBGPU_DTYPE+ : DEFAULT_WHISPER_WEBGPU_DTYPE;+};diff --git a/packages/whisper-webgpu/src/remove-whisper-model.ts b/packages/whisper-webgpu/src/remove-whisper-model.tsindex e5e05dc0ce..49e563ebba 100644--- a/packages/whisper-webgpu/src/remove-whisper-model.ts+++ b/packages/whisper-webgpu/src/remove-whisper-model.ts@@ -1,7 +1,7 @@import {disposeWhisperModel} from './load-whisper-model';import {getHostedModelId,- WHISPER_WEBGPU_DTYPE,+ getWhisperWebGpuDtype,type WhisperWebGpuModel,} from './models';import {withRemotionModelHost} from './with-remotion-model-host';@@ -20,7 +20,7 @@ export const removeWhisperModel = async ({getHostedModelId(model),{device: 'webgpu',- dtype: WHISPER_WEBGPU_DTYPE,+ dtype: getWhisperWebGpuDtype(model),},);});diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.tsindex 5aed3ca4b8..d81f198826 100644--- a/packages/whisper-webgpu/src/test/transcribe.test.ts+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts@@ -332,6 +332,7 @@ test('transcribes with word timestamps using WebGPU', async () => {{multilingual: true,supportsTranslation: false,+ webGpuDownloadSize: 1_608_611_679,},);expect(await isWhisperModelCached({model: 'small.en'})).toBe(true);@@ -434,6 +435,10 @@ test('loads every public model from its explicit immutable hosted path', async ('onnx-community/whisper-medium.en_timestamped','onnx-community/whisper-large-v3-turbo_timestamped',]);+ expect(pipelineInitialization?.options).toMatchObject({+ device: 'webgpu',+ dtype: {encoder_model: 'fp16', decoder_model_merged: 'q4'},+ });expect(transformersEnvironment).toEqual(originalTransformersEnvironment);});@@ -677,18 +682,18 @@ test('shares concurrent initialization and keeps the model host scoped until evetest('removes a downloaded model from memory and the persistent cache', async () => {const {loadWhisperModel, removeWhisperModel} = await import('../index');const disposalsBeforeRemoval = disposeCalls;- await loadWhisperModel({model: 'tiny.en'});- await removeWhisperModel({model: 'tiny.en'});+ await loadWhisperModel({model: 'large-v3-turbo'});+ await removeWhisperModel({model: 'large-v3-turbo'});expect(disposeCalls).toBe(disposalsBeforeRemoval + 1);expect(cacheClear).toEqual({task: 'automatic-speech-recognition',- modelId: 'whisper-tiny.en_timestamped-v1',+ modelId: 'whisper-large-v3-turbo_timestamped-v1',options: {device: 'webgpu',dtype: {decoder_model_merged: 'q4',- encoder_model: 'fp32',+ encoder_model: 'fp16',},},});