Skip to content

🎥 Make videos programmatically with React

Support large Whisper model in browsers

JonnyBurger38756ff78c91
diff --git a/packages/whisper-webgpu/src/is-whisper-model-cached.ts b/packages/whisper-webgpu/src/is-whisper-model-cached.ts
index 0b7f41252e..a1c04d7c8e 100644
--- a/packages/whisper-webgpu/src/is-whisper-model-cached.ts
+++ b/packages/whisper-webgpu/src/is-whisper-model-cached.ts
@@ -1,6 +1,6 @@
import {
getHostedModelId,
- WHISPER_WEBGPU_DTYPE,
+ getWhisperWebGpuDtype,
type WhisperWebGpuModel,
} from './models';
import {withRemotionModelHost} from './with-remotion-model-host';
@@ -18,7 +18,7 @@ export const isWhisperModelCached = ({
getHostedModelId(model),
{
device: 'webgpu',
- dtype: WHISPER_WEBGPU_DTYPE,
+ dtype: getWhisperWebGpuDtype(model),
},
);
});
diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.ts
index ce18e1d560..1eb39610d5 100644
--- a/packages/whisper-webgpu/src/load-whisper-model.ts
+++ b/packages/whisper-webgpu/src/load-whisper-model.ts
@@ -1,7 +1,7 @@
import {
getHostedModelId,
getModelInfo,
- WHISPER_WEBGPU_DTYPE,
+ getWhisperWebGpuDtype,
type WhisperWebGpuModel,
} from './models';
import {withRemotionModelHost} from './with-remotion-model-host';
@@ -107,7 +107,7 @@ const getOrCreateWhisperPipeline = ({
let lastLoadedBytes = 0;
return pipeline('automatic-speech-recognition', hostedModelId, {
device: 'webgpu',
- dtype: WHISPER_WEBGPU_DTYPE,
+ dtype: getWhisperWebGpuDtype(model),
progress_callback: (event) => {
const record = event as Record<string, unknown>;
if (
diff --git a/packages/whisper-webgpu/src/models.ts b/packages/whisper-webgpu/src/models.ts
index 623f3ff2bd..c8e3615c78 100644
--- a/packages/whisper-webgpu/src/models.ts
+++ b/packages/whisper-webgpu/src/models.ts
@@ -92,7 +92,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
parameters: 809_000_000,
multilingual: true,
supportsTranslation: false,
- webGpuDownloadSize: 2_882_584_170,
+ webGpuDownloadSize: 1_608_611_679,
},
};
@@ -122,7 +122,20 @@ export const getHostedModelId = (model: WhisperWebGpuModel): string => {
return HOSTED_MODEL_IDS[model];
};
-export const WHISPER_WEBGPU_DTYPE = {
+const DEFAULT_WHISPER_WEBGPU_DTYPE = {
encoder_model: 'fp32',
decoder_model_merged: 'q4',
} as const;
+
+// The FP32 encoder is 2.55 GB and Transformers.js reads it into one ArrayBuffer,
+// which exceeds the browser allocation limit.
+const LARGE_V3_TURBO_WHISPER_WEBGPU_DTYPE = {
+ encoder_model: 'fp16',
+ decoder_model_merged: 'q4',
+} as const;
+
+export const getWhisperWebGpuDtype = (model: WhisperWebGpuModel) => {
+ return model === 'large-v3-turbo'
+ ? LARGE_V3_TURBO_WHISPER_WEBGPU_DTYPE
+ : DEFAULT_WHISPER_WEBGPU_DTYPE;
+};
diff --git a/packages/whisper-webgpu/src/remove-whisper-model.ts b/packages/whisper-webgpu/src/remove-whisper-model.ts
index e5e05dc0ce..49e563ebba 100644
--- a/packages/whisper-webgpu/src/remove-whisper-model.ts
+++ b/packages/whisper-webgpu/src/remove-whisper-model.ts
@@ -1,7 +1,7 @@
import {disposeWhisperModel} from './load-whisper-model';
import {
getHostedModelId,
- WHISPER_WEBGPU_DTYPE,
+ getWhisperWebGpuDtype,
type WhisperWebGpuModel,
} from './models';
import {withRemotionModelHost} from './with-remotion-model-host';
@@ -20,7 +20,7 @@ export const removeWhisperModel = async ({
getHostedModelId(model),
{
device: 'webgpu',
- dtype: WHISPER_WEBGPU_DTYPE,
+ dtype: getWhisperWebGpuDtype(model),
},
);
});
diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.ts
index 5aed3ca4b8..d81f198826 100644
--- a/packages/whisper-webgpu/src/test/transcribe.test.ts
+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts
@@ -332,6 +332,7 @@ test('transcribes with word timestamps using WebGPU', async () => {
{
multilingual: true,
supportsTranslation: false,
+ webGpuDownloadSize: 1_608_611_679,
},
);
expect(await isWhisperModelCached({model: 'small.en'})).toBe(true);
@@ -434,6 +435,10 @@ test('loads every public model from its explicit immutable hosted path', async (
'onnx-community/whisper-medium.en_timestamped',
'onnx-community/whisper-large-v3-turbo_timestamped',
]);
+ expect(pipelineInitialization?.options).toMatchObject({
+ device: 'webgpu',
+ dtype: {encoder_model: 'fp16', decoder_model_merged: 'q4'},
+ });
expect(transformersEnvironment).toEqual(originalTransformersEnvironment);
});
@@ -677,18 +682,18 @@ test('shares concurrent initialization and keeps the model host scoped until eve
test('removes a downloaded model from memory and the persistent cache', async () => {
const {loadWhisperModel, removeWhisperModel} = await import('../index');
const disposalsBeforeRemoval = disposeCalls;
- await loadWhisperModel({model: 'tiny.en'});
- await removeWhisperModel({model: 'tiny.en'});
+ await loadWhisperModel({model: 'large-v3-turbo'});
+ await removeWhisperModel({model: 'large-v3-turbo'});
expect(disposeCalls).toBe(disposalsBeforeRemoval + 1);
expect(cacheClear).toEqual({
task: 'automatic-speech-recognition',
- modelId: 'whisper-tiny.en_timestamped-v1',
+ modelId: 'whisper-large-v3-turbo_timestamped-v1',
options: {
device: 'webgpu',
dtype: {
decoder_model_merged: 'q4',
- encoder_model: 'fp32',
+ encoder_model: 'fp16',
},
},
});