Skip to content

🎥 Make videos programmatically with React

`@remotion/whisper-webgpu`: Fix model lifecycle and validation

Jonny Burger87a7f2b8bf60
diff --git a/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx
index c65bc6cf42..9becd0555f 100644
--- a/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx
+++ b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx
@@ -8,6 +8,8 @@ crumb: '@remotion/whisper-webgpu'
Releases the memory used by initialized Whisper pipelines. Persistent browser-cached model files are not deleted.
+If a model is loading or transcribing, disposal waits until initialization and active transcriptions have completed.
+
```ts twoslash title="dispose.ts"
import {disposeWhisperModel} from '@remotion/whisper-webgpu';
diff --git a/packages/docs/docs/whisper-webgpu/transcribe.mdx b/packages/docs/docs/whisper-webgpu/transcribe.mdx
index 1a93b7d1a6..e0f5d43b17 100644
--- a/packages/docs/docs/whisper-webgpu/transcribe.mdx
+++ b/packages/docs/docs/whisper-webgpu/transcribe.mdx
@@ -17,7 +17,6 @@ const channelWaveform = await resampleTo16Khz({file});
const result = await transcribe({
channelWaveform,
model: 'small.en',
- language: 'en',
});
console.log(result.text, result.words);
@@ -35,15 +34,15 @@ One of the timestamped models returned by [`getAvailableModels()`](/docs/whisper
### language?
-The spoken language name or language code. Default: `'auto'`. Set it when known to reduce language-detection errors.
+The spoken language name or language code. This option is required for multilingual models because automatic language detection is not supported. Omit it for English-only models such as `small.en`.
### chunkLengthInSeconds?
-Length of long-audio chunks. Default: `30`.
+Length of long-audio chunks. Default: `30`. Must be a finite number greater than `0`.
### strideLengthInSeconds?
-Overlap on both sides of a chunk. Default: `5`. It must be less than half of `chunkLengthInSeconds`.
+Overlap on both sides of a chunk. Default: `5`. Must be finite, non-negative, and less than half of `chunkLengthInSeconds`.
### onModelLoadProgress?
diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.ts
index 520ba80964..1c3861e271 100644
--- a/packages/whisper-webgpu/src/load-whisper-model.ts
+++ b/packages/whisper-webgpu/src/load-whisper-model.ts
@@ -21,7 +21,13 @@ type LoadedWhisperPipeline = {
dispose: () => Promise<void>;
};
-const pipelines = new Map<WhisperWebGpuModel, Promise<LoadedWhisperPipeline>>();
+type LoadedWhisperPipelineState = {
+ loading: Promise<LoadedWhisperPipeline>;
+ activeTranscriptions: number;
+ onIdle: Array<() => void>;
+};
+
+const pipelines = new Map<WhisperWebGpuModel, LoadedWhisperPipelineState>();
export type LoadWhisperModelOptions = {
model: WhisperWebGpuModel;
@@ -32,113 +38,155 @@ export type LoadWhisperModelResult = {
alreadyLoaded: boolean;
};
-export const loadWhisperModel = async ({
+const getOrCreateWhisperPipeline = ({
model,
onProgress,
-}: LoadWhisperModelOptions): Promise<LoadWhisperModelResult> => {
+}: LoadWhisperModelOptions): {
+ state: LoadedWhisperPipelineState;
+ alreadyLoaded: boolean;
+ totalBytes: number;
+} => {
const modelInfo = getModelInfo(model);
const totalBytes = modelInfo.webGpuDownloadSize;
const existing = pipelines.get(model);
if (existing) {
- await existing;
+ return {state: existing, alreadyLoaded: true, totalBytes};
+ }
+
+ const loading = Promise.resolve().then(async () => {
+ const {pipeline} = await import('@huggingface/transformers');
+
onProgress?.({
- status: 'ready',
+ status: 'loading',
file: null,
- progress: 1,
- loadedBytes: totalBytes,
+ progress: 0,
+ loadedBytes: 0,
totalBytes,
});
- return {alreadyLoaded: true};
- }
-
- const {pipeline} = await import('@huggingface/transformers');
- onProgress?.({
- status: 'loading',
- file: null,
- progress: 0,
- loadedBytes: 0,
- totalBytes,
- });
-
- const {modelId} = modelInfo;
- const loadedByFile = new Map<string, number>();
- let lastProgress = 0;
- let lastLoadedBytes = 0;
- const loading = pipeline('automatic-speech-recognition', modelId, {
- device: 'webgpu',
- dtype: WHISPER_WEBGPU_DTYPE,
- progress_callback: onProgress
- ? (event) => {
- const record = event as Record<string, unknown>;
- if (
- record.status === 'progress' &&
- typeof record.file === 'string' &&
- typeof record.loaded === 'number' &&
- Number.isFinite(record.loaded)
- ) {
- loadedByFile.set(
- record.file,
- Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),
- );
- const loadedBytes = [...loadedByFile.values()].reduce(
- (sum, loaded) => sum + loaded,
- 0,
- );
- lastProgress = Math.max(
- lastProgress,
- Math.min(loadedBytes / totalBytes, 0.99),
- );
- lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);
- onProgress({
- status: 'loading',
- file: null,
- progress: lastProgress,
- loadedBytes: lastLoadedBytes,
- totalBytes,
- });
+ const {modelId} = modelInfo;
+ const loadedByFile = new Map<string, number>();
+ let lastProgress = 0;
+ let lastLoadedBytes = 0;
+ return pipeline('automatic-speech-recognition', modelId, {
+ device: 'webgpu',
+ dtype: WHISPER_WEBGPU_DTYPE,
+ progress_callback: onProgress
+ ? (event) => {
+ const record = event as Record<string, unknown>;
+ if (
+ record.status === 'progress' &&
+ typeof record.file === 'string' &&
+ typeof record.loaded === 'number' &&
+ Number.isFinite(record.loaded)
+ ) {
+ loadedByFile.set(
+ record.file,
+ Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),
+ );
+ const loadedBytes = [...loadedByFile.values()].reduce(
+ (sum, loaded) => sum + loaded,
+ 0,
+ );
+ lastProgress = Math.max(
+ lastProgress,
+ Math.min(loadedBytes / totalBytes, 0.99),
+ );
+ lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);
+ onProgress({
+ status: 'loading',
+ file: null,
+ progress: lastProgress,
+ loadedBytes: lastLoadedBytes,
+ totalBytes,
+ });
+ }
+
+ if (record.status === 'ready') {
+ onProgress({
+ status: 'ready',
+ file: null,
+ progress: 1,
+ loadedBytes: Math.max(lastLoadedBytes, totalBytes),
+ totalBytes,
+ });
+ }
}
+ : undefined,
+ }) as Promise<LoadedWhisperPipeline>;
+ });
+ const state: LoadedWhisperPipelineState = {
+ loading,
+ activeTranscriptions: 0,
+ onIdle: [],
+ };
+ pipelines.set(model, state);
+ loading.catch(() => {
+ if (pipelines.get(model) === state) {
+ pipelines.delete(model);
+ }
+ });
- if (record.status === 'ready') {
- onProgress({
- status: 'ready',
- file: null,
- progress: 1,
- loadedBytes: Math.max(lastLoadedBytes, totalBytes),
- totalBytes,
- });
- }
- }
- : undefined,
- }) as Promise<LoadedWhisperPipeline>;
+ return {state, alreadyLoaded: false, totalBytes};
+};
- pipelines.set(model, loading);
- try {
- await loading;
- return {alreadyLoaded: false};
- } catch (error) {
- pipelines.delete(model);
- throw error;
+export const loadWhisperModel = async ({
+ model,
+ onProgress,
+}: LoadWhisperModelOptions): Promise<LoadWhisperModelResult> => {
+ const {state, alreadyLoaded, totalBytes} = getOrCreateWhisperPipeline({
+ model,
+ onProgress,
+ });
+ await state.loading;
+ if (alreadyLoaded) {
+ onProgress?.({
+ status: 'ready',
+ file: null,
+ progress: 1,
+ loadedBytes: totalBytes,
+ totalBytes,
+ });
}
+
+ return {alreadyLoaded};
};
-export const getLoadedWhisperPipeline = async ({
+export const withLoadedWhisperPipeline = async <ReturnValue>({
model,
onProgress,
+ run,
}: {
model: WhisperWebGpuModel;
onProgress?: OnWhisperWebGpuModelLoadProgress;
-}) => {
- await loadWhisperModel({
+ run: (pipeline: LoadedWhisperPipeline) => Promise<ReturnValue>;
+}): Promise<ReturnValue> => {
+ const {state, alreadyLoaded, totalBytes} = getOrCreateWhisperPipeline({
model,
onProgress,
});
- const loaded = pipelines.get(model);
- if (!loaded) {
- throw new Error(`Whisper model ${model} was not loaded.`);
+ state.activeTranscriptions++;
+ try {
+ const loaded = await state.loading;
+ if (alreadyLoaded) {
+ onProgress?.({
+ status: 'ready',
+ file: null,
+ progress: 1,
+ loadedBytes: totalBytes,
+ totalBytes,
+ });
+ }
+
+ return await run(loaded);
+ } finally {
+ state.activeTranscriptions--;
+ if (state.activeTranscriptions === 0) {
+ for (const resolve of state.onIdle.splice(0)) {
+ resolve();
+ }
+ }
}
-
- return loaded;
};
export type DisposeWhisperModelOptions = {
@@ -151,11 +199,21 @@ export const disposeWhisperModel = async ({
const matching = [...pipelines.entries()].filter(([loadedModel]) => {
return model === undefined || loadedModel === model;
});
+ for (const [key, state] of matching) {
+ if (pipelines.get(key) === state) {
+ pipelines.delete(key);
+ }
+ }
await Promise.all(
- matching.map(async ([key, loaded]) => {
- pipelines.delete(key);
- const loadedPipeline = await loaded;
+ matching.map(async ([, state]) => {
+ const loadedPipeline = await state.loading;
+ if (state.activeTranscriptions > 0) {
+ await new Promise<void>((resolve) => {
+ state.onIdle.push(resolve);
+ });
+ }
+
await loadedPipeline.dispose();
}),
);
diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.ts
index fedc7f8346..d0b04f34d0 100644
--- a/packages/whisper-webgpu/src/test/transcribe.test.ts
+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts
@@ -13,6 +13,12 @@ let transcriptionCall:
}
| undefined;
let disposed = false;
+let disposeCalls = 0;
+let pipelineInitializationCount = 0;
+let pipelineInitializationGate: Promise<void> | null = null;
+let onPipelineInitializationStarted: (() => void) | null = null;
+let transcriptionGate: Promise<void> | null = null;
+let onTranscriptionStarted: (() => void) | null = null;
let cacheCheck:
| {
task: string;
@@ -22,9 +28,11 @@ let cacheCheck:
| undefined;
const fakePipeline = Object.assign(
- (audio: Float32Array, options: Record<string, unknown>) => {
+ async (audio: Float32Array, options: Record<string, unknown>) => {
transcriptionCall = {audio, options};
- return Promise.resolve({
+ onTranscriptionStarted?.();
+ await transcriptionGate;
+ return {
text: ' Hello world free today.',
chunks: [
{text: ' Hello', timestamp: [0.25, 0.75]},
@@ -32,11 +40,12 @@ const fakePipeline = Object.assign(
{text: ' free', timestamp: [1.8, 1.7]},
{text: ' today.', timestamp: [2.2, null]},
],
- });
+ };
},
{
dispose: () => {
disposed = true;
+ disposeCalls++;
return Promise.resolve();
},
},
@@ -53,12 +62,15 @@ mock.module('@huggingface/transformers', () => ({
return Promise.resolve(true);
},
},
- pipeline: (
+ pipeline: async (
_task: string,
modelId: string,
options: Record<string, unknown>,
) => {
+ pipelineInitializationCount++;
pipelineInitialization = {modelId, options};
+ onPipelineInitializationStarted?.();
+ await pipelineInitializationGate;
const onProgress = options.progress_callback as
| ((progress: Record<string, unknown>) => void)
| undefined;
@@ -98,7 +110,7 @@ mock.module('@huggingface/transformers', () => ({
});
onProgress?.({status: 'done', file: 'encoder_model.onnx'});
onProgress?.({status: 'ready'});
- return Promise.resolve(fakePipeline);
+ return fakePipeline;
},
}));
@@ -130,11 +142,10 @@ test('transcribes with word timestamps using WebGPU', async () => {
dtype: {encoder_model: 'fp32', decoder_model_merged: 'q4'},
});
expect(transcriptionCall?.audio).toBe(channelWaveform);
- expect(transcriptionCall?.options).toMatchObject({
+ expect(transcriptionCall?.options).toEqual({
return_timestamps: 'word',
chunk_length_s: 30,
stride_length_s: 5,
- language: 'en',
});
expect(result).toEqual({
text: 'Hello world free today.',
@@ -301,3 +312,105 @@ test('reports model progress without reaching 100% before every file loads', asy
);
await disposeWhisperModel({model: 'medium.en'});
});
+
+test('validates language and finite chunk settings through transcribe()', async () => {
+ const {disposeWhisperModel, transcribe} = await import('../index');
+ const channelWaveform = new Float32Array(16_000);
+
+ await expect(
+ transcribe({channelWaveform, model: 'tiny', language: 'auto'}),
+ ).rejects.toThrow(
+ 'The language option is required for the multilingual model "tiny" because automatic language detection is not supported.',
+ );
+ await expect(
+ transcribe({channelWaveform, model: 'tiny.en', language: 'de'}),
+ ).rejects.toThrow(
+ 'The English-only model "tiny.en" does not support the language "de".',
+ );
+ await expect(
+ transcribe({
+ channelWaveform,
+ model: 'tiny.en',
+ chunkLengthInSeconds: Number.NaN,
+ }),
+ ).rejects.toThrow(
+ 'chunkLengthInSeconds must be a finite number greater than 0.',
+ );
+ await expect(
+ transcribe({
+ channelWaveform,
+ model: 'tiny.en',
+ strideLengthInSeconds: Number.NaN,
+ }),
+ ).rejects.toThrow(
+ 'strideLengthInSeconds must be a finite, non-negative number and less than half of chunkLengthInSeconds.',
+ );
+
+ await transcribe({channelWaveform, model: 'tiny', language: 'de'});
+ expect(transcriptionCall?.options).toEqual({
+ return_timestamps: 'word',
+ chunk_length_s: 30,
+ stride_length_s: 5,
+ language: 'de',
+ });
+ await disposeWhisperModel({model: 'tiny'});
+});
+
+test('shares concurrent initialization and waits for it before disposal', async () => {
+ const {disposeWhisperModel, loadWhisperModel} = await import('../index');
+ let releaseInitialization: () => void = () => {};
+ pipelineInitializationGate = new Promise<void>((resolve) => {
+ releaseInitialization = resolve;
+ });
+ const initializationStarted = new Promise<void>((resolve) => {
+ onPipelineInitializationStarted = resolve;
+ });
+ const initializationsBeforeLoading = pipelineInitializationCount;
+ const disposalsBeforeLoading = disposeCalls;
+
+ const firstLoad = loadWhisperModel({model: 'base'});
+ const secondLoad = loadWhisperModel({model: 'base'});
+ await initializationStarted;
+ expect(pipelineInitializationCount - initializationsBeforeLoading).toBe(1);
+ const disposal = disposeWhisperModel({model: 'base'});
+ await Promise.resolve();
+ expect(disposeCalls).toBe(disposalsBeforeLoading);
+
+ releaseInitialization();
+ await expect(Promise.all([firstLoad, secondLoad])).resolves.toEqual([
+ {alreadyLoaded: false},
+ {alreadyLoaded: true},
+ ]);
+ await disposal;
+ expect(disposeCalls).toBe(disposalsBeforeLoading + 1);
+ pipelineInitializationGate = null;
+ onPipelineInitializationStarted = null;
+});
+
+test('waits for active transcriptions before disposal', async () => {
+ const {disposeWhisperModel, transcribe} = await import('../index');
+ let releaseTranscription: () => void = () => {};
+ transcriptionGate = new Promise<void>((resolve) => {
+ releaseTranscription = resolve;
+ });
+ const transcriptionStarted = new Promise<void>((resolve) => {
+ onTranscriptionStarted = resolve;
+ });
+ const disposalsBeforeTranscription = disposeCalls;
+
+ const transcription = transcribe({
+ channelWaveform: new Float32Array(16_000),
+ model: 'base.en',
+ });
+ await transcriptionStarted;
+ const disposal = disposeWhisperModel({model: 'base.en'});
+ await Promise.resolve();
+ expect(disposeCalls).toBe(disposalsBeforeTranscription);
+
+ releaseTranscription();
+ await expect(transcription).resolves.toMatchObject({model: 'base.en'});
+ await disposal;
+ expect(disposeCalls).toBe(disposalsBeforeTranscription + 1);
+ transcriptionGate = null;
+ onTranscriptionStarted = null;
+});
diff --git a/packages/whisper-webgpu/src/transcribe.ts b/packages/whisper-webgpu/src/transcribe.ts
index d1a7123224..95c287df04 100644
--- a/packages/whisper-webgpu/src/transcribe.ts
+++ b/packages/whisper-webgpu/src/transcribe.ts
@@ -1,8 +1,8 @@
import {
- getLoadedWhisperPipeline,
+ withLoadedWhisperPipeline,
type OnWhisperWebGpuModelLoadProgress,
} from './load-whisper-model';
-import type {WhisperWebGpuModel} from './models';
+import {getModelInfo, type WhisperWebGpuModel} from './models';
export type WhisperWebGpuWord = {
text: string;
@@ -38,7 +38,7 @@ type TransformersJsTranscription = {
export const transcribe = async ({
channelWaveform,
model,
- language = 'auto',
+ language,
chunkLengthInSeconds = 30,
strideLengthInSeconds = 5,
onModelLoadProgress,
@@ -47,31 +47,54 @@ export const transcribe = async ({
throw new Error('The audio waveform is empty.');
}
- if (chunkLengthInSeconds <= 0) {
- throw new Error('chunkLengthInSeconds must be greater than 0.');
+ if (!Number.isFinite(chunkLengthInSeconds) || chunkLengthInSeconds <= 0) {
+ throw new Error(
+ 'chunkLengthInSeconds must be a finite number greater than 0.',
+ );
}
if (
+ !Number.isFinite(strideLengthInSeconds) ||
strideLengthInSeconds < 0 ||
strideLengthInSeconds * 2 >= chunkLengthInSeconds
) {
throw new Error(
- 'strideLengthInSeconds must be non-negative and less than half of chunkLengthInSeconds.',
+ 'strideLengthInSeconds must be a finite, non-negative number and less than half of chunkLengthInSeconds.',
);
}
- const transcriber = await getLoadedWhisperPipeline({
+ const {multilingual} = getModelInfo(model);
+ if (multilingual && (language === undefined || language === 'auto')) {
+ throw new Error(
+ `The language option is required for the multilingual model "${model}" because automatic language detection is not supported.`,
+ );
+ }
+
+ if (
+ !multilingual &&
+ language !== undefined &&
+ language !== 'auto' &&
+ language !== 'en' &&
+ language !== 'english'
+ ) {
+ throw new Error(
+ `The English-only model "${model}" does not support the language "${language}".`,
+ );
+ }
+
+ const output = await withLoadedWhisperPipeline({
model,
onProgress: onModelLoadProgress,
+ run: async (transcriber) => {
+ return (await transcriber(channelWaveform, {
+ return_timestamps: 'word',
+ chunk_length_s: chunkLengthInSeconds,
+ stride_length_s: strideLengthInSeconds,
+ ...(multilingual ? {language} : {}),
+ })) as TransformersJsTranscription;
+ },
});
- const output = (await transcriber(channelWaveform, {
- return_timestamps: 'word',
- chunk_length_s: chunkLengthInSeconds,
- stride_length_s: strideLengthInSeconds,
- ...(language === 'auto' ? {} : {language}),
- })) as TransformersJsTranscription;
-
if (!output.chunks) {
throw new Error(
'The model did not return word-level timestamps. Use one of the timestamped models returned by getAvailableModels().',