🎥 Make videos programmatically with React
`@remotion/whisper-webgpu`: Fix model lifecycle and validation
Jonny Burger87a7f2b8bf60
diff --git a/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdxindex c65bc6cf42..9becd0555f 100644--- a/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx+++ b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx@@ -8,6 +8,8 @@ crumb: '@remotion/whisper-webgpu'Releases the memory used by initialized Whisper pipelines. Persistent browser-cached model files are not deleted.+If a model is loading or transcribing, disposal waits until initialization and active transcriptions have completed.+```ts twoslash title="dispose.ts"import {disposeWhisperModel} from '@remotion/whisper-webgpu';diff --git a/packages/docs/docs/whisper-webgpu/transcribe.mdx b/packages/docs/docs/whisper-webgpu/transcribe.mdxindex 1a93b7d1a6..e0f5d43b17 100644--- a/packages/docs/docs/whisper-webgpu/transcribe.mdx+++ b/packages/docs/docs/whisper-webgpu/transcribe.mdx@@ -17,7 +17,6 @@ const channelWaveform = await resampleTo16Khz({file});const result = await transcribe({channelWaveform,model: 'small.en',- language: 'en',});console.log(result.text, result.words);@@ -35,15 +34,15 @@ One of the timestamped models returned by [`getAvailableModels()`](/docs/whisper### language?-The spoken language name or language code. Default: `'auto'`. Set it when known to reduce language-detection errors.+The spoken language name or language code. This option is required for multilingual models because automatic language detection is not supported. Omit it for English-only models such as `small.en`.### chunkLengthInSeconds?-Length of long-audio chunks. Default: `30`.+Length of long-audio chunks. Default: `30`. Must be a finite number greater than `0`.### strideLengthInSeconds?-Overlap on both sides of a chunk. Default: `5`. It must be less than half of `chunkLengthInSeconds`.+Overlap on both sides of a chunk. Default: `5`. Must be finite, non-negative, and less than half of `chunkLengthInSeconds`.### onModelLoadProgress?diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.tsindex 520ba80964..1c3861e271 100644--- a/packages/whisper-webgpu/src/load-whisper-model.ts+++ b/packages/whisper-webgpu/src/load-whisper-model.ts@@ -21,7 +21,13 @@ type LoadedWhisperPipeline = {dispose: () => Promise<void>;};-const pipelines = new Map<WhisperWebGpuModel, Promise<LoadedWhisperPipeline>>();+type LoadedWhisperPipelineState = {+ loading: Promise<LoadedWhisperPipeline>;+ activeTranscriptions: number;+ onIdle: Array<() => void>;+};++const pipelines = new Map<WhisperWebGpuModel, LoadedWhisperPipelineState>();export type LoadWhisperModelOptions = {model: WhisperWebGpuModel;@@ -32,113 +38,155 @@ export type LoadWhisperModelResult = {alreadyLoaded: boolean;};-export const loadWhisperModel = async ({+const getOrCreateWhisperPipeline = ({model,onProgress,-}: LoadWhisperModelOptions): Promise<LoadWhisperModelResult> => {+}: LoadWhisperModelOptions): {+ state: LoadedWhisperPipelineState;+ alreadyLoaded: boolean;+ totalBytes: number;+} => {const modelInfo = getModelInfo(model);const totalBytes = modelInfo.webGpuDownloadSize;const existing = pipelines.get(model);if (existing) {- await existing;+ return {state: existing, alreadyLoaded: true, totalBytes};+ }++ const loading = Promise.resolve().then(async () => {+ const {pipeline} = await import('@huggingface/transformers');+onProgress?.({- status: 'ready',+ status: 'loading',file: null,- progress: 1,- loadedBytes: totalBytes,+ progress: 0,+ loadedBytes: 0,totalBytes,});- return {alreadyLoaded: true};- }-- const {pipeline} = await import('@huggingface/transformers');- onProgress?.({- status: 'loading',- file: null,- progress: 0,- loadedBytes: 0,- totalBytes,- });-- const {modelId} = modelInfo;- const loadedByFile = new Map<string, number>();- let lastProgress = 0;- let lastLoadedBytes = 0;- const loading = pipeline('automatic-speech-recognition', modelId, {- device: 'webgpu',- dtype: WHISPER_WEBGPU_DTYPE,- progress_callback: onProgress- ? (event) => {- const record = event as Record<string, unknown>;- if (- record.status === 'progress' &&- typeof record.file === 'string' &&- typeof record.loaded === 'number' &&- Number.isFinite(record.loaded)- ) {- loadedByFile.set(- record.file,- Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),- );- const loadedBytes = [...loadedByFile.values()].reduce(- (sum, loaded) => sum + loaded,- 0,- );- lastProgress = Math.max(- lastProgress,- Math.min(loadedBytes / totalBytes, 0.99),- );- lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);- onProgress({- status: 'loading',- file: null,- progress: lastProgress,- loadedBytes: lastLoadedBytes,- totalBytes,- });+ const {modelId} = modelInfo;+ const loadedByFile = new Map<string, number>();+ let lastProgress = 0;+ let lastLoadedBytes = 0;+ return pipeline('automatic-speech-recognition', modelId, {+ device: 'webgpu',+ dtype: WHISPER_WEBGPU_DTYPE,+ progress_callback: onProgress+ ? (event) => {+ const record = event as Record<string, unknown>;+ if (+ record.status === 'progress' &&+ typeof record.file === 'string' &&+ typeof record.loaded === 'number' &&+ Number.isFinite(record.loaded)+ ) {+ loadedByFile.set(+ record.file,+ Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),+ );+ const loadedBytes = [...loadedByFile.values()].reduce(+ (sum, loaded) => sum + loaded,+ 0,+ );+ lastProgress = Math.max(+ lastProgress,+ Math.min(loadedBytes / totalBytes, 0.99),+ );+ lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);+ onProgress({+ status: 'loading',+ file: null,+ progress: lastProgress,+ loadedBytes: lastLoadedBytes,+ totalBytes,+ });+ }++ if (record.status === 'ready') {+ onProgress({+ status: 'ready',+ file: null,+ progress: 1,+ loadedBytes: Math.max(lastLoadedBytes, totalBytes),+ totalBytes,+ });+ }}+ : undefined,+ }) as Promise<LoadedWhisperPipeline>;+ });+ const state: LoadedWhisperPipelineState = {+ loading,+ activeTranscriptions: 0,+ onIdle: [],+ };+ pipelines.set(model, state);+ loading.catch(() => {+ if (pipelines.get(model) === state) {+ pipelines.delete(model);+ }+ });- if (record.status === 'ready') {- onProgress({- status: 'ready',- file: null,- progress: 1,- loadedBytes: Math.max(lastLoadedBytes, totalBytes),- totalBytes,- });- }- }- : undefined,- }) as Promise<LoadedWhisperPipeline>;+ return {state, alreadyLoaded: false, totalBytes};+};- pipelines.set(model, loading);- try {- await loading;- return {alreadyLoaded: false};- } catch (error) {- pipelines.delete(model);- throw error;+export const loadWhisperModel = async ({+ model,+ onProgress,+}: LoadWhisperModelOptions): Promise<LoadWhisperModelResult> => {+ const {state, alreadyLoaded, totalBytes} = getOrCreateWhisperPipeline({+ model,+ onProgress,+ });+ await state.loading;+ if (alreadyLoaded) {+ onProgress?.({+ status: 'ready',+ file: null,+ progress: 1,+ loadedBytes: totalBytes,+ totalBytes,+ });}++ return {alreadyLoaded};};-export const getLoadedWhisperPipeline = async ({+export const withLoadedWhisperPipeline = async <ReturnValue>({model,onProgress,+ run,}: {model: WhisperWebGpuModel;onProgress?: OnWhisperWebGpuModelLoadProgress;-}) => {- await loadWhisperModel({+ run: (pipeline: LoadedWhisperPipeline) => Promise<ReturnValue>;+}): Promise<ReturnValue> => {+ const {state, alreadyLoaded, totalBytes} = getOrCreateWhisperPipeline({model,onProgress,});- const loaded = pipelines.get(model);- if (!loaded) {- throw new Error(`Whisper model ${model} was not loaded.`);+ state.activeTranscriptions++;+ try {+ const loaded = await state.loading;+ if (alreadyLoaded) {+ onProgress?.({+ status: 'ready',+ file: null,+ progress: 1,+ loadedBytes: totalBytes,+ totalBytes,+ });+ }++ return await run(loaded);+ } finally {+ state.activeTranscriptions--;+ if (state.activeTranscriptions === 0) {+ for (const resolve of state.onIdle.splice(0)) {+ resolve();+ }+ }}-- return loaded;};export type DisposeWhisperModelOptions = {@@ -151,11 +199,21 @@ export const disposeWhisperModel = async ({const matching = [...pipelines.entries()].filter(([loadedModel]) => {return model === undefined || loadedModel === model;});+ for (const [key, state] of matching) {+ if (pipelines.get(key) === state) {+ pipelines.delete(key);+ }+ }await Promise.all(- matching.map(async ([key, loaded]) => {- pipelines.delete(key);- const loadedPipeline = await loaded;+ matching.map(async ([, state]) => {+ const loadedPipeline = await state.loading;+ if (state.activeTranscriptions > 0) {+ await new Promise<void>((resolve) => {+ state.onIdle.push(resolve);+ });+ }+await loadedPipeline.dispose();}),);diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.tsindex fedc7f8346..d0b04f34d0 100644--- a/packages/whisper-webgpu/src/test/transcribe.test.ts+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts@@ -13,6 +13,12 @@ let transcriptionCall:}| undefined;let disposed = false;+let disposeCalls = 0;+let pipelineInitializationCount = 0;+let pipelineInitializationGate: Promise<void> | null = null;+let onPipelineInitializationStarted: (() => void) | null = null;+let transcriptionGate: Promise<void> | null = null;+let onTranscriptionStarted: (() => void) | null = null;let cacheCheck:| {task: string;@@ -22,9 +28,11 @@ let cacheCheck:| undefined;const fakePipeline = Object.assign(- (audio: Float32Array, options: Record<string, unknown>) => {+ async (audio: Float32Array, options: Record<string, unknown>) => {transcriptionCall = {audio, options};- return Promise.resolve({+ onTranscriptionStarted?.();+ await transcriptionGate;+ return {text: ' Hello world free today.',chunks: [{text: ' Hello', timestamp: [0.25, 0.75]},@@ -32,11 +40,12 @@ const fakePipeline = Object.assign({text: ' free', timestamp: [1.8, 1.7]},{text: ' today.', timestamp: [2.2, null]},],- });+ };},{dispose: () => {disposed = true;+ disposeCalls++;return Promise.resolve();},},@@ -53,12 +62,15 @@ mock.module('@huggingface/transformers', () => ({return Promise.resolve(true);},},- pipeline: (+ pipeline: async (_task: string,modelId: string,options: Record<string, unknown>,) => {+ pipelineInitializationCount++;pipelineInitialization = {modelId, options};+ onPipelineInitializationStarted?.();+ await pipelineInitializationGate;const onProgress = options.progress_callback as| ((progress: Record<string, unknown>) => void)| undefined;@@ -98,7 +110,7 @@ mock.module('@huggingface/transformers', () => ({});onProgress?.({status: 'done', file: 'encoder_model.onnx'});onProgress?.({status: 'ready'});- return Promise.resolve(fakePipeline);+ return fakePipeline;},}));@@ -130,11 +142,10 @@ test('transcribes with word timestamps using WebGPU', async () => {dtype: {encoder_model: 'fp32', decoder_model_merged: 'q4'},});expect(transcriptionCall?.audio).toBe(channelWaveform);- expect(transcriptionCall?.options).toMatchObject({+ expect(transcriptionCall?.options).toEqual({return_timestamps: 'word',chunk_length_s: 30,stride_length_s: 5,- language: 'en',});expect(result).toEqual({text: 'Hello world free today.',@@ -301,3 +312,105 @@ test('reports model progress without reaching 100% before every file loads', asy);await disposeWhisperModel({model: 'medium.en'});});++test('validates language and finite chunk settings through transcribe()', async () => {+ const {disposeWhisperModel, transcribe} = await import('../index');+ const channelWaveform = new Float32Array(16_000);++ await expect(+ transcribe({channelWaveform, model: 'tiny', language: 'auto'}),+ ).rejects.toThrow(+ 'The language option is required for the multilingual model "tiny" because automatic language detection is not supported.',+ );+ await expect(+ transcribe({channelWaveform, model: 'tiny.en', language: 'de'}),+ ).rejects.toThrow(+ 'The English-only model "tiny.en" does not support the language "de".',+ );+ await expect(+ transcribe({+ channelWaveform,+ model: 'tiny.en',+ chunkLengthInSeconds: Number.NaN,+ }),+ ).rejects.toThrow(+ 'chunkLengthInSeconds must be a finite number greater than 0.',+ );+ await expect(+ transcribe({+ channelWaveform,+ model: 'tiny.en',+ strideLengthInSeconds: Number.NaN,+ }),+ ).rejects.toThrow(+ 'strideLengthInSeconds must be a finite, non-negative number and less than half of chunkLengthInSeconds.',+ );++ await transcribe({channelWaveform, model: 'tiny', language: 'de'});+ expect(transcriptionCall?.options).toEqual({+ return_timestamps: 'word',+ chunk_length_s: 30,+ stride_length_s: 5,+ language: 'de',+ });+ await disposeWhisperModel({model: 'tiny'});+});++test('shares concurrent initialization and waits for it before disposal', async () => {+ const {disposeWhisperModel, loadWhisperModel} = await import('../index');+ let releaseInitialization: () => void = () => {};+ pipelineInitializationGate = new Promise<void>((resolve) => {+ releaseInitialization = resolve;+ });+ const initializationStarted = new Promise<void>((resolve) => {+ onPipelineInitializationStarted = resolve;+ });+ const initializationsBeforeLoading = pipelineInitializationCount;+ const disposalsBeforeLoading = disposeCalls;++ const firstLoad = loadWhisperModel({model: 'base'});+ const secondLoad = loadWhisperModel({model: 'base'});+ await initializationStarted;+ expect(pipelineInitializationCount - initializationsBeforeLoading).toBe(1);+ const disposal = disposeWhisperModel({model: 'base'});+ await Promise.resolve();+ expect(disposeCalls).toBe(disposalsBeforeLoading);++ releaseInitialization();+ await expect(Promise.all([firstLoad, secondLoad])).resolves.toEqual([+ {alreadyLoaded: false},+ {alreadyLoaded: true},+ ]);+ await disposal;+ expect(disposeCalls).toBe(disposalsBeforeLoading + 1);+ pipelineInitializationGate = null;+ onPipelineInitializationStarted = null;+});++test('waits for active transcriptions before disposal', async () => {+ const {disposeWhisperModel, transcribe} = await import('../index');+ let releaseTranscription: () => void = () => {};+ transcriptionGate = new Promise<void>((resolve) => {+ releaseTranscription = resolve;+ });+ const transcriptionStarted = new Promise<void>((resolve) => {+ onTranscriptionStarted = resolve;+ });+ const disposalsBeforeTranscription = disposeCalls;++ const transcription = transcribe({+ channelWaveform: new Float32Array(16_000),+ model: 'base.en',+ });+ await transcriptionStarted;+ const disposal = disposeWhisperModel({model: 'base.en'});+ await Promise.resolve();+ expect(disposeCalls).toBe(disposalsBeforeTranscription);++ releaseTranscription();+ await expect(transcription).resolves.toMatchObject({model: 'base.en'});+ await disposal;+ expect(disposeCalls).toBe(disposalsBeforeTranscription + 1);+ transcriptionGate = null;+ onTranscriptionStarted = null;+});diff --git a/packages/whisper-webgpu/src/transcribe.ts b/packages/whisper-webgpu/src/transcribe.tsindex d1a7123224..95c287df04 100644--- a/packages/whisper-webgpu/src/transcribe.ts+++ b/packages/whisper-webgpu/src/transcribe.ts@@ -1,8 +1,8 @@import {- getLoadedWhisperPipeline,+ withLoadedWhisperPipeline,type OnWhisperWebGpuModelLoadProgress,} from './load-whisper-model';-import type {WhisperWebGpuModel} from './models';+import {getModelInfo, type WhisperWebGpuModel} from './models';export type WhisperWebGpuWord = {text: string;@@ -38,7 +38,7 @@ type TransformersJsTranscription = {export const transcribe = async ({channelWaveform,model,- language = 'auto',+ language,chunkLengthInSeconds = 30,strideLengthInSeconds = 5,onModelLoadProgress,@@ -47,31 +47,54 @@ export const transcribe = async ({throw new Error('The audio waveform is empty.');}- if (chunkLengthInSeconds <= 0) {- throw new Error('chunkLengthInSeconds must be greater than 0.');+ if (!Number.isFinite(chunkLengthInSeconds) || chunkLengthInSeconds <= 0) {+ throw new Error(+ 'chunkLengthInSeconds must be a finite number greater than 0.',+ );}if (+ !Number.isFinite(strideLengthInSeconds) ||strideLengthInSeconds < 0 ||strideLengthInSeconds * 2 >= chunkLengthInSeconds) {throw new Error(- 'strideLengthInSeconds must be non-negative and less than half of chunkLengthInSeconds.',+ 'strideLengthInSeconds must be a finite, non-negative number and less than half of chunkLengthInSeconds.',);}- const transcriber = await getLoadedWhisperPipeline({+ const {multilingual} = getModelInfo(model);+ if (multilingual && (language === undefined || language === 'auto')) {+ throw new Error(+ `The language option is required for the multilingual model "${model}" because automatic language detection is not supported.`,+ );+ }++ if (+ !multilingual &&+ language !== undefined &&+ language !== 'auto' &&+ language !== 'en' &&+ language !== 'english'+ ) {+ throw new Error(+ `The English-only model "${model}" does not support the language "${language}".`,+ );+ }++ const output = await withLoadedWhisperPipeline({model,onProgress: onModelLoadProgress,+ run: async (transcriber) => {+ return (await transcriber(channelWaveform, {+ return_timestamps: 'word',+ chunk_length_s: chunkLengthInSeconds,+ stride_length_s: strideLengthInSeconds,+ ...(multilingual ? {language} : {}),+ })) as TransformersJsTranscription;+ },});- const output = (await transcriber(channelWaveform, {- return_timestamps: 'word',- chunk_length_s: chunkLengthInSeconds,- stride_length_s: strideLengthInSeconds,- ...(language === 'auto' ? {} : {language}),- })) as TransformersJsTranscription;-if (!output.chunks) {throw new Error('The model did not return word-level timestamps. Use one of the timestamped models returned by getAvailableModels().',