🎥 Make videos programmatically with React
Address Whisper WebGPU review feedback
Jonny Burgera590e6ee8095
diff --git a/packages/docs/docs/whisper-web/index.mdx b/packages/docs/docs/whisper-web/index.mdxindex 6b7487ea25..27e0e24c0a 100644--- a/packages/docs/docs/whisper-web/index.mdx+++ b/packages/docs/docs/whisper-web/index.mdx@@ -8,7 +8,7 @@ import Tabs from '@theme/Tabs';import TabItem from '@theme/TabItem';:::warning-**Unstable API**: This package is experimental for the moment. As we test it, we might make a few changes to the API and switch to a WebGPU-based backend in the future.+Unstable API: This package is experimental for the moment. It remains supported and is not deprecated. For new browser transcription integrations, prefer [`@remotion/whisper-webgpu`](/docs/whisper-webgpu), which uses WebGPU when available and does not require cross-origin isolation.:::Similar to [@remotion/install-whisper-cpp](/docs/install-whisper-cpp) but for the browser. Allows you to transcribe audio locally in the browser, with the help of WASM.diff --git a/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdxindex 10d4caa7c3..ffe394d1e8 100644--- a/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx+++ b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx@@ -6,7 +6,7 @@ crumb: '@remotion/whisper-webgpu'# canUseWhisperWebGpu()<AvailableFrom v="4.0.517" />-Checks whether WebGPU or the WASM fallback can run in the current browser.+Asynchronously checks whether a usable WebGPU adapter or the WASM fallback can run in the current browser.```ts twoslash title="support.ts"import {canUseWhisperWebGpu} from '@remotion/whisper-webgpu';@@ -23,9 +23,9 @@ console.log(result.backend, result.wasmThreads);### backend?-`'auto'`, `'webgpu'`, or `'wasm'`. Default: `'auto'`, which selects WebGPU when available and otherwise uses WASM.+`'auto'`, `'webgpu'`, or `'wasm'`. Default: `'auto'`, which requests a WebGPU adapter and uses WASM when no adapter is available.-Requesting `'webgpu'` reports an unsupported result when WebGPU is unavailable or the page is not a secure context.+Requesting `'webgpu'` reports an unsupported result when the WebGPU API or a usable adapter is unavailable, or when the page is not a secure context.## Cross-origin isolationdiff --git a/packages/docs/docs/whisper-webgpu/index.mdx b/packages/docs/docs/whisper-webgpu/index.mdxindex 6828e09f08..965540635b 100644--- a/packages/docs/docs/whisper-webgpu/index.mdx+++ b/packages/docs/docs/whisper-webgpu/index.mdx@@ -10,9 +10,9 @@ crumb: 'Transcribe audio in browser'<p>This package is experimental.</p></ExperimentalBadge>-Transcribe audio locally in the browser using timestamped Whisper models from Transformers.js. WebGPU is used when available and single-threaded WASM is the fallback.+Transcribe audio locally in the browser using timestamped Whisper models from Transformers.js. WebGPU is used when a usable adapter is available and single-threaded WASM is the fallback.-Unlike [`@remotion/whisper-web`](/docs/whisper-web), cross-origin isolation is not required.+For new browser transcription integrations, prefer `@remotion/whisper-webgpu`. [`@remotion/whisper-web`](/docs/whisper-web) remains supported and is not deprecated, but requires cross-origin isolation and does not use WebGPU.import {TableOfContents} from './table-of-contents';diff --git a/packages/example/src/WhisperWebGpu/index.tsx b/packages/example/src/WhisperWebGpu/index.tsxindex e03fa3cb71..3a47fc774a 100644--- a/packages/example/src/WhisperWebGpu/index.tsx+++ b/packages/example/src/WhisperWebGpu/index.tsx@@ -30,7 +30,7 @@ export const WhisperWebGpu = () => {setResult('');try {- const support = canUseWhisperWebGpu({backend: 'auto'});+ const support = await canUseWhisperWebGpu({backend: 'auto'});if (!support.supported) {throw new Error(support.detailedReason);}diff --git a/packages/whisper-webgpu/src/backend.ts b/packages/whisper-webgpu/src/backend.tsindex 0e710405ee..a2e54ce758 100644--- a/packages/whisper-webgpu/src/backend.ts+++ b/packages/whisper-webgpu/src/backend.ts@@ -4,24 +4,41 @@ export type ResolvedWhisperWebGpuBackend = Exclude<'auto'>;-const hasWebGpu = () => {- return (- typeof navigator !== 'undefined' &&- 'gpu' in navigator &&- globalThis.isSecureContext !== false- );+const requestWebGpuAdapter = async (): Promise<unknown | null> => {+ if (+ typeof navigator === 'undefined' ||+ !('gpu' in navigator) ||+ globalThis.isSecureContext === false+ ) {+ return null;+ }++ const {gpu} = navigator as unknown as {+ gpu: {requestAdapter: () => Promise<unknown | null>};+ };++ try {+ return await gpu.requestAdapter();+ } catch {+ return null;+ }};-export const resolveBackend = (+export const resolveBackend = async (backend: WhisperWebGpuBackend,-): ResolvedWhisperWebGpuBackend => {- if (backend === 'webgpu' && !hasWebGpu()) {- throw new Error('WebGPU is not available in this browser.');+): Promise<ResolvedWhisperWebGpuBackend> => {+ if (backend === 'wasm') {+ return 'wasm';+ }++ const adapter = await requestWebGpuAdapter();+ if (adapter) {+ return 'webgpu';}- if (backend === 'auto') {- return hasWebGpu() ? 'webgpu' : 'wasm';+ if (backend === 'webgpu') {+ throw new Error('No usable WebGPU adapter is available in this browser.');}- return backend;+ return 'wasm';};diff --git a/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts b/packages/whisper-webgpu/src/can-use-whisper-webgpu.tsindex 2357ba9ec4..9b055927c5 100644--- a/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts+++ b/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts@@ -27,9 +27,9 @@ export type CanUseWhisperWebGpuOptions = {backend?: WhisperWebGpuBackend;};-export const canUseWhisperWebGpu = ({+export const canUseWhisperWebGpu = async ({backend = 'auto',-}: CanUseWhisperWebGpuOptions = {}): CanUseWhisperWebGpuResult => {+}: CanUseWhisperWebGpuOptions = {}): Promise<CanUseWhisperWebGpuResult> => {if (typeof window === 'undefined') {return {supported: false,@@ -39,7 +39,10 @@ export const canUseWhisperWebGpu = ({};}- if (backend === 'webgpu' && !('gpu' in navigator)) {+ if (+ backend === 'webgpu' &&+ (typeof navigator === 'undefined' || !('gpu' in navigator))+ ) {return {supported: false,reason: WhisperWebGpuUnsupportedReason.WebGpuUnavailable,@@ -56,7 +59,17 @@ export const canUseWhisperWebGpu = ({};}- const resolvedBackend = resolveBackend(backend);+ let resolvedBackend: ResolvedWhisperWebGpuBackend;+ try {+ resolvedBackend = await resolveBackend(backend);+ } catch {+ return {+ supported: false,+ reason: WhisperWebGpuUnsupportedReason.WebGpuUnavailable,+ detailedReason: 'No usable WebGPU adapter is available in this browser.',+ };+ }+if (resolvedBackend === 'wasm' && typeof WebAssembly === 'undefined') {return {supported: false,diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.tsindex c8c85b751e..6e5d211a5c 100644--- a/packages/whisper-webgpu/src/load-whisper-model.ts+++ b/packages/whisper-webgpu/src/load-whisper-model.ts@@ -66,7 +66,7 @@ export const loadWhisperModel = async ({backend = 'auto',onProgress,}: LoadWhisperModelOptions): Promise<LoadWhisperModelResult> => {- const resolvedBackend = resolveBackend(backend);+ const resolvedBackend = await resolveBackend(backend);const cacheKey = getCacheKey(model, resolvedBackend);const existing = pipelines.get(cacheKey);if (existing) {diff --git a/packages/whisper-webgpu/src/resample-to-16khz.ts b/packages/whisper-webgpu/src/resample-to-16khz.tsindex c41d74f109..3fb50d0c6f 100644--- a/packages/whisper-webgpu/src/resample-to-16khz.ts+++ b/packages/whisper-webgpu/src/resample-to-16khz.ts@@ -9,10 +9,6 @@ export const resampleTo16Khz = async ({file,onProgress,}: ResampleTo16KhzOptions): Promise<Float32Array> => {- if (typeof AudioContext === 'undefined') {- throw new Error('AudioContext is not available in this environment.');- }-if (typeof OfflineAudioContext === 'undefined') {throw new Error('OfflineAudioContext is not available in this environment.',@@ -27,30 +23,23 @@ export const resampleTo16Khz = async ({const arrayBuffer = await file.arrayBuffer();onProgress?.(0.25);- const audioContext = new AudioContext({- sampleRate: WHISPER_WEBGPU_SAMPLE_RATE,- });-- try {- const decoded = await audioContext.decodeAudioData(arrayBuffer);- onProgress?.(0.6);- const length = Math.max(- 1,- Math.ceil(decoded.duration * WHISPER_WEBGPU_SAMPLE_RATE),- );- const offlineContext = new OfflineAudioContext(- 1,- length,- WHISPER_WEBGPU_SAMPLE_RATE,- );- const source = offlineContext.createBufferSource();- source.buffer = decoded;- source.connect(offlineContext.destination);- source.start();- const rendered = await offlineContext.startRendering();- onProgress?.(1);- return rendered.getChannelData(0).slice();- } finally {- await audioContext.close();- }+ const decodingContext = new OfflineAudioContext(1, 1, 44_100);+ const decoded = await decodingContext.decodeAudioData(arrayBuffer);+ onProgress?.(0.6);+ const length = Math.max(+ 1,+ Math.ceil(decoded.duration * WHISPER_WEBGPU_SAMPLE_RATE),+ );+ const offlineContext = new OfflineAudioContext(+ 1,+ length,+ WHISPER_WEBGPU_SAMPLE_RATE,+ );+ const source = offlineContext.createBufferSource();+ source.buffer = decoded;+ source.connect(offlineContext.destination);+ source.start();+ const rendered = await offlineContext.startRendering();+ onProgress?.(1);+ return rendered.getChannelData(0).slice();};diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.tsindex 1171b0fd60..2ac57877db 100644--- a/packages/whisper-webgpu/src/test/transcribe.test.ts+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts@@ -45,9 +45,16 @@ mock.module('@huggingface/transformers', () => ({},}));-test('transcribes with the selected timestamped model and returns Remotion captions', async () => {+test('transcribes with timestamps and selects a usable backend', async () => {const api = await import('../index');- const {disposeWhisperModel, getAvailableModels, toCaptions, transcribe} = api;+ const {+ canUseWhisperWebGpu,+ disposeWhisperModel,+ getAvailableModels,+ toCaptions,+ transcribe,+ WhisperWebGpuUnsupportedReason,+ } = api;const channelWaveform = new Float32Array(16_000 * 3);const result = await transcribe({backend: 'wasm',@@ -117,6 +124,82 @@ test('transcribes with the selected timestamped model and returns Remotion captiwebGpuDownloadSize: 586_209_938,});- await disposeWhisperModel({model: 'small.en', backend: 'wasm'});+ const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');+ const originalNavigator = Object.getOwnPropertyDescriptor(+ globalThis,+ 'navigator',+ );+ let adapter: object | null = null;+ Object.defineProperty(globalThis, 'window', {+ configurable: true,+ value: {+ crossOriginIsolated: false,+ isSecureContext: true,+ },+ });+ Object.defineProperty(globalThis, 'navigator', {+ configurable: true,+ value: {+ gpu: {+ requestAdapter: () => Promise.resolve(adapter),+ },+ hardwareConcurrency: 8,+ },+ });++ try {+ expect(await canUseWhisperWebGpu()).toEqual({+ supported: true,+ backend: 'wasm',+ wasmThreads: 1,+ });+ expect(await canUseWhisperWebGpu({backend: 'webgpu'})).toEqual({+ supported: false,+ reason: WhisperWebGpuUnsupportedReason.WebGpuUnavailable,+ detailedReason: 'No usable WebGPU adapter is available in this browser.',+ });++ const fallbackResult = await transcribe({+ backend: 'auto',+ channelWaveform,+ model: 'base.en',+ });+ expect(fallbackResult.backend).toBe('wasm');+ expect(pipelineInitialization?.options).toMatchObject({+ device: 'wasm',+ dtype: 'q8',+ });++ adapter = {};+ expect(await canUseWhisperWebGpu()).toEqual({+ supported: true,+ backend: 'webgpu',+ wasmThreads: null,+ });+ const webGpuResult = await transcribe({+ backend: 'auto',+ channelWaveform,+ model: 'tiny.en',+ });+ expect(webGpuResult.backend).toBe('webgpu');+ expect(pipelineInitialization?.options).toMatchObject({+ device: 'webgpu',+ dtype: {encoder_model: 'fp32', decoder_model_merged: 'q4'},+ });+ } finally {+ if (originalWindow) {+ Object.defineProperty(globalThis, 'window', originalWindow);+ } else {+ delete (globalThis as {window?: unknown}).window;+ }++ if (originalNavigator) {+ Object.defineProperty(globalThis, 'navigator', originalNavigator);+ } else {+ delete (globalThis as {navigator?: unknown}).navigator;+ }+ }++ await disposeWhisperModel();expect(disposed).toBe(true);});