Skip to content

🎥 Make videos programmatically with React

Address Whisper WebGPU review feedback

Jonny Burgera590e6ee8095
diff --git a/packages/docs/docs/whisper-web/index.mdx b/packages/docs/docs/whisper-web/index.mdx
index 6b7487ea25..27e0e24c0a 100644
--- a/packages/docs/docs/whisper-web/index.mdx
+++ b/packages/docs/docs/whisper-web/index.mdx
@@ -8,7 +8,7 @@ import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
:::warning
-**Unstable API**: This package is experimental for the moment. As we test it, we might make a few changes to the API and switch to a WebGPU-based backend in the future.
+Unstable API: This package is experimental for the moment. It remains supported and is not deprecated. For new browser transcription integrations, prefer [`@remotion/whisper-webgpu`](/docs/whisper-webgpu), which uses WebGPU when available and does not require cross-origin isolation.
:::
Similar to [@remotion/install-whisper-cpp](/docs/install-whisper-cpp) but for the browser. Allows you to transcribe audio locally in the browser, with the help of WASM.
diff --git a/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx
index 10d4caa7c3..ffe394d1e8 100644
--- a/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx
+++ b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx
@@ -6,7 +6,7 @@ crumb: '@remotion/whisper-webgpu'
# canUseWhisperWebGpu()<AvailableFrom v="4.0.517" />
-Checks whether WebGPU or the WASM fallback can run in the current browser.
+Asynchronously checks whether a usable WebGPU adapter or the WASM fallback can run in the current browser.
```ts twoslash title="support.ts"
import {canUseWhisperWebGpu} from '@remotion/whisper-webgpu';
@@ -23,9 +23,9 @@ console.log(result.backend, result.wasmThreads);
### backend?
-`'auto'`, `'webgpu'`, or `'wasm'`. Default: `'auto'`, which selects WebGPU when available and otherwise uses WASM.
+`'auto'`, `'webgpu'`, or `'wasm'`. Default: `'auto'`, which requests a WebGPU adapter and uses WASM when no adapter is available.
-Requesting `'webgpu'` reports an unsupported result when WebGPU is unavailable or the page is not a secure context.
+Requesting `'webgpu'` reports an unsupported result when the WebGPU API or a usable adapter is unavailable, or when the page is not a secure context.
## Cross-origin isolation
diff --git a/packages/docs/docs/whisper-webgpu/index.mdx b/packages/docs/docs/whisper-webgpu/index.mdx
index 6828e09f08..965540635b 100644
--- a/packages/docs/docs/whisper-webgpu/index.mdx
+++ b/packages/docs/docs/whisper-webgpu/index.mdx
@@ -10,9 +10,9 @@ crumb: 'Transcribe audio in browser'
<p>This package is experimental.</p>
</ExperimentalBadge>
-Transcribe audio locally in the browser using timestamped Whisper models from Transformers.js. WebGPU is used when available and single-threaded WASM is the fallback.
+Transcribe audio locally in the browser using timestamped Whisper models from Transformers.js. WebGPU is used when a usable adapter is available and single-threaded WASM is the fallback.
-Unlike [`@remotion/whisper-web`](/docs/whisper-web), cross-origin isolation is not required.
+For new browser transcription integrations, prefer `@remotion/whisper-webgpu`. [`@remotion/whisper-web`](/docs/whisper-web) remains supported and is not deprecated, but requires cross-origin isolation and does not use WebGPU.
import {TableOfContents} from './table-of-contents';
diff --git a/packages/example/src/WhisperWebGpu/index.tsx b/packages/example/src/WhisperWebGpu/index.tsx
index e03fa3cb71..3a47fc774a 100644
--- a/packages/example/src/WhisperWebGpu/index.tsx
+++ b/packages/example/src/WhisperWebGpu/index.tsx
@@ -30,7 +30,7 @@ export const WhisperWebGpu = () => {
setResult('');
try {
- const support = canUseWhisperWebGpu({backend: 'auto'});
+ const support = await canUseWhisperWebGpu({backend: 'auto'});
if (!support.supported) {
throw new Error(support.detailedReason);
}
diff --git a/packages/whisper-webgpu/src/backend.ts b/packages/whisper-webgpu/src/backend.ts
index 0e710405ee..a2e54ce758 100644
--- a/packages/whisper-webgpu/src/backend.ts
+++ b/packages/whisper-webgpu/src/backend.ts
@@ -4,24 +4,41 @@ export type ResolvedWhisperWebGpuBackend = Exclude<
'auto'
>;
-const hasWebGpu = () => {
- return (
- typeof navigator !== 'undefined' &&
- 'gpu' in navigator &&
- globalThis.isSecureContext !== false
- );
+const requestWebGpuAdapter = async (): Promise<unknown | null> => {
+ if (
+ typeof navigator === 'undefined' ||
+ !('gpu' in navigator) ||
+ globalThis.isSecureContext === false
+ ) {
+ return null;
+ }
+
+ const {gpu} = navigator as unknown as {
+ gpu: {requestAdapter: () => Promise<unknown | null>};
+ };
+
+ try {
+ return await gpu.requestAdapter();
+ } catch {
+ return null;
+ }
};
-export const resolveBackend = (
+export const resolveBackend = async (
backend: WhisperWebGpuBackend,
-): ResolvedWhisperWebGpuBackend => {
- if (backend === 'webgpu' && !hasWebGpu()) {
- throw new Error('WebGPU is not available in this browser.');
+): Promise<ResolvedWhisperWebGpuBackend> => {
+ if (backend === 'wasm') {
+ return 'wasm';
+ }
+
+ const adapter = await requestWebGpuAdapter();
+ if (adapter) {
+ return 'webgpu';
}
- if (backend === 'auto') {
- return hasWebGpu() ? 'webgpu' : 'wasm';
+ if (backend === 'webgpu') {
+ throw new Error('No usable WebGPU adapter is available in this browser.');
}
- return backend;
+ return 'wasm';
};
diff --git a/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts b/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts
index 2357ba9ec4..9b055927c5 100644
--- a/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts
+++ b/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts
@@ -27,9 +27,9 @@ export type CanUseWhisperWebGpuOptions = {
backend?: WhisperWebGpuBackend;
};
-export const canUseWhisperWebGpu = ({
+export const canUseWhisperWebGpu = async ({
backend = 'auto',
-}: CanUseWhisperWebGpuOptions = {}): CanUseWhisperWebGpuResult => {
+}: CanUseWhisperWebGpuOptions = {}): Promise<CanUseWhisperWebGpuResult> => {
if (typeof window === 'undefined') {
return {
supported: false,
@@ -39,7 +39,10 @@ export const canUseWhisperWebGpu = ({
};
}
- if (backend === 'webgpu' && !('gpu' in navigator)) {
+ if (
+ backend === 'webgpu' &&
+ (typeof navigator === 'undefined' || !('gpu' in navigator))
+ ) {
return {
supported: false,
reason: WhisperWebGpuUnsupportedReason.WebGpuUnavailable,
@@ -56,7 +59,17 @@ export const canUseWhisperWebGpu = ({
};
}
- const resolvedBackend = resolveBackend(backend);
+ let resolvedBackend: ResolvedWhisperWebGpuBackend;
+ try {
+ resolvedBackend = await resolveBackend(backend);
+ } catch {
+ return {
+ supported: false,
+ reason: WhisperWebGpuUnsupportedReason.WebGpuUnavailable,
+ detailedReason: 'No usable WebGPU adapter is available in this browser.',
+ };
+ }
+
if (resolvedBackend === 'wasm' && typeof WebAssembly === 'undefined') {
return {
supported: false,
diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.ts
index c8c85b751e..6e5d211a5c 100644
--- a/packages/whisper-webgpu/src/load-whisper-model.ts
+++ b/packages/whisper-webgpu/src/load-whisper-model.ts
@@ -66,7 +66,7 @@ export const loadWhisperModel = async ({
backend = 'auto',
onProgress,
}: LoadWhisperModelOptions): Promise<LoadWhisperModelResult> => {
- const resolvedBackend = resolveBackend(backend);
+ const resolvedBackend = await resolveBackend(backend);
const cacheKey = getCacheKey(model, resolvedBackend);
const existing = pipelines.get(cacheKey);
if (existing) {
diff --git a/packages/whisper-webgpu/src/resample-to-16khz.ts b/packages/whisper-webgpu/src/resample-to-16khz.ts
index c41d74f109..3fb50d0c6f 100644
--- a/packages/whisper-webgpu/src/resample-to-16khz.ts
+++ b/packages/whisper-webgpu/src/resample-to-16khz.ts
@@ -9,10 +9,6 @@ export const resampleTo16Khz = async ({
file,
onProgress,
}: ResampleTo16KhzOptions): Promise<Float32Array> => {
- if (typeof AudioContext === 'undefined') {
- throw new Error('AudioContext is not available in this environment.');
- }
-
if (typeof OfflineAudioContext === 'undefined') {
throw new Error(
'OfflineAudioContext is not available in this environment.',
@@ -27,30 +23,23 @@ export const resampleTo16Khz = async ({
const arrayBuffer = await file.arrayBuffer();
onProgress?.(0.25);
- const audioContext = new AudioContext({
- sampleRate: WHISPER_WEBGPU_SAMPLE_RATE,
- });
-
- try {
- const decoded = await audioContext.decodeAudioData(arrayBuffer);
- onProgress?.(0.6);
- const length = Math.max(
- 1,
- Math.ceil(decoded.duration * WHISPER_WEBGPU_SAMPLE_RATE),
- );
- const offlineContext = new OfflineAudioContext(
- 1,
- length,
- WHISPER_WEBGPU_SAMPLE_RATE,
- );
- const source = offlineContext.createBufferSource();
- source.buffer = decoded;
- source.connect(offlineContext.destination);
- source.start();
- const rendered = await offlineContext.startRendering();
- onProgress?.(1);
- return rendered.getChannelData(0).slice();
- } finally {
- await audioContext.close();
- }
+ const decodingContext = new OfflineAudioContext(1, 1, 44_100);
+ const decoded = await decodingContext.decodeAudioData(arrayBuffer);
+ onProgress?.(0.6);
+ const length = Math.max(
+ 1,
+ Math.ceil(decoded.duration * WHISPER_WEBGPU_SAMPLE_RATE),
+ );
+ const offlineContext = new OfflineAudioContext(
+ 1,
+ length,
+ WHISPER_WEBGPU_SAMPLE_RATE,
+ );
+ const source = offlineContext.createBufferSource();
+ source.buffer = decoded;
+ source.connect(offlineContext.destination);
+ source.start();
+ const rendered = await offlineContext.startRendering();
+ onProgress?.(1);
+ return rendered.getChannelData(0).slice();
};
diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.ts
index 1171b0fd60..2ac57877db 100644
--- a/packages/whisper-webgpu/src/test/transcribe.test.ts
+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts
@@ -45,9 +45,16 @@ mock.module('@huggingface/transformers', () => ({
},
}));
-test('transcribes with the selected timestamped model and returns Remotion captions', async () => {
+test('transcribes with timestamps and selects a usable backend', async () => {
const api = await import('../index');
- const {disposeWhisperModel, getAvailableModels, toCaptions, transcribe} = api;
+ const {
+ canUseWhisperWebGpu,
+ disposeWhisperModel,
+ getAvailableModels,
+ toCaptions,
+ transcribe,
+ WhisperWebGpuUnsupportedReason,
+ } = api;
const channelWaveform = new Float32Array(16_000 * 3);
const result = await transcribe({
backend: 'wasm',
@@ -117,6 +124,82 @@ test('transcribes with the selected timestamped model and returns Remotion capti
webGpuDownloadSize: 586_209_938,
});
- await disposeWhisperModel({model: 'small.en', backend: 'wasm'});
+ const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');
+ const originalNavigator = Object.getOwnPropertyDescriptor(
+ globalThis,
+ 'navigator',
+ );
+ let adapter: object | null = null;
+ Object.defineProperty(globalThis, 'window', {
+ configurable: true,
+ value: {
+ crossOriginIsolated: false,
+ isSecureContext: true,
+ },
+ });
+ Object.defineProperty(globalThis, 'navigator', {
+ configurable: true,
+ value: {
+ gpu: {
+ requestAdapter: () => Promise.resolve(adapter),
+ },
+ hardwareConcurrency: 8,
+ },
+ });
+
+ try {
+ expect(await canUseWhisperWebGpu()).toEqual({
+ supported: true,
+ backend: 'wasm',
+ wasmThreads: 1,
+ });
+ expect(await canUseWhisperWebGpu({backend: 'webgpu'})).toEqual({
+ supported: false,
+ reason: WhisperWebGpuUnsupportedReason.WebGpuUnavailable,
+ detailedReason: 'No usable WebGPU adapter is available in this browser.',
+ });
+
+ const fallbackResult = await transcribe({
+ backend: 'auto',
+ channelWaveform,
+ model: 'base.en',
+ });
+ expect(fallbackResult.backend).toBe('wasm');
+ expect(pipelineInitialization?.options).toMatchObject({
+ device: 'wasm',
+ dtype: 'q8',
+ });
+
+ adapter = {};
+ expect(await canUseWhisperWebGpu()).toEqual({
+ supported: true,
+ backend: 'webgpu',
+ wasmThreads: null,
+ });
+ const webGpuResult = await transcribe({
+ backend: 'auto',
+ channelWaveform,
+ model: 'tiny.en',
+ });
+ expect(webGpuResult.backend).toBe('webgpu');
+ expect(pipelineInitialization?.options).toMatchObject({
+ device: 'webgpu',
+ dtype: {encoder_model: 'fp32', decoder_model_merged: 'q4'},
+ });
+ } finally {
+ if (originalWindow) {
+ Object.defineProperty(globalThis, 'window', originalWindow);
+ } else {
+ delete (globalThis as {window?: unknown}).window;
+ }
+
+ if (originalNavigator) {
+ Object.defineProperty(globalThis, 'navigator', originalNavigator);
+ } else {
+ delete (globalThis as {navigator?: unknown}).navigator;
+ }
+ }
+
+ await disposeWhisperModel();
expect(disposed).toBe(true);
});