Skip to content

🎥 Make videos programmatically with React

Add timestamped WebGPU transcription

Jonny Burgerecd8d66c1fd2
diff --git a/bun.lock b/bun.lock
index 1a7525224b..6cbf17276c 100644
--- a/bun.lock
+++ b/bun.lock
@@ -582,6 +582,7 @@
"@remotion/web-renderer": "workspace:*",
"@remotion/webcodecs": "workspace:*",
"@remotion/whisper-web": "workspace:*",
+ "@remotion/whisper-webgpu": "workspace:*",
"@remotion/zod-types": "workspace:*",
"@remotion/zod-types-v3": "workspace:*",
"@rive-app/canvas-advanced": "2.31.5",
@@ -827,6 +828,7 @@
"@remotion/transitions": "workspace:*",
"@remotion/webcodecs": "workspace:*",
"@remotion/whisper-web": "workspace:*",
+ "@remotion/whisper-webgpu": "workspace:*",
"@remotion/zod-types": "workspace:*",
"@remotion/zod-types-v3": "workspace:*",
"@shopify/react-native-skia": "2.0.0",
@@ -2821,6 +2823,19 @@
"eslint": "catalog:",
},
},
+ "packages/whisper-webgpu": {
+ "name": "@remotion/whisper-webgpu",
+ "version": "4.0.516",
+ "dependencies": {
+ "@huggingface/transformers": "catalog:",
+ "@remotion/captions": "workspace:*",
+ },
+ "devDependencies": {
+ "@remotion/eslint-config-internal": "workspace:*",
+ "@typescript/native-preview": "catalog:",
+ "eslint": "catalog:",
+ },
+ },
"packages/zod-types": {
"name": "@remotion/zod-types",
"version": "4.0.516",
@@ -2873,6 +2888,7 @@
"@aws-sdk/lib-storage": "3.986.0",
"@aws-sdk/middleware-flexible-checksums": "3.972.5",
"@aws-sdk/s3-request-presigner": "3.986.0",
+ "@huggingface/transformers": "4.2.0",
"@mediabunny/aac-encoder": "1.55.1",
"@mediabunny/ac3": "1.55.1",
"@mediabunny/dts": "1.55.1",
@@ -3673,6 +3689,12 @@
"@hono/node-server": ["@hono/[email protected]", "", { "peerDependencies": { "hono": "^4" } }, "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw=="],
+ "@huggingface/jinja": ["@huggingface/[email protected]", "", {}, "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw=="],
+
+ "@huggingface/tokenizers": ["@huggingface/[email protected]", "", {}, "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA=="],
+
+ "@huggingface/transformers": ["@huggingface/[email protected]", "", { "dependencies": { "@huggingface/jinja": "^0.5.6", "@huggingface/tokenizers": "^0.1.3", "onnxruntime-node": "1.24.3", "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", "sharp": "^0.34.5" } }, "sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ=="],
+
"@humanfs/core": ["@humanfs/[email protected]", "", {}, "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA=="],
"@humanfs/node": ["@humanfs/[email protected]", "", { "dependencies": { "@humanfs/core": "0.19.1", "@humanwhocodes/retry": "0.3.1" } }, "sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw=="],
@@ -4561,6 +4583,8 @@
"@remotion/whisper-web": ["@remotion/whisper-web@workspace:packages/whisper-web"],
+ "@remotion/whisper-webgpu": ["@remotion/whisper-webgpu@workspace:packages/whisper-webgpu"],
+
"@remotion/zod-types": ["@remotion/zod-types@workspace:packages/zod-types"],
"@remotion/zod-types-v3": ["@remotion/zod-types-v3@workspace:packages/zod-types-v3"],
@@ -6499,6 +6523,8 @@
"flat-cache": ["[email protected]", "", { "dependencies": { "flatted": "3.3.1", "keyv": "4.5.4" } }, "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw=="],
+ "flatbuffers": ["[email protected]", "", {}, "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ=="],
+
"flatted": ["[email protected]", "", {}, "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw=="],
"flattie": ["[email protected]", "", {}, "sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ=="],
@@ -6655,6 +6681,8 @@
"gtoken": ["[email protected]", "", { "dependencies": { "gaxios": "5.1.0", "google-p12-pem": "4.0.1", "jws": "4.0.0" } }, "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ=="],
+ "guid-typescript": ["[email protected]", "", {}, "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ=="],
+
"gunzip-maybe": ["[email protected]", "", { "dependencies": { "browserify-zlib": "0.1.4", "is-deflate": "1.0.0", "is-gzip": "1.0.0", "peek-stream": "1.1.3", "pumpify": "1.5.1", "through2": "2.0.5" }, "bin": { "gunzip-maybe": "bin.js" } }, "sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw=="],
"gzip-size": ["[email protected]", "", { "dependencies": { "duplexer": "0.1.2" } }, "sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q=="],
@@ -7535,6 +7563,12 @@
"oniguruma-to-es": ["[email protected]", "", { "dependencies": { "oniguruma-parser": "^0.12.1", "regex": "^6.0.1", "regex-recursion": "^6.0.2" } }, "sha512-3VhUGN3w2eYxnTzHn+ikMI+fp/96KoRSVK9/kMTcFqj1NRDh2IhQCKvYxDnWePKRXY/AqH+Fuiyb7VHSzBjHfA=="],
+ "onnxruntime-common": ["[email protected]", "", {}, "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA=="],
+
+ "onnxruntime-node": ["[email protected]", "", { "dependencies": { "adm-zip": "^0.5.16", "global-agent": "^3.0.0", "onnxruntime-common": "1.24.3" }, "os": [ "linux", "win32", "darwin", ] }, "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg=="],
+
+ "onnxruntime-web": ["[email protected]", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw=="],
+
"open": ["[email protected]", "", { "dependencies": { "define-lazy-prop": "2.0.0", "is-docker": "2.2.1", "is-wsl": "2.2.0" } }, "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ=="],
"openai": ["[email protected]", "", { "dependencies": { "@types/node": "18.14.6", "@types/node-fetch": "2.6.11", "abort-controller": "3.0.0", "agentkeepalive": "4.5.0", "form-data-encoder": "1.7.2", "formdata-node": "4.4.1", "node-fetch": "2.6.9" }, "peerDependencies": { "zod": "3.22.3" }, "optionalPeers": ["zod"], "bin": { "openai": "bin/cli" } }, "sha512-2YbRFy6qaYRJabK2zLMn4txrB2xBy0KP5g/eoqeSPTT31mIJMnkT75toagvfE555IKa2RdrzJrZwdDsUipsAMw=="],
@@ -7689,6 +7723,8 @@
"pkg-types": ["[email protected]", "", { "dependencies": { "confbox": "^0.2.4", "exsolve": "^1.0.8", "pathe": "^2.0.3" } }, "sha512-y+ichcgc2LrADuhLNAx8DFjVfgz91pRxfZdI3UDhxHvcVEZsenLO+7XaU5vOp0u/7V/wZ+plyuQxtrDlZJ+yeg=="],
+ "platform": ["[email protected]", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="],
+
"playwright": ["[email protected]", "", { "dependencies": { "playwright-core": "1.55.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-cJW4Xd/G3v5ovXtJJ52MAOclqeac9S/aGGgRzLabuF8TnIb6xHvMzKIa6JmrRzUkeXJgfL1MhukP0NK6l39h3A=="],
"playwright-core": ["[email protected]", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-Z6Mh9mkwX+zxSlHqdr5AOcJnfp+xUWLCt9uKV18fhzA8eyxUd8NUWzAjxUh55RZKSYwDGX0cfaySdhZJGMoJ+w=="],
@@ -12059,6 +12095,10 @@
"ofetch/ufo": ["[email protected]", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="],
+ "onnxruntime-node/adm-zip": ["[email protected]", "", {}, "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng=="],
+
+ "onnxruntime-web/onnxruntime-common": ["[email protected]", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="],
+
"openai/@types/node": ["@types/[email protected]", "", {}, "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA=="],
"ora/chalk": ["[email protected]", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="],
diff --git a/package.json b/package.json
index 554dd5cac7..78b151194f 100644
--- a/package.json
+++ b/package.json
@@ -109,6 +109,7 @@
"!packages/bugs"
],
"catalog": {
+ "@huggingface/transformers": "4.2.0",
"@aws-sdk/s3-request-presigner": "3.986.0",
"@aws-sdk/credential-provider-ini": "3.972.5",
"@aws-sdk/client-s3": "3.986.0",
diff --git a/packages/cli/src/list-of-remotion-packages.ts b/packages/cli/src/list-of-remotion-packages.ts
index 9913a6e553..194410863b 100644
--- a/packages/cli/src/list-of-remotion-packages.ts
+++ b/packages/cli/src/list-of-remotion-packages.ts
@@ -92,6 +92,7 @@ export const listOfRemotionPackages = [
'@remotion/compositor',
'@remotion/example-videos',
'@remotion/whisper-web',
+ '@remotion/whisper-webgpu',
'@remotion/media',
'@remotion/remotion-media',
'@remotion/web-renderer',
diff --git a/packages/create-video/src/list-of-remotion-packages.ts b/packages/create-video/src/list-of-remotion-packages.ts
index 9913a6e553..194410863b 100644
--- a/packages/create-video/src/list-of-remotion-packages.ts
+++ b/packages/create-video/src/list-of-remotion-packages.ts
@@ -92,6 +92,7 @@ export const listOfRemotionPackages = [
'@remotion/compositor',
'@remotion/example-videos',
'@remotion/whisper-web',
+ '@remotion/whisper-webgpu',
'@remotion/media',
'@remotion/remotion-media',
'@remotion/web-renderer',
diff --git a/packages/docs/components/TableOfContents/api.tsx b/packages/docs/components/TableOfContents/api.tsx
index 4de30ed848..55d4892cb9 100644
--- a/packages/docs/components/TableOfContents/api.tsx
+++ b/packages/docs/components/TableOfContents/api.tsx
@@ -42,6 +42,7 @@ import {TableOfContents as ThreeTableOfContents} from '../../docs/three/TableOfC
import {TableOfContents as TransitionsTableOfContents} from '../../docs/transitions/table-of-contents';
import {TableOfContents as VercelTableOfContents} from '../../docs/vercel/table-of-contents';
import {TableOfContents as WebcodecsTableOfContents} from '../../docs/webcodecs/TableOfContents';
+import {TableOfContents as WhisperWebGpuTableOfContents} from '../../docs/whisper-webgpu/table-of-contents';
import {ZodTypesTableOfContents} from '../../docs/zod-types/TableOfContents';
import {Grid} from './Grid';
import {TOCItem} from './TOCItem';
@@ -198,6 +199,9 @@ export const TableOfContents: React.FC = () => {
<h2>@remotion/webcodecs</h2>
<p>Converting media using WebCodecs</p>
<WebcodecsTableOfContents />
+ <h2>@remotion/whisper-webgpu</h2>
+ <p>Transcribe audio in the browser using Whisper and WebGPU</p>
+ <WhisperWebGpuTableOfContents />
<h2>@remotion/zod-types</h2>
<p>Zod types enabling Remotion Studio UI</p>
<ZodTypesTableOfContents />
diff --git a/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx
new file mode 100644
index 0000000000..10d4caa7c3
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx
@@ -0,0 +1,40 @@
+---
+image: /generated/articles-docs-whisper-webgpu-can-use-whisper-webgpu.png
+title: canUseWhisperWebGpu()
+crumb: '@remotion/whisper-webgpu'
+---
+
+# canUseWhisperWebGpu()<AvailableFrom v="4.0.517" />
+
+Checks whether WebGPU or the WASM fallback can run in the current browser.
+
+```ts twoslash title="support.ts"
+import {canUseWhisperWebGpu} from '@remotion/whisper-webgpu';
+
+const result = await canUseWhisperWebGpu();
+if (!result.supported) {
+ throw new Error(result.detailedReason);
+}
+
+console.log(result.backend, result.wasmThreads);
+```
+
+## Options
+
+### backend?
+
+`'auto'`, `'webgpu'`, or `'wasm'`. Default: `'auto'`, which selects WebGPU when available and otherwise uses WASM.
+
+Requesting `'webgpu'` reports an unsupported result when WebGPU is unavailable or the page is not a secure context.
+
+## Cross-origin isolation
+
+Cross-origin isolation is not required. It only changes the WASM thread count: isolated pages may use multiple threads, while other pages use one.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+
+## See also
+
+- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)
diff --git a/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx
new file mode 100644
index 0000000000..f3d252b99b
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx
@@ -0,0 +1,33 @@
+---
+image: /generated/articles-docs-whisper-webgpu-dispose-whisper-model.png
+title: disposeWhisperModel()
+crumb: '@remotion/whisper-webgpu'
+---
+
+# disposeWhisperModel()<AvailableFrom v="4.0.517" />
+
+Releases the memory used by initialized Whisper pipelines. Persistent browser-cached model files are not deleted.
+
+```ts twoslash title="dispose.ts"
+import {disposeWhisperModel} from '@remotion/whisper-webgpu';
+
+await disposeWhisperModel({model: 'small.en'});
+```
+
+## Options
+
+### model?
+
+Only dispose this model. Default: all loaded models.
+
+### backend?
+
+Only dispose the `'webgpu'` or `'wasm'` instance. Default: both backends.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+
+## See also
+
+- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)
diff --git a/packages/docs/docs/whisper-webgpu/get-available-models.mdx b/packages/docs/docs/whisper-webgpu/get-available-models.mdx
new file mode 100644
index 0000000000..1163b015f2
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/get-available-models.mdx
@@ -0,0 +1,31 @@
+---
+image: /generated/articles-docs-whisper-webgpu-get-available-models.png
+title: getAvailableModels()
+crumb: '@remotion/whisper-webgpu'
+---
+
+# getAvailableModels()<AvailableFrom v="4.0.517" />
+
+Returns the timestamped Whisper models supported by [`transcribe()`](/docs/whisper-webgpu/transcribe).
+
+```ts twoslash title="models.ts"
+import {getAvailableModels} from '@remotion/whisper-webgpu';
+
+const models = getAvailableModels();
+console.log(models);
+```
+
+Each item contains the short `name`, Hugging Face `modelId`, parameter count, whether the model is multilingual, and estimated downloads for the default WebGPU and WASM configurations.
+
+Available sizes are `tiny`, `base`, `small`, `medium`, and `large-v3-turbo`. English-only `.en` variants are available through `medium`.
+
+The estimates cover model weights. Browser caches and supporting model files may add a small amount.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+
+## See also
+
+- [`transcribe()`](/docs/whisper-webgpu/transcribe)
+- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)
diff --git a/packages/docs/docs/whisper-webgpu/index.mdx b/packages/docs/docs/whisper-webgpu/index.mdx
new file mode 100644
index 0000000000..6828e09f08
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/index.mdx
@@ -0,0 +1,59 @@
+---
+image: /generated/articles-docs-whisper-webgpu-index.png
+title: '@remotion/whisper-webgpu'
+crumb: 'Transcribe audio in browser'
+---
+
+# @remotion/whisper-webgpu<AvailableFrom v="4.0.517" />
+
+<ExperimentalBadge>
+<p>This package is experimental.</p>
+</ExperimentalBadge>
+
+Transcribe audio locally in the browser using timestamped Whisper models from Transformers.js. WebGPU is used when available and single-threaded WASM is the fallback.
+
+Unlike [`@remotion/whisper-web`](/docs/whisper-web), cross-origin isolation is not required.
+
+import {TableOfContents} from './table-of-contents';
+
+## Installation
+
+<Installation pkg="@remotion/whisper-webgpu" />
+
+## Example
+
+```ts twoslash title="transcribe.ts"
+import {
+ resampleTo16Khz,
+ toCaptions,
+ transcribe,
+} from '@remotion/whisper-webgpu';
+
+const file = new File([], 'audio.wav');
+const channelWaveform = await resampleTo16Khz({file});
+
+const transcription = await transcribe({
+ channelWaveform,
+ model: 'small.en',
+ language: 'en',
+});
+
+const {captions} = toCaptions({whisperWebGpuOutput: transcription});
+console.log(captions);
+```
+
+The model is downloaded on the first call and cached by the browser. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) first to display model download progress.
+
+## APIs
+
+<TableOfContents />
+
+## Requirements
+
+WebGPU requires HTTPS in production or `localhost` during development. Browsers without WebGPU fall back to WASM.
+
+Cross-origin isolation enables multithreaded WASM but is not required. Without it, the WASM fallback uses one thread.
+
+## License
+
+MIT
diff --git a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
new file mode 100644
index 0000000000..4f9b56eef0
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
@@ -0,0 +1,51 @@
+---
+image: /generated/articles-docs-whisper-webgpu-load-whisper-model.png
+title: loadWhisperModel()
+crumb: '@remotion/whisper-webgpu'
+---
+
+# loadWhisperModel()<AvailableFrom v="4.0.517" />
+
+Downloads and initializes a timestamped Whisper model. Calling it before [`transcribe()`](/docs/whisper-webgpu/transcribe) lets you show model-loading progress.
+
+```ts twoslash title="load-model.ts"
+import {loadWhisperModel} from '@remotion/whisper-webgpu';
+
+const result = await loadWhisperModel({
+ model: 'small.en',
+ onProgress: ({progress, file}) => {
+ console.log(file, progress);
+ },
+});
+
+console.log(result.backend, result.alreadyLoaded);
+```
+
+## Options
+
+### model
+
+One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models).
+
+### backend?
+
+`'auto'`, `'webgpu'`, or `'wasm'`. Default: `'auto'`.
+
+### onProgress?
+
+Called while model files are resolved and downloaded. `progress` is between `0` and `1` when available.
+
+## Return value
+
+Returns the selected `backend` and whether the same model pipeline was already loaded in memory.
+
+The browser manages its persistent Transformers.js cache. This function also retains the initialized model in memory until [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model) is called or the page closes.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+
+## See also
+
+- [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model)
+- [`transcribe()`](/docs/whisper-webgpu/transcribe)
diff --git a/packages/docs/docs/whisper-webgpu/resample-to-16khz.mdx b/packages/docs/docs/whisper-webgpu/resample-to-16khz.mdx
new file mode 100644
index 0000000000..250200287b
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/resample-to-16khz.mdx
@@ -0,0 +1,39 @@
+---
+image: /generated/articles-docs-whisper-webgpu-resample-to-16khz.png
+title: resampleTo16Khz()
+crumb: '@remotion/whisper-webgpu'
+---
+
+# resampleTo16Khz()<AvailableFrom v="4.0.517" />
+
+Decodes an audio `Blob`, mixes it to mono, and resamples it to the 16kHz waveform expected by [`transcribe()`](/docs/whisper-webgpu/transcribe).
+
+```ts twoslash title="resample.ts"
+import {resampleTo16Khz} from '@remotion/whisper-webgpu';
+
+const file = new File([], 'audio.wav');
+const waveform = await resampleTo16Khz({
+ file,
+ onProgress: (progress) => console.log(progress),
+});
+
+console.log(waveform);
+```
+
+## Options
+
+### file
+
+An audio `Blob` or `File` supported by the browser's Web Audio decoder.
+
+### onProgress?
+
+Called with values between `0` and `1` while the file is read, decoded, and resampled.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+
+## See also
+
+- [`transcribe()`](/docs/whisper-webgpu/transcribe)
diff --git a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
new file mode 100644
index 0000000000..52664f7b21
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
@@ -0,0 +1,38 @@
+import React from 'react';
+import {Grid} from '../../components/TableOfContents/Grid';
+import {TOCItem} from '../../components/TableOfContents/TOCItem';
+
+export const TableOfContents: React.FC = () => {
+ return (
+ <Grid>
+ <TOCItem link="/docs/whisper-webgpu/can-use-whisper-webgpu">
+ <strong>canUseWhisperWebGpu()</strong>
+ <div>Check which inference backend is available</div>
+ </TOCItem>
+ <TOCItem link="/docs/whisper-webgpu/get-available-models">
+ <strong>getAvailableModels()</strong>
+ <div>List timestamped models and their download sizes</div>
+ </TOCItem>
+ <TOCItem link="/docs/whisper-webgpu/load-whisper-model">
+ <strong>loadWhisperModel()</strong>
+ <div>Download and initialize a model</div>
+ </TOCItem>
+ <TOCItem link="/docs/whisper-webgpu/transcribe">
+ <strong>transcribe()</strong>
+ <div>Transcribe a waveform with word-level timestamps</div>
+ </TOCItem>
+ <TOCItem link="/docs/whisper-webgpu/to-captions">
+ <strong>toCaptions()</strong>
+ <div>Convert a transcription to Remotion captions</div>
+ </TOCItem>
+ <TOCItem link="/docs/whisper-webgpu/resample-to-16khz">
+ <strong>resampleTo16Khz()</strong>
+ <div>Decode and resample browser audio</div>
+ </TOCItem>
+ <TOCItem link="/docs/whisper-webgpu/dispose-whisper-model">
+ <strong>disposeWhisperModel()</strong>
+ <div>Release model memory</div>
+ </TOCItem>
+ </Grid>
+ );
+};
diff --git a/packages/docs/docs/whisper-webgpu/to-captions.mdx b/packages/docs/docs/whisper-webgpu/to-captions.mdx
new file mode 100644
index 0000000000..fe6d9f0cba
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/to-captions.mdx
@@ -0,0 +1,39 @@
+---
+image: /generated/articles-docs-whisper-webgpu-to-captions.png
+title: toCaptions()
+crumb: '@remotion/whisper-webgpu'
+---
+
+# toCaptions()<AvailableFrom v="4.0.517" />
+
+Converts [`transcribe()`](/docs/whisper-webgpu/transcribe) output into [`Caption`](/docs/captions/caption) objects from `@remotion/captions`.
+
+```ts twoslash title="captions.ts"
+import {toCaptions, transcribe} from '@remotion/whisper-webgpu';
+
+const channelWaveform = new Float32Array(16_000);
+const transcription = await transcribe({
+ channelWaveform,
+ model: 'tiny.en',
+});
+
+const {captions} = toCaptions({whisperWebGpuOutput: transcription});
+console.log(captions);
+```
+
+`startMs` and `endMs` use the model's word boundaries. `timestampMs` is the midpoint of each word. `confidence` is `null` because Transformers.js does not expose calibrated word confidence values for this pipeline.
+
+## Options
+
+### whisperWebGpuOutput
+
+A complete transcription or an array of timestamped words returned by `transcribe()`.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering={false} serverSideRendering player studio />
+
+## See also
+
+- [`@remotion/captions`](/docs/captions)
+- [`transcribe()`](/docs/whisper-webgpu/transcribe)
diff --git a/packages/docs/docs/whisper-webgpu/transcribe.mdx b/packages/docs/docs/whisper-webgpu/transcribe.mdx
new file mode 100644
index 0000000000..33f899d50a
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/transcribe.mdx
@@ -0,0 +1,69 @@
+---
+image: /generated/articles-docs-whisper-webgpu-transcribe.png
+title: transcribe()
+crumb: '@remotion/whisper-webgpu'
+---
+
+# transcribe()<AvailableFrom v="4.0.517" />
+
+Transcribes a 16kHz mono waveform and returns words with start and end timestamps.
+
+```ts twoslash title="transcribe.ts"
+import {resampleTo16Khz, transcribe} from '@remotion/whisper-webgpu';
+
+const file = new File([], 'audio.wav');
+const channelWaveform = await resampleTo16Khz({file});
+
+const result = await transcribe({
+ channelWaveform,
+ model: 'small.en',
+ language: 'en',
+});
+
+console.log(result.text, result.words, result.backend);
+```
+
+## Options
+
+### channelWaveform
+
+A mono `Float32Array` sampled at 16kHz. Use [`resampleTo16Khz()`](/docs/whisper-webgpu/resample-to-16khz) for browser audio files.
+
+### model
+
+One of the timestamped models returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models). `small` or `small.en` is the recommended default.
+
+### backend?
+
+`'auto'`, `'webgpu'`, or `'wasm'`. Default: `'auto'`.
+
+### language?
+
+The spoken language name or language code. Default: `'auto'`. Set it when known to reduce language-detection errors.
+
+### chunkLengthInSeconds?
+
+Length of long-audio chunks. Default: `30`.
+
+### strideLengthInSeconds?
+
+Overlap on both sides of a chunk. Default: `5`. It must be less than half of `chunkLengthInSeconds`.
+
+### onModelLoadProgress?
+
+Called while the model is loaded. To load independently, use [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model).
+
+## Return value
+
+Returns the full `text`, timestamped `words`, selected `model`, and resolved `backend`.
+
+Word-level timestamps are returned after transcription completes. Transformers.js does not currently stream word-aligned updates while inference is running.
+
+## Compatibility
+
+<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+
+## See also
+
+- [`toCaptions()`](/docs/whisper-webgpu/to-captions)
+- [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models)
diff --git a/packages/docs/package.json b/packages/docs/package.json
index f6c3771919..e11d55e9c7 100644
--- a/packages/docs/package.json
+++ b/packages/docs/package.json
@@ -99,6 +99,7 @@
"@remotion/web-renderer": "workspace:*",
"@remotion/webcodecs": "workspace:*",
"@remotion/whisper-web": "workspace:*",
+ "@remotion/whisper-webgpu": "workspace:*",
"@remotion/zod-types": "workspace:*",
"@remotion/zod-types-v3": "workspace:*",
"@rive-app/canvas-advanced": "2.31.5",
diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.ts
index 99b5c30d52..836afb476f 100644
--- a/packages/docs/sidebars.ts
+++ b/packages/docs/sidebars.ts
@@ -979,6 +979,23 @@ const sidebars: SidebarsConfig = {
'whisper-web/to-captions',
],
},
+ {
+ type: 'category',
+ label: '@remotion/whisper-webgpu',
+ link: {
+ type: 'doc',
+ id: 'whisper-webgpu/index',
+ },
+ items: [
+ 'whisper-webgpu/can-use-whisper-webgpu',
+ 'whisper-webgpu/get-available-models',
+ 'whisper-webgpu/load-whisper-model',
+ 'whisper-webgpu/transcribe',
+ 'whisper-webgpu/to-captions',
+ 'whisper-webgpu/resample-to-16khz',
+ 'whisper-webgpu/dispose-whisper-model',
+ ],
+ },
{
type: 'category',
label: '@remotion/zod-types',
diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.ts
index fafe845400..43ca5f8692 100644
--- a/packages/docs/src/data/articles.ts
+++ b/packages/docs/src/data/articles.ts
@@ -8975,6 +8975,78 @@ export const articles = [
noAi: false,
slug: 'whisper-web/transcribe',
},
+ {
+ id: 'whisper-webgpu/can-use-whisper-webgpu',
+ title: 'canUseWhisperWebGpu()',
+ relativePath: 'docs/whisper-webgpu/can-use-whisper-webgpu.mdx',
+ compId: 'articles-docs-whisper-webgpu-can-use-whisper-webgpu',
+ crumb: '@remotion/whisper-webgpu',
+ noAi: false,
+ slug: 'whisper-webgpu/can-use-whisper-webgpu',
+ },
+ {
+ id: 'whisper-webgpu/dispose-whisper-model',
+ title: 'disposeWhisperModel()',
+ relativePath: 'docs/whisper-webgpu/dispose-whisper-model.mdx',
+ compId: 'articles-docs-whisper-webgpu-dispose-whisper-model',
+ crumb: '@remotion/whisper-webgpu',
+ noAi: false,
+ slug: 'whisper-webgpu/dispose-whisper-model',
+ },
+ {
+ id: 'whisper-webgpu/get-available-models',
+ title: 'getAvailableModels()',
+ relativePath: 'docs/whisper-webgpu/get-available-models.mdx',
+ compId: 'articles-docs-whisper-webgpu-get-available-models',
+ crumb: '@remotion/whisper-webgpu',
+ noAi: false,
+ slug: 'whisper-webgpu/get-available-models',
+ },
+ {
+ id: 'whisper-webgpu/index',
+ title: '@remotion/whisper-webgpu',
+ relativePath: 'docs/whisper-webgpu/index.mdx',
+ compId: 'articles-docs-whisper-webgpu-index',
+ crumb: 'Transcribe audio in browser',
+ noAi: false,
+ slug: 'whisper-webgpu/index',
+ },
+ {
+ id: 'whisper-webgpu/load-whisper-model',
+ title: 'loadWhisperModel()',
+ relativePath: 'docs/whisper-webgpu/load-whisper-model.mdx',
+ compId: 'articles-docs-whisper-webgpu-load-whisper-model',
+ crumb: '@remotion/whisper-webgpu',
+ noAi: false,
+ slug: 'whisper-webgpu/load-whisper-model',
+ },
+ {
+ id: 'whisper-webgpu/resample-to-16khz',
+ title: 'resampleTo16Khz()',
+ relativePath: 'docs/whisper-webgpu/resample-to-16khz.mdx',
+ compId: 'articles-docs-whisper-webgpu-resample-to-16khz',
+ crumb: '@remotion/whisper-webgpu',
+ noAi: false,
+ slug: 'whisper-webgpu/resample-to-16khz',
+ },
+ {
+ id: 'whisper-webgpu/to-captions',
+ title: 'toCaptions()',
+ relativePath: 'docs/whisper-webgpu/to-captions.mdx',
+ compId: 'articles-docs-whisper-webgpu-to-captions',
+ crumb: '@remotion/whisper-webgpu',
+ noAi: false,
+ slug: 'whisper-webgpu/to-captions',
+ },
+ {
+ id: 'whisper-webgpu/transcribe',
+ title: 'transcribe()',
+ relativePath: 'docs/whisper-webgpu/transcribe.mdx',
+ compId: 'articles-docs-whisper-webgpu-transcribe',
+ crumb: '@remotion/whisper-webgpu',
+ noAi: false,
+ slug: 'whisper-webgpu/transcribe',
+ },
{
id: 'wrong-composition-mount',
title: 'Wrongly mounted <Composition>',
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-can-use-whisper-webgpu.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-can-use-whisper-webgpu.png
new file mode 100644
index 0000000000..3fb19eeb8a
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-can-use-whisper-webgpu.png differ
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-dispose-whisper-model.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-dispose-whisper-model.png
new file mode 100644
index 0000000000..fcc6be76ae
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-dispose-whisper-model.png differ
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-get-available-models.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-get-available-models.png
new file mode 100644
index 0000000000..c32e096dd8
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-get-available-models.png differ
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-index.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-index.png
new file mode 100644
index 0000000000..b6d6e8b3ea
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-index.png differ
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-load-whisper-model.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-load-whisper-model.png
new file mode 100644
index 0000000000..d1537682c2
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-load-whisper-model.png differ
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-resample-to-16khz.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-resample-to-16khz.png
new file mode 100644
index 0000000000..5dc4392baf
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-resample-to-16khz.png differ
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-to-captions.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-to-captions.png
new file mode 100644
index 0000000000..1667ed1c95
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-to-captions.png differ
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-transcribe.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-transcribe.png
new file mode 100644
index 0000000000..46287b498a
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-transcribe.png differ
diff --git a/packages/example/package.json b/packages/example/package.json
index d78c6235b2..47aee9a7cd 100644
--- a/packages/example/package.json
+++ b/packages/example/package.json
@@ -84,6 +84,7 @@
"@remotion/transitions": "workspace:*",
"@remotion/webcodecs": "workspace:*",
"@remotion/whisper-web": "workspace:*",
+ "@remotion/whisper-webgpu": "workspace:*",
"@remotion/zod-types": "workspace:*",
"@remotion/zod-types-v3": "workspace:*",
"@shopify/react-native-skia": "2.0.0",
diff --git a/packages/example/src/Root.tsx b/packages/example/src/Root.tsx
index 0c59524046..79622fe077 100644
--- a/packages/example/src/Root.tsx
+++ b/packages/example/src/Root.tsx
@@ -323,6 +323,7 @@ import {SvgPaintSchema} from './VisualModeTests/SvgPaintSchema';
import {VideoConfigExpressions} from './VisualModeTests/VideoConfigExpressions';
import {VoiceVisualization} from './voice-visualization';
import {WhisperWeb} from './WhisperWeb';
+import {WhisperWebGpu} from './WhisperWebGpu';
class Vector2 {
readonly x: number;
@@ -2702,6 +2703,12 @@ export const Index: React.FC = () => {
/>
</Folder>
<Still id="whisper-web" component={WhisperWeb} width={800} height={800} />
+ <Still
+ id="whisper-webgpu"
+ component={WhisperWebGpu}
+ width={900}
+ height={900}
+ />
<Composition
id="empty"
component={Empty}
diff --git a/packages/example/src/WhisperWebGpu/index.tsx b/packages/example/src/WhisperWebGpu/index.tsx
new file mode 100644
index 0000000000..e03fa3cb71
--- /dev/null
+++ b/packages/example/src/WhisperWebGpu/index.tsx
@@ -0,0 +1,142 @@
+import {
+ canUseWhisperWebGpu,
+ getAvailableModels,
+ resampleTo16Khz,
+ toCaptions,
+ transcribe,
+ type WhisperWebGpuModel,
+} from '@remotion/whisper-webgpu';
+import {useCallback, useMemo, useState} from 'react';
+
+const audioFileUrl = 'https://remotion.media/16khz.wav';
+
+const formatMegabytes = (bytes: number) => {
+ return `${Math.round(bytes / 1_000_000)} MB`;
+};
+
+export const WhisperWebGpu = () => {
+ const models = useMemo(() => getAvailableModels(), []);
+ const [selectedModel, setSelectedModel] =
+ useState<WhisperWebGpuModel>('small.en');
+ const [status, setStatus] = useState('Ready to transcribe the sample audio.');
+ const [result, setResult] = useState('');
+ const [isTranscribing, setIsTranscribing] = useState(false);
+ const selectedModelInfo = models.find(
+ (model) => model.name === selectedModel,
+ );
+
+ const onTranscribe = useCallback(async () => {
+ setIsTranscribing(true);
+ setResult('');
+
+ try {
+ const support = canUseWhisperWebGpu({backend: 'auto'});
+ if (!support.supported) {
+ throw new Error(support.detailedReason);
+ }
+
+ setStatus(`Fetching audio for the ${support.backend} backend...`);
+ const response = await fetch(audioFileUrl);
+ if (!response.ok) {
+ throw new Error(`Could not fetch audio: HTTP ${response.status}`);
+ }
+
+ setStatus('Decoding and resampling audio...');
+ const channelWaveform = await resampleTo16Khz({
+ file: await response.blob(),
+ onProgress: (progress) => {
+ setStatus(`Resampling audio... ${Math.round(progress * 100)}%`);
+ },
+ });
+
+ setStatus(`Loading ${selectedModel} and transcribing...`);
+ const transcription = await transcribe({
+ channelWaveform,
+ model: selectedModel,
+ onModelLoadProgress: (progress) => {
+ if (progress.progress !== null) {
+ setStatus(
+ `Loading ${selectedModel}... ${Math.round(progress.progress * 100)}%`,
+ );
+ }
+ },
+ });
+ const {captions} = toCaptions({whisperWebGpuOutput: transcription});
+ setStatus(
+ `Done using ${transcription.backend}: ${captions.length} timestamped words.`,
+ );
+ setResult(
+ captions
+ .map(
+ (caption) =>
+ `[${(caption.startMs / 1000).toFixed(2)}–${(caption.endMs / 1000).toFixed(2)}]${caption.text}`,
+ )
+ .join('\n'),
+ );
+ } catch (error) {
+ setStatus(error instanceof Error ? error.message : String(error));
+ } finally {
+ setIsTranscribing(false);
+ }
+ }, [selectedModel]);
+
+ return (
+ <div className="flex min-h-screen items-center justify-center bg-slate-950 p-8 text-slate-900">
+ <div className="w-full max-w-2xl space-y-5 rounded-xl bg-white p-7 shadow-2xl">
+ <div>
+ <h1 className="text-2xl font-semibold">Whisper WebGPU</h1>
+ <p className="mt-1 text-sm text-slate-600">
+ Timestamped Transformers.js transcription for Remotion captions.
+ </p>
+ </div>
+
+ <label className="block text-sm font-medium" htmlFor="webgpu-model">
+ Model
+ <select
+ className="mt-2 block w-full rounded-md border border-slate-300 p-2"
+ disabled={isTranscribing}
+ id="webgpu-model"
+ onChange={(event) => {
+ setSelectedModel(event.target.value as WhisperWebGpuModel);
+ }}
+ value={selectedModel}
+ >
+ {models.map((model) => (
+ <option key={model.name} value={model.name}>
+ {model.name} · {Math.round(model.parameters / 1_000_000)}M ·{' '}
+ {formatMegabytes(model.webGpuDownloadSize)} WebGPU
+ </option>
+ ))}
+ </select>
+ </label>
+
+ {selectedModelInfo ? (
+ <p className="rounded-md bg-slate-100 p-3 text-sm text-slate-700">
+ {selectedModelInfo.multilingual ? 'Multilingual' : 'English-only'} ·{' '}
+ {formatMegabytes(selectedModelInfo.webGpuDownloadSize)} WebGPU
+ download · {formatMegabytes(selectedModelInfo.wasmDownloadSize)}{' '}
+ WASM download
+ </p>
+ ) : null}
+
+ <button
+ className="w-full rounded-md bg-indigo-600 px-4 py-3 font-semibold text-white disabled:opacity-50"
+ disabled={isTranscribing}
+ onClick={onTranscribe}
+ type="button"
+ >
+ {isTranscribing ? 'Transcribing…' : 'Transcribe sample audio'}
+ </button>
+
+ <div className="rounded-md bg-blue-50 p-3 text-sm text-blue-900">
+ {status}
+ </div>
+ {result ? (
+ <pre className="max-h-80 overflow-auto whitespace-pre-wrap rounded-md bg-slate-950 p-4 text-xs text-slate-100">
+ {result}
+ </pre>
+ ) : null}
+ </div>
+ </div>
+ );
+};
diff --git a/packages/example/tsconfig.json b/packages/example/tsconfig.json
index 0e2830b518..23ced969fa 100644
--- a/packages/example/tsconfig.json
+++ b/packages/example/tsconfig.json
@@ -42,6 +42,7 @@
{"path": "../media-parser"},
{"path": "../webcodecs"},
{"path": "../whisper-web"},
+ {"path": "../whisper-webgpu"},
{"path": "../studio-protocol"},
{"path": "../light-leaks"},
{"path": "../mac-cursors"},
diff --git a/packages/studio-shared/src/package-info.ts b/packages/studio-shared/src/package-info.ts
index eab6167430..d0a1d15106 100644
--- a/packages/studio-shared/src/package-info.ts
+++ b/packages/studio-shared/src/package-info.ts
@@ -94,6 +94,7 @@ const allPackages = [
'compositor',
'example-videos',
'whisper-web',
+ 'whisper-webgpu',
'media',
'remotion-media',
'web-renderer',
@@ -176,6 +177,7 @@ export const descriptions: {[key in Pkgs]: string | null} = {
'studio-server': 'Run a Remotion Studio with a server backend',
'install-whisper-cpp': 'Helpers for installing and using Whisper.cpp',
'whisper-web': 'Helpers for using Whisper.cpp in browser using WASM',
+ 'whisper-webgpu': 'Transcribe audio in the browser using Whisper and WebGPU',
'google-fonts': 'Use Google Fonts in Remotion',
gsap: 'Use GSAP timelines in Remotion',
mcp: "Remotion's Model Context Protocol",
@@ -314,6 +316,7 @@ export const installableMap: {[key in Pkgs]: boolean} = {
gsap: true,
'install-whisper-cpp': true,
'whisper-web': true,
+ 'whisper-webgpu': true,
'it-tests': false,
'react18-tests': false,
'lambda-go-example': false,
@@ -413,6 +416,7 @@ export const apiDocs: {[key in Pkgs]: string | null} = {
'studio-server': null,
'install-whisper-cpp': 'https://www.remotion.dev/docs/install-whisper-cpp',
'whisper-web': 'https://www.remotion.dev/docs/whisper-web',
+ 'whisper-webgpu': 'https://www.remotion.dev/docs/whisper-webgpu',
'google-fonts': 'https://www.remotion.dev/docs/google-fonts',
gsap: 'https://www.remotion.dev/docs/gsap',
'media-utils': 'https://www.remotion.dev/docs/media-utils',
diff --git a/packages/whisper-webgpu/.npmignore b/packages/whisper-webgpu/.npmignore
new file mode 100644
index 0000000000..eb76a5ae81
--- /dev/null
+++ b/packages/whisper-webgpu/.npmignore
@@ -0,0 +1,10 @@
+dist/test/**
+src
+eslint.config.mjs
+tsconfig.tsbuildinfo
+*.tgz
+.turbo
+tsconfig.json
+.prettierrc.js
+bundle.ts
+dist/cjs/test
diff --git a/packages/whisper-webgpu/README.md b/packages/whisper-webgpu/README.md
new file mode 100644
index 0000000000..ae068a1fe3
--- /dev/null
+++ b/packages/whisper-webgpu/README.md
@@ -0,0 +1,9 @@
+# @remotion/whisper-webgpu
+
+Transcribe audio in the browser using timestamped Whisper models and WebGPU.
+
+```bash
+npx remotion add @remotion/whisper-webgpu
+```
+
+See the [documentation](https://www.remotion.dev/docs/whisper-webgpu).
diff --git a/packages/whisper-webgpu/bundle.ts b/packages/whisper-webgpu/bundle.ts
new file mode 100644
index 0000000000..a0a1a34b77
--- /dev/null
+++ b/packages/whisper-webgpu/bundle.ts
@@ -0,0 +1,36 @@
+import path from 'path';
+import {build} from 'bun';
+
+if (process.env.NODE_ENV !== 'production') {
+ throw new Error('This script must be run using NODE_ENV=production');
+}
+
+console.time('Generated.');
+const output = await build({
+ entrypoints: ['src/index.ts'],
+ naming: '[name].mjs',
+ external: [
+ '@huggingface/transformers',
+ '@remotion/captions',
+ 'remotion',
+ 'remotion/no-react',
+ 'react',
+ 'react/jsx-runtime',
+ 'react/jsx-dev-runtime',
+ 'react-dom',
+ ],
+});
+
+if (!output.success) {
+ console.log(output.logs.join('\n'));
+ process.exit(1);
+}
+
+for (const file of output.outputs) {
+ const str = await file.text();
+ const out = path.join('dist', 'esm', file.path);
+
+ await Bun.write(out, str);
+}
+
+console.timeEnd('Generated.');
diff --git a/packages/whisper-webgpu/eslint.config.mjs b/packages/whisper-webgpu/eslint.config.mjs
new file mode 100644
index 0000000000..ce2fc683dd
--- /dev/null
+++ b/packages/whisper-webgpu/eslint.config.mjs
@@ -0,0 +1,5 @@
+import {remotionFlatConfig} from '@remotion/eslint-config-internal';
+
+const config = remotionFlatConfig({react: false});
+
+export default [config];
diff --git a/packages/whisper-webgpu/package.json b/packages/whisper-webgpu/package.json
new file mode 100644
index 0000000000..1de2bb4de7
--- /dev/null
+++ b/packages/whisper-webgpu/package.json
@@ -0,0 +1,54 @@
+{
+ "name": "@remotion/whisper-webgpu",
+ "version": "4.0.516",
+ "description": "Transcribe audio in the browser using Whisper and WebGPU",
+ "main": "dist/index.js",
+ "types": "dist/index.d.ts",
+ "module": "dist/esm/index.mjs",
+ "repository": {
+ "url": "https://github.com/remotion-dev/remotion/tree/main/packages/whisper-webgpu"
+ },
+ "type": "module",
+ "scripts": {
+ "test": "bun test src/test",
+ "formatting": "oxfmt src --check",
+ "format": "oxfmt src",
+ "lint": "eslint src",
+ "watch": "tsgo -w",
+ "make": "tsgo && bun --env-file=../.env.bundle bundle.ts"
+ },
+ "author": "Jonny Burger <[email protected]>",
+ "contributors": [],
+ "license": "MIT",
+ "bugs": {
+ "url": "https://github.com/remotion-dev/remotion/issues"
+ },
+ "dependencies": {
+ "@huggingface/transformers": "catalog:",
+ "@remotion/captions": "workspace:*"
+ },
+ "exports": {
+ ".": {
+ "types": "./dist/index.d.ts",
+ "module": "./dist/esm/index.mjs",
+ "import": "./dist/esm/index.mjs"
+ },
+ "./package.json": "./package.json"
+ },
+ "devDependencies": {
+ "@remotion/eslint-config-internal": "workspace:*",
+ "@typescript/native-preview": "catalog:",
+ "eslint": "catalog:"
+ },
+ "keywords": [
+ "remotion",
+ "whisper",
+ "webgpu",
+ "captions",
+ "speech-to-text"
+ ],
+ "publishConfig": {
+ "access": "public"
+ },
+ "homepage": "https://www.remotion.dev/docs/whisper-webgpu"
+}
diff --git a/packages/whisper-webgpu/src/backend.ts b/packages/whisper-webgpu/src/backend.ts
new file mode 100644
index 0000000000..0e710405ee
--- /dev/null
+++ b/packages/whisper-webgpu/src/backend.ts
@@ -0,0 +1,27 @@
+export type WhisperWebGpuBackend = 'auto' | 'webgpu' | 'wasm';
+export type ResolvedWhisperWebGpuBackend = Exclude<
+ WhisperWebGpuBackend,
+ 'auto'
+>;
+
+const hasWebGpu = () => {
+ return (
+ typeof navigator !== 'undefined' &&
+ 'gpu' in navigator &&
+ globalThis.isSecureContext !== false
+ );
+};
+
+export const resolveBackend = (
+ backend: WhisperWebGpuBackend,
+): ResolvedWhisperWebGpuBackend => {
+ if (backend === 'webgpu' && !hasWebGpu()) {
+ throw new Error('WebGPU is not available in this browser.');
+ }
+
+ if (backend === 'auto') {
+ return hasWebGpu() ? 'webgpu' : 'wasm';
+ }
+
+ return backend;
+};
diff --git a/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts b/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts
new file mode 100644
index 0000000000..2357ba9ec4
--- /dev/null
+++ b/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts
@@ -0,0 +1,78 @@
+import type {
+ WhisperWebGpuBackend,
+ ResolvedWhisperWebGpuBackend,
+} from './backend';
+import {resolveBackend} from './backend';
+
+export enum WhisperWebGpuUnsupportedReason {
+ WindowUndefined = 'window-undefined',
+ WebAssemblyUnavailable = 'webassembly-unavailable',
+ WebGpuUnavailable = 'webgpu-unavailable',
+ WebGpuRequiresSecureContext = 'webgpu-requires-secure-context',
+}
+
+export type CanUseWhisperWebGpuResult =
+ | {
+ supported: true;
+ backend: ResolvedWhisperWebGpuBackend;
+ wasmThreads: number | null;
+ }
+ | {
+ supported: false;
+ reason: WhisperWebGpuUnsupportedReason;
+ detailedReason: string;
+ };
+
+export type CanUseWhisperWebGpuOptions = {
+ backend?: WhisperWebGpuBackend;
+};
+
+export const canUseWhisperWebGpu = ({
+ backend = 'auto',
+}: CanUseWhisperWebGpuOptions = {}): CanUseWhisperWebGpuResult => {
+ if (typeof window === 'undefined') {
+ return {
+ supported: false,
+ reason: WhisperWebGpuUnsupportedReason.WindowUndefined,
+ detailedReason:
+ '`window` is not defined. @remotion/whisper-webgpu is intended for browser environments.',
+ };
+ }
+
+ if (backend === 'webgpu' && !('gpu' in navigator)) {
+ return {
+ supported: false,
+ reason: WhisperWebGpuUnsupportedReason.WebGpuUnavailable,
+ detailedReason: 'WebGPU is not available in this browser.',
+ };
+ }
+
+ if (backend === 'webgpu' && !window.isSecureContext) {
+ return {
+ supported: false,
+ reason: WhisperWebGpuUnsupportedReason.WebGpuRequiresSecureContext,
+ detailedReason:
+ 'WebGPU requires HTTPS in production or localhost during development.',
+ };
+ }
+
+ const resolvedBackend = resolveBackend(backend);
+ if (resolvedBackend === 'wasm' && typeof WebAssembly === 'undefined') {
+ return {
+ supported: false,
+ reason: WhisperWebGpuUnsupportedReason.WebAssemblyUnavailable,
+ detailedReason: 'WebAssembly is not available in this browser.',
+ };
+ }
+
+ return {
+ supported: true,
+ backend: resolvedBackend,
+ wasmThreads:
+ resolvedBackend === 'wasm'
+ ? window.crossOriginIsolated
+ ? Math.max(1, navigator.hardwareConcurrency || 1)
+ : 1
+ : null,
+ };
+};
diff --git a/packages/whisper-webgpu/src/index.ts b/packages/whisper-webgpu/src/index.ts
new file mode 100644
index 0000000000..c39445dcfc
--- /dev/null
+++ b/packages/whisper-webgpu/src/index.ts
@@ -0,0 +1,32 @@
+export type {
+ ResolvedWhisperWebGpuBackend,
+ WhisperWebGpuBackend,
+} from './backend';
+export {
+ canUseWhisperWebGpu,
+ WhisperWebGpuUnsupportedReason,
+} from './can-use-whisper-webgpu';
+export type {
+ CanUseWhisperWebGpuOptions,
+ CanUseWhisperWebGpuResult,
+} from './can-use-whisper-webgpu';
+export {disposeWhisperModel, loadWhisperModel} from './load-whisper-model';
+export type {
+ DisposeWhisperModelOptions,
+ LoadWhisperModelOptions,
+ LoadWhisperModelResult,
+ OnWhisperWebGpuModelLoadProgress,
+ WhisperWebGpuModelLoadProgress,
+} from './load-whisper-model';
+export {getAvailableModels, WHISPER_WEBGPU_MODELS} from './models';
+export type {WhisperWebGpuModel, WhisperWebGpuModelInfo} from './models';
+export {resampleTo16Khz, WHISPER_WEBGPU_SAMPLE_RATE} from './resample-to-16khz';
+export type {ResampleTo16KhzOptions} from './resample-to-16khz';
+export {toCaptions} from './to-captions';
+export type {ToCaptionsOptions, ToCaptionsResult} from './to-captions';
+export {transcribe} from './transcribe';
+export type {
+ TranscribeOptions,
+ WhisperWebGpuTranscription,
+ WhisperWebGpuWord,
+} from './transcribe';
diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.ts
new file mode 100644
index 0000000000..3890eaf1c0
--- /dev/null
+++ b/packages/whisper-webgpu/src/load-whisper-model.ts
@@ -0,0 +1,156 @@
+import {env, pipeline} from '@huggingface/transformers';
+import type {
+ ResolvedWhisperWebGpuBackend,
+ WhisperWebGpuBackend,
+} from './backend';
+import {resolveBackend} from './backend';
+import {getModelInfo, type WhisperWebGpuModel} from './models';
+
+export type WhisperWebGpuModelLoadProgress = {
+ status: string;
+ file: string | null;
+ progress: number | null;
+ loadedBytes: number | null;
+ totalBytes: number | null;
+};
+
+export type OnWhisperWebGpuModelLoadProgress = (
+ progress: WhisperWebGpuModelLoadProgress,
+) => void;
+
+type LoadedWhisperPipeline = Awaited<
+ ReturnType<typeof pipeline<'automatic-speech-recognition'>>
+>;
+
+const pipelines = new Map<string, Promise<LoadedWhisperPipeline>>();
+
+const getCacheKey = (
+ model: WhisperWebGpuModel,
+ backend: ResolvedWhisperWebGpuBackend,
+) => `${model}:${backend}`;
+
+const normalizeProgress = (
+ event: Record<string, unknown>,
+): WhisperWebGpuModelLoadProgress => {
+ const rawProgress =
+ typeof event.progress === 'number' ? event.progress : null;
+
+ return {
+ status: typeof event.status === 'string' ? event.status : 'progress',
+ file: typeof event.file === 'string' ? event.file : null,
+ progress:
+ rawProgress === null
+ ? null
+ : Math.max(
+ 0,
+ Math.min(1, rawProgress > 1 ? rawProgress / 100 : rawProgress),
+ ),
+ loadedBytes: typeof event.loaded === 'number' ? event.loaded : null,
+ totalBytes: typeof event.total === 'number' ? event.total : null,
+ };
+};
+
+export type LoadWhisperModelOptions = {
+ model: WhisperWebGpuModel;
+ backend?: WhisperWebGpuBackend;
+ onProgress?: OnWhisperWebGpuModelLoadProgress;
+};
+
+export type LoadWhisperModelResult = {
+ backend: ResolvedWhisperWebGpuBackend;
+ alreadyLoaded: boolean;
+};
+
+export const loadWhisperModel = async ({
+ model,
+ backend = 'auto',
+ onProgress,
+}: LoadWhisperModelOptions): Promise<LoadWhisperModelResult> => {
+ const resolvedBackend = resolveBackend(backend);
+ const cacheKey = getCacheKey(model, resolvedBackend);
+ const existing = pipelines.get(cacheKey);
+ if (existing) {
+ await existing;
+ onProgress?.({
+ status: 'ready',
+ file: null,
+ progress: 1,
+ loadedBytes: null,
+ totalBytes: null,
+ });
+ return {backend: resolvedBackend, alreadyLoaded: true};
+ }
+
+ if (resolvedBackend === 'wasm') {
+ env.backends.onnx.wasm!.numThreads = globalThis.crossOriginIsolated ? 0 : 1;
+ }
+
+ const {modelId} = getModelInfo(model);
+ const loading = pipeline('automatic-speech-recognition', modelId, {
+ device: resolvedBackend,
+ dtype:
+ resolvedBackend === 'webgpu'
+ ? {encoder_model: 'fp32', decoder_model_merged: 'q4'}
+ : 'q8',
+ progress_callback: (event) => {
+ onProgress?.(normalizeProgress(event as Record<string, unknown>));
+ },
+ });
+
+ pipelines.set(cacheKey, loading);
+ try {
+ await loading;
+ return {backend: resolvedBackend, alreadyLoaded: false};
+ } catch (error) {
+ pipelines.delete(cacheKey);
+ throw error;
+ }
+};
+
+export const getLoadedWhisperPipeline = async ({
+ model,
+ backend,
+ onProgress,
+}: {
+ model: WhisperWebGpuModel;
+ backend: WhisperWebGpuBackend;
+ onProgress?: OnWhisperWebGpuModelLoadProgress;
+}) => {
+ const {backend: resolvedBackend} = await loadWhisperModel({
+ model,
+ backend,
+ onProgress,
+ });
+ const loaded = pipelines.get(getCacheKey(model, resolvedBackend));
+ if (!loaded) {
+ throw new Error(`Whisper model ${model} was not loaded.`);
+ }
+
+ return {pipeline: await loaded, backend: resolvedBackend};
+};
+
+export type DisposeWhisperModelOptions = {
+ model?: WhisperWebGpuModel;
+ backend?: Exclude<WhisperWebGpuBackend, 'auto'>;
+};
+
+export const disposeWhisperModel = async ({
+ model,
+ backend,
+}: DisposeWhisperModelOptions = {}): Promise<void> => {
+ const matching = [...pipelines.entries()].filter(([key]) => {
+ const [loadedModel, loadedBackend] = key.split(':');
+ return (
+ (model === undefined || loadedModel === model) &&
+ (backend === undefined || loadedBackend === backend)
+ );
+ });
+
+ await Promise.all(
+ matching.map(async ([key, loaded]) => {
+ pipelines.delete(key);
+ const loadedPipeline = await loaded;
+ await loadedPipeline.dispose();
+ }),
+ );
+};
diff --git a/packages/whisper-webgpu/src/models.ts b/packages/whisper-webgpu/src/models.ts
new file mode 100644
index 0000000000..d83efe778d
--- /dev/null
+++ b/packages/whisper-webgpu/src/models.ts
@@ -0,0 +1,107 @@
+export const WHISPER_WEBGPU_MODELS = [
+ 'tiny',
+ 'tiny.en',
+ 'base',
+ 'base.en',
+ 'small',
+ 'small.en',
+ 'medium',
+ 'medium.en',
+ 'large-v3-turbo',
+] as const;
+
+export type WhisperWebGpuModel = (typeof WHISPER_WEBGPU_MODELS)[number];
+
+export type WhisperWebGpuModelInfo = {
+ name: WhisperWebGpuModel;
+ modelId: string;
+ parameters: number;
+ multilingual: boolean;
+ webGpuDownloadSize: number;
+ wasmDownloadSize: number;
+};
+
+const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
+ tiny: {
+ name: 'tiny',
+ modelId: 'onnx-community/whisper-tiny_timestamped',
+ parameters: 39_000_000,
+ multilingual: true,
+ webGpuDownloadSize: 119_699_015,
+ wasmDownloadSize: 40_827_373,
+ },
+ 'tiny.en': {
+ name: 'tiny.en',
+ modelId: 'onnx-community/whisper-tiny.en_timestamped',
+ parameters: 39_000_000,
+ multilingual: false,
+ webGpuDownloadSize: 119_697_479,
+ wasmDownloadSize: 40_826_993,
+ },
+ base: {
+ name: 'base',
+ modelId: 'onnx-community/whisper-base_timestamped',
+ parameters: 74_000_000,
+ multilingual: true,
+ webGpuDownloadSize: 206_190_057,
+ wasmDownloadSize: 76_871_875,
+ },
+ 'base.en': {
+ name: 'base.en',
+ modelId: 'onnx-community/whisper-base.en_timestamped',
+ parameters: 74_000_000,
+ multilingual: false,
+ webGpuDownloadSize: 206_188_009,
+ wasmDownloadSize: 76_871_369,
+ },
+ small: {
+ name: 'small',
+ modelId: 'onnx-community/whisper-small_timestamped',
+ parameters: 244_000_000,
+ multilingual: true,
+ webGpuDownloadSize: 586_213_010,
+ wasmDownloadSize: 249_036_248,
+ },
+ 'small.en': {
+ name: 'small.en',
+ modelId: 'onnx-community/whisper-small.en_timestamped',
+ parameters: 244_000_000,
+ multilingual: false,
+ webGpuDownloadSize: 586_209_938,
+ wasmDownloadSize: 249_035_489,
+ },
+ medium: {
+ name: 'medium',
+ modelId: 'onnx-community/whisper-medium_timestamped',
+ parameters: 769_000_000,
+ multilingual: true,
+ webGpuDownloadSize: 1_698_508_143,
+ wasmDownloadSize: 985_716_093,
+ },
+ 'medium.en': {
+ name: 'medium.en',
+ modelId: 'onnx-community/whisper-medium.en_timestamped',
+ parameters: 769_000_000,
+ multilingual: false,
+ webGpuDownloadSize: 1_698_504_047,
+ wasmDownloadSize: 985_710_974,
+ },
+ 'large-v3-turbo': {
+ name: 'large-v3-turbo',
+ modelId: 'onnx-community/whisper-large-v3-turbo_timestamped',
+ parameters: 809_000_000,
+ multilingual: true,
+ webGpuDownloadSize: 2_882_584_170,
+ wasmDownloadSize: 1_084_880_793,
+ },
+};
+
+export const getAvailableModels = (): WhisperWebGpuModelInfo[] => {
+ return WHISPER_WEBGPU_MODELS.map((model) => ({...MODEL_INFO[model]}));
+};
+
+export const getModelInfo = (
+ model: WhisperWebGpuModel,
+): WhisperWebGpuModelInfo => {
+ return MODEL_INFO[model];
+};
diff --git a/packages/whisper-webgpu/src/resample-to-16khz.ts b/packages/whisper-webgpu/src/resample-to-16khz.ts
new file mode 100644
index 0000000000..c41d74f109
--- /dev/null
+++ b/packages/whisper-webgpu/src/resample-to-16khz.ts
@@ -0,0 +1,56 @@
+export const WHISPER_WEBGPU_SAMPLE_RATE = 16_000;
+
+export type ResampleTo16KhzOptions = {
+ file: Blob;
+ onProgress?: (progress: number) => void;
+};
+
+export const resampleTo16Khz = async ({
+ file,
+ onProgress,
+}: ResampleTo16KhzOptions): Promise<Float32Array> => {
+ if (typeof AudioContext === 'undefined') {
+ throw new Error('AudioContext is not available in this environment.');
+ }
+
+ if (typeof OfflineAudioContext === 'undefined') {
+ throw new Error(
+ 'OfflineAudioContext is not available in this environment.',
+ );
+ }
+
+ if (file.size === 0) {
+ throw new Error('The audio file is empty.');
+ }
+
+ onProgress?.(0);
+ const arrayBuffer = await file.arrayBuffer();
+ onProgress?.(0.25);
+
+ const audioContext = new AudioContext({
+ sampleRate: WHISPER_WEBGPU_SAMPLE_RATE,
+ });
+
+ try {
+ const decoded = await audioContext.decodeAudioData(arrayBuffer);
+ onProgress?.(0.6);
+ const length = Math.max(
+ 1,
+ Math.ceil(decoded.duration * WHISPER_WEBGPU_SAMPLE_RATE),
+ );
+ const offlineContext = new OfflineAudioContext(
+ 1,
+ length,
+ WHISPER_WEBGPU_SAMPLE_RATE,
+ );
+ const source = offlineContext.createBufferSource();
+ source.buffer = decoded;
+ source.connect(offlineContext.destination);
+ source.start();
+ const rendered = await offlineContext.startRendering();
+ onProgress?.(1);
+ return rendered.getChannelData(0).slice();
+ } finally {
+ await audioContext.close();
+ }
+};
diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.ts
new file mode 100644
index 0000000000..1171b0fd60
--- /dev/null
+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts
@@ -0,0 +1,122 @@
+import {expect, mock, test} from 'bun:test';
+
+let pipelineInitialization:
+ | {
+ modelId: string;
+ options: Record<string, unknown>;
+ }
+ | undefined;
+let transcriptionCall:
+ | {
+ audio: Float32Array;
+ options: Record<string, unknown>;
+ }
+ | undefined;
+let disposed = false;
+
+const fakePipeline = Object.assign(
+ (audio: Float32Array, options: Record<string, unknown>) => {
+ transcriptionCall = {audio, options};
+ return Promise.resolve({
+ text: ' Hello world.',
+ chunks: [
+ {text: ' Hello', timestamp: [0.25, 0.75]},
+ {text: ' world.', timestamp: [1, null]},
+ ],
+ });
+ },
+ {
+ dispose: () => {
+ disposed = true;
+ return Promise.resolve();
+ },
+ },
+);
+
+mock.module('@huggingface/transformers', () => ({
+ env: {backends: {onnx: {wasm: {numThreads: 0}}}},
+ pipeline: (
+ _task: string,
+ modelId: string,
+ options: Record<string, unknown>,
+ ) => {
+ pipelineInitialization = {modelId, options};
+ return Promise.resolve(fakePipeline);
+ },
+}));
+
+test('transcribes with the selected timestamped model and returns Remotion captions', async () => {
+ const api = await import('../index');
+ const {disposeWhisperModel, getAvailableModels, toCaptions, transcribe} = api;
+ const channelWaveform = new Float32Array(16_000 * 3);
+ const result = await transcribe({
+ backend: 'wasm',
+ channelWaveform,
+ language: 'en',
+ model: 'small.en',
+ });
+
+ expect(pipelineInitialization?.modelId).toBe(
+ 'onnx-community/whisper-small.en_timestamped',
+ );
+ expect(pipelineInitialization?.options).toMatchObject({
+ device: 'wasm',
+ dtype: 'q8',
+ });
+ expect(transcriptionCall?.audio).toBe(channelWaveform);
+ expect(transcriptionCall?.options).toMatchObject({
+ return_timestamps: 'word',
+ chunk_length_s: 30,
+ stride_length_s: 5,
+ language: 'en',
+ });
+ expect(result).toEqual({
+ text: 'Hello world.',
+ model: 'small.en',
+ backend: 'wasm',
+ words: [
+ {text: 'Hello', startInSeconds: 0.25, endInSeconds: 0.75},
+ {text: ' world.', startInSeconds: 1, endInSeconds: 3},
+ ],
+ });
+
+ expect(toCaptions({whisperWebGpuOutput: result})).toEqual({
+ captions: [
+ {
+ text: 'Hello',
+ startMs: 250,
+ endMs: 750,
+ timestampMs: 500,
+ confidence: null,
+ },
+ {
+ text: ' world.',
+ startMs: 1000,
+ endMs: 3000,
+ timestampMs: 2000,
+ confidence: null,
+ },
+ ],
+ });
+
+ const models = getAvailableModels();
+ expect(models.map((model) => model.name)).toEqual([
+ 'tiny',
+ 'tiny.en',
+ 'base',
+ 'base.en',
+ 'small',
+ 'small.en',
+ 'medium',
+ 'medium.en',
+ 'large-v3-turbo',
+ ]);
+ expect(models.find((model) => model.name === 'small.en')).toMatchObject({
+ parameters: 244_000_000,
+ multilingual: false,
+ webGpuDownloadSize: 586_209_938,
+ });
+
+ await disposeWhisperModel({model: 'small.en', backend: 'wasm'});
+ expect(disposed).toBe(true);
+});
diff --git a/packages/whisper-webgpu/src/to-captions.ts b/packages/whisper-webgpu/src/to-captions.ts
new file mode 100644
index 0000000000..7aa1b0db69
--- /dev/null
+++ b/packages/whisper-webgpu/src/to-captions.ts
@@ -0,0 +1,30 @@
+import type {Caption} from '@remotion/captions';
+import type {WhisperWebGpuTranscription, WhisperWebGpuWord} from './transcribe';
+
+export type ToCaptionsOptions = {
+ whisperWebGpuOutput: WhisperWebGpuTranscription | WhisperWebGpuWord[];
+};
+
+export type ToCaptionsResult = {
+ captions: Caption[];
+};
+
+export const toCaptions = ({
+ whisperWebGpuOutput,
+}: ToCaptionsOptions): ToCaptionsResult => {
+ const words = Array.isArray(whisperWebGpuOutput)
+ ? whisperWebGpuOutput
+ : whisperWebGpuOutput.words;
+
+ return {
+ captions: words.map((word, index) => ({
+ text: index === 0 ? word.text.trimStart() : word.text,
+ startMs: Math.round(word.startInSeconds * 1000),
+ endMs: Math.round(word.endInSeconds * 1000),
+ timestampMs: Math.round(
+ ((word.startInSeconds + word.endInSeconds) / 2) * 1000,
+ ),
+ confidence: null,
+ })),
+ };
+};
diff --git a/packages/whisper-webgpu/src/transcribe.ts b/packages/whisper-webgpu/src/transcribe.ts
new file mode 100644
index 0000000000..7f21742c18
--- /dev/null
+++ b/packages/whisper-webgpu/src/transcribe.ts
@@ -0,0 +1,119 @@
+import type {
+ ResolvedWhisperWebGpuBackend,
+ WhisperWebGpuBackend,
+} from './backend';
+import {
+ getLoadedWhisperPipeline,
+ type OnWhisperWebGpuModelLoadProgress,
+} from './load-whisper-model';
+import type {WhisperWebGpuModel} from './models';
+
+export type WhisperWebGpuWord = {
+ text: string;
+ startInSeconds: number;
+ endInSeconds: number;
+};
+
+export type WhisperWebGpuTranscription = {
+ text: string;
+ words: WhisperWebGpuWord[];
+ model: WhisperWebGpuModel;
+ backend: ResolvedWhisperWebGpuBackend;
+};
+
+export type TranscribeOptions = {
+ channelWaveform: Float32Array;
+ model: WhisperWebGpuModel;
+ backend?: WhisperWebGpuBackend;
+ language?: string;
+ chunkLengthInSeconds?: number;
+ strideLengthInSeconds?: number;
+ onModelLoadProgress?: OnWhisperWebGpuModelLoadProgress;
+};
+
+type TransformersJsWord = {
+ text: string;
+ timestamp: [number | null, number | null];
+};
+
+type TransformersJsTranscription = {
+ text: string;
+ chunks?: TransformersJsWord[];
+};
+
+export const transcribe = async ({
+ channelWaveform,
+ model,
+ backend = 'auto',
+ language = 'auto',
+ chunkLengthInSeconds = 30,
+ strideLengthInSeconds = 5,
+ onModelLoadProgress,
+}: TranscribeOptions): Promise<WhisperWebGpuTranscription> => {
+ if (channelWaveform.length === 0) {
+ throw new Error('The audio waveform is empty.');
+ }
+
+ if (chunkLengthInSeconds <= 0) {
+ throw new Error('chunkLengthInSeconds must be greater than 0.');
+ }
+
+ if (
+ strideLengthInSeconds < 0 ||
+ strideLengthInSeconds * 2 >= chunkLengthInSeconds
+ ) {
+ throw new Error(
+ 'strideLengthInSeconds must be non-negative and less than half of chunkLengthInSeconds.',
+ );
+ }
+
+ const {pipeline: transcriber, backend: resolvedBackend} =
+ await getLoadedWhisperPipeline({
+ model,
+ backend,
+ onProgress: onModelLoadProgress,
+ });
+
+ const output = (await transcriber(channelWaveform, {
+ return_timestamps: 'word',
+ chunk_length_s: chunkLengthInSeconds,
+ stride_length_s: strideLengthInSeconds,
+ ...(language === 'auto' ? {} : {language}),
+ })) as TransformersJsTranscription;
+
+ if (!output.chunks) {
+ throw new Error(
+ 'The model did not return word-level timestamps. Use one of the timestamped models returned by getAvailableModels().',
+ );
+ }
+
+ const audioDuration = channelWaveform.length / 16_000;
+ const words = output.chunks.map((chunk, index): WhisperWebGpuWord => {
+ const start = chunk.timestamp[0];
+ const end = Math.min(chunk.timestamp[1] ?? audioDuration, audioDuration);
+ if (start === null || !Number.isFinite(start) || !Number.isFinite(end)) {
+ throw new Error(
+ `The model returned an invalid timestamp for "${chunk.text}".`,
+ );
+ }
+
+ if (end < start) {
+ throw new Error(
+ `The model returned a backwards timestamp for "${chunk.text}".`,
+ );
+ }
+
+ return {
+ text: index === 0 ? chunk.text.trimStart() : chunk.text,
+ startInSeconds: start,
+ endInSeconds: end,
+ };
+ });
+
+ return {
+ text: output.text.trimStart(),
+ words,
+ model,
+ backend: resolvedBackend,
+ };
+};
diff --git a/packages/whisper-webgpu/tsconfig.json b/packages/whisper-webgpu/tsconfig.json
new file mode 100644
index 0000000000..446cea780b
--- /dev/null
+++ b/packages/whisper-webgpu/tsconfig.json
@@ -0,0 +1,15 @@
+{
+ "extends": "../tsconfig.settings.json",
+ "compilerOptions": {
+ "rootDir": "src",
+ "outDir": "dist",
+ "skipLibCheck": true,
+ "target": "ES2022",
+ "module": "es2020",
+ "moduleResolution": "bundler",
+ "declaration": true,
+ "emitDeclarationOnly": true
+ },
+ "include": ["src"],
+ "references": [{"path": "../captions"}]
+}
diff --git a/tsconfig.json b/tsconfig.json
index de3eea01a2..bcf5580d63 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -279,6 +279,9 @@
{
"path": "./packages/light-leaks"
},
+ {
+ "path": "./packages/whisper-webgpu"
+ },
{
"path": "./packages/maptiler"
},