🎥 Make videos programmatically with React
Add timestamped WebGPU transcription
Jonny Burgerecd8d66c1fd2
diff --git a/bun.lock b/bun.lockindex 1a7525224b..6cbf17276c 100644--- a/bun.lock+++ b/bun.lock@@ -582,6 +582,7 @@"@remotion/web-renderer": "workspace:*","@remotion/webcodecs": "workspace:*","@remotion/whisper-web": "workspace:*",+ "@remotion/whisper-webgpu": "workspace:*","@remotion/zod-types": "workspace:*","@remotion/zod-types-v3": "workspace:*","@rive-app/canvas-advanced": "2.31.5",@@ -827,6 +828,7 @@"@remotion/transitions": "workspace:*","@remotion/webcodecs": "workspace:*","@remotion/whisper-web": "workspace:*",+ "@remotion/whisper-webgpu": "workspace:*","@remotion/zod-types": "workspace:*","@remotion/zod-types-v3": "workspace:*","@shopify/react-native-skia": "2.0.0",@@ -2821,6 +2823,19 @@"eslint": "catalog:",},},+ "packages/whisper-webgpu": {+ "name": "@remotion/whisper-webgpu",+ "version": "4.0.516",+ "dependencies": {+ "@huggingface/transformers": "catalog:",+ "@remotion/captions": "workspace:*",+ },+ "devDependencies": {+ "@remotion/eslint-config-internal": "workspace:*",+ "@typescript/native-preview": "catalog:",+ "eslint": "catalog:",+ },+ },"packages/zod-types": {"name": "@remotion/zod-types","version": "4.0.516",@@ -2873,6 +2888,7 @@"@aws-sdk/lib-storage": "3.986.0","@aws-sdk/middleware-flexible-checksums": "3.972.5","@aws-sdk/s3-request-presigner": "3.986.0",+ "@huggingface/transformers": "4.2.0","@mediabunny/aac-encoder": "1.55.1","@mediabunny/ac3": "1.55.1","@mediabunny/dts": "1.55.1",@@ -3673,6 +3689,12 @@"@hono/node-server": ["@hono/[email protected]", "", { "peerDependencies": { "hono": "^4" } }, "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw=="],+ "@huggingface/jinja": ["@huggingface/[email protected]", "", {}, "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw=="],++ "@huggingface/tokenizers": ["@huggingface/[email protected]", "", {}, "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA=="],++ "@huggingface/transformers": ["@huggingface/[email protected]", "", { "dependencies": { "@huggingface/jinja": "^0.5.6", "@huggingface/tokenizers": "^0.1.3", "onnxruntime-node": "1.24.3", "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", "sharp": "^0.34.5" } }, "sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ=="],+"@humanfs/core": ["@humanfs/[email protected]", "", {}, "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA=="],"@humanfs/node": ["@humanfs/[email protected]", "", { "dependencies": { "@humanfs/core": "0.19.1", "@humanwhocodes/retry": "0.3.1" } }, "sha512-YuI2ZHQL78Q5HbhDiBA1X4LmYdXCKCMQIfw0pw7piHJwyREFebJUvrQN4cMssyES6x+vfUbx1CIpaQUKYdQZOw=="],@@ -4561,6 +4583,8 @@"@remotion/whisper-web": ["@remotion/whisper-web@workspace:packages/whisper-web"],+ "@remotion/whisper-webgpu": ["@remotion/whisper-webgpu@workspace:packages/whisper-webgpu"],+"@remotion/zod-types": ["@remotion/zod-types@workspace:packages/zod-types"],"@remotion/zod-types-v3": ["@remotion/zod-types-v3@workspace:packages/zod-types-v3"],@@ -6499,6 +6523,8 @@"flat-cache": ["[email protected]", "", { "dependencies": { "flatted": "3.3.1", "keyv": "4.5.4" } }, "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw=="],+ "flatbuffers": ["[email protected]", "", {}, "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ=="],+"flatted": ["[email protected]", "", {}, "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw=="],"flattie": ["[email protected]", "", {}, "sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ=="],@@ -6655,6 +6681,8 @@"gtoken": ["[email protected]", "", { "dependencies": { "gaxios": "5.1.0", "google-p12-pem": "4.0.1", "jws": "4.0.0" } }, "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ=="],+ "guid-typescript": ["[email protected]", "", {}, "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ=="],+"gunzip-maybe": ["[email protected]", "", { "dependencies": { "browserify-zlib": "0.1.4", "is-deflate": "1.0.0", "is-gzip": "1.0.0", "peek-stream": "1.1.3", "pumpify": "1.5.1", "through2": "2.0.5" }, "bin": { "gunzip-maybe": "bin.js" } }, "sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw=="],"gzip-size": ["[email protected]", "", { "dependencies": { "duplexer": "0.1.2" } }, "sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q=="],@@ -7535,6 +7563,12 @@"oniguruma-to-es": ["[email protected]", "", { "dependencies": { "oniguruma-parser": "^0.12.1", "regex": "^6.0.1", "regex-recursion": "^6.0.2" } }, "sha512-3VhUGN3w2eYxnTzHn+ikMI+fp/96KoRSVK9/kMTcFqj1NRDh2IhQCKvYxDnWePKRXY/AqH+Fuiyb7VHSzBjHfA=="],+ "onnxruntime-common": ["[email protected]", "", {}, "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA=="],++ "onnxruntime-node": ["[email protected]", "", { "dependencies": { "adm-zip": "^0.5.16", "global-agent": "^3.0.0", "onnxruntime-common": "1.24.3" }, "os": [ "linux", "win32", "darwin", ] }, "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg=="],++ "onnxruntime-web": ["[email protected]", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw=="],+"open": ["[email protected]", "", { "dependencies": { "define-lazy-prop": "2.0.0", "is-docker": "2.2.1", "is-wsl": "2.2.0" } }, "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ=="],"openai": ["[email protected]", "", { "dependencies": { "@types/node": "18.14.6", "@types/node-fetch": "2.6.11", "abort-controller": "3.0.0", "agentkeepalive": "4.5.0", "form-data-encoder": "1.7.2", "formdata-node": "4.4.1", "node-fetch": "2.6.9" }, "peerDependencies": { "zod": "3.22.3" }, "optionalPeers": ["zod"], "bin": { "openai": "bin/cli" } }, "sha512-2YbRFy6qaYRJabK2zLMn4txrB2xBy0KP5g/eoqeSPTT31mIJMnkT75toagvfE555IKa2RdrzJrZwdDsUipsAMw=="],@@ -7689,6 +7723,8 @@"pkg-types": ["[email protected]", "", { "dependencies": { "confbox": "^0.2.4", "exsolve": "^1.0.8", "pathe": "^2.0.3" } }, "sha512-y+ichcgc2LrADuhLNAx8DFjVfgz91pRxfZdI3UDhxHvcVEZsenLO+7XaU5vOp0u/7V/wZ+plyuQxtrDlZJ+yeg=="],+ "platform": ["[email protected]", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="],+"playwright": ["[email protected]", "", { "dependencies": { "playwright-core": "1.55.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-cJW4Xd/G3v5ovXtJJ52MAOclqeac9S/aGGgRzLabuF8TnIb6xHvMzKIa6JmrRzUkeXJgfL1MhukP0NK6l39h3A=="],"playwright-core": ["[email protected]", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-Z6Mh9mkwX+zxSlHqdr5AOcJnfp+xUWLCt9uKV18fhzA8eyxUd8NUWzAjxUh55RZKSYwDGX0cfaySdhZJGMoJ+w=="],@@ -12059,6 +12095,10 @@"ofetch/ufo": ["[email protected]", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="],+ "onnxruntime-node/adm-zip": ["[email protected]", "", {}, "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng=="],++ "onnxruntime-web/onnxruntime-common": ["[email protected]", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="],+"openai/@types/node": ["@types/[email protected]", "", {}, "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA=="],"ora/chalk": ["[email protected]", "", {}, "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA=="],diff --git a/package.json b/package.jsonindex 554dd5cac7..78b151194f 100644--- a/package.json+++ b/package.json@@ -109,6 +109,7 @@"!packages/bugs"],"catalog": {+ "@huggingface/transformers": "4.2.0","@aws-sdk/s3-request-presigner": "3.986.0","@aws-sdk/credential-provider-ini": "3.972.5","@aws-sdk/client-s3": "3.986.0",diff --git a/packages/cli/src/list-of-remotion-packages.ts b/packages/cli/src/list-of-remotion-packages.tsindex 9913a6e553..194410863b 100644--- a/packages/cli/src/list-of-remotion-packages.ts+++ b/packages/cli/src/list-of-remotion-packages.ts@@ -92,6 +92,7 @@ export const listOfRemotionPackages = ['@remotion/compositor','@remotion/example-videos','@remotion/whisper-web',+ '@remotion/whisper-webgpu','@remotion/media','@remotion/remotion-media','@remotion/web-renderer',diff --git a/packages/create-video/src/list-of-remotion-packages.ts b/packages/create-video/src/list-of-remotion-packages.tsindex 9913a6e553..194410863b 100644--- a/packages/create-video/src/list-of-remotion-packages.ts+++ b/packages/create-video/src/list-of-remotion-packages.ts@@ -92,6 +92,7 @@ export const listOfRemotionPackages = ['@remotion/compositor','@remotion/example-videos','@remotion/whisper-web',+ '@remotion/whisper-webgpu','@remotion/media','@remotion/remotion-media','@remotion/web-renderer',diff --git a/packages/docs/components/TableOfContents/api.tsx b/packages/docs/components/TableOfContents/api.tsxindex 4de30ed848..55d4892cb9 100644--- a/packages/docs/components/TableOfContents/api.tsx+++ b/packages/docs/components/TableOfContents/api.tsx@@ -42,6 +42,7 @@ import {TableOfContents as ThreeTableOfContents} from '../../docs/three/TableOfCimport {TableOfContents as TransitionsTableOfContents} from '../../docs/transitions/table-of-contents';import {TableOfContents as VercelTableOfContents} from '../../docs/vercel/table-of-contents';import {TableOfContents as WebcodecsTableOfContents} from '../../docs/webcodecs/TableOfContents';+import {TableOfContents as WhisperWebGpuTableOfContents} from '../../docs/whisper-webgpu/table-of-contents';import {ZodTypesTableOfContents} from '../../docs/zod-types/TableOfContents';import {Grid} from './Grid';import {TOCItem} from './TOCItem';@@ -198,6 +199,9 @@ export const TableOfContents: React.FC = () => {<h2>@remotion/webcodecs</h2><p>Converting media using WebCodecs</p><WebcodecsTableOfContents />+ <h2>@remotion/whisper-webgpu</h2>+ <p>Transcribe audio in the browser using Whisper and WebGPU</p>+ <WhisperWebGpuTableOfContents /><h2>@remotion/zod-types</h2><p>Zod types enabling Remotion Studio UI</p><ZodTypesTableOfContents />diff --git a/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdxnew file mode 100644index 0000000000..10d4caa7c3--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx@@ -0,0 +1,40 @@+---+image: /generated/articles-docs-whisper-webgpu-can-use-whisper-webgpu.png+title: canUseWhisperWebGpu()+crumb: '@remotion/whisper-webgpu'+---++# canUseWhisperWebGpu()<AvailableFrom v="4.0.517" />++Checks whether WebGPU or the WASM fallback can run in the current browser.++```ts twoslash title="support.ts"+import {canUseWhisperWebGpu} from '@remotion/whisper-webgpu';++const result = await canUseWhisperWebGpu();+if (!result.supported) {+ throw new Error(result.detailedReason);+}++console.log(result.backend, result.wasmThreads);+```++## Options++### backend?++`'auto'`, `'webgpu'`, or `'wasm'`. Default: `'auto'`, which selects WebGPU when available and otherwise uses WASM.++Requesting `'webgpu'` reports an unsupported result when WebGPU is unavailable or the page is not a secure context.++## Cross-origin isolation++Cross-origin isolation is not required. It only changes the WASM thread count: isolated pages may use multiple threads, while other pages use one.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)diff --git a/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdxnew file mode 100644index 0000000000..f3d252b99b--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx@@ -0,0 +1,33 @@+---+image: /generated/articles-docs-whisper-webgpu-dispose-whisper-model.png+title: disposeWhisperModel()+crumb: '@remotion/whisper-webgpu'+---++# disposeWhisperModel()<AvailableFrom v="4.0.517" />++Releases the memory used by initialized Whisper pipelines. Persistent browser-cached model files are not deleted.++```ts twoslash title="dispose.ts"+import {disposeWhisperModel} from '@remotion/whisper-webgpu';++await disposeWhisperModel({model: 'small.en'});+```++## Options++### model?++Only dispose this model. Default: all loaded models.++### backend?++Only dispose the `'webgpu'` or `'wasm'` instance. Default: both backends.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)diff --git a/packages/docs/docs/whisper-webgpu/get-available-models.mdx b/packages/docs/docs/whisper-webgpu/get-available-models.mdxnew file mode 100644index 0000000000..1163b015f2--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/get-available-models.mdx@@ -0,0 +1,31 @@+---+image: /generated/articles-docs-whisper-webgpu-get-available-models.png+title: getAvailableModels()+crumb: '@remotion/whisper-webgpu'+---++# getAvailableModels()<AvailableFrom v="4.0.517" />++Returns the timestamped Whisper models supported by [`transcribe()`](/docs/whisper-webgpu/transcribe).++```ts twoslash title="models.ts"+import {getAvailableModels} from '@remotion/whisper-webgpu';++const models = getAvailableModels();+console.log(models);+```++Each item contains the short `name`, Hugging Face `modelId`, parameter count, whether the model is multilingual, and estimated downloads for the default WebGPU and WASM configurations.++Available sizes are `tiny`, `base`, `small`, `medium`, and `large-v3-turbo`. English-only `.en` variants are available through `medium`.++The estimates cover model weights. Browser caches and supporting model files may add a small amount.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`transcribe()`](/docs/whisper-webgpu/transcribe)+- [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model)diff --git a/packages/docs/docs/whisper-webgpu/index.mdx b/packages/docs/docs/whisper-webgpu/index.mdxnew file mode 100644index 0000000000..6828e09f08--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/index.mdx@@ -0,0 +1,59 @@+---+image: /generated/articles-docs-whisper-webgpu-index.png+title: '@remotion/whisper-webgpu'+crumb: 'Transcribe audio in browser'+---++# @remotion/whisper-webgpu<AvailableFrom v="4.0.517" />++<ExperimentalBadge>+<p>This package is experimental.</p>+</ExperimentalBadge>++Transcribe audio locally in the browser using timestamped Whisper models from Transformers.js. WebGPU is used when available and single-threaded WASM is the fallback.++Unlike [`@remotion/whisper-web`](/docs/whisper-web), cross-origin isolation is not required.++import {TableOfContents} from './table-of-contents';++## Installation++<Installation pkg="@remotion/whisper-webgpu" />++## Example++```ts twoslash title="transcribe.ts"+import {+ resampleTo16Khz,+ toCaptions,+ transcribe,+} from '@remotion/whisper-webgpu';++const file = new File([], 'audio.wav');+const channelWaveform = await resampleTo16Khz({file});++const transcription = await transcribe({+ channelWaveform,+ model: 'small.en',+ language: 'en',+});++const {captions} = toCaptions({whisperWebGpuOutput: transcription});+console.log(captions);+```++The model is downloaded on the first call and cached by the browser. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) first to display model download progress.++## APIs++<TableOfContents />++## Requirements++WebGPU requires HTTPS in production or `localhost` during development. Browsers without WebGPU fall back to WASM.++Cross-origin isolation enables multithreaded WASM but is not required. Without it, the WASM fallback uses one thread.++## License++MITdiff --git a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdxnew file mode 100644index 0000000000..4f9b56eef0--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx@@ -0,0 +1,51 @@+---+image: /generated/articles-docs-whisper-webgpu-load-whisper-model.png+title: loadWhisperModel()+crumb: '@remotion/whisper-webgpu'+---++# loadWhisperModel()<AvailableFrom v="4.0.517" />++Downloads and initializes a timestamped Whisper model. Calling it before [`transcribe()`](/docs/whisper-webgpu/transcribe) lets you show model-loading progress.++```ts twoslash title="load-model.ts"+import {loadWhisperModel} from '@remotion/whisper-webgpu';++const result = await loadWhisperModel({+ model: 'small.en',+ onProgress: ({progress, file}) => {+ console.log(file, progress);+ },+});++console.log(result.backend, result.alreadyLoaded);+```++## Options++### model++One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models).++### backend?++`'auto'`, `'webgpu'`, or `'wasm'`. Default: `'auto'`.++### onProgress?++Called while model files are resolved and downloaded. `progress` is between `0` and `1` when available.++## Return value++Returns the selected `backend` and whether the same model pipeline was already loaded in memory.++The browser manages its persistent Transformers.js cache. This function also retains the initialized model in memory until [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model) is called or the page closes.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model)+- [`transcribe()`](/docs/whisper-webgpu/transcribe)diff --git a/packages/docs/docs/whisper-webgpu/resample-to-16khz.mdx b/packages/docs/docs/whisper-webgpu/resample-to-16khz.mdxnew file mode 100644index 0000000000..250200287b--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/resample-to-16khz.mdx@@ -0,0 +1,39 @@+---+image: /generated/articles-docs-whisper-webgpu-resample-to-16khz.png+title: resampleTo16Khz()+crumb: '@remotion/whisper-webgpu'+---++# resampleTo16Khz()<AvailableFrom v="4.0.517" />++Decodes an audio `Blob`, mixes it to mono, and resamples it to the 16kHz waveform expected by [`transcribe()`](/docs/whisper-webgpu/transcribe).++```ts twoslash title="resample.ts"+import {resampleTo16Khz} from '@remotion/whisper-webgpu';++const file = new File([], 'audio.wav');+const waveform = await resampleTo16Khz({+ file,+ onProgress: (progress) => console.log(progress),+});++console.log(waveform);+```++## Options++### file++An audio `Blob` or `File` supported by the browser's Web Audio decoder.++### onProgress?++Called with values between `0` and `1` while the file is read, decoded, and resampled.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`transcribe()`](/docs/whisper-webgpu/transcribe)diff --git a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx b/packages/docs/docs/whisper-webgpu/table-of-contents.tsxnew file mode 100644index 0000000000..52664f7b21--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx@@ -0,0 +1,38 @@+import React from 'react';+import {Grid} from '../../components/TableOfContents/Grid';+import {TOCItem} from '../../components/TableOfContents/TOCItem';++export const TableOfContents: React.FC = () => {+ return (+ <Grid>+ <TOCItem link="/docs/whisper-webgpu/can-use-whisper-webgpu">+ <strong>canUseWhisperWebGpu()</strong>+ <div>Check which inference backend is available</div>+ </TOCItem>+ <TOCItem link="/docs/whisper-webgpu/get-available-models">+ <strong>getAvailableModels()</strong>+ <div>List timestamped models and their download sizes</div>+ </TOCItem>+ <TOCItem link="/docs/whisper-webgpu/load-whisper-model">+ <strong>loadWhisperModel()</strong>+ <div>Download and initialize a model</div>+ </TOCItem>+ <TOCItem link="/docs/whisper-webgpu/transcribe">+ <strong>transcribe()</strong>+ <div>Transcribe a waveform with word-level timestamps</div>+ </TOCItem>+ <TOCItem link="/docs/whisper-webgpu/to-captions">+ <strong>toCaptions()</strong>+ <div>Convert a transcription to Remotion captions</div>+ </TOCItem>+ <TOCItem link="/docs/whisper-webgpu/resample-to-16khz">+ <strong>resampleTo16Khz()</strong>+ <div>Decode and resample browser audio</div>+ </TOCItem>+ <TOCItem link="/docs/whisper-webgpu/dispose-whisper-model">+ <strong>disposeWhisperModel()</strong>+ <div>Release model memory</div>+ </TOCItem>+ </Grid>+ );+};diff --git a/packages/docs/docs/whisper-webgpu/to-captions.mdx b/packages/docs/docs/whisper-webgpu/to-captions.mdxnew file mode 100644index 0000000000..fe6d9f0cba--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/to-captions.mdx@@ -0,0 +1,39 @@+---+image: /generated/articles-docs-whisper-webgpu-to-captions.png+title: toCaptions()+crumb: '@remotion/whisper-webgpu'+---++# toCaptions()<AvailableFrom v="4.0.517" />++Converts [`transcribe()`](/docs/whisper-webgpu/transcribe) output into [`Caption`](/docs/captions/caption) objects from `@remotion/captions`.++```ts twoslash title="captions.ts"+import {toCaptions, transcribe} from '@remotion/whisper-webgpu';++const channelWaveform = new Float32Array(16_000);+const transcription = await transcribe({+ channelWaveform,+ model: 'tiny.en',+});++const {captions} = toCaptions({whisperWebGpuOutput: transcription});+console.log(captions);+```++`startMs` and `endMs` use the model's word boundaries. `timestampMs` is the midpoint of each word. `confidence` is `null` because Transformers.js does not expose calibrated word confidence values for this pipeline.++## Options++### whisperWebGpuOutput++A complete transcription or an array of timestamped words returned by `transcribe()`.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering={false} serverSideRendering player studio />++## See also++- [`@remotion/captions`](/docs/captions)+- [`transcribe()`](/docs/whisper-webgpu/transcribe)diff --git a/packages/docs/docs/whisper-webgpu/transcribe.mdx b/packages/docs/docs/whisper-webgpu/transcribe.mdxnew file mode 100644index 0000000000..33f899d50a--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/transcribe.mdx@@ -0,0 +1,69 @@+---+image: /generated/articles-docs-whisper-webgpu-transcribe.png+title: transcribe()+crumb: '@remotion/whisper-webgpu'+---++# transcribe()<AvailableFrom v="4.0.517" />++Transcribes a 16kHz mono waveform and returns words with start and end timestamps.++```ts twoslash title="transcribe.ts"+import {resampleTo16Khz, transcribe} from '@remotion/whisper-webgpu';++const file = new File([], 'audio.wav');+const channelWaveform = await resampleTo16Khz({file});++const result = await transcribe({+ channelWaveform,+ model: 'small.en',+ language: 'en',+});++console.log(result.text, result.words, result.backend);+```++## Options++### channelWaveform++A mono `Float32Array` sampled at 16kHz. Use [`resampleTo16Khz()`](/docs/whisper-webgpu/resample-to-16khz) for browser audio files.++### model++One of the timestamped models returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models). `small` or `small.en` is the recommended default.++### backend?++`'auto'`, `'webgpu'`, or `'wasm'`. Default: `'auto'`.++### language?++The spoken language name or language code. Default: `'auto'`. Set it when known to reduce language-detection errors.++### chunkLengthInSeconds?++Length of long-audio chunks. Default: `30`.++### strideLengthInSeconds?++Overlap on both sides of a chunk. Default: `5`. It must be less than half of `chunkLengthInSeconds`.++### onModelLoadProgress?++Called while the model is loaded. To load independently, use [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model).++## Return value++Returns the full `text`, timestamped `words`, selected `model`, and resolved `backend`.++Word-level timestamps are returned after transcription completes. Transformers.js does not currently stream word-aligned updates while inference is running.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />++## See also++- [`toCaptions()`](/docs/whisper-webgpu/to-captions)+- [`getAvailableModels()`](/docs/whisper-webgpu/get-available-models)diff --git a/packages/docs/package.json b/packages/docs/package.jsonindex f6c3771919..e11d55e9c7 100644--- a/packages/docs/package.json+++ b/packages/docs/package.json@@ -99,6 +99,7 @@"@remotion/web-renderer": "workspace:*","@remotion/webcodecs": "workspace:*","@remotion/whisper-web": "workspace:*",+ "@remotion/whisper-webgpu": "workspace:*","@remotion/zod-types": "workspace:*","@remotion/zod-types-v3": "workspace:*","@rive-app/canvas-advanced": "2.31.5",diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.tsindex 99b5c30d52..836afb476f 100644--- a/packages/docs/sidebars.ts+++ b/packages/docs/sidebars.ts@@ -979,6 +979,23 @@ const sidebars: SidebarsConfig = {'whisper-web/to-captions',],},+ {+ type: 'category',+ label: '@remotion/whisper-webgpu',+ link: {+ type: 'doc',+ id: 'whisper-webgpu/index',+ },+ items: [+ 'whisper-webgpu/can-use-whisper-webgpu',+ 'whisper-webgpu/get-available-models',+ 'whisper-webgpu/load-whisper-model',+ 'whisper-webgpu/transcribe',+ 'whisper-webgpu/to-captions',+ 'whisper-webgpu/resample-to-16khz',+ 'whisper-webgpu/dispose-whisper-model',+ ],+ },{type: 'category',label: '@remotion/zod-types',diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.tsindex fafe845400..43ca5f8692 100644--- a/packages/docs/src/data/articles.ts+++ b/packages/docs/src/data/articles.ts@@ -8975,6 +8975,78 @@ export const articles = [noAi: false,slug: 'whisper-web/transcribe',},+ {+ id: 'whisper-webgpu/can-use-whisper-webgpu',+ title: 'canUseWhisperWebGpu()',+ relativePath: 'docs/whisper-webgpu/can-use-whisper-webgpu.mdx',+ compId: 'articles-docs-whisper-webgpu-can-use-whisper-webgpu',+ crumb: '@remotion/whisper-webgpu',+ noAi: false,+ slug: 'whisper-webgpu/can-use-whisper-webgpu',+ },+ {+ id: 'whisper-webgpu/dispose-whisper-model',+ title: 'disposeWhisperModel()',+ relativePath: 'docs/whisper-webgpu/dispose-whisper-model.mdx',+ compId: 'articles-docs-whisper-webgpu-dispose-whisper-model',+ crumb: '@remotion/whisper-webgpu',+ noAi: false,+ slug: 'whisper-webgpu/dispose-whisper-model',+ },+ {+ id: 'whisper-webgpu/get-available-models',+ title: 'getAvailableModels()',+ relativePath: 'docs/whisper-webgpu/get-available-models.mdx',+ compId: 'articles-docs-whisper-webgpu-get-available-models',+ crumb: '@remotion/whisper-webgpu',+ noAi: false,+ slug: 'whisper-webgpu/get-available-models',+ },+ {+ id: 'whisper-webgpu/index',+ title: '@remotion/whisper-webgpu',+ relativePath: 'docs/whisper-webgpu/index.mdx',+ compId: 'articles-docs-whisper-webgpu-index',+ crumb: 'Transcribe audio in browser',+ noAi: false,+ slug: 'whisper-webgpu/index',+ },+ {+ id: 'whisper-webgpu/load-whisper-model',+ title: 'loadWhisperModel()',+ relativePath: 'docs/whisper-webgpu/load-whisper-model.mdx',+ compId: 'articles-docs-whisper-webgpu-load-whisper-model',+ crumb: '@remotion/whisper-webgpu',+ noAi: false,+ slug: 'whisper-webgpu/load-whisper-model',+ },+ {+ id: 'whisper-webgpu/resample-to-16khz',+ title: 'resampleTo16Khz()',+ relativePath: 'docs/whisper-webgpu/resample-to-16khz.mdx',+ compId: 'articles-docs-whisper-webgpu-resample-to-16khz',+ crumb: '@remotion/whisper-webgpu',+ noAi: false,+ slug: 'whisper-webgpu/resample-to-16khz',+ },+ {+ id: 'whisper-webgpu/to-captions',+ title: 'toCaptions()',+ relativePath: 'docs/whisper-webgpu/to-captions.mdx',+ compId: 'articles-docs-whisper-webgpu-to-captions',+ crumb: '@remotion/whisper-webgpu',+ noAi: false,+ slug: 'whisper-webgpu/to-captions',+ },+ {+ id: 'whisper-webgpu/transcribe',+ title: 'transcribe()',+ relativePath: 'docs/whisper-webgpu/transcribe.mdx',+ compId: 'articles-docs-whisper-webgpu-transcribe',+ crumb: '@remotion/whisper-webgpu',+ noAi: false,+ slug: 'whisper-webgpu/transcribe',+ },{id: 'wrong-composition-mount',title: 'Wrongly mounted <Composition>',diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-can-use-whisper-webgpu.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-can-use-whisper-webgpu.pngnew file mode 100644index 0000000000..3fb19eeb8aBinary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-can-use-whisper-webgpu.png differdiff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-dispose-whisper-model.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-dispose-whisper-model.pngnew file mode 100644index 0000000000..fcc6be76aeBinary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-dispose-whisper-model.png differdiff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-get-available-models.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-get-available-models.pngnew file mode 100644index 0000000000..c32e096dd8Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-get-available-models.png differdiff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-index.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-index.pngnew file mode 100644index 0000000000..b6d6e8b3eaBinary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-index.png differdiff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-load-whisper-model.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-load-whisper-model.pngnew file mode 100644index 0000000000..d1537682c2Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-load-whisper-model.png differdiff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-resample-to-16khz.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-resample-to-16khz.pngnew file mode 100644index 0000000000..5dc4392bafBinary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-resample-to-16khz.png differdiff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-to-captions.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-to-captions.pngnew file mode 100644index 0000000000..1667ed1c95Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-to-captions.png differdiff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-transcribe.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-transcribe.pngnew file mode 100644index 0000000000..46287b498aBinary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-transcribe.png differdiff --git a/packages/example/package.json b/packages/example/package.jsonindex d78c6235b2..47aee9a7cd 100644--- a/packages/example/package.json+++ b/packages/example/package.json@@ -84,6 +84,7 @@"@remotion/transitions": "workspace:*","@remotion/webcodecs": "workspace:*","@remotion/whisper-web": "workspace:*",+ "@remotion/whisper-webgpu": "workspace:*","@remotion/zod-types": "workspace:*","@remotion/zod-types-v3": "workspace:*","@shopify/react-native-skia": "2.0.0",diff --git a/packages/example/src/Root.tsx b/packages/example/src/Root.tsxindex 0c59524046..79622fe077 100644--- a/packages/example/src/Root.tsx+++ b/packages/example/src/Root.tsx@@ -323,6 +323,7 @@ import {SvgPaintSchema} from './VisualModeTests/SvgPaintSchema';import {VideoConfigExpressions} from './VisualModeTests/VideoConfigExpressions';import {VoiceVisualization} from './voice-visualization';import {WhisperWeb} from './WhisperWeb';+import {WhisperWebGpu} from './WhisperWebGpu';class Vector2 {readonly x: number;@@ -2702,6 +2703,12 @@ export const Index: React.FC = () => {/></Folder><Still id="whisper-web" component={WhisperWeb} width={800} height={800} />+ <Still+ id="whisper-webgpu"+ component={WhisperWebGpu}+ width={900}+ height={900}+ /><Compositionid="empty"component={Empty}diff --git a/packages/example/src/WhisperWebGpu/index.tsx b/packages/example/src/WhisperWebGpu/index.tsxnew file mode 100644index 0000000000..e03fa3cb71--- /dev/null+++ b/packages/example/src/WhisperWebGpu/index.tsx@@ -0,0 +1,142 @@+import {+ canUseWhisperWebGpu,+ getAvailableModels,+ resampleTo16Khz,+ toCaptions,+ transcribe,+ type WhisperWebGpuModel,+} from '@remotion/whisper-webgpu';+import {useCallback, useMemo, useState} from 'react';++const audioFileUrl = 'https://remotion.media/16khz.wav';++const formatMegabytes = (bytes: number) => {+ return `${Math.round(bytes / 1_000_000)} MB`;+};++export const WhisperWebGpu = () => {+ const models = useMemo(() => getAvailableModels(), []);+ const [selectedModel, setSelectedModel] =+ useState<WhisperWebGpuModel>('small.en');+ const [status, setStatus] = useState('Ready to transcribe the sample audio.');+ const [result, setResult] = useState('');+ const [isTranscribing, setIsTranscribing] = useState(false);+ const selectedModelInfo = models.find(+ (model) => model.name === selectedModel,+ );++ const onTranscribe = useCallback(async () => {+ setIsTranscribing(true);+ setResult('');++ try {+ const support = canUseWhisperWebGpu({backend: 'auto'});+ if (!support.supported) {+ throw new Error(support.detailedReason);+ }++ setStatus(`Fetching audio for the ${support.backend} backend...`);+ const response = await fetch(audioFileUrl);+ if (!response.ok) {+ throw new Error(`Could not fetch audio: HTTP ${response.status}`);+ }++ setStatus('Decoding and resampling audio...');+ const channelWaveform = await resampleTo16Khz({+ file: await response.blob(),+ onProgress: (progress) => {+ setStatus(`Resampling audio... ${Math.round(progress * 100)}%`);+ },+ });++ setStatus(`Loading ${selectedModel} and transcribing...`);+ const transcription = await transcribe({+ channelWaveform,+ model: selectedModel,+ onModelLoadProgress: (progress) => {+ if (progress.progress !== null) {+ setStatus(+ `Loading ${selectedModel}... ${Math.round(progress.progress * 100)}%`,+ );+ }+ },+ });+ const {captions} = toCaptions({whisperWebGpuOutput: transcription});+ setStatus(+ `Done using ${transcription.backend}: ${captions.length} timestamped words.`,+ );+ setResult(+ captions+ .map(+ (caption) =>+ `[${(caption.startMs / 1000).toFixed(2)}–${(caption.endMs / 1000).toFixed(2)}]${caption.text}`,+ )+ .join('\n'),+ );+ } catch (error) {+ setStatus(error instanceof Error ? error.message : String(error));+ } finally {+ setIsTranscribing(false);+ }+ }, [selectedModel]);++ return (+ <div className="flex min-h-screen items-center justify-center bg-slate-950 p-8 text-slate-900">+ <div className="w-full max-w-2xl space-y-5 rounded-xl bg-white p-7 shadow-2xl">+ <div>+ <h1 className="text-2xl font-semibold">Whisper WebGPU</h1>+ <p className="mt-1 text-sm text-slate-600">+ Timestamped Transformers.js transcription for Remotion captions.+ </p>+ </div>++ <label className="block text-sm font-medium" htmlFor="webgpu-model">+ Model+ <select+ className="mt-2 block w-full rounded-md border border-slate-300 p-2"+ disabled={isTranscribing}+ id="webgpu-model"+ onChange={(event) => {+ setSelectedModel(event.target.value as WhisperWebGpuModel);+ }}+ value={selectedModel}+ >+ {models.map((model) => (+ <option key={model.name} value={model.name}>+ {model.name} · {Math.round(model.parameters / 1_000_000)}M ·{' '}+ {formatMegabytes(model.webGpuDownloadSize)} WebGPU+ </option>+ ))}+ </select>+ </label>++ {selectedModelInfo ? (+ <p className="rounded-md bg-slate-100 p-3 text-sm text-slate-700">+ {selectedModelInfo.multilingual ? 'Multilingual' : 'English-only'} ·{' '}+ {formatMegabytes(selectedModelInfo.webGpuDownloadSize)} WebGPU+ download · {formatMegabytes(selectedModelInfo.wasmDownloadSize)}{' '}+ WASM download+ </p>+ ) : null}++ <button+ className="w-full rounded-md bg-indigo-600 px-4 py-3 font-semibold text-white disabled:opacity-50"+ disabled={isTranscribing}+ onClick={onTranscribe}+ type="button"+ >+ {isTranscribing ? 'Transcribing…' : 'Transcribe sample audio'}+ </button>++ <div className="rounded-md bg-blue-50 p-3 text-sm text-blue-900">+ {status}+ </div>+ {result ? (+ <pre className="max-h-80 overflow-auto whitespace-pre-wrap rounded-md bg-slate-950 p-4 text-xs text-slate-100">+ {result}+ </pre>+ ) : null}+ </div>+ </div>+ );+};diff --git a/packages/example/tsconfig.json b/packages/example/tsconfig.jsonindex 0e2830b518..23ced969fa 100644--- a/packages/example/tsconfig.json+++ b/packages/example/tsconfig.json@@ -42,6 +42,7 @@{"path": "../media-parser"},{"path": "../webcodecs"},{"path": "../whisper-web"},+ {"path": "../whisper-webgpu"},{"path": "../studio-protocol"},{"path": "../light-leaks"},{"path": "../mac-cursors"},diff --git a/packages/studio-shared/src/package-info.ts b/packages/studio-shared/src/package-info.tsindex eab6167430..d0a1d15106 100644--- a/packages/studio-shared/src/package-info.ts+++ b/packages/studio-shared/src/package-info.ts@@ -94,6 +94,7 @@ const allPackages = ['compositor','example-videos','whisper-web',+ 'whisper-webgpu','media','remotion-media','web-renderer',@@ -176,6 +177,7 @@ export const descriptions: {[key in Pkgs]: string | null} = {'studio-server': 'Run a Remotion Studio with a server backend','install-whisper-cpp': 'Helpers for installing and using Whisper.cpp','whisper-web': 'Helpers for using Whisper.cpp in browser using WASM',+ 'whisper-webgpu': 'Transcribe audio in the browser using Whisper and WebGPU','google-fonts': 'Use Google Fonts in Remotion',gsap: 'Use GSAP timelines in Remotion',mcp: "Remotion's Model Context Protocol",@@ -314,6 +316,7 @@ export const installableMap: {[key in Pkgs]: boolean} = {gsap: true,'install-whisper-cpp': true,'whisper-web': true,+ 'whisper-webgpu': true,'it-tests': false,'react18-tests': false,'lambda-go-example': false,@@ -413,6 +416,7 @@ export const apiDocs: {[key in Pkgs]: string | null} = {'studio-server': null,'install-whisper-cpp': 'https://www.remotion.dev/docs/install-whisper-cpp','whisper-web': 'https://www.remotion.dev/docs/whisper-web',+ 'whisper-webgpu': 'https://www.remotion.dev/docs/whisper-webgpu','google-fonts': 'https://www.remotion.dev/docs/google-fonts',gsap: 'https://www.remotion.dev/docs/gsap','media-utils': 'https://www.remotion.dev/docs/media-utils',diff --git a/packages/whisper-webgpu/.npmignore b/packages/whisper-webgpu/.npmignorenew file mode 100644index 0000000000..eb76a5ae81--- /dev/null+++ b/packages/whisper-webgpu/.npmignore@@ -0,0 +1,10 @@+dist/test/**+src+eslint.config.mjs+tsconfig.tsbuildinfo+*.tgz+.turbo+tsconfig.json+.prettierrc.js+bundle.ts+dist/cjs/testdiff --git a/packages/whisper-webgpu/README.md b/packages/whisper-webgpu/README.mdnew file mode 100644index 0000000000..ae068a1fe3--- /dev/null+++ b/packages/whisper-webgpu/README.md@@ -0,0 +1,9 @@+# @remotion/whisper-webgpu++Transcribe audio in the browser using timestamped Whisper models and WebGPU.++```bash+npx remotion add @remotion/whisper-webgpu+```++See the [documentation](https://www.remotion.dev/docs/whisper-webgpu).diff --git a/packages/whisper-webgpu/bundle.ts b/packages/whisper-webgpu/bundle.tsnew file mode 100644index 0000000000..a0a1a34b77--- /dev/null+++ b/packages/whisper-webgpu/bundle.ts@@ -0,0 +1,36 @@+import path from 'path';+import {build} from 'bun';++if (process.env.NODE_ENV !== 'production') {+ throw new Error('This script must be run using NODE_ENV=production');+}++console.time('Generated.');+const output = await build({+ entrypoints: ['src/index.ts'],+ naming: '[name].mjs',+ external: [+ '@huggingface/transformers',+ '@remotion/captions',+ 'remotion',+ 'remotion/no-react',+ 'react',+ 'react/jsx-runtime',+ 'react/jsx-dev-runtime',+ 'react-dom',+ ],+});++if (!output.success) {+ console.log(output.logs.join('\n'));+ process.exit(1);+}++for (const file of output.outputs) {+ const str = await file.text();+ const out = path.join('dist', 'esm', file.path);++ await Bun.write(out, str);+}++console.timeEnd('Generated.');diff --git a/packages/whisper-webgpu/eslint.config.mjs b/packages/whisper-webgpu/eslint.config.mjsnew file mode 100644index 0000000000..ce2fc683dd--- /dev/null+++ b/packages/whisper-webgpu/eslint.config.mjs@@ -0,0 +1,5 @@+import {remotionFlatConfig} from '@remotion/eslint-config-internal';++const config = remotionFlatConfig({react: false});++export default [config];diff --git a/packages/whisper-webgpu/package.json b/packages/whisper-webgpu/package.jsonnew file mode 100644index 0000000000..1de2bb4de7--- /dev/null+++ b/packages/whisper-webgpu/package.json@@ -0,0 +1,54 @@+{+ "name": "@remotion/whisper-webgpu",+ "version": "4.0.516",+ "description": "Transcribe audio in the browser using Whisper and WebGPU",+ "main": "dist/index.js",+ "types": "dist/index.d.ts",+ "module": "dist/esm/index.mjs",+ "repository": {+ "url": "https://github.com/remotion-dev/remotion/tree/main/packages/whisper-webgpu"+ },+ "type": "module",+ "scripts": {+ "test": "bun test src/test",+ "formatting": "oxfmt src --check",+ "format": "oxfmt src",+ "lint": "eslint src",+ "watch": "tsgo -w",+ "make": "tsgo && bun --env-file=../.env.bundle bundle.ts"+ },+ "author": "Jonny Burger <[email protected]>",+ "contributors": [],+ "license": "MIT",+ "bugs": {+ "url": "https://github.com/remotion-dev/remotion/issues"+ },+ "dependencies": {+ "@huggingface/transformers": "catalog:",+ "@remotion/captions": "workspace:*"+ },+ "exports": {+ ".": {+ "types": "./dist/index.d.ts",+ "module": "./dist/esm/index.mjs",+ "import": "./dist/esm/index.mjs"+ },+ "./package.json": "./package.json"+ },+ "devDependencies": {+ "@remotion/eslint-config-internal": "workspace:*",+ "@typescript/native-preview": "catalog:",+ "eslint": "catalog:"+ },+ "keywords": [+ "remotion",+ "whisper",+ "webgpu",+ "captions",+ "speech-to-text"+ ],+ "publishConfig": {+ "access": "public"+ },+ "homepage": "https://www.remotion.dev/docs/whisper-webgpu"+}diff --git a/packages/whisper-webgpu/src/backend.ts b/packages/whisper-webgpu/src/backend.tsnew file mode 100644index 0000000000..0e710405ee--- /dev/null+++ b/packages/whisper-webgpu/src/backend.ts@@ -0,0 +1,27 @@+export type WhisperWebGpuBackend = 'auto' | 'webgpu' | 'wasm';+export type ResolvedWhisperWebGpuBackend = Exclude<+ WhisperWebGpuBackend,+ 'auto'+>;++const hasWebGpu = () => {+ return (+ typeof navigator !== 'undefined' &&+ 'gpu' in navigator &&+ globalThis.isSecureContext !== false+ );+};++export const resolveBackend = (+ backend: WhisperWebGpuBackend,+): ResolvedWhisperWebGpuBackend => {+ if (backend === 'webgpu' && !hasWebGpu()) {+ throw new Error('WebGPU is not available in this browser.');+ }++ if (backend === 'auto') {+ return hasWebGpu() ? 'webgpu' : 'wasm';+ }++ return backend;+};diff --git a/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts b/packages/whisper-webgpu/src/can-use-whisper-webgpu.tsnew file mode 100644index 0000000000..2357ba9ec4--- /dev/null+++ b/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts@@ -0,0 +1,78 @@+import type {+ WhisperWebGpuBackend,+ ResolvedWhisperWebGpuBackend,+} from './backend';+import {resolveBackend} from './backend';++export enum WhisperWebGpuUnsupportedReason {+ WindowUndefined = 'window-undefined',+ WebAssemblyUnavailable = 'webassembly-unavailable',+ WebGpuUnavailable = 'webgpu-unavailable',+ WebGpuRequiresSecureContext = 'webgpu-requires-secure-context',+}++export type CanUseWhisperWebGpuResult =+ | {+ supported: true;+ backend: ResolvedWhisperWebGpuBackend;+ wasmThreads: number | null;+ }+ | {+ supported: false;+ reason: WhisperWebGpuUnsupportedReason;+ detailedReason: string;+ };++export type CanUseWhisperWebGpuOptions = {+ backend?: WhisperWebGpuBackend;+};++export const canUseWhisperWebGpu = ({+ backend = 'auto',+}: CanUseWhisperWebGpuOptions = {}): CanUseWhisperWebGpuResult => {+ if (typeof window === 'undefined') {+ return {+ supported: false,+ reason: WhisperWebGpuUnsupportedReason.WindowUndefined,+ detailedReason:+ '`window` is not defined. @remotion/whisper-webgpu is intended for browser environments.',+ };+ }++ if (backend === 'webgpu' && !('gpu' in navigator)) {+ return {+ supported: false,+ reason: WhisperWebGpuUnsupportedReason.WebGpuUnavailable,+ detailedReason: 'WebGPU is not available in this browser.',+ };+ }++ if (backend === 'webgpu' && !window.isSecureContext) {+ return {+ supported: false,+ reason: WhisperWebGpuUnsupportedReason.WebGpuRequiresSecureContext,+ detailedReason:+ 'WebGPU requires HTTPS in production or localhost during development.',+ };+ }++ const resolvedBackend = resolveBackend(backend);+ if (resolvedBackend === 'wasm' && typeof WebAssembly === 'undefined') {+ return {+ supported: false,+ reason: WhisperWebGpuUnsupportedReason.WebAssemblyUnavailable,+ detailedReason: 'WebAssembly is not available in this browser.',+ };+ }++ return {+ supported: true,+ backend: resolvedBackend,+ wasmThreads:+ resolvedBackend === 'wasm'+ ? window.crossOriginIsolated+ ? Math.max(1, navigator.hardwareConcurrency || 1)+ : 1+ : null,+ };+};diff --git a/packages/whisper-webgpu/src/index.ts b/packages/whisper-webgpu/src/index.tsnew file mode 100644index 0000000000..c39445dcfc--- /dev/null+++ b/packages/whisper-webgpu/src/index.ts@@ -0,0 +1,32 @@+export type {+ ResolvedWhisperWebGpuBackend,+ WhisperWebGpuBackend,+} from './backend';+export {+ canUseWhisperWebGpu,+ WhisperWebGpuUnsupportedReason,+} from './can-use-whisper-webgpu';+export type {+ CanUseWhisperWebGpuOptions,+ CanUseWhisperWebGpuResult,+} from './can-use-whisper-webgpu';+export {disposeWhisperModel, loadWhisperModel} from './load-whisper-model';+export type {+ DisposeWhisperModelOptions,+ LoadWhisperModelOptions,+ LoadWhisperModelResult,+ OnWhisperWebGpuModelLoadProgress,+ WhisperWebGpuModelLoadProgress,+} from './load-whisper-model';+export {getAvailableModels, WHISPER_WEBGPU_MODELS} from './models';+export type {WhisperWebGpuModel, WhisperWebGpuModelInfo} from './models';+export {resampleTo16Khz, WHISPER_WEBGPU_SAMPLE_RATE} from './resample-to-16khz';+export type {ResampleTo16KhzOptions} from './resample-to-16khz';+export {toCaptions} from './to-captions';+export type {ToCaptionsOptions, ToCaptionsResult} from './to-captions';+export {transcribe} from './transcribe';+export type {+ TranscribeOptions,+ WhisperWebGpuTranscription,+ WhisperWebGpuWord,+} from './transcribe';diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.tsnew file mode 100644index 0000000000..3890eaf1c0--- /dev/null+++ b/packages/whisper-webgpu/src/load-whisper-model.ts@@ -0,0 +1,156 @@+import {env, pipeline} from '@huggingface/transformers';+import type {+ ResolvedWhisperWebGpuBackend,+ WhisperWebGpuBackend,+} from './backend';+import {resolveBackend} from './backend';+import {getModelInfo, type WhisperWebGpuModel} from './models';++export type WhisperWebGpuModelLoadProgress = {+ status: string;+ file: string | null;+ progress: number | null;+ loadedBytes: number | null;+ totalBytes: number | null;+};++export type OnWhisperWebGpuModelLoadProgress = (+ progress: WhisperWebGpuModelLoadProgress,+) => void;++type LoadedWhisperPipeline = Awaited<+ ReturnType<typeof pipeline<'automatic-speech-recognition'>>+>;++const pipelines = new Map<string, Promise<LoadedWhisperPipeline>>();++const getCacheKey = (+ model: WhisperWebGpuModel,+ backend: ResolvedWhisperWebGpuBackend,+) => `${model}:${backend}`;++const normalizeProgress = (+ event: Record<string, unknown>,+): WhisperWebGpuModelLoadProgress => {+ const rawProgress =+ typeof event.progress === 'number' ? event.progress : null;++ return {+ status: typeof event.status === 'string' ? event.status : 'progress',+ file: typeof event.file === 'string' ? event.file : null,+ progress:+ rawProgress === null+ ? null+ : Math.max(+ 0,+ Math.min(1, rawProgress > 1 ? rawProgress / 100 : rawProgress),+ ),+ loadedBytes: typeof event.loaded === 'number' ? event.loaded : null,+ totalBytes: typeof event.total === 'number' ? event.total : null,+ };+};++export type LoadWhisperModelOptions = {+ model: WhisperWebGpuModel;+ backend?: WhisperWebGpuBackend;+ onProgress?: OnWhisperWebGpuModelLoadProgress;+};++export type LoadWhisperModelResult = {+ backend: ResolvedWhisperWebGpuBackend;+ alreadyLoaded: boolean;+};++export const loadWhisperModel = async ({+ model,+ backend = 'auto',+ onProgress,+}: LoadWhisperModelOptions): Promise<LoadWhisperModelResult> => {+ const resolvedBackend = resolveBackend(backend);+ const cacheKey = getCacheKey(model, resolvedBackend);+ const existing = pipelines.get(cacheKey);+ if (existing) {+ await existing;+ onProgress?.({+ status: 'ready',+ file: null,+ progress: 1,+ loadedBytes: null,+ totalBytes: null,+ });+ return {backend: resolvedBackend, alreadyLoaded: true};+ }++ if (resolvedBackend === 'wasm') {+ env.backends.onnx.wasm!.numThreads = globalThis.crossOriginIsolated ? 0 : 1;+ }++ const {modelId} = getModelInfo(model);+ const loading = pipeline('automatic-speech-recognition', modelId, {+ device: resolvedBackend,+ dtype:+ resolvedBackend === 'webgpu'+ ? {encoder_model: 'fp32', decoder_model_merged: 'q4'}+ : 'q8',+ progress_callback: (event) => {+ onProgress?.(normalizeProgress(event as Record<string, unknown>));+ },+ });++ pipelines.set(cacheKey, loading);+ try {+ await loading;+ return {backend: resolvedBackend, alreadyLoaded: false};+ } catch (error) {+ pipelines.delete(cacheKey);+ throw error;+ }+};++export const getLoadedWhisperPipeline = async ({+ model,+ backend,+ onProgress,+}: {+ model: WhisperWebGpuModel;+ backend: WhisperWebGpuBackend;+ onProgress?: OnWhisperWebGpuModelLoadProgress;+}) => {+ const {backend: resolvedBackend} = await loadWhisperModel({+ model,+ backend,+ onProgress,+ });+ const loaded = pipelines.get(getCacheKey(model, resolvedBackend));+ if (!loaded) {+ throw new Error(`Whisper model ${model} was not loaded.`);+ }++ return {pipeline: await loaded, backend: resolvedBackend};+};++export type DisposeWhisperModelOptions = {+ model?: WhisperWebGpuModel;+ backend?: Exclude<WhisperWebGpuBackend, 'auto'>;+};++export const disposeWhisperModel = async ({+ model,+ backend,+}: DisposeWhisperModelOptions = {}): Promise<void> => {+ const matching = [...pipelines.entries()].filter(([key]) => {+ const [loadedModel, loadedBackend] = key.split(':');+ return (+ (model === undefined || loadedModel === model) &&+ (backend === undefined || loadedBackend === backend)+ );+ });++ await Promise.all(+ matching.map(async ([key, loaded]) => {+ pipelines.delete(key);+ const loadedPipeline = await loaded;+ await loadedPipeline.dispose();+ }),+ );+};diff --git a/packages/whisper-webgpu/src/models.ts b/packages/whisper-webgpu/src/models.tsnew file mode 100644index 0000000000..d83efe778d--- /dev/null+++ b/packages/whisper-webgpu/src/models.ts@@ -0,0 +1,107 @@+export const WHISPER_WEBGPU_MODELS = [+ 'tiny',+ 'tiny.en',+ 'base',+ 'base.en',+ 'small',+ 'small.en',+ 'medium',+ 'medium.en',+ 'large-v3-turbo',+] as const;++export type WhisperWebGpuModel = (typeof WHISPER_WEBGPU_MODELS)[number];++export type WhisperWebGpuModelInfo = {+ name: WhisperWebGpuModel;+ modelId: string;+ parameters: number;+ multilingual: boolean;+ webGpuDownloadSize: number;+ wasmDownloadSize: number;+};++const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {+ tiny: {+ name: 'tiny',+ modelId: 'onnx-community/whisper-tiny_timestamped',+ parameters: 39_000_000,+ multilingual: true,+ webGpuDownloadSize: 119_699_015,+ wasmDownloadSize: 40_827_373,+ },+ 'tiny.en': {+ name: 'tiny.en',+ modelId: 'onnx-community/whisper-tiny.en_timestamped',+ parameters: 39_000_000,+ multilingual: false,+ webGpuDownloadSize: 119_697_479,+ wasmDownloadSize: 40_826_993,+ },+ base: {+ name: 'base',+ modelId: 'onnx-community/whisper-base_timestamped',+ parameters: 74_000_000,+ multilingual: true,+ webGpuDownloadSize: 206_190_057,+ wasmDownloadSize: 76_871_875,+ },+ 'base.en': {+ name: 'base.en',+ modelId: 'onnx-community/whisper-base.en_timestamped',+ parameters: 74_000_000,+ multilingual: false,+ webGpuDownloadSize: 206_188_009,+ wasmDownloadSize: 76_871_369,+ },+ small: {+ name: 'small',+ modelId: 'onnx-community/whisper-small_timestamped',+ parameters: 244_000_000,+ multilingual: true,+ webGpuDownloadSize: 586_213_010,+ wasmDownloadSize: 249_036_248,+ },+ 'small.en': {+ name: 'small.en',+ modelId: 'onnx-community/whisper-small.en_timestamped',+ parameters: 244_000_000,+ multilingual: false,+ webGpuDownloadSize: 586_209_938,+ wasmDownloadSize: 249_035_489,+ },+ medium: {+ name: 'medium',+ modelId: 'onnx-community/whisper-medium_timestamped',+ parameters: 769_000_000,+ multilingual: true,+ webGpuDownloadSize: 1_698_508_143,+ wasmDownloadSize: 985_716_093,+ },+ 'medium.en': {+ name: 'medium.en',+ modelId: 'onnx-community/whisper-medium.en_timestamped',+ parameters: 769_000_000,+ multilingual: false,+ webGpuDownloadSize: 1_698_504_047,+ wasmDownloadSize: 985_710_974,+ },+ 'large-v3-turbo': {+ name: 'large-v3-turbo',+ modelId: 'onnx-community/whisper-large-v3-turbo_timestamped',+ parameters: 809_000_000,+ multilingual: true,+ webGpuDownloadSize: 2_882_584_170,+ wasmDownloadSize: 1_084_880_793,+ },+};++export const getAvailableModels = (): WhisperWebGpuModelInfo[] => {+ return WHISPER_WEBGPU_MODELS.map((model) => ({...MODEL_INFO[model]}));+};++export const getModelInfo = (+ model: WhisperWebGpuModel,+): WhisperWebGpuModelInfo => {+ return MODEL_INFO[model];+};diff --git a/packages/whisper-webgpu/src/resample-to-16khz.ts b/packages/whisper-webgpu/src/resample-to-16khz.tsnew file mode 100644index 0000000000..c41d74f109--- /dev/null+++ b/packages/whisper-webgpu/src/resample-to-16khz.ts@@ -0,0 +1,56 @@+export const WHISPER_WEBGPU_SAMPLE_RATE = 16_000;++export type ResampleTo16KhzOptions = {+ file: Blob;+ onProgress?: (progress: number) => void;+};++export const resampleTo16Khz = async ({+ file,+ onProgress,+}: ResampleTo16KhzOptions): Promise<Float32Array> => {+ if (typeof AudioContext === 'undefined') {+ throw new Error('AudioContext is not available in this environment.');+ }++ if (typeof OfflineAudioContext === 'undefined') {+ throw new Error(+ 'OfflineAudioContext is not available in this environment.',+ );+ }++ if (file.size === 0) {+ throw new Error('The audio file is empty.');+ }++ onProgress?.(0);+ const arrayBuffer = await file.arrayBuffer();+ onProgress?.(0.25);++ const audioContext = new AudioContext({+ sampleRate: WHISPER_WEBGPU_SAMPLE_RATE,+ });++ try {+ const decoded = await audioContext.decodeAudioData(arrayBuffer);+ onProgress?.(0.6);+ const length = Math.max(+ 1,+ Math.ceil(decoded.duration * WHISPER_WEBGPU_SAMPLE_RATE),+ );+ const offlineContext = new OfflineAudioContext(+ 1,+ length,+ WHISPER_WEBGPU_SAMPLE_RATE,+ );+ const source = offlineContext.createBufferSource();+ source.buffer = decoded;+ source.connect(offlineContext.destination);+ source.start();+ const rendered = await offlineContext.startRendering();+ onProgress?.(1);+ return rendered.getChannelData(0).slice();+ } finally {+ await audioContext.close();+ }+};diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.tsnew file mode 100644index 0000000000..1171b0fd60--- /dev/null+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts@@ -0,0 +1,122 @@+import {expect, mock, test} from 'bun:test';++let pipelineInitialization:+ | {+ modelId: string;+ options: Record<string, unknown>;+ }+ | undefined;+let transcriptionCall:+ | {+ audio: Float32Array;+ options: Record<string, unknown>;+ }+ | undefined;+let disposed = false;++const fakePipeline = Object.assign(+ (audio: Float32Array, options: Record<string, unknown>) => {+ transcriptionCall = {audio, options};+ return Promise.resolve({+ text: ' Hello world.',+ chunks: [+ {text: ' Hello', timestamp: [0.25, 0.75]},+ {text: ' world.', timestamp: [1, null]},+ ],+ });+ },+ {+ dispose: () => {+ disposed = true;+ return Promise.resolve();+ },+ },+);++mock.module('@huggingface/transformers', () => ({+ env: {backends: {onnx: {wasm: {numThreads: 0}}}},+ pipeline: (+ _task: string,+ modelId: string,+ options: Record<string, unknown>,+ ) => {+ pipelineInitialization = {modelId, options};+ return Promise.resolve(fakePipeline);+ },+}));++test('transcribes with the selected timestamped model and returns Remotion captions', async () => {+ const api = await import('../index');+ const {disposeWhisperModel, getAvailableModels, toCaptions, transcribe} = api;+ const channelWaveform = new Float32Array(16_000 * 3);+ const result = await transcribe({+ backend: 'wasm',+ channelWaveform,+ language: 'en',+ model: 'small.en',+ });++ expect(pipelineInitialization?.modelId).toBe(+ 'onnx-community/whisper-small.en_timestamped',+ );+ expect(pipelineInitialization?.options).toMatchObject({+ device: 'wasm',+ dtype: 'q8',+ });+ expect(transcriptionCall?.audio).toBe(channelWaveform);+ expect(transcriptionCall?.options).toMatchObject({+ return_timestamps: 'word',+ chunk_length_s: 30,+ stride_length_s: 5,+ language: 'en',+ });+ expect(result).toEqual({+ text: 'Hello world.',+ model: 'small.en',+ backend: 'wasm',+ words: [+ {text: 'Hello', startInSeconds: 0.25, endInSeconds: 0.75},+ {text: ' world.', startInSeconds: 1, endInSeconds: 3},+ ],+ });++ expect(toCaptions({whisperWebGpuOutput: result})).toEqual({+ captions: [+ {+ text: 'Hello',+ startMs: 250,+ endMs: 750,+ timestampMs: 500,+ confidence: null,+ },+ {+ text: ' world.',+ startMs: 1000,+ endMs: 3000,+ timestampMs: 2000,+ confidence: null,+ },+ ],+ });++ const models = getAvailableModels();+ expect(models.map((model) => model.name)).toEqual([+ 'tiny',+ 'tiny.en',+ 'base',+ 'base.en',+ 'small',+ 'small.en',+ 'medium',+ 'medium.en',+ 'large-v3-turbo',+ ]);+ expect(models.find((model) => model.name === 'small.en')).toMatchObject({+ parameters: 244_000_000,+ multilingual: false,+ webGpuDownloadSize: 586_209_938,+ });++ await disposeWhisperModel({model: 'small.en', backend: 'wasm'});+ expect(disposed).toBe(true);+});diff --git a/packages/whisper-webgpu/src/to-captions.ts b/packages/whisper-webgpu/src/to-captions.tsnew file mode 100644index 0000000000..7aa1b0db69--- /dev/null+++ b/packages/whisper-webgpu/src/to-captions.ts@@ -0,0 +1,30 @@+import type {Caption} from '@remotion/captions';+import type {WhisperWebGpuTranscription, WhisperWebGpuWord} from './transcribe';++export type ToCaptionsOptions = {+ whisperWebGpuOutput: WhisperWebGpuTranscription | WhisperWebGpuWord[];+};++export type ToCaptionsResult = {+ captions: Caption[];+};++export const toCaptions = ({+ whisperWebGpuOutput,+}: ToCaptionsOptions): ToCaptionsResult => {+ const words = Array.isArray(whisperWebGpuOutput)+ ? whisperWebGpuOutput+ : whisperWebGpuOutput.words;++ return {+ captions: words.map((word, index) => ({+ text: index === 0 ? word.text.trimStart() : word.text,+ startMs: Math.round(word.startInSeconds * 1000),+ endMs: Math.round(word.endInSeconds * 1000),+ timestampMs: Math.round(+ ((word.startInSeconds + word.endInSeconds) / 2) * 1000,+ ),+ confidence: null,+ })),+ };+};diff --git a/packages/whisper-webgpu/src/transcribe.ts b/packages/whisper-webgpu/src/transcribe.tsnew file mode 100644index 0000000000..7f21742c18--- /dev/null+++ b/packages/whisper-webgpu/src/transcribe.ts@@ -0,0 +1,119 @@+import type {+ ResolvedWhisperWebGpuBackend,+ WhisperWebGpuBackend,+} from './backend';+import {+ getLoadedWhisperPipeline,+ type OnWhisperWebGpuModelLoadProgress,+} from './load-whisper-model';+import type {WhisperWebGpuModel} from './models';++export type WhisperWebGpuWord = {+ text: string;+ startInSeconds: number;+ endInSeconds: number;+};++export type WhisperWebGpuTranscription = {+ text: string;+ words: WhisperWebGpuWord[];+ model: WhisperWebGpuModel;+ backend: ResolvedWhisperWebGpuBackend;+};++export type TranscribeOptions = {+ channelWaveform: Float32Array;+ model: WhisperWebGpuModel;+ backend?: WhisperWebGpuBackend;+ language?: string;+ chunkLengthInSeconds?: number;+ strideLengthInSeconds?: number;+ onModelLoadProgress?: OnWhisperWebGpuModelLoadProgress;+};++type TransformersJsWord = {+ text: string;+ timestamp: [number | null, number | null];+};++type TransformersJsTranscription = {+ text: string;+ chunks?: TransformersJsWord[];+};++export const transcribe = async ({+ channelWaveform,+ model,+ backend = 'auto',+ language = 'auto',+ chunkLengthInSeconds = 30,+ strideLengthInSeconds = 5,+ onModelLoadProgress,+}: TranscribeOptions): Promise<WhisperWebGpuTranscription> => {+ if (channelWaveform.length === 0) {+ throw new Error('The audio waveform is empty.');+ }++ if (chunkLengthInSeconds <= 0) {+ throw new Error('chunkLengthInSeconds must be greater than 0.');+ }++ if (+ strideLengthInSeconds < 0 ||+ strideLengthInSeconds * 2 >= chunkLengthInSeconds+ ) {+ throw new Error(+ 'strideLengthInSeconds must be non-negative and less than half of chunkLengthInSeconds.',+ );+ }++ const {pipeline: transcriber, backend: resolvedBackend} =+ await getLoadedWhisperPipeline({+ model,+ backend,+ onProgress: onModelLoadProgress,+ });++ const output = (await transcriber(channelWaveform, {+ return_timestamps: 'word',+ chunk_length_s: chunkLengthInSeconds,+ stride_length_s: strideLengthInSeconds,+ ...(language === 'auto' ? {} : {language}),+ })) as TransformersJsTranscription;++ if (!output.chunks) {+ throw new Error(+ 'The model did not return word-level timestamps. Use one of the timestamped models returned by getAvailableModels().',+ );+ }++ const audioDuration = channelWaveform.length / 16_000;+ const words = output.chunks.map((chunk, index): WhisperWebGpuWord => {+ const start = chunk.timestamp[0];+ const end = Math.min(chunk.timestamp[1] ?? audioDuration, audioDuration);+ if (start === null || !Number.isFinite(start) || !Number.isFinite(end)) {+ throw new Error(+ `The model returned an invalid timestamp for "${chunk.text}".`,+ );+ }++ if (end < start) {+ throw new Error(+ `The model returned a backwards timestamp for "${chunk.text}".`,+ );+ }++ return {+ text: index === 0 ? chunk.text.trimStart() : chunk.text,+ startInSeconds: start,+ endInSeconds: end,+ };+ });++ return {+ text: output.text.trimStart(),+ words,+ model,+ backend: resolvedBackend,+ };+};diff --git a/packages/whisper-webgpu/tsconfig.json b/packages/whisper-webgpu/tsconfig.jsonnew file mode 100644index 0000000000..446cea780b--- /dev/null+++ b/packages/whisper-webgpu/tsconfig.json@@ -0,0 +1,15 @@+{+ "extends": "../tsconfig.settings.json",+ "compilerOptions": {+ "rootDir": "src",+ "outDir": "dist",+ "skipLibCheck": true,+ "target": "ES2022",+ "module": "es2020",+ "moduleResolution": "bundler",+ "declaration": true,+ "emitDeclarationOnly": true+ },+ "include": ["src"],+ "references": [{"path": "../captions"}]+}diff --git a/tsconfig.json b/tsconfig.jsonindex de3eea01a2..bcf5580d63 100644--- a/tsconfig.json+++ b/tsconfig.json@@ -279,6 +279,9 @@{"path": "./packages/light-leaks"},+ {+ "path": "./packages/whisper-webgpu"+ },{"path": "./packages/maptiler"},