🎥 Make videos programmatically with React
Expand Whisper WebGPU transcription
JonnyBurgerfdd6acbed917
diff --git a/bun.lock b/bun.lockindex 920628bce2..9e978a9bd8 100644--- a/bun.lock+++ b/bun.lock@@ -2837,10 +2837,11 @@"name": "@remotion/whisper-webgpu","version": "4.0.522","dependencies": {- "@huggingface/transformers": "catalog:","@remotion/captions": "workspace:*",+ "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c",},"devDependencies": {+ "@huggingface/transformers": "catalog:","@remotion/eslint-config-internal": "workspace:*","@typescript/native-preview": "catalog:","eslint": "catalog:",@@ -7629,7 +7630,7 @@"oniguruma-to-es": ["[email protected]", "", { "dependencies": { "oniguruma-parser": "^0.12.1", "regex": "^6.0.1", "regex-recursion": "^6.0.2" } }, "sha512-3VhUGN3w2eYxnTzHn+ikMI+fp/96KoRSVK9/kMTcFqj1NRDh2IhQCKvYxDnWePKRXY/AqH+Fuiyb7VHSzBjHfA=="],- "onnxruntime-common": ["[email protected]", "", {}, "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA=="],+ "onnxruntime-common": ["[email protected]", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="],"onnxruntime-node": ["[email protected]", "", { "dependencies": { "adm-zip": "^0.5.16", "global-agent": "^3.0.0", "onnxruntime-common": "1.24.3" }, "os": [ "linux", "win32", "darwin", ] }, "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg=="],@@ -12127,7 +12128,7 @@"onnxruntime-node/adm-zip": ["[email protected]", "", {}, "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng=="],- "onnxruntime-web/onnxruntime-common": ["[email protected]", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="],+ "onnxruntime-node/onnxruntime-common": ["[email protected]", "", {}, "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA=="],"openai/@types/node": ["@types/[email protected]", "", {}, "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA=="],diff --git a/packages/whisper-webgpu/THIRD_PARTY_LICENSES.txt b/packages/whisper-webgpu/THIRD_PARTY_LICENSES.txtnew file mode 100644index 0000000000..a8e44398c8--- /dev/null+++ b/packages/whisper-webgpu/THIRD_PARTY_LICENSES.txt@@ -0,0 +1,263 @@+Third-party licenses for bundled code++The private Transformers.js runtime in this package includes code from the+following projects.++@huggingface/transformers 4.2.0+@huggingface/tokenizers 0.1.3+License: Apache-2.0+++ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright [yyyy] [name of copyright owner]++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.++-------------------------------------------------------------------------------++@huggingface/jinja 0.5.6+License: MIT++MIT License++Copyright (c) 2023 Hugging Face++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.+-------------------------------------------------------------------------------++onnxruntime-common 1.24.0-dev.20251116-b39e144322+Source revision: b39e14432259d5f579f34c3e338f1f70aa368f01+License: MIT++MIT License++Copyright (c) Microsoft Corporation++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:+The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/packages/whisper-webgpu/bundle.ts b/packages/whisper-webgpu/bundle.tsindex a0a1a34b77..da2b6902b3 100644--- a/packages/whisper-webgpu/bundle.ts+++ b/packages/whisper-webgpu/bundle.ts@@ -6,19 +6,22 @@ if (process.env.NODE_ENV !== 'production') {}console.time('Generated.');+const privateTransformersPackage =+ '@remotion/whisper-webgpu/private-transformers';+const sharedExternals = [+ '@remotion/captions',+ 'onnxruntime-web/webgpu',+ 'remotion',+ 'remotion/no-react',+ 'react',+ 'react/jsx-runtime',+ 'react/jsx-dev-runtime',+ 'react-dom',+];const output = await build({entrypoints: ['src/index.ts'],naming: '[name].mjs',- external: [- '@huggingface/transformers',- '@remotion/captions',- 'remotion',- 'remotion/no-react',- 'react',- 'react/jsx-runtime',- 'react/jsx-dev-runtime',- 'react-dom',- ],+ external: ['@huggingface/transformers', ...sharedExternals],});if (!output.success) {@@ -26,8 +29,46 @@ if (!output.success) {process.exit(1);}-for (const file of output.outputs) {- const str = await file.text();+const privateTransformersOutput = await build({+ entrypoints: ['src/private-transformers.ts'],+ define: {__dirname: 'undefined'},+ naming: '[name].mjs',+ target: 'browser',+ external: sharedExternals,+});++if (!privateTransformersOutput.success) {+ console.log(privateTransformersOutput.logs.join('\n'));+ process.exit(1);+}++for (const file of [...output.outputs, ...privateTransformersOutput.outputs]) {+ // This equivalent expression avoids a critical-dependency warning when a+ // downstream bundler processes the generated Transformers.js browser chunk.+ let str = (await file.text()).replaceAll(+ 'Object(import.meta).url',+ 'import.meta.url',+ );+ if (path.basename(file.path) === 'index.mjs') {+ str = str.replaceAll(+ 'import("@huggingface/transformers")',+ `import(${JSON.stringify(privateTransformersPackage)})`,+ );+ if (+ !str.includes(`import(${JSON.stringify(privateTransformersPackage)})`)+ ) {+ throw new Error(+ 'Transformers.js must be loaded through the private package entry.',+ );+ }++ if (str.includes('import("@huggingface/transformers")')) {+ throw new Error(+ 'The public entry must not share the consumer Transformers.js environment.',+ );+ }+ }+const out = path.join('dist', 'esm', file.path);await Bun.write(out, str);diff --git a/packages/whisper-webgpu/package.json b/packages/whisper-webgpu/package.jsonindex c0a33e72fc..d1ea988e75 100644--- a/packages/whisper-webgpu/package.json+++ b/packages/whisper-webgpu/package.json@@ -24,8 +24,8 @@"url": "https://github.com/remotion-dev/remotion/issues"},"dependencies": {- "@huggingface/transformers": "catalog:",- "@remotion/captions": "workspace:*"+ "@remotion/captions": "workspace:*",+ "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c"},"exports": {".": {@@ -33,9 +33,14 @@"module": "./dist/esm/index.mjs","import": "./dist/esm/index.mjs"},+ "./private-transformers": {+ "module": "./dist/esm/private-transformers.mjs",+ "import": "./dist/esm/private-transformers.mjs"+ },"./package.json": "./package.json"},"devDependencies": {+ "@huggingface/transformers": "catalog:","@remotion/eslint-config-internal": "workspace:*","@typescript/native-preview": "catalog:","eslint": "catalog:"diff --git a/packages/whisper-webgpu/src/index.ts b/packages/whisper-webgpu/src/index.tsindex a020d03828..49b9910d8a 100644--- a/packages/whisper-webgpu/src/index.ts+++ b/packages/whisper-webgpu/src/index.ts@@ -24,5 +24,6 @@ export {transcribe} from './transcribe';export type {TranscribeOptions,WhisperWebGpuTranscription,+ WhisperWebGpuTask,WhisperWebGpuWord,} from './transcribe';diff --git a/packages/whisper-webgpu/src/is-whisper-model-cached.ts b/packages/whisper-webgpu/src/is-whisper-model-cached.tsindex fe67b30e56..0b7f41252e 100644--- a/packages/whisper-webgpu/src/is-whisper-model-cached.ts+++ b/packages/whisper-webgpu/src/is-whisper-model-cached.ts@@ -1,25 +1,25 @@import {- getModelInfo,+ getHostedModelId,WHISPER_WEBGPU_DTYPE,type WhisperWebGpuModel,} from './models';+import {withRemotionModelHost} from './with-remotion-model-host';export type IsWhisperModelCachedOptions = {model: WhisperWebGpuModel;};-export const isWhisperModelCached = async ({+export const isWhisperModelCached = ({model,}: IsWhisperModelCachedOptions): Promise<boolean> => {- const {ModelRegistry} = await import('@huggingface/transformers');- const {modelId} = getModelInfo(model);-- return ModelRegistry.is_pipeline_cached(- 'automatic-speech-recognition',- modelId,- {- device: 'webgpu',- dtype: WHISPER_WEBGPU_DTYPE,- },- );+ return withRemotionModelHost(({ModelRegistry}) => {+ return ModelRegistry.is_pipeline_cached(+ 'automatic-speech-recognition',+ getHostedModelId(model),+ {+ device: 'webgpu',+ dtype: WHISPER_WEBGPU_DTYPE,+ },+ );+ });};diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.tsindex 1c3861e271..17f5d79884 100644--- a/packages/whisper-webgpu/src/load-whisper-model.ts+++ b/packages/whisper-webgpu/src/load-whisper-model.ts@@ -1,8 +1,10 @@import {+ getHostedModelId,getModelInfo,WHISPER_WEBGPU_DTYPE,type WhisperWebGpuModel,} from './models';+import {withRemotionModelHost} from './with-remotion-model-host';export type WhisperWebGpuModelLoadProgress = {status: string;@@ -53,67 +55,67 @@ const getOrCreateWhisperPipeline = ({return {state: existing, alreadyLoaded: true, totalBytes};}- const loading = Promise.resolve().then(async () => {- const {pipeline} = await import('@huggingface/transformers');-- onProgress?.({- status: 'loading',- file: null,- progress: 0,- loadedBytes: 0,- totalBytes,- });-- const {modelId} = modelInfo;- const loadedByFile = new Map<string, number>();- let lastProgress = 0;- let lastLoadedBytes = 0;- return pipeline('automatic-speech-recognition', modelId, {- device: 'webgpu',- dtype: WHISPER_WEBGPU_DTYPE,- progress_callback: onProgress- ? (event) => {- const record = event as Record<string, unknown>;- if (- record.status === 'progress' &&- typeof record.file === 'string' &&- typeof record.loaded === 'number' &&- Number.isFinite(record.loaded)- ) {- loadedByFile.set(- record.file,- Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),- );- const loadedBytes = [...loadedByFile.values()].reduce(- (sum, loaded) => sum + loaded,- 0,- );- lastProgress = Math.max(- lastProgress,- Math.min(loadedBytes / totalBytes, 0.99),- );- lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);- onProgress({- status: 'loading',- file: null,- progress: lastProgress,- loadedBytes: lastLoadedBytes,- totalBytes,- });- }+ const loading = Promise.resolve().then(() => {+ return withRemotionModelHost(({pipeline}) => {+ onProgress?.({+ status: 'loading',+ file: null,+ progress: 0,+ loadedBytes: 0,+ totalBytes,+ });- if (record.status === 'ready') {- onProgress({- status: 'ready',- file: null,- progress: 1,- loadedBytes: Math.max(lastLoadedBytes, totalBytes),- totalBytes,- });+ const hostedModelId = getHostedModelId(model);+ const loadedByFile = new Map<string, number>();+ let lastProgress = 0;+ let lastLoadedBytes = 0;+ return pipeline('automatic-speech-recognition', hostedModelId, {+ device: 'webgpu',+ dtype: WHISPER_WEBGPU_DTYPE,+ progress_callback: onProgress+ ? (event) => {+ const record = event as Record<string, unknown>;+ if (+ record.status === 'progress' &&+ typeof record.file === 'string' &&+ typeof record.loaded === 'number' &&+ Number.isFinite(record.loaded)+ ) {+ loadedByFile.set(+ record.file,+ Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),+ );+ const loadedBytes = [...loadedByFile.values()].reduce(+ (sum, loaded) => sum + loaded,+ 0,+ );+ lastProgress = Math.max(+ lastProgress,+ Math.min(loadedBytes / totalBytes, 0.99),+ );+ lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);+ onProgress({+ status: 'loading',+ file: null,+ progress: lastProgress,+ loadedBytes: lastLoadedBytes,+ totalBytes,+ });+ }++ if (record.status === 'ready') {+ onProgress({+ status: 'ready',+ file: null,+ progress: 1,+ loadedBytes: Math.max(lastLoadedBytes, totalBytes),+ totalBytes,+ });+ }}- }- : undefined,- }) as Promise<LoadedWhisperPipeline>;+ : undefined,+ }) as Promise<LoadedWhisperPipeline>;+ });});const state: LoadedWhisperPipelineState = {loading,diff --git a/packages/whisper-webgpu/src/models.ts b/packages/whisper-webgpu/src/models.tsindex 4e3f3e7dc4..623f3ff2bd 100644--- a/packages/whisper-webgpu/src/models.ts+++ b/packages/whisper-webgpu/src/models.ts@@ -17,6 +17,7 @@ export type WhisperWebGpuModelInfo = {modelId: string;parameters: number;multilingual: boolean;+ supportsTranslation: boolean;webGpuDownloadSize: number;};@@ -26,6 +27,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {modelId: 'onnx-community/whisper-tiny_timestamped',parameters: 39_000_000,multilingual: true,+ supportsTranslation: true,webGpuDownloadSize: 119_699_015,},'tiny.en': {@@ -33,6 +35,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {modelId: 'onnx-community/whisper-tiny.en_timestamped',parameters: 39_000_000,multilingual: false,+ supportsTranslation: false,webGpuDownloadSize: 119_697_479,},base: {@@ -40,6 +43,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {modelId: 'onnx-community/whisper-base_timestamped',parameters: 74_000_000,multilingual: true,+ supportsTranslation: true,webGpuDownloadSize: 206_190_057,},'base.en': {@@ -47,6 +51,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {modelId: 'onnx-community/whisper-base.en_timestamped',parameters: 74_000_000,multilingual: false,+ supportsTranslation: false,webGpuDownloadSize: 206_188_009,},small: {@@ -54,6 +59,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {modelId: 'onnx-community/whisper-small_timestamped',parameters: 244_000_000,multilingual: true,+ supportsTranslation: true,webGpuDownloadSize: 586_213_010,},'small.en': {@@ -61,6 +67,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {modelId: 'onnx-community/whisper-small.en_timestamped',parameters: 244_000_000,multilingual: false,+ supportsTranslation: false,webGpuDownloadSize: 586_209_938,},medium: {@@ -68,6 +75,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {modelId: 'onnx-community/whisper-medium_timestamped',parameters: 769_000_000,multilingual: true,+ supportsTranslation: true,webGpuDownloadSize: 1_698_508_143,},'medium.en': {@@ -75,6 +83,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {modelId: 'onnx-community/whisper-medium.en_timestamped',parameters: 769_000_000,multilingual: false,+ supportsTranslation: false,webGpuDownloadSize: 1_698_504_047,},'large-v3-turbo': {@@ -82,10 +91,23 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {modelId: 'onnx-community/whisper-large-v3-turbo_timestamped',parameters: 809_000_000,multilingual: true,+ supportsTranslation: false,webGpuDownloadSize: 2_882_584_170,},};+const HOSTED_MODEL_IDS: Record<WhisperWebGpuModel, string> = {+ tiny: 'whisper-tiny_timestamped-v1',+ 'tiny.en': 'whisper-tiny.en_timestamped-v1',+ base: 'whisper-base_timestamped-v1',+ 'base.en': 'whisper-base.en_timestamped-v1',+ small: 'whisper-small_timestamped-v1',+ 'small.en': 'whisper-small.en_timestamped-v1',+ medium: 'whisper-medium_timestamped-v1',+ 'medium.en': 'whisper-medium.en_timestamped-v1',+ 'large-v3-turbo': 'whisper-large-v3-turbo_timestamped-v1',+};+export const getAvailableModels = (): WhisperWebGpuModelInfo[] => {return WHISPER_WEBGPU_MODELS.map((model) => ({...MODEL_INFO[model]}));};@@ -96,6 +118,10 @@ export const getModelInfo = (return MODEL_INFO[model];};+export const getHostedModelId = (model: WhisperWebGpuModel): string => {+ return HOSTED_MODEL_IDS[model];+};+export const WHISPER_WEBGPU_DTYPE = {encoder_model: 'fp32',decoder_model_merged: 'q4',diff --git a/packages/whisper-webgpu/src/private-transformers.ts b/packages/whisper-webgpu/src/private-transformers.tsnew file mode 100644index 0000000000..bb03ecc2a5--- /dev/null+++ b/packages/whisper-webgpu/src/private-transformers.ts@@ -0,0 +1 @@+export * from '@huggingface/transformers';diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.tsindex d0b04f34d0..780c3ab669 100644--- a/packages/whisper-webgpu/src/test/transcribe.test.ts+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts@@ -6,6 +6,18 @@ let pipelineInitialization:options: Record<string, unknown>;}| undefined;+let pipelineEnvironment:+ | {+ remoteHost: string;+ remotePathTemplate: string;+ }+ | undefined;+let pipelineEnvironmentAfterLoading:+ | {+ remoteHost: string;+ remotePathTemplate: string;+ }+ | undefined;let transcriptionCall:| {audio: Float32Array;@@ -15,6 +27,7 @@ let transcriptionCall:let disposed = false;let disposeCalls = 0;let pipelineInitializationCount = 0;+const pipelineInitializationModelIds: string[] = [];let pipelineInitializationGate: Promise<void> | null = null;let onPipelineInitializationStarted: (() => void) | null = null;let transcriptionGate: Promise<void> | null = null;@@ -26,6 +39,19 @@ let cacheCheck:options: Record<string, unknown>;}| undefined;+let cacheCheckEnvironment:+ | {+ remoteHost: string;+ remotePathTemplate: string;+ }+ | undefined;+let cacheCheckError: Error | null = null;++const originalTransformersEnvironment = {+ remoteHost: 'https://huggingface.co/',+ remotePathTemplate: '{model}/resolve/{revision}/',+};+const transformersEnvironment = {...originalTransformersEnvironment};const fakePipeline = Object.assign(async (audio: Float32Array, options: Record<string, unknown>) => {@@ -52,6 +78,7 @@ const fakePipeline = Object.assign();mock.module('@huggingface/transformers', () => ({+ env: transformersEnvironment,ModelRegistry: {is_pipeline_cached: (task: string,@@ -59,6 +86,11 @@ mock.module('@huggingface/transformers', () => ({options: Record<string, unknown>,) => {cacheCheck = {task, modelId, options};+ cacheCheckEnvironment = {...transformersEnvironment};+ if (cacheCheckError) {+ return Promise.reject(cacheCheckError);+ }+return Promise.resolve(true);},},@@ -68,9 +100,12 @@ mock.module('@huggingface/transformers', () => ({options: Record<string, unknown>,) => {pipelineInitializationCount++;+ pipelineInitializationModelIds.push(modelId);pipelineInitialization = {modelId, options};+ pipelineEnvironment = {...transformersEnvironment};onPipelineInitializationStarted?.();await pipelineInitializationGate;+ pipelineEnvironmentAfterLoading = {...transformersEnvironment};const onProgress = options.progress_callback as| ((progress: Record<string, unknown>) => void)| undefined;@@ -135,8 +170,13 @@ test('transcribes with word timestamps using WebGPU', async () => {});expect(pipelineInitialization?.modelId).toBe(- 'onnx-community/whisper-small.en_timestamped',+ 'whisper-small.en_timestamped-v1',);+ expect(pipelineEnvironment).toEqual({+ remoteHost: 'https://remotion.media/',+ remotePathTemplate: 'models/{model}/',+ });+ expect(pipelineEnvironmentAfterLoading).toEqual(pipelineEnvironment);expect(pipelineInitialization?.options).toMatchObject({device: 'webgpu',dtype: {encoder_model: 'fp32', decoder_model_merged: 'q4'},@@ -146,6 +186,12 @@ test('transcribes with word timestamps using WebGPU', async () => {return_timestamps: 'word',chunk_length_s: 30,stride_length_s: 5,+ force_full_sequences: false,+ do_sample: false,+ temperature: 1,+ top_k: 50,+ repetition_penalty: 1,+ no_repeat_ngram_size: 0,});expect(result).toEqual({text: 'Hello world free today.',@@ -208,17 +254,29 @@ test('transcribes with word timestamps using WebGPU', async () => {modelId: 'onnx-community/whisper-small.en_timestamped',parameters: 244_000_000,multilingual: false,+ supportsTranslation: false,webGpuDownloadSize: 586_209_938,});+ expect(models.find((model) => model.name === 'large-v3-turbo')).toMatchObject(+ {+ multilingual: true,+ supportsTranslation: false,+ },+ );expect(await isWhisperModelCached({model: 'small.en'})).toBe(true);expect(cacheCheck).toEqual({task: 'automatic-speech-recognition',- modelId: 'onnx-community/whisper-small.en_timestamped',+ modelId: 'whisper-small.en_timestamped-v1',options: {device: 'webgpu',dtype: {encoder_model: 'fp32', decoder_model_merged: 'q4'},},});+ expect(cacheCheckEnvironment).toEqual({+ remoteHost: 'https://remotion.media/',+ remotePathTemplate: 'models/{model}/',+ });+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');const originalNavigator = Object.getOwnPropertyDescriptor(@@ -270,6 +328,57 @@ test('transcribes with word timestamps using WebGPU', async () => {expect(disposed).toBe(true);});+test('loads every public model from its explicit immutable hosted path', async () => {+ const {disposeWhisperModel, getAvailableModels, loadWhisperModel} =+ await import('../index');+ const models = getAvailableModels();+ const initializationsBeforeLoading = pipelineInitializationModelIds.length;++ for (const model of models) {+ await loadWhisperModel({model: model.name});+ await disposeWhisperModel({model: model.name});+ }++ expect(+ pipelineInitializationModelIds.slice(initializationsBeforeLoading),+ ).toEqual([+ 'whisper-tiny_timestamped-v1',+ 'whisper-tiny.en_timestamped-v1',+ 'whisper-base_timestamped-v1',+ 'whisper-base.en_timestamped-v1',+ 'whisper-small_timestamped-v1',+ 'whisper-small.en_timestamped-v1',+ 'whisper-medium_timestamped-v1',+ 'whisper-medium.en_timestamped-v1',+ 'whisper-large-v3-turbo_timestamped-v1',+ ]);+ expect(models.map(({modelId}) => modelId)).toEqual([+ 'onnx-community/whisper-tiny_timestamped',+ 'onnx-community/whisper-tiny.en_timestamped',+ 'onnx-community/whisper-base_timestamped',+ 'onnx-community/whisper-base.en_timestamped',+ 'onnx-community/whisper-small_timestamped',+ 'onnx-community/whisper-small.en_timestamped',+ 'onnx-community/whisper-medium_timestamped',+ 'onnx-community/whisper-medium.en_timestamped',+ 'onnx-community/whisper-large-v3-turbo_timestamped',+ ]);+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);+});++test('restores the Transformers environment if a hosted model operation fails', async () => {+ const {isWhisperModelCached} = await import('../index');+ cacheCheckError = new Error('Could not read the model cache');+ try {+ await expect(isWhisperModelCached({model: 'tiny'})).rejects.toThrow(+ 'Could not read the model cache',+ );+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);+ } finally {+ cacheCheckError = null;+ }+});+test('reports model progress without reaching 100% before every file loads', async () => {const {disposeWhisperModel, loadWhisperModel} = await import('../index');const progressValues: Array<{@@ -313,7 +422,7 @@ test('reports model progress without reaching 100% before every file loads', asyawait disposeWhisperModel({model: 'medium.en'});});-test('validates language and finite chunk settings through transcribe()', async () => {+test('validates and forwards transcription settings through transcribe()', async () => {const {disposeWhisperModel, transcribe} = await import('../index');const channelWaveform = new Float32Array(16_000);@@ -327,6 +436,21 @@ test('validates language and finite chunk settings through transcribe()', async).rejects.toThrow('The English-only model "tiny.en" does not support the language "de".',);+ await expect(+ transcribe({+ channelWaveform,+ model: 'tiny.en',+ task: 'translate',+ }),+ ).rejects.toThrow('does not support translation');+ await expect(+ transcribe({+ channelWaveform,+ language: 'en',+ model: 'large-v3-turbo',+ task: 'translate',+ }),+ ).rejects.toThrow('does not support translation');await expect(transcribe({channelWaveform,@@ -346,18 +470,66 @@ test('validates language and finite chunk settings through transcribe()', async'strideLengthInSeconds must be a finite, non-negative number and less than half of chunkLengthInSeconds.',);- await transcribe({channelWaveform, model: 'tiny', language: 'de'});+ const invalidOptions: Array<{+ options: Record<string, unknown>;+ optionName: string;+ }> = [+ {options: {forceFullSequences: 'yes'}, optionName: 'forceFullSequences'},+ {options: {doSample: 'yes'}, optionName: 'doSample'},+ {options: {temperature: Number.NaN}, optionName: 'temperature'},+ {options: {temperature: 0}, optionName: 'temperature'},+ {options: {topK: Number.NaN}, optionName: 'topK'},+ {options: {topK: -1}, optionName: 'topK'},+ {options: {topK: 1.5}, optionName: 'topK'},+ {options: {repetitionPenalty: Number.NaN}, optionName: 'repetitionPenalty'},+ {options: {repetitionPenalty: 0}, optionName: 'repetitionPenalty'},+ {options: {noRepeatNgramSize: Number.NaN}, optionName: 'noRepeatNgramSize'},+ {options: {noRepeatNgramSize: -1}, optionName: 'noRepeatNgramSize'},+ {options: {noRepeatNgramSize: 1.5}, optionName: 'noRepeatNgramSize'},+ {options: {task: 'summarize'}, optionName: 'task'},+ ];++ for (const {options, optionName} of invalidOptions) {+ await expect(+ transcribe({+ channelWaveform,+ model: 'tiny.en',+ ...options,+ } as never),+ ).rejects.toThrow(optionName);+ }++ await transcribe({+ channelWaveform,+ model: 'tiny',+ language: 'de',+ task: 'translate',+ forceFullSequences: true,+ doSample: true,+ temperature: 0.7,+ topK: 25,+ repetitionPenalty: 1.2,+ noRepeatNgramSize: 3,+ });expect(transcriptionCall?.options).toEqual({return_timestamps: 'word',chunk_length_s: 30,stride_length_s: 5,language: 'de',+ task: 'translate',+ force_full_sequences: true,+ do_sample: true,+ temperature: 0.7,+ top_k: 25,+ repetition_penalty: 1.2,+ no_repeat_ngram_size: 3,});await disposeWhisperModel({model: 'tiny'});});-test('shares concurrent initialization and waits for it before disposal', async () => {- const {disposeWhisperModel, loadWhisperModel} = await import('../index');+test('shares concurrent initialization and keeps the model host scoped until every operation completes', async () => {+ const {disposeWhisperModel, isWhisperModelCached, loadWhisperModel} =+ await import('../index');let releaseInitialization: () => void = () => {};pipelineInitializationGate = new Promise<void>((resolve) => {releaseInitialization = resolve;@@ -372,6 +544,27 @@ test('shares concurrent initialization and waits for it before disposal', asyncconst secondLoad = loadWhisperModel({model: 'base'});await initializationStarted;expect(pipelineInitializationCount - initializationsBeforeLoading).toBe(1);+ expect(transformersEnvironment).toEqual({+ remoteHost: 'https://remotion.media/',+ remotePathTemplate: 'models/{model}/',+ });+ expect(+ (+ transformersEnvironment as typeof transformersEnvironment &+ Record<symbol, unknown>+ )[Symbol.for('@remotion/whisper-webgpu/model-host-state')],+ ).toMatchObject({+ activeOperations: 1,+ previousRemoteConfiguration: originalTransformersEnvironment,+ });+ await expect(isWhisperModelCached({model: 'tiny.en'})).resolves.toBe(true);+ expect(cacheCheck).toMatchObject({+ modelId: 'whisper-tiny.en_timestamped-v1',+ });+ expect(transformersEnvironment).toEqual({+ remoteHost: 'https://remotion.media/',+ remotePathTemplate: 'models/{model}/',+ });const disposal = disposeWhisperModel({model: 'base'});await Promise.resolve();expect(disposeCalls).toBe(disposalsBeforeLoading);@@ -383,10 +576,40 @@ test('shares concurrent initialization and waits for it before disposal', async]);await disposal;expect(disposeCalls).toBe(disposalsBeforeLoading + 1);+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);pipelineInitializationGate = null;onPipelineInitializationStarted = null;});+test('only restores Transformers environment fields left unchanged by the consumer', async () => {+ const {disposeWhisperModel, loadWhisperModel} = await import('../index');+ let releaseInitialization: () => void = () => {};+ pipelineInitializationGate = new Promise<void>((resolve) => {+ releaseInitialization = resolve;+ });+ const initializationStarted = new Promise<void>((resolve) => {+ onPipelineInitializationStarted = resolve;+ });+ const loading = loadWhisperModel({model: 'small'});++ try {+ await initializationStarted;+ transformersEnvironment.remoteHost = 'https://models.example.com/';+ releaseInitialization();+ await loading;+ expect(transformersEnvironment).toEqual({+ remoteHost: 'https://models.example.com/',+ remotePathTemplate: '{model}/resolve/{revision}/',+ });+ await disposeWhisperModel({model: 'small'});+ } finally {+ releaseInitialization();+ pipelineInitializationGate = null;+ onPipelineInitializationStarted = null;+ Object.assign(transformersEnvironment, originalTransformersEnvironment);+ }+});+test('waits for active transcriptions before disposal', async () => {const {disposeWhisperModel, transcribe} = await import('../index');let releaseTranscription: () => void = () => {};diff --git a/packages/whisper-webgpu/src/transcribe.ts b/packages/whisper-webgpu/src/transcribe.tsindex 95c287df04..6299453fc8 100644--- a/packages/whisper-webgpu/src/transcribe.ts+++ b/packages/whisper-webgpu/src/transcribe.ts@@ -16,12 +16,21 @@ export type WhisperWebGpuTranscription = {model: WhisperWebGpuModel;};+export type WhisperWebGpuTask = 'transcribe' | 'translate';+export type TranscribeOptions = {channelWaveform: Float32Array;model: WhisperWebGpuModel;language?: string;+ task?: WhisperWebGpuTask;chunkLengthInSeconds?: number;strideLengthInSeconds?: number;+ forceFullSequences?: boolean;+ doSample?: boolean;+ temperature?: number;+ topK?: number;+ repetitionPenalty?: number;+ noRepeatNgramSize?: number;onModelLoadProgress?: OnWhisperWebGpuModelLoadProgress;};@@ -39,8 +48,15 @@ export const transcribe = async ({channelWaveform,model,language,+ task = 'transcribe',chunkLengthInSeconds = 30,strideLengthInSeconds = 5,+ forceFullSequences = false,+ doSample = false,+ temperature = 1,+ topK = 50,+ repetitionPenalty = 1,+ noRepeatNgramSize = 0,onModelLoadProgress,}: TranscribeOptions): Promise<WhisperWebGpuTranscription> => {if (channelWaveform.length === 0) {@@ -63,7 +79,41 @@ export const transcribe = async ({);}- const {multilingual} = getModelInfo(model);+ if (typeof forceFullSequences !== 'boolean') {+ throw new TypeError('forceFullSequences must be a boolean.');+ }++ if (typeof doSample !== 'boolean') {+ throw new TypeError('doSample must be a boolean.');+ }++ if (!Number.isFinite(temperature) || temperature <= 0) {+ throw new TypeError('temperature must be a finite number greater than 0.');+ }++ if (!Number.isInteger(topK) || topK < 0) {+ throw new TypeError('topK must be a non-negative integer.');+ }++ if (!Number.isFinite(repetitionPenalty) || repetitionPenalty <= 0) {+ throw new TypeError(+ 'repetitionPenalty must be a finite number greater than 0.',+ );+ }++ if (!Number.isInteger(noRepeatNgramSize) || noRepeatNgramSize < 0) {+ throw new TypeError('noRepeatNgramSize must be a non-negative integer.');+ }++ if (task !== 'transcribe' && task !== 'translate') {+ throw new TypeError('task must be either "transcribe" or "translate".');+ }++ const {multilingual, supportsTranslation} = getModelInfo(model);+ if (task === 'translate' && !supportsTranslation) {+ throw new Error(`The model "${model}" does not support translation.`);+ }+if (multilingual && (language === undefined || language === 'auto')) {throw new Error(`The language option is required for the multilingual model "${model}" because automatic language detection is not supported.`,@@ -90,7 +140,13 @@ export const transcribe = async ({return_timestamps: 'word',chunk_length_s: chunkLengthInSeconds,stride_length_s: strideLengthInSeconds,- ...(multilingual ? {language} : {}),+ force_full_sequences: forceFullSequences,+ do_sample: doSample,+ temperature,+ top_k: topK,+ repetition_penalty: repetitionPenalty,+ no_repeat_ngram_size: noRepeatNgramSize,+ ...(multilingual ? {language, task} : {}),})) as TransformersJsTranscription;},});diff --git a/packages/whisper-webgpu/src/with-remotion-model-host.ts b/packages/whisper-webgpu/src/with-remotion-model-host.tsnew file mode 100644index 0000000000..e1e201788c--- /dev/null+++ b/packages/whisper-webgpu/src/with-remotion-model-host.ts@@ -0,0 +1,71 @@+import type {env, ModelRegistry, pipeline} from '@huggingface/transformers';++const REMOTION_MODEL_HOST = 'https://remotion.media/';+const REMOTION_MODEL_PATH_TEMPLATE = 'models/{model}/';+const REMOTION_MODEL_HOST_STATE = Symbol.for(+ '@remotion/whisper-webgpu/model-host-state',+);++type TransformersModule = {+ readonly env: typeof env;+ readonly ModelRegistry: typeof ModelRegistry;+ readonly pipeline: typeof pipeline;+};++type ModelHostState = {+ activeOperations: number;+ previousRemoteConfiguration: {+ remoteHost: string;+ remotePathTemplate: string;+ } | null;+};++export const withRemotionModelHost = async <ReturnValue>(+ operation: (transformers: TransformersModule) => Promise<ReturnValue>,+): Promise<ReturnValue> => {+ // bundle.ts puts this import into a private lazy chunk. This keeps the mutable+ // Transformers.js environment isolated from other pipelines in the application.+ const transformers = await import('@huggingface/transformers');+ const {env} = transformers;+ const environmentWithState = env as typeof env &+ Record<symbol, ModelHostState | undefined>;+ let state = environmentWithState[REMOTION_MODEL_HOST_STATE];+ if (!state) {+ state = {+ activeOperations: 0,+ previousRemoteConfiguration: null,+ };+ Object.defineProperty(environmentWithState, REMOTION_MODEL_HOST_STATE, {+ value: state,+ });+ }++ if (state.activeOperations === 0) {+ state.previousRemoteConfiguration = {+ remoteHost: env.remoteHost,+ remotePathTemplate: env.remotePathTemplate,+ };+ env.remoteHost = REMOTION_MODEL_HOST;+ env.remotePathTemplate = REMOTION_MODEL_PATH_TEMPLATE;+ }++ state.activeOperations++;+ try {+ return await operation(transformers);+ } finally {+ state.activeOperations--;+ if (state.activeOperations === 0) {+ const configurationToRestore = state.previousRemoteConfiguration;+ state.previousRemoteConfiguration = null;+ if (configurationToRestore) {+ if (env.remoteHost === REMOTION_MODEL_HOST) {+ env.remoteHost = configurationToRestore.remoteHost;+ }++ if (env.remotePathTemplate === REMOTION_MODEL_PATH_TEMPLATE) {+ env.remotePathTemplate = configurationToRestore.remotePathTemplate;+ }+ }+ }+ }+};