Skip to content

🎥 Make videos programmatically with React

Expand Whisper WebGPU transcription

JonnyBurgerfdd6acbed917
diff --git a/bun.lock b/bun.lock
index 920628bce2..9e978a9bd8 100644
--- a/bun.lock
+++ b/bun.lock
@@ -2837,10 +2837,11 @@
"name": "@remotion/whisper-webgpu",
"version": "4.0.522",
"dependencies": {
- "@huggingface/transformers": "catalog:",
"@remotion/captions": "workspace:*",
+ "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c",
},
"devDependencies": {
+ "@huggingface/transformers": "catalog:",
"@remotion/eslint-config-internal": "workspace:*",
"@typescript/native-preview": "catalog:",
"eslint": "catalog:",
@@ -7629,7 +7630,7 @@
"oniguruma-to-es": ["[email protected]", "", { "dependencies": { "oniguruma-parser": "^0.12.1", "regex": "^6.0.1", "regex-recursion": "^6.0.2" } }, "sha512-3VhUGN3w2eYxnTzHn+ikMI+fp/96KoRSVK9/kMTcFqj1NRDh2IhQCKvYxDnWePKRXY/AqH+Fuiyb7VHSzBjHfA=="],
- "onnxruntime-common": ["[email protected]", "", {}, "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA=="],
+ "onnxruntime-common": ["[email protected]", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="],
"onnxruntime-node": ["[email protected]", "", { "dependencies": { "adm-zip": "^0.5.16", "global-agent": "^3.0.0", "onnxruntime-common": "1.24.3" }, "os": [ "linux", "win32", "darwin", ] }, "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg=="],
@@ -12127,7 +12128,7 @@
"onnxruntime-node/adm-zip": ["[email protected]", "", {}, "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng=="],
- "onnxruntime-web/onnxruntime-common": ["[email protected]", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="],
+ "onnxruntime-node/onnxruntime-common": ["[email protected]", "", {}, "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA=="],
"openai/@types/node": ["@types/[email protected]", "", {}, "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA=="],
diff --git a/packages/whisper-webgpu/THIRD_PARTY_LICENSES.txt b/packages/whisper-webgpu/THIRD_PARTY_LICENSES.txt
new file mode 100644
index 0000000000..a8e44398c8
--- /dev/null
+++ b/packages/whisper-webgpu/THIRD_PARTY_LICENSES.txt
@@ -0,0 +1,263 @@
+Third-party licenses for bundled code
+
+The private Transformers.js runtime in this package includes code from the
+following projects.
+
+@huggingface/transformers 4.2.0
+@huggingface/tokenizers 0.1.3
+License: Apache-2.0
+
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-------------------------------------------------------------------------------
+
+@huggingface/jinja 0.5.6
+License: MIT
+
+MIT License
+
+Copyright (c) 2023 Hugging Face
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+-------------------------------------------------------------------------------
+
+onnxruntime-common 1.24.0-dev.20251116-b39e144322
+Source revision: b39e14432259d5f579f34c3e338f1f70aa368f01
+License: MIT
+
+MIT License
+
+Copyright (c) Microsoft Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/packages/whisper-webgpu/bundle.ts b/packages/whisper-webgpu/bundle.ts
index a0a1a34b77..da2b6902b3 100644
--- a/packages/whisper-webgpu/bundle.ts
+++ b/packages/whisper-webgpu/bundle.ts
@@ -6,19 +6,22 @@ if (process.env.NODE_ENV !== 'production') {
}
console.time('Generated.');
+const privateTransformersPackage =
+ '@remotion/whisper-webgpu/private-transformers';
+const sharedExternals = [
+ '@remotion/captions',
+ 'onnxruntime-web/webgpu',
+ 'remotion',
+ 'remotion/no-react',
+ 'react',
+ 'react/jsx-runtime',
+ 'react/jsx-dev-runtime',
+ 'react-dom',
+];
const output = await build({
entrypoints: ['src/index.ts'],
naming: '[name].mjs',
- external: [
- '@huggingface/transformers',
- '@remotion/captions',
- 'remotion',
- 'remotion/no-react',
- 'react',
- 'react/jsx-runtime',
- 'react/jsx-dev-runtime',
- 'react-dom',
- ],
+ external: ['@huggingface/transformers', ...sharedExternals],
});
if (!output.success) {
@@ -26,8 +29,46 @@ if (!output.success) {
process.exit(1);
}
-for (const file of output.outputs) {
- const str = await file.text();
+const privateTransformersOutput = await build({
+ entrypoints: ['src/private-transformers.ts'],
+ define: {__dirname: 'undefined'},
+ naming: '[name].mjs',
+ target: 'browser',
+ external: sharedExternals,
+});
+
+if (!privateTransformersOutput.success) {
+ console.log(privateTransformersOutput.logs.join('\n'));
+ process.exit(1);
+}
+
+for (const file of [...output.outputs, ...privateTransformersOutput.outputs]) {
+ // This equivalent expression avoids a critical-dependency warning when a
+ // downstream bundler processes the generated Transformers.js browser chunk.
+ let str = (await file.text()).replaceAll(
+ 'Object(import.meta).url',
+ 'import.meta.url',
+ );
+ if (path.basename(file.path) === 'index.mjs') {
+ str = str.replaceAll(
+ 'import("@huggingface/transformers")',
+ `import(${JSON.stringify(privateTransformersPackage)})`,
+ );
+ if (
+ !str.includes(`import(${JSON.stringify(privateTransformersPackage)})`)
+ ) {
+ throw new Error(
+ 'Transformers.js must be loaded through the private package entry.',
+ );
+ }
+
+ if (str.includes('import("@huggingface/transformers")')) {
+ throw new Error(
+ 'The public entry must not share the consumer Transformers.js environment.',
+ );
+ }
+ }
+
const out = path.join('dist', 'esm', file.path);
await Bun.write(out, str);
diff --git a/packages/whisper-webgpu/package.json b/packages/whisper-webgpu/package.json
index c0a33e72fc..d1ea988e75 100644
--- a/packages/whisper-webgpu/package.json
+++ b/packages/whisper-webgpu/package.json
@@ -24,8 +24,8 @@
"url": "https://github.com/remotion-dev/remotion/issues"
},
"dependencies": {
- "@huggingface/transformers": "catalog:",
- "@remotion/captions": "workspace:*"
+ "@remotion/captions": "workspace:*",
+ "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c"
},
"exports": {
".": {
@@ -33,9 +33,14 @@
"module": "./dist/esm/index.mjs",
"import": "./dist/esm/index.mjs"
},
+ "./private-transformers": {
+ "module": "./dist/esm/private-transformers.mjs",
+ "import": "./dist/esm/private-transformers.mjs"
+ },
"./package.json": "./package.json"
},
"devDependencies": {
+ "@huggingface/transformers": "catalog:",
"@remotion/eslint-config-internal": "workspace:*",
"@typescript/native-preview": "catalog:",
"eslint": "catalog:"
diff --git a/packages/whisper-webgpu/src/index.ts b/packages/whisper-webgpu/src/index.ts
index a020d03828..49b9910d8a 100644
--- a/packages/whisper-webgpu/src/index.ts
+++ b/packages/whisper-webgpu/src/index.ts
@@ -24,5 +24,6 @@ export {transcribe} from './transcribe';
export type {
TranscribeOptions,
WhisperWebGpuTranscription,
+ WhisperWebGpuTask,
WhisperWebGpuWord,
} from './transcribe';
diff --git a/packages/whisper-webgpu/src/is-whisper-model-cached.ts b/packages/whisper-webgpu/src/is-whisper-model-cached.ts
index fe67b30e56..0b7f41252e 100644
--- a/packages/whisper-webgpu/src/is-whisper-model-cached.ts
+++ b/packages/whisper-webgpu/src/is-whisper-model-cached.ts
@@ -1,25 +1,25 @@
import {
- getModelInfo,
+ getHostedModelId,
WHISPER_WEBGPU_DTYPE,
type WhisperWebGpuModel,
} from './models';
+import {withRemotionModelHost} from './with-remotion-model-host';
export type IsWhisperModelCachedOptions = {
model: WhisperWebGpuModel;
};
-export const isWhisperModelCached = async ({
+export const isWhisperModelCached = ({
model,
}: IsWhisperModelCachedOptions): Promise<boolean> => {
- const {ModelRegistry} = await import('@huggingface/transformers');
- const {modelId} = getModelInfo(model);
-
- return ModelRegistry.is_pipeline_cached(
- 'automatic-speech-recognition',
- modelId,
- {
- device: 'webgpu',
- dtype: WHISPER_WEBGPU_DTYPE,
- },
- );
+ return withRemotionModelHost(({ModelRegistry}) => {
+ return ModelRegistry.is_pipeline_cached(
+ 'automatic-speech-recognition',
+ getHostedModelId(model),
+ {
+ device: 'webgpu',
+ dtype: WHISPER_WEBGPU_DTYPE,
+ },
+ );
+ });
};
diff --git a/packages/whisper-webgpu/src/load-whisper-model.ts b/packages/whisper-webgpu/src/load-whisper-model.ts
index 1c3861e271..17f5d79884 100644
--- a/packages/whisper-webgpu/src/load-whisper-model.ts
+++ b/packages/whisper-webgpu/src/load-whisper-model.ts
@@ -1,8 +1,10 @@
import {
+ getHostedModelId,
getModelInfo,
WHISPER_WEBGPU_DTYPE,
type WhisperWebGpuModel,
} from './models';
+import {withRemotionModelHost} from './with-remotion-model-host';
export type WhisperWebGpuModelLoadProgress = {
status: string;
@@ -53,67 +55,67 @@ const getOrCreateWhisperPipeline = ({
return {state: existing, alreadyLoaded: true, totalBytes};
}
- const loading = Promise.resolve().then(async () => {
- const {pipeline} = await import('@huggingface/transformers');
-
- onProgress?.({
- status: 'loading',
- file: null,
- progress: 0,
- loadedBytes: 0,
- totalBytes,
- });
-
- const {modelId} = modelInfo;
- const loadedByFile = new Map<string, number>();
- let lastProgress = 0;
- let lastLoadedBytes = 0;
- return pipeline('automatic-speech-recognition', modelId, {
- device: 'webgpu',
- dtype: WHISPER_WEBGPU_DTYPE,
- progress_callback: onProgress
- ? (event) => {
- const record = event as Record<string, unknown>;
- if (
- record.status === 'progress' &&
- typeof record.file === 'string' &&
- typeof record.loaded === 'number' &&
- Number.isFinite(record.loaded)
- ) {
- loadedByFile.set(
- record.file,
- Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),
- );
- const loadedBytes = [...loadedByFile.values()].reduce(
- (sum, loaded) => sum + loaded,
- 0,
- );
- lastProgress = Math.max(
- lastProgress,
- Math.min(loadedBytes / totalBytes, 0.99),
- );
- lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);
- onProgress({
- status: 'loading',
- file: null,
- progress: lastProgress,
- loadedBytes: lastLoadedBytes,
- totalBytes,
- });
- }
+ const loading = Promise.resolve().then(() => {
+ return withRemotionModelHost(({pipeline}) => {
+ onProgress?.({
+ status: 'loading',
+ file: null,
+ progress: 0,
+ loadedBytes: 0,
+ totalBytes,
+ });
- if (record.status === 'ready') {
- onProgress({
- status: 'ready',
- file: null,
- progress: 1,
- loadedBytes: Math.max(lastLoadedBytes, totalBytes),
- totalBytes,
- });
+ const hostedModelId = getHostedModelId(model);
+ const loadedByFile = new Map<string, number>();
+ let lastProgress = 0;
+ let lastLoadedBytes = 0;
+ return pipeline('automatic-speech-recognition', hostedModelId, {
+ device: 'webgpu',
+ dtype: WHISPER_WEBGPU_DTYPE,
+ progress_callback: onProgress
+ ? (event) => {
+ const record = event as Record<string, unknown>;
+ if (
+ record.status === 'progress' &&
+ typeof record.file === 'string' &&
+ typeof record.loaded === 'number' &&
+ Number.isFinite(record.loaded)
+ ) {
+ loadedByFile.set(
+ record.file,
+ Math.max(loadedByFile.get(record.file) ?? 0, record.loaded),
+ );
+ const loadedBytes = [...loadedByFile.values()].reduce(
+ (sum, loaded) => sum + loaded,
+ 0,
+ );
+ lastProgress = Math.max(
+ lastProgress,
+ Math.min(loadedBytes / totalBytes, 0.99),
+ );
+ lastLoadedBytes = Math.max(lastLoadedBytes, loadedBytes);
+ onProgress({
+ status: 'loading',
+ file: null,
+ progress: lastProgress,
+ loadedBytes: lastLoadedBytes,
+ totalBytes,
+ });
+ }
+
+ if (record.status === 'ready') {
+ onProgress({
+ status: 'ready',
+ file: null,
+ progress: 1,
+ loadedBytes: Math.max(lastLoadedBytes, totalBytes),
+ totalBytes,
+ });
+ }
}
- }
- : undefined,
- }) as Promise<LoadedWhisperPipeline>;
+ : undefined,
+ }) as Promise<LoadedWhisperPipeline>;
+ });
});
const state: LoadedWhisperPipelineState = {
loading,
diff --git a/packages/whisper-webgpu/src/models.ts b/packages/whisper-webgpu/src/models.ts
index 4e3f3e7dc4..623f3ff2bd 100644
--- a/packages/whisper-webgpu/src/models.ts
+++ b/packages/whisper-webgpu/src/models.ts
@@ -17,6 +17,7 @@ export type WhisperWebGpuModelInfo = {
modelId: string;
parameters: number;
multilingual: boolean;
+ supportsTranslation: boolean;
webGpuDownloadSize: number;
};
@@ -26,6 +27,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
modelId: 'onnx-community/whisper-tiny_timestamped',
parameters: 39_000_000,
multilingual: true,
+ supportsTranslation: true,
webGpuDownloadSize: 119_699_015,
},
'tiny.en': {
@@ -33,6 +35,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
modelId: 'onnx-community/whisper-tiny.en_timestamped',
parameters: 39_000_000,
multilingual: false,
+ supportsTranslation: false,
webGpuDownloadSize: 119_697_479,
},
base: {
@@ -40,6 +43,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
modelId: 'onnx-community/whisper-base_timestamped',
parameters: 74_000_000,
multilingual: true,
+ supportsTranslation: true,
webGpuDownloadSize: 206_190_057,
},
'base.en': {
@@ -47,6 +51,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
modelId: 'onnx-community/whisper-base.en_timestamped',
parameters: 74_000_000,
multilingual: false,
+ supportsTranslation: false,
webGpuDownloadSize: 206_188_009,
},
small: {
@@ -54,6 +59,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
modelId: 'onnx-community/whisper-small_timestamped',
parameters: 244_000_000,
multilingual: true,
+ supportsTranslation: true,
webGpuDownloadSize: 586_213_010,
},
'small.en': {
@@ -61,6 +67,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
modelId: 'onnx-community/whisper-small.en_timestamped',
parameters: 244_000_000,
multilingual: false,
+ supportsTranslation: false,
webGpuDownloadSize: 586_209_938,
},
medium: {
@@ -68,6 +75,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
modelId: 'onnx-community/whisper-medium_timestamped',
parameters: 769_000_000,
multilingual: true,
+ supportsTranslation: true,
webGpuDownloadSize: 1_698_508_143,
},
'medium.en': {
@@ -75,6 +83,7 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
modelId: 'onnx-community/whisper-medium.en_timestamped',
parameters: 769_000_000,
multilingual: false,
+ supportsTranslation: false,
webGpuDownloadSize: 1_698_504_047,
},
'large-v3-turbo': {
@@ -82,10 +91,23 @@ const MODEL_INFO: Record<WhisperWebGpuModel, WhisperWebGpuModelInfo> = {
modelId: 'onnx-community/whisper-large-v3-turbo_timestamped',
parameters: 809_000_000,
multilingual: true,
+ supportsTranslation: false,
webGpuDownloadSize: 2_882_584_170,
},
};
+const HOSTED_MODEL_IDS: Record<WhisperWebGpuModel, string> = {
+ tiny: 'whisper-tiny_timestamped-v1',
+ 'tiny.en': 'whisper-tiny.en_timestamped-v1',
+ base: 'whisper-base_timestamped-v1',
+ 'base.en': 'whisper-base.en_timestamped-v1',
+ small: 'whisper-small_timestamped-v1',
+ 'small.en': 'whisper-small.en_timestamped-v1',
+ medium: 'whisper-medium_timestamped-v1',
+ 'medium.en': 'whisper-medium.en_timestamped-v1',
+ 'large-v3-turbo': 'whisper-large-v3-turbo_timestamped-v1',
+};
+
export const getAvailableModels = (): WhisperWebGpuModelInfo[] => {
return WHISPER_WEBGPU_MODELS.map((model) => ({...MODEL_INFO[model]}));
};
@@ -96,6 +118,10 @@ export const getModelInfo = (
return MODEL_INFO[model];
};
+export const getHostedModelId = (model: WhisperWebGpuModel): string => {
+ return HOSTED_MODEL_IDS[model];
+};
+
export const WHISPER_WEBGPU_DTYPE = {
encoder_model: 'fp32',
decoder_model_merged: 'q4',
diff --git a/packages/whisper-webgpu/src/private-transformers.ts b/packages/whisper-webgpu/src/private-transformers.ts
new file mode 100644
index 0000000000..bb03ecc2a5
--- /dev/null
+++ b/packages/whisper-webgpu/src/private-transformers.ts
@@ -0,0 +1 @@
+export * from '@huggingface/transformers';
diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.ts
index d0b04f34d0..780c3ab669 100644
--- a/packages/whisper-webgpu/src/test/transcribe.test.ts
+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts
@@ -6,6 +6,18 @@ let pipelineInitialization:
options: Record<string, unknown>;
}
| undefined;
+let pipelineEnvironment:
+ | {
+ remoteHost: string;
+ remotePathTemplate: string;
+ }
+ | undefined;
+let pipelineEnvironmentAfterLoading:
+ | {
+ remoteHost: string;
+ remotePathTemplate: string;
+ }
+ | undefined;
let transcriptionCall:
| {
audio: Float32Array;
@@ -15,6 +27,7 @@ let transcriptionCall:
let disposed = false;
let disposeCalls = 0;
let pipelineInitializationCount = 0;
+const pipelineInitializationModelIds: string[] = [];
let pipelineInitializationGate: Promise<void> | null = null;
let onPipelineInitializationStarted: (() => void) | null = null;
let transcriptionGate: Promise<void> | null = null;
@@ -26,6 +39,19 @@ let cacheCheck:
options: Record<string, unknown>;
}
| undefined;
+let cacheCheckEnvironment:
+ | {
+ remoteHost: string;
+ remotePathTemplate: string;
+ }
+ | undefined;
+let cacheCheckError: Error | null = null;
+
+const originalTransformersEnvironment = {
+ remoteHost: 'https://huggingface.co/',
+ remotePathTemplate: '{model}/resolve/{revision}/',
+};
+const transformersEnvironment = {...originalTransformersEnvironment};
const fakePipeline = Object.assign(
async (audio: Float32Array, options: Record<string, unknown>) => {
@@ -52,6 +78,7 @@ const fakePipeline = Object.assign(
);
mock.module('@huggingface/transformers', () => ({
+ env: transformersEnvironment,
ModelRegistry: {
is_pipeline_cached: (
task: string,
@@ -59,6 +86,11 @@ mock.module('@huggingface/transformers', () => ({
options: Record<string, unknown>,
) => {
cacheCheck = {task, modelId, options};
+ cacheCheckEnvironment = {...transformersEnvironment};
+ if (cacheCheckError) {
+ return Promise.reject(cacheCheckError);
+ }
+
return Promise.resolve(true);
},
},
@@ -68,9 +100,12 @@ mock.module('@huggingface/transformers', () => ({
options: Record<string, unknown>,
) => {
pipelineInitializationCount++;
+ pipelineInitializationModelIds.push(modelId);
pipelineInitialization = {modelId, options};
+ pipelineEnvironment = {...transformersEnvironment};
onPipelineInitializationStarted?.();
await pipelineInitializationGate;
+ pipelineEnvironmentAfterLoading = {...transformersEnvironment};
const onProgress = options.progress_callback as
| ((progress: Record<string, unknown>) => void)
| undefined;
@@ -135,8 +170,13 @@ test('transcribes with word timestamps using WebGPU', async () => {
});
expect(pipelineInitialization?.modelId).toBe(
- 'onnx-community/whisper-small.en_timestamped',
+ 'whisper-small.en_timestamped-v1',
);
+ expect(pipelineEnvironment).toEqual({
+ remoteHost: 'https://remotion.media/',
+ remotePathTemplate: 'models/{model}/',
+ });
+ expect(pipelineEnvironmentAfterLoading).toEqual(pipelineEnvironment);
expect(pipelineInitialization?.options).toMatchObject({
device: 'webgpu',
dtype: {encoder_model: 'fp32', decoder_model_merged: 'q4'},
@@ -146,6 +186,12 @@ test('transcribes with word timestamps using WebGPU', async () => {
return_timestamps: 'word',
chunk_length_s: 30,
stride_length_s: 5,
+ force_full_sequences: false,
+ do_sample: false,
+ temperature: 1,
+ top_k: 50,
+ repetition_penalty: 1,
+ no_repeat_ngram_size: 0,
});
expect(result).toEqual({
text: 'Hello world free today.',
@@ -208,17 +254,29 @@ test('transcribes with word timestamps using WebGPU', async () => {
modelId: 'onnx-community/whisper-small.en_timestamped',
parameters: 244_000_000,
multilingual: false,
+ supportsTranslation: false,
webGpuDownloadSize: 586_209_938,
});
+ expect(models.find((model) => model.name === 'large-v3-turbo')).toMatchObject(
+ {
+ multilingual: true,
+ supportsTranslation: false,
+ },
+ );
expect(await isWhisperModelCached({model: 'small.en'})).toBe(true);
expect(cacheCheck).toEqual({
task: 'automatic-speech-recognition',
- modelId: 'onnx-community/whisper-small.en_timestamped',
+ modelId: 'whisper-small.en_timestamped-v1',
options: {
device: 'webgpu',
dtype: {encoder_model: 'fp32', decoder_model_merged: 'q4'},
},
});
+ expect(cacheCheckEnvironment).toEqual({
+ remoteHost: 'https://remotion.media/',
+ remotePathTemplate: 'models/{model}/',
+ });
+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);
const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');
const originalNavigator = Object.getOwnPropertyDescriptor(
@@ -270,6 +328,57 @@ test('transcribes with word timestamps using WebGPU', async () => {
expect(disposed).toBe(true);
});
+test('loads every public model from its explicit immutable hosted path', async () => {
+ const {disposeWhisperModel, getAvailableModels, loadWhisperModel} =
+ await import('../index');
+ const models = getAvailableModels();
+ const initializationsBeforeLoading = pipelineInitializationModelIds.length;
+
+ for (const model of models) {
+ await loadWhisperModel({model: model.name});
+ await disposeWhisperModel({model: model.name});
+ }
+
+ expect(
+ pipelineInitializationModelIds.slice(initializationsBeforeLoading),
+ ).toEqual([
+ 'whisper-tiny_timestamped-v1',
+ 'whisper-tiny.en_timestamped-v1',
+ 'whisper-base_timestamped-v1',
+ 'whisper-base.en_timestamped-v1',
+ 'whisper-small_timestamped-v1',
+ 'whisper-small.en_timestamped-v1',
+ 'whisper-medium_timestamped-v1',
+ 'whisper-medium.en_timestamped-v1',
+ 'whisper-large-v3-turbo_timestamped-v1',
+ ]);
+ expect(models.map(({modelId}) => modelId)).toEqual([
+ 'onnx-community/whisper-tiny_timestamped',
+ 'onnx-community/whisper-tiny.en_timestamped',
+ 'onnx-community/whisper-base_timestamped',
+ 'onnx-community/whisper-base.en_timestamped',
+ 'onnx-community/whisper-small_timestamped',
+ 'onnx-community/whisper-small.en_timestamped',
+ 'onnx-community/whisper-medium_timestamped',
+ 'onnx-community/whisper-medium.en_timestamped',
+ 'onnx-community/whisper-large-v3-turbo_timestamped',
+ ]);
+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);
+});
+
+test('restores the Transformers environment if a hosted model operation fails', async () => {
+ const {isWhisperModelCached} = await import('../index');
+ cacheCheckError = new Error('Could not read the model cache');
+ try {
+ await expect(isWhisperModelCached({model: 'tiny'})).rejects.toThrow(
+ 'Could not read the model cache',
+ );
+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);
+ } finally {
+ cacheCheckError = null;
+ }
+});
+
test('reports model progress without reaching 100% before every file loads', async () => {
const {disposeWhisperModel, loadWhisperModel} = await import('../index');
const progressValues: Array<{
@@ -313,7 +422,7 @@ test('reports model progress without reaching 100% before every file loads', asy
await disposeWhisperModel({model: 'medium.en'});
});
-test('validates language and finite chunk settings through transcribe()', async () => {
+test('validates and forwards transcription settings through transcribe()', async () => {
const {disposeWhisperModel, transcribe} = await import('../index');
const channelWaveform = new Float32Array(16_000);
@@ -327,6 +436,21 @@ test('validates language and finite chunk settings through transcribe()', async
).rejects.toThrow(
'The English-only model "tiny.en" does not support the language "de".',
);
+ await expect(
+ transcribe({
+ channelWaveform,
+ model: 'tiny.en',
+ task: 'translate',
+ }),
+ ).rejects.toThrow('does not support translation');
+ await expect(
+ transcribe({
+ channelWaveform,
+ language: 'en',
+ model: 'large-v3-turbo',
+ task: 'translate',
+ }),
+ ).rejects.toThrow('does not support translation');
await expect(
transcribe({
channelWaveform,
@@ -346,18 +470,66 @@ test('validates language and finite chunk settings through transcribe()', async
'strideLengthInSeconds must be a finite, non-negative number and less than half of chunkLengthInSeconds.',
);
- await transcribe({channelWaveform, model: 'tiny', language: 'de'});
+ const invalidOptions: Array<{
+ options: Record<string, unknown>;
+ optionName: string;
+ }> = [
+ {options: {forceFullSequences: 'yes'}, optionName: 'forceFullSequences'},
+ {options: {doSample: 'yes'}, optionName: 'doSample'},
+ {options: {temperature: Number.NaN}, optionName: 'temperature'},
+ {options: {temperature: 0}, optionName: 'temperature'},
+ {options: {topK: Number.NaN}, optionName: 'topK'},
+ {options: {topK: -1}, optionName: 'topK'},
+ {options: {topK: 1.5}, optionName: 'topK'},
+ {options: {repetitionPenalty: Number.NaN}, optionName: 'repetitionPenalty'},
+ {options: {repetitionPenalty: 0}, optionName: 'repetitionPenalty'},
+ {options: {noRepeatNgramSize: Number.NaN}, optionName: 'noRepeatNgramSize'},
+ {options: {noRepeatNgramSize: -1}, optionName: 'noRepeatNgramSize'},
+ {options: {noRepeatNgramSize: 1.5}, optionName: 'noRepeatNgramSize'},
+ {options: {task: 'summarize'}, optionName: 'task'},
+ ];
+
+ for (const {options, optionName} of invalidOptions) {
+ await expect(
+ transcribe({
+ channelWaveform,
+ model: 'tiny.en',
+ ...options,
+ } as never),
+ ).rejects.toThrow(optionName);
+ }
+
+ await transcribe({
+ channelWaveform,
+ model: 'tiny',
+ language: 'de',
+ task: 'translate',
+ forceFullSequences: true,
+ doSample: true,
+ temperature: 0.7,
+ topK: 25,
+ repetitionPenalty: 1.2,
+ noRepeatNgramSize: 3,
+ });
expect(transcriptionCall?.options).toEqual({
return_timestamps: 'word',
chunk_length_s: 30,
stride_length_s: 5,
language: 'de',
+ task: 'translate',
+ force_full_sequences: true,
+ do_sample: true,
+ temperature: 0.7,
+ top_k: 25,
+ repetition_penalty: 1.2,
+ no_repeat_ngram_size: 3,
});
await disposeWhisperModel({model: 'tiny'});
});
-test('shares concurrent initialization and waits for it before disposal', async () => {
- const {disposeWhisperModel, loadWhisperModel} = await import('../index');
+test('shares concurrent initialization and keeps the model host scoped until every operation completes', async () => {
+ const {disposeWhisperModel, isWhisperModelCached, loadWhisperModel} =
+ await import('../index');
let releaseInitialization: () => void = () => {};
pipelineInitializationGate = new Promise<void>((resolve) => {
releaseInitialization = resolve;
@@ -372,6 +544,27 @@ test('shares concurrent initialization and waits for it before disposal', async
const secondLoad = loadWhisperModel({model: 'base'});
await initializationStarted;
expect(pipelineInitializationCount - initializationsBeforeLoading).toBe(1);
+ expect(transformersEnvironment).toEqual({
+ remoteHost: 'https://remotion.media/',
+ remotePathTemplate: 'models/{model}/',
+ });
+ expect(
+ (
+ transformersEnvironment as typeof transformersEnvironment &
+ Record<symbol, unknown>
+ )[Symbol.for('@remotion/whisper-webgpu/model-host-state')],
+ ).toMatchObject({
+ activeOperations: 1,
+ previousRemoteConfiguration: originalTransformersEnvironment,
+ });
+ await expect(isWhisperModelCached({model: 'tiny.en'})).resolves.toBe(true);
+ expect(cacheCheck).toMatchObject({
+ modelId: 'whisper-tiny.en_timestamped-v1',
+ });
+ expect(transformersEnvironment).toEqual({
+ remoteHost: 'https://remotion.media/',
+ remotePathTemplate: 'models/{model}/',
+ });
const disposal = disposeWhisperModel({model: 'base'});
await Promise.resolve();
expect(disposeCalls).toBe(disposalsBeforeLoading);
@@ -383,10 +576,40 @@ test('shares concurrent initialization and waits for it before disposal', async
]);
await disposal;
expect(disposeCalls).toBe(disposalsBeforeLoading + 1);
+ expect(transformersEnvironment).toEqual(originalTransformersEnvironment);
pipelineInitializationGate = null;
onPipelineInitializationStarted = null;
});
+test('only restores Transformers environment fields left unchanged by the consumer', async () => {
+ const {disposeWhisperModel, loadWhisperModel} = await import('../index');
+ let releaseInitialization: () => void = () => {};
+ pipelineInitializationGate = new Promise<void>((resolve) => {
+ releaseInitialization = resolve;
+ });
+ const initializationStarted = new Promise<void>((resolve) => {
+ onPipelineInitializationStarted = resolve;
+ });
+ const loading = loadWhisperModel({model: 'small'});
+
+ try {
+ await initializationStarted;
+ transformersEnvironment.remoteHost = 'https://models.example.com/';
+ releaseInitialization();
+ await loading;
+ expect(transformersEnvironment).toEqual({
+ remoteHost: 'https://models.example.com/',
+ remotePathTemplate: '{model}/resolve/{revision}/',
+ });
+ await disposeWhisperModel({model: 'small'});
+ } finally {
+ releaseInitialization();
+ pipelineInitializationGate = null;
+ onPipelineInitializationStarted = null;
+ Object.assign(transformersEnvironment, originalTransformersEnvironment);
+ }
+});
+
test('waits for active transcriptions before disposal', async () => {
const {disposeWhisperModel, transcribe} = await import('../index');
let releaseTranscription: () => void = () => {};
diff --git a/packages/whisper-webgpu/src/transcribe.ts b/packages/whisper-webgpu/src/transcribe.ts
index 95c287df04..6299453fc8 100644
--- a/packages/whisper-webgpu/src/transcribe.ts
+++ b/packages/whisper-webgpu/src/transcribe.ts
@@ -16,12 +16,21 @@ export type WhisperWebGpuTranscription = {
model: WhisperWebGpuModel;
};
+export type WhisperWebGpuTask = 'transcribe' | 'translate';
+
export type TranscribeOptions = {
channelWaveform: Float32Array;
model: WhisperWebGpuModel;
language?: string;
+ task?: WhisperWebGpuTask;
chunkLengthInSeconds?: number;
strideLengthInSeconds?: number;
+ forceFullSequences?: boolean;
+ doSample?: boolean;
+ temperature?: number;
+ topK?: number;
+ repetitionPenalty?: number;
+ noRepeatNgramSize?: number;
onModelLoadProgress?: OnWhisperWebGpuModelLoadProgress;
};
@@ -39,8 +48,15 @@ export const transcribe = async ({
channelWaveform,
model,
language,
+ task = 'transcribe',
chunkLengthInSeconds = 30,
strideLengthInSeconds = 5,
+ forceFullSequences = false,
+ doSample = false,
+ temperature = 1,
+ topK = 50,
+ repetitionPenalty = 1,
+ noRepeatNgramSize = 0,
onModelLoadProgress,
}: TranscribeOptions): Promise<WhisperWebGpuTranscription> => {
if (channelWaveform.length === 0) {
@@ -63,7 +79,41 @@ export const transcribe = async ({
);
}
- const {multilingual} = getModelInfo(model);
+ if (typeof forceFullSequences !== 'boolean') {
+ throw new TypeError('forceFullSequences must be a boolean.');
+ }
+
+ if (typeof doSample !== 'boolean') {
+ throw new TypeError('doSample must be a boolean.');
+ }
+
+ if (!Number.isFinite(temperature) || temperature <= 0) {
+ throw new TypeError('temperature must be a finite number greater than 0.');
+ }
+
+ if (!Number.isInteger(topK) || topK < 0) {
+ throw new TypeError('topK must be a non-negative integer.');
+ }
+
+ if (!Number.isFinite(repetitionPenalty) || repetitionPenalty <= 0) {
+ throw new TypeError(
+ 'repetitionPenalty must be a finite number greater than 0.',
+ );
+ }
+
+ if (!Number.isInteger(noRepeatNgramSize) || noRepeatNgramSize < 0) {
+ throw new TypeError('noRepeatNgramSize must be a non-negative integer.');
+ }
+
+ if (task !== 'transcribe' && task !== 'translate') {
+ throw new TypeError('task must be either "transcribe" or "translate".');
+ }
+
+ const {multilingual, supportsTranslation} = getModelInfo(model);
+ if (task === 'translate' && !supportsTranslation) {
+ throw new Error(`The model "${model}" does not support translation.`);
+ }
+
if (multilingual && (language === undefined || language === 'auto')) {
throw new Error(
`The language option is required for the multilingual model "${model}" because automatic language detection is not supported.`,
@@ -90,7 +140,13 @@ export const transcribe = async ({
return_timestamps: 'word',
chunk_length_s: chunkLengthInSeconds,
stride_length_s: strideLengthInSeconds,
- ...(multilingual ? {language} : {}),
+ force_full_sequences: forceFullSequences,
+ do_sample: doSample,
+ temperature,
+ top_k: topK,
+ repetition_penalty: repetitionPenalty,
+ no_repeat_ngram_size: noRepeatNgramSize,
+ ...(multilingual ? {language, task} : {}),
})) as TransformersJsTranscription;
},
});
diff --git a/packages/whisper-webgpu/src/with-remotion-model-host.ts b/packages/whisper-webgpu/src/with-remotion-model-host.ts
new file mode 100644
index 0000000000..e1e201788c
--- /dev/null
+++ b/packages/whisper-webgpu/src/with-remotion-model-host.ts
@@ -0,0 +1,71 @@
+import type {env, ModelRegistry, pipeline} from '@huggingface/transformers';
+
+const REMOTION_MODEL_HOST = 'https://remotion.media/';
+const REMOTION_MODEL_PATH_TEMPLATE = 'models/{model}/';
+const REMOTION_MODEL_HOST_STATE = Symbol.for(
+ '@remotion/whisper-webgpu/model-host-state',
+);
+
+type TransformersModule = {
+ readonly env: typeof env;
+ readonly ModelRegistry: typeof ModelRegistry;
+ readonly pipeline: typeof pipeline;
+};
+
+type ModelHostState = {
+ activeOperations: number;
+ previousRemoteConfiguration: {
+ remoteHost: string;
+ remotePathTemplate: string;
+ } | null;
+};
+
+export const withRemotionModelHost = async <ReturnValue>(
+ operation: (transformers: TransformersModule) => Promise<ReturnValue>,
+): Promise<ReturnValue> => {
+ // bundle.ts puts this import into a private lazy chunk. This keeps the mutable
+ // Transformers.js environment isolated from other pipelines in the application.
+ const transformers = await import('@huggingface/transformers');
+ const {env} = transformers;
+ const environmentWithState = env as typeof env &
+ Record<symbol, ModelHostState | undefined>;
+ let state = environmentWithState[REMOTION_MODEL_HOST_STATE];
+ if (!state) {
+ state = {
+ activeOperations: 0,
+ previousRemoteConfiguration: null,
+ };
+ Object.defineProperty(environmentWithState, REMOTION_MODEL_HOST_STATE, {
+ value: state,
+ });
+ }
+
+ if (state.activeOperations === 0) {
+ state.previousRemoteConfiguration = {
+ remoteHost: env.remoteHost,
+ remotePathTemplate: env.remotePathTemplate,
+ };
+ env.remoteHost = REMOTION_MODEL_HOST;
+ env.remotePathTemplate = REMOTION_MODEL_PATH_TEMPLATE;
+ }
+
+ state.activeOperations++;
+ try {
+ return await operation(transformers);
+ } finally {
+ state.activeOperations--;
+ if (state.activeOperations === 0) {
+ const configurationToRestore = state.previousRemoteConfiguration;
+ state.previousRemoteConfiguration = null;
+ if (configurationToRestore) {
+ if (env.remoteHost === REMOTION_MODEL_HOST) {
+ env.remoteHost = configurationToRestore.remoteHost;
+ }
+
+ if (env.remotePathTemplate === REMOTION_MODEL_PATH_TEMPLATE) {
+ env.remotePathTemplate = configurationToRestore.remotePathTemplate;
+ }
+ }
+ }
+ }
+};