Skip to content

🎥 Make videos programmatically with React

@remotion/whisper-webgpu: Add Node.js transcription

JonnyBurger1d54f7520c4e
diff --git a/bun.lock b/bun.lock
index 7a6ef295b6..9c2d4ae6ec 100644
--- a/bun.lock
+++ b/bun.lock
@@ -552,6 +552,7 @@
"@mediabunny/ac3": "catalog:",
"@mediabunny/dts": "catalog:",
"@mediabunny/prores": "catalog:",
+ "@mediabunny/server": "catalog:",
"@radix-ui/react-dropdown-menu": "2.1.16",
"@react-three/fiber": "catalog:",
"@remotion-dev/cube-presentation": "https://www.remotion.pro/pkg/cube-transition/1.0.2/docs",
diff --git a/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx
index 050a5bac4d..1902c23e8f 100644
--- a/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx
+++ b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx
@@ -6,7 +6,9 @@ crumb: '@remotion/whisper-webgpu'
# canUseWhisperWebGpu()<AvailableFrom v="4.0.518" />
-Asynchronously checks whether WebGPU can be used in the current browser. WebGPU, a usable adapter, and a secure context (or localhost) is required.
+Asynchronously checks whether WebGPU transcription is supported in the current environment.
+
+In a browser, WebGPU, a usable adapter, and a secure context (or localhost) are required. Node.js support is available from <AvailableFrom v="4.0.527" inline />. In Node.js, the function checks whether the current operating system and architecture are supported by the native WebGPU backend.
```ts twoslash title="support.ts"
import {canUseWhisperWebGpu} from '@remotion/whisper-webgpu';
@@ -43,7 +45,7 @@ If `supported` is `false`, a human-readable explanation of why WebGPU transcript
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx
index 9ddea4e6b5..247e04da72 100644
--- a/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx
+++ b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx
@@ -6,7 +6,7 @@ crumb: '@remotion/whisper-webgpu'
# disposeWhisperModel()<AvailableFrom v="4.0.518" />
-Releases the memory used by initialized Whisper pipelines. Persistent browser-cached model files are not deleted.
+Releases the memory used by initialized Whisper pipelines. Persistently cached model files are not deleted.
```ts twoslash title="dispose.ts"
import {disposeWhisperModel} from '@remotion/whisper-webgpu';
@@ -22,7 +22,7 @@ Only dispose this model. Default: all loaded models.
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/whisper-webgpu/get-available-models.mdx b/packages/docs/docs/whisper-webgpu/get-available-models.mdx
index a449359d5e..cf18b13add 100644
--- a/packages/docs/docs/whisper-webgpu/get-available-models.mdx
+++ b/packages/docs/docs/whisper-webgpu/get-available-models.mdx
@@ -55,7 +55,7 @@ We noticed that hosting the models on R2 leads to faster downloading than throug
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/whisper-webgpu/index.mdx b/packages/docs/docs/whisper-webgpu/index.mdx
index aeeed1647f..f6518a4954 100644
--- a/packages/docs/docs/whisper-webgpu/index.mdx
+++ b/packages/docs/docs/whisper-webgpu/index.mdx
@@ -1,12 +1,12 @@
---
image: /generated/articles-docs-whisper-webgpu-index.png
title: '@remotion/whisper-webgpu'
-crumb: 'Transcribe audio in browser'
+crumb: 'Transcribe audio locally'
---
# @remotion/whisper-webgpu<AvailableFrom v="4.0.518" />
-Transcribe audio locally in the browser using timestamped Whisper models and WebGPU through [Transformers.js](https://huggingface.co/docs/transformers.js), and convert the result to [`@remotion/captions`](/docs/captions).
+Transcribe audio locally in the browser or Node.js using timestamped Whisper models and WebGPU through [Transformers.js](https://huggingface.co/docs/transformers.js), and convert the result to [`@remotion/captions`](/docs/captions).
import {TableOfContents} from './table-of-contents';
@@ -34,7 +34,9 @@ const {captions} = toCaptions({whisperWebGpuOutput: transcription});
console.log(captions);
```
-The model is downloaded on the first call and cached by the browser. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) first to display model download progress.
+The model is downloaded on the first call and cached by the runtime. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) first to display model download progress.
+
+See [Using `@remotion/whisper-webgpu` in Node.js](/docs/whisper-webgpu/node) for a complete Node.js example.
## Model hosting
diff --git a/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx
index 835704d66c..a66c0e3b62 100644
--- a/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx
+++ b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx
@@ -6,7 +6,7 @@ crumb: '@remotion/whisper-webgpu'
# isWhisperModelCached()<AvailableFrom v="4.0.518" />
-Checks whether all files needed for a timestamped Whisper model from `remotion.media` are present in the browser cache. The check does not download or initialize the model.
+Checks whether all files needed for a timestamped Whisper model from `remotion.media` are present in the persistent Transformers.js cache. The check does not download or initialize the model.
```ts twoslash title="model-cache.ts"
import {isWhisperModelCached} from '@remotion/whisper-webgpu';
@@ -30,7 +30,7 @@ Models cached from Hugging Face by versions before 4.0.522 do not satisfy this c
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
index b99a7eaf08..a9fa0e8f18 100644
--- a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
+++ b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx
@@ -35,13 +35,13 @@ Called while model files are resolved and downloaded. `progress` is the aggregat
Returns whether the same model pipeline was already loaded in memory.
-The browser manages its persistent [Transformers.js](https://huggingface.co/docs/transformers.js) cache. This function also retains the initialized model in memory until [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model) is called or the page closes.
+[Transformers.js](https://huggingface.co/docs/transformers.js) manages the persistent model cache. This function also retains the initialized model in memory until [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model) is called or the process or page closes.
Versions before 4.0.522 downloaded models from Hugging Face. A model cached by an earlier version is downloaded once more from `remotion.media`, then reused from the new cache entry when using the default Transformers.js cache settings.
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/whisper-webgpu/node.mdx b/packages/docs/docs/whisper-webgpu/node.mdx
new file mode 100644
index 0000000000..f8bf717f67
--- /dev/null
+++ b/packages/docs/docs/whisper-webgpu/node.mdx
@@ -0,0 +1,140 @@
+---
+image: /generated/articles-docs-whisper-webgpu-node.png
+title: 'Using @remotion/whisper-webgpu in Node.js'
+crumb: '@remotion/whisper-webgpu'
+---
+
+# Using @remotion/whisper-webgpu in Node.js<AvailableFrom v="4.0.527" />
+
+`@remotion/whisper-webgpu` can transcribe a 16kHz mono waveform in Node.js using the native WebGPU backend from [Transformers.js](https://huggingface.co/docs/transformers.js).
+
+The following example uses [`@mediabunny/server`](https://mediabunny.dev/guide/extensions/server) to decode and resample a local audio or video file before transcribing it.
+
+## Installation
+
+<Installation pkg="@remotion/whisper-webgpu @huggingface/transformers mediabunny @mediabunny/server" />
+
+## Example
+
+```ts twoslash title="transcribe.ts"
+import {registerMediabunnyServer} from '@mediabunny/server';
+import {
+ WHISPER_WEBGPU_SAMPLE_RATE,
+ disposeWhisperModel,
+ toCaptions,
+ transcribe,
+} from '@remotion/whisper-webgpu';
+import {
+ ALL_FORMATS,
+ Conversion,
+ FilePathSource,
+ Input,
+ NullTarget,
+ Output,
+ WavOutputFormat,
+} from 'mediabunny';
+import {writeFile} from 'node:fs/promises';
+
+registerMediabunnyServer();
+
+const input = new Input({
+ formats: ALL_FORMATS,
+ source: new FilePathSource('audio.mp3'),
+});
+const chunks: Array<{startFrame: number; waveform: Float32Array}> = [];
+
+try {
+ const audioTrack = await input.getPrimaryAudioTrack();
+ if (audioTrack === null) {
+ throw new Error('The media does not contain an audio track.');
+ }
+
+ const conversion = await Conversion.init({
+ input,
+ output: new Output({
+ format: new WavOutputFormat(),
+ target: new NullTarget(),
+ }),
+ video: {discard: true},
+ audio: (track) => {
+ if (track.id !== audioTrack.id) {
+ return {discard: true};
+ }
+
+ return {
+ codec: 'pcm-f32',
+ forceTranscode: true,
+ numberOfChannels: 1,
+ sampleFormat: 'f32',
+ sampleRate: WHISPER_WEBGPU_SAMPLE_RATE,
+ process: (sample) => {
+ const waveform = new Float32Array(
+ sample.allocationSize({format: 'f32', planeIndex: 0}) /
+ Float32Array.BYTES_PER_ELEMENT,
+ );
+ sample.copyTo(waveform, {format: 'f32', planeIndex: 0});
+ chunks.push({
+ startFrame: Math.round(
+ sample.timestamp * WHISPER_WEBGPU_SAMPLE_RATE,
+ ),
+ waveform,
+ });
+ return sample;
+ },
+ };
+ },
+ });
+
+ if (!conversion.isValid) {
+ throw new Error('The audio track cannot be decoded.');
+ }
+
+ await conversion.execute();
+} finally {
+ input.dispose();
+}
+
+const waveformLength = chunks.reduce(
+ (max, chunk) => Math.max(max, chunk.startFrame + chunk.waveform.length),
+ 0,
+);
+const channelWaveform = new Float32Array(waveformLength);
+for (const chunk of chunks) {
+ const destinationStart = Math.max(0, chunk.startFrame);
+ const sourceStart = Math.max(0, -chunk.startFrame);
+ const availableLength = Math.min(
+ chunk.waveform.length - sourceStart,
+ channelWaveform.length - destinationStart,
+ );
+
+ if (availableLength > 0) {
+ channelWaveform.set(
+ chunk.waveform.subarray(sourceStart, sourceStart + availableLength),
+ destinationStart,
+ );
+ }
+}
+
+const model = 'small.en';
+try {
+ const transcription = await transcribe({channelWaveform, model});
+ const {captions} = toCaptions({whisperWebGpuOutput: transcription});
+ await writeFile('captions.json', JSON.stringify(captions, null, 2));
+} finally {
+ await disposeWhisperModel({model});
+}
+```
+
+The model is downloaded on the first run and cached on the file system. The input to [`transcribe()`](/docs/whisper-webgpu/transcribe) must be a mono `Float32Array` sampled at 16kHz.
+
+Call [`canUseWhisperWebGpu()`](/docs/whisper-webgpu/can-use-whisper-webgpu) before loading the model if the script may run on different operating systems or CPU architectures.
+
+## Supported platforms
+
+The native WebGPU backend supports Node.js on macOS x64 and Arm64, Windows x64 and Arm64, and Linux x64.
+
+## See also
+
+- [`transcribe()`](/docs/whisper-webgpu/transcribe)
+- [`toCaptions()`](/docs/whisper-webgpu/to-captions)
+- [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model)
diff --git a/packages/docs/docs/whisper-webgpu/remove-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/remove-whisper-model.mdx
index bd06eafe2b..9c0056c0a5 100644
--- a/packages/docs/docs/whisper-webgpu/remove-whisper-model.mdx
+++ b/packages/docs/docs/whisper-webgpu/remove-whisper-model.mdx
@@ -6,7 +6,7 @@ crumb: '@remotion/whisper-webgpu'
# removeWhisperModel()<AvailableFrom v="4.0.523" />
-Removes a Whisper model from memory and the persistent browser cache.
+Removes a Whisper model from memory and the persistent Transformers.js cache.
If the model is still loading or transcribing, removal waits for the operation to finish.
@@ -24,7 +24,7 @@ One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-a
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
index 5baa367f2d..5537c3ede0 100644
--- a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
+++ b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx
@@ -5,6 +5,10 @@ import {TOCItem} from '../../components/TableOfContents/TOCItem';
export const TableOfContents: React.FC = () => {
return (
<Grid>
+ <TOCItem link="/docs/whisper-webgpu/node">
+ <strong>Using in Node.js</strong>
+ <div>Decode media and transcribe it in Node.js</div>
+ </TOCItem>
<TOCItem link="/docs/whisper-webgpu/can-use-whisper-webgpu">
<strong>canUseWhisperWebGpu()</strong>
<div>Check whether WebGPU is available</div>
@@ -27,7 +31,7 @@ export const TableOfContents: React.FC = () => {
</TOCItem>
<TOCItem link="/docs/whisper-webgpu/remove-whisper-model">
<strong>removeWhisperModel()</strong>
- <div>Remove a model from the browser cache</div>
+ <div>Remove a model from the persistent cache</div>
</TOCItem>
<TOCItem link="/docs/whisper-webgpu/transcribe">
<strong>transcribe()</strong>
diff --git a/packages/docs/docs/whisper-webgpu/transcribe.mdx b/packages/docs/docs/whisper-webgpu/transcribe.mdx
index e806506022..a04cb7707a 100644
--- a/packages/docs/docs/whisper-webgpu/transcribe.mdx
+++ b/packages/docs/docs/whisper-webgpu/transcribe.mdx
@@ -86,7 +86,7 @@ Word-level timestamps are returned after transcription completes. [Transformers.
## Compatibility
-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />
+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />
## See also
diff --git a/packages/docs/package.json b/packages/docs/package.json
index 625321c73d..20abe53b7a 100644
--- a/packages/docs/package.json
+++ b/packages/docs/package.json
@@ -39,6 +39,7 @@
"@mediabunny/ac3": "catalog:",
"@mediabunny/dts": "catalog:",
"@mediabunny/prores": "catalog:",
+ "@mediabunny/server": "catalog:",
"@radix-ui/react-dropdown-menu": "2.1.16",
"@react-three/fiber": "catalog:",
"@remotion-dev/cube-presentation": "https://www.remotion.pro/pkg/cube-transition/1.0.2/docs",
diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.ts
index d04b94b83c..7acda71fb0 100644
--- a/packages/docs/sidebars.ts
+++ b/packages/docs/sidebars.ts
@@ -1015,6 +1015,7 @@ const sidebars: SidebarsConfig = {
id: 'whisper-webgpu/index',
},
items: [
+ 'whisper-webgpu/node',
'whisper-webgpu/can-use-whisper-webgpu',
'whisper-webgpu/get-available-models',
'whisper-webgpu/clear-stale-models',
diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.ts
index 3e4d23052e..b08d38b964 100644
--- a/packages/docs/src/data/articles.ts
+++ b/packages/docs/src/data/articles.ts
@@ -9305,7 +9305,7 @@ export const articles = [
title: '@remotion/whisper-webgpu',
relativePath: 'docs/whisper-webgpu/index.mdx',
compId: 'articles-docs-whisper-webgpu-index',
- crumb: 'Transcribe audio in browser',
+ crumb: 'Transcribe audio locally',
noAi: false,
slug: 'whisper-webgpu/index',
},
@@ -9327,6 +9327,15 @@ export const articles = [
noAi: false,
slug: 'whisper-webgpu/load-whisper-model',
},
+ {
+ id: 'whisper-webgpu/node',
+ title: 'Using @remotion/whisper-webgpu in Node.js',
+ relativePath: 'docs/whisper-webgpu/node.mdx',
+ compId: 'articles-docs-whisper-webgpu-node',
+ crumb: '@remotion/whisper-webgpu',
+ noAi: false,
+ slug: 'whisper-webgpu/node',
+ },
{
id: 'whisper-webgpu/remove-whisper-model',
title: 'removeWhisperModel()',
diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-node.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-node.png
new file mode 100644
index 0000000000..d4be72f1aa
Binary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-node.png differ
diff --git a/packages/whisper-webgpu/README.md b/packages/whisper-webgpu/README.md
index 335bed9b24..afc9dd9ab4 100644
--- a/packages/whisper-webgpu/README.md
+++ b/packages/whisper-webgpu/README.md
@@ -1,6 +1,6 @@
# @remotion/whisper-webgpu
-Transcribe audio in the browser using Whisper and WebGPU
+Transcribe audio in the browser or Node.js using Whisper and WebGPU
[![NPM Downloads](https://img.shields.io/npm/dm/@remotion/whisper-webgpu.svg?style=flat&color=black&label=Downloads)](https://npmcharts.com/compare/@remotion/whisper-webgpu?minimal=true)
diff --git a/packages/whisper-webgpu/package.json b/packages/whisper-webgpu/package.json
index f5d5a95b27..cdb8764a0a 100644
--- a/packages/whisper-webgpu/package.json
+++ b/packages/whisper-webgpu/package.json
@@ -1,7 +1,7 @@
{
"name": "@remotion/whisper-webgpu",
"version": "4.0.526",
- "description": "Transcribe audio in the browser using Whisper and WebGPU",
+ "description": "Transcribe audio using Whisper and WebGPU",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"module": "dist/esm/index.mjs",
diff --git a/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts b/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts
index 1c71d634b8..f2c6c09b2d 100644
--- a/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts
+++ b/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts
@@ -17,11 +17,29 @@ export type CanUseWhisperWebGpuResult =
export const canUseWhisperWebGpu =
async (): Promise<CanUseWhisperWebGpuResult> => {
if (typeof window === 'undefined') {
+ if (typeof process !== 'undefined' && process.release?.name === 'node') {
+ const supportsNativeWebGpu =
+ (process.platform === 'darwin' &&
+ (process.arch === 'arm64' || process.arch === 'x64')) ||
+ (process.platform === 'win32' &&
+ (process.arch === 'arm64' || process.arch === 'x64')) ||
+ (process.platform === 'linux' && process.arch === 'x64');
+
+ if (supportsNativeWebGpu) {
+ return {supported: true};
+ }
+
+ return {
+ supported: false,
+ reason: WhisperWebGpuUnsupportedReason.WebGpuUnavailable,
+ detailedReason: `The native WebGPU backend is not available on ${process.platform} ${process.arch}.`,
+ };
+ }
+
return {
supported: false,
reason: WhisperWebGpuUnsupportedReason.WindowUndefined,
- detailedReason:
- '`window` is not defined. @remotion/whisper-webgpu is intended for browser environments.',
+ detailedReason: 'WebGPU is not available in this environment.',
};
}
diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.ts
index d81f198826..2289fe2c9d 100644
--- a/packages/whisper-webgpu/src/test/transcribe.test.ts
+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts
@@ -349,6 +349,7 @@ test('transcribes with word timestamps using WebGPU', async () => {
remotePathTemplate: 'models/{model}/',
});
expect(transformersEnvironment).toEqual(originalTransformersEnvironment);
+ expect(await canUseWhisperWebGpu()).toEqual({supported: true});
const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');
const originalNavigator = Object.getOwnPropertyDescriptor(