🎥 Make videos programmatically with React
@remotion/whisper-webgpu: Add Node.js transcription
JonnyBurger1d54f7520c4e
diff --git a/bun.lock b/bun.lockindex 7a6ef295b6..9c2d4ae6ec 100644--- a/bun.lock+++ b/bun.lock@@ -552,6 +552,7 @@"@mediabunny/ac3": "catalog:","@mediabunny/dts": "catalog:","@mediabunny/prores": "catalog:",+ "@mediabunny/server": "catalog:","@radix-ui/react-dropdown-menu": "2.1.16","@react-three/fiber": "catalog:","@remotion-dev/cube-presentation": "https://www.remotion.pro/pkg/cube-transition/1.0.2/docs",diff --git a/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdxindex 050a5bac4d..1902c23e8f 100644--- a/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx+++ b/packages/docs/docs/whisper-webgpu/can-use-whisper-webgpu.mdx@@ -6,7 +6,9 @@ crumb: '@remotion/whisper-webgpu'# canUseWhisperWebGpu()<AvailableFrom v="4.0.518" />-Asynchronously checks whether WebGPU can be used in the current browser. WebGPU, a usable adapter, and a secure context (or localhost) is required.+Asynchronously checks whether WebGPU transcription is supported in the current environment.++In a browser, WebGPU, a usable adapter, and a secure context (or localhost) are required. Node.js support is available from <AvailableFrom v="4.0.527" inline />. In Node.js, the function checks whether the current operating system and architecture are supported by the native WebGPU backend.```ts twoslash title="support.ts"import {canUseWhisperWebGpu} from '@remotion/whisper-webgpu';@@ -43,7 +45,7 @@ If `supported` is `false`, a human-readable explanation of why WebGPU transcript## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdxindex 9ddea4e6b5..247e04da72 100644--- a/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx+++ b/packages/docs/docs/whisper-webgpu/dispose-whisper-model.mdx@@ -6,7 +6,7 @@ crumb: '@remotion/whisper-webgpu'# disposeWhisperModel()<AvailableFrom v="4.0.518" />-Releases the memory used by initialized Whisper pipelines. Persistent browser-cached model files are not deleted.+Releases the memory used by initialized Whisper pipelines. Persistently cached model files are not deleted.```ts twoslash title="dispose.ts"import {disposeWhisperModel} from '@remotion/whisper-webgpu';@@ -22,7 +22,7 @@ Only dispose this model. Default: all loaded models.## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/whisper-webgpu/get-available-models.mdx b/packages/docs/docs/whisper-webgpu/get-available-models.mdxindex a449359d5e..cf18b13add 100644--- a/packages/docs/docs/whisper-webgpu/get-available-models.mdx+++ b/packages/docs/docs/whisper-webgpu/get-available-models.mdx@@ -55,7 +55,7 @@ We noticed that hosting the models on R2 leads to faster downloading than throug## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/whisper-webgpu/index.mdx b/packages/docs/docs/whisper-webgpu/index.mdxindex aeeed1647f..f6518a4954 100644--- a/packages/docs/docs/whisper-webgpu/index.mdx+++ b/packages/docs/docs/whisper-webgpu/index.mdx@@ -1,12 +1,12 @@---image: /generated/articles-docs-whisper-webgpu-index.pngtitle: '@remotion/whisper-webgpu'-crumb: 'Transcribe audio in browser'+crumb: 'Transcribe audio locally'---# @remotion/whisper-webgpu<AvailableFrom v="4.0.518" />-Transcribe audio locally in the browser using timestamped Whisper models and WebGPU through [Transformers.js](https://huggingface.co/docs/transformers.js), and convert the result to [`@remotion/captions`](/docs/captions).+Transcribe audio locally in the browser or Node.js using timestamped Whisper models and WebGPU through [Transformers.js](https://huggingface.co/docs/transformers.js), and convert the result to [`@remotion/captions`](/docs/captions).import {TableOfContents} from './table-of-contents';@@ -34,7 +34,9 @@ const {captions} = toCaptions({whisperWebGpuOutput: transcription});console.log(captions);```-The model is downloaded on the first call and cached by the browser. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) first to display model download progress.+The model is downloaded on the first call and cached by the runtime. Call [`loadWhisperModel()`](/docs/whisper-webgpu/load-whisper-model) first to display model download progress.++See [Using `@remotion/whisper-webgpu` in Node.js](/docs/whisper-webgpu/node) for a complete Node.js example.## Model hostingdiff --git a/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdxindex 835704d66c..a66c0e3b62 100644--- a/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx+++ b/packages/docs/docs/whisper-webgpu/is-whisper-model-cached.mdx@@ -6,7 +6,7 @@ crumb: '@remotion/whisper-webgpu'# isWhisperModelCached()<AvailableFrom v="4.0.518" />-Checks whether all files needed for a timestamped Whisper model from `remotion.media` are present in the browser cache. The check does not download or initialize the model.+Checks whether all files needed for a timestamped Whisper model from `remotion.media` are present in the persistent Transformers.js cache. The check does not download or initialize the model.```ts twoslash title="model-cache.ts"import {isWhisperModelCached} from '@remotion/whisper-webgpu';@@ -30,7 +30,7 @@ Models cached from Hugging Face by versions before 4.0.522 do not satisfy this c## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdxindex b99a7eaf08..a9fa0e8f18 100644--- a/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx+++ b/packages/docs/docs/whisper-webgpu/load-whisper-model.mdx@@ -35,13 +35,13 @@ Called while model files are resolved and downloaded. `progress` is the aggregatReturns whether the same model pipeline was already loaded in memory.-The browser manages its persistent [Transformers.js](https://huggingface.co/docs/transformers.js) cache. This function also retains the initialized model in memory until [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model) is called or the page closes.+[Transformers.js](https://huggingface.co/docs/transformers.js) manages the persistent model cache. This function also retains the initialized model in memory until [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model) is called or the process or page closes.Versions before 4.0.522 downloaded models from Hugging Face. A model cached by an earlier version is downloaded once more from `remotion.media`, then reused from the new cache entry when using the default Transformers.js cache settings.## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/whisper-webgpu/node.mdx b/packages/docs/docs/whisper-webgpu/node.mdxnew file mode 100644index 0000000000..f8bf717f67--- /dev/null+++ b/packages/docs/docs/whisper-webgpu/node.mdx@@ -0,0 +1,140 @@+---+image: /generated/articles-docs-whisper-webgpu-node.png+title: 'Using @remotion/whisper-webgpu in Node.js'+crumb: '@remotion/whisper-webgpu'+---++# Using @remotion/whisper-webgpu in Node.js<AvailableFrom v="4.0.527" />++`@remotion/whisper-webgpu` can transcribe a 16kHz mono waveform in Node.js using the native WebGPU backend from [Transformers.js](https://huggingface.co/docs/transformers.js).++The following example uses [`@mediabunny/server`](https://mediabunny.dev/guide/extensions/server) to decode and resample a local audio or video file before transcribing it.++## Installation++<Installation pkg="@remotion/whisper-webgpu @huggingface/transformers mediabunny @mediabunny/server" />++## Example++```ts twoslash title="transcribe.ts"+import {registerMediabunnyServer} from '@mediabunny/server';+import {+ WHISPER_WEBGPU_SAMPLE_RATE,+ disposeWhisperModel,+ toCaptions,+ transcribe,+} from '@remotion/whisper-webgpu';+import {+ ALL_FORMATS,+ Conversion,+ FilePathSource,+ Input,+ NullTarget,+ Output,+ WavOutputFormat,+} from 'mediabunny';+import {writeFile} from 'node:fs/promises';++registerMediabunnyServer();++const input = new Input({+ formats: ALL_FORMATS,+ source: new FilePathSource('audio.mp3'),+});+const chunks: Array<{startFrame: number; waveform: Float32Array}> = [];++try {+ const audioTrack = await input.getPrimaryAudioTrack();+ if (audioTrack === null) {+ throw new Error('The media does not contain an audio track.');+ }++ const conversion = await Conversion.init({+ input,+ output: new Output({+ format: new WavOutputFormat(),+ target: new NullTarget(),+ }),+ video: {discard: true},+ audio: (track) => {+ if (track.id !== audioTrack.id) {+ return {discard: true};+ }++ return {+ codec: 'pcm-f32',+ forceTranscode: true,+ numberOfChannels: 1,+ sampleFormat: 'f32',+ sampleRate: WHISPER_WEBGPU_SAMPLE_RATE,+ process: (sample) => {+ const waveform = new Float32Array(+ sample.allocationSize({format: 'f32', planeIndex: 0}) /+ Float32Array.BYTES_PER_ELEMENT,+ );+ sample.copyTo(waveform, {format: 'f32', planeIndex: 0});+ chunks.push({+ startFrame: Math.round(+ sample.timestamp * WHISPER_WEBGPU_SAMPLE_RATE,+ ),+ waveform,+ });+ return sample;+ },+ };+ },+ });++ if (!conversion.isValid) {+ throw new Error('The audio track cannot be decoded.');+ }++ await conversion.execute();+} finally {+ input.dispose();+}++const waveformLength = chunks.reduce(+ (max, chunk) => Math.max(max, chunk.startFrame + chunk.waveform.length),+ 0,+);+const channelWaveform = new Float32Array(waveformLength);+for (const chunk of chunks) {+ const destinationStart = Math.max(0, chunk.startFrame);+ const sourceStart = Math.max(0, -chunk.startFrame);+ const availableLength = Math.min(+ chunk.waveform.length - sourceStart,+ channelWaveform.length - destinationStart,+ );++ if (availableLength > 0) {+ channelWaveform.set(+ chunk.waveform.subarray(sourceStart, sourceStart + availableLength),+ destinationStart,+ );+ }+}++const model = 'small.en';+try {+ const transcription = await transcribe({channelWaveform, model});+ const {captions} = toCaptions({whisperWebGpuOutput: transcription});+ await writeFile('captions.json', JSON.stringify(captions, null, 2));+} finally {+ await disposeWhisperModel({model});+}+```++The model is downloaded on the first run and cached on the file system. The input to [`transcribe()`](/docs/whisper-webgpu/transcribe) must be a mono `Float32Array` sampled at 16kHz.++Call [`canUseWhisperWebGpu()`](/docs/whisper-webgpu/can-use-whisper-webgpu) before loading the model if the script may run on different operating systems or CPU architectures.++## Supported platforms++The native WebGPU backend supports Node.js on macOS x64 and Arm64, Windows x64 and Arm64, and Linux x64.++## See also++- [`transcribe()`](/docs/whisper-webgpu/transcribe)+- [`toCaptions()`](/docs/whisper-webgpu/to-captions)+- [`disposeWhisperModel()`](/docs/whisper-webgpu/dispose-whisper-model)diff --git a/packages/docs/docs/whisper-webgpu/remove-whisper-model.mdx b/packages/docs/docs/whisper-webgpu/remove-whisper-model.mdxindex bd06eafe2b..9c0056c0a5 100644--- a/packages/docs/docs/whisper-webgpu/remove-whisper-model.mdx+++ b/packages/docs/docs/whisper-webgpu/remove-whisper-model.mdx@@ -6,7 +6,7 @@ crumb: '@remotion/whisper-webgpu'# removeWhisperModel()<AvailableFrom v="4.0.523" />-Removes a Whisper model from memory and the persistent browser cache.+Removes a Whisper model from memory and the persistent Transformers.js cache.If the model is still loading or transcribing, removal waits for the operation to finish.@@ -24,7 +24,7 @@ One of the names returned by [`getAvailableModels()`](/docs/whisper-webgpu/get-a## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx b/packages/docs/docs/whisper-webgpu/table-of-contents.tsxindex 5baa367f2d..5537c3ede0 100644--- a/packages/docs/docs/whisper-webgpu/table-of-contents.tsx+++ b/packages/docs/docs/whisper-webgpu/table-of-contents.tsx@@ -5,6 +5,10 @@ import {TOCItem} from '../../components/TableOfContents/TOCItem';export const TableOfContents: React.FC = () => {return (<Grid>+ <TOCItem link="/docs/whisper-webgpu/node">+ <strong>Using in Node.js</strong>+ <div>Decode media and transcribe it in Node.js</div>+ </TOCItem><TOCItem link="/docs/whisper-webgpu/can-use-whisper-webgpu"><strong>canUseWhisperWebGpu()</strong><div>Check whether WebGPU is available</div>@@ -27,7 +31,7 @@ export const TableOfContents: React.FC = () => {</TOCItem><TOCItem link="/docs/whisper-webgpu/remove-whisper-model"><strong>removeWhisperModel()</strong>- <div>Remove a model from the browser cache</div>+ <div>Remove a model from the persistent cache</div></TOCItem><TOCItem link="/docs/whisper-webgpu/transcribe"><strong>transcribe()</strong>diff --git a/packages/docs/docs/whisper-webgpu/transcribe.mdx b/packages/docs/docs/whisper-webgpu/transcribe.mdxindex e806506022..a04cb7707a 100644--- a/packages/docs/docs/whisper-webgpu/transcribe.mdx+++ b/packages/docs/docs/whisper-webgpu/transcribe.mdx@@ -86,7 +86,7 @@ Word-level timestamps are returned after transcription completes. [Transformers.## Compatibility-<CompatibilityTable chrome firefox safari nodejs="" bun="" serverlessFunctions="" clientSideRendering={false} serverSideRendering player studio hideServers />+<CompatibilityTable chrome firefox safari nodejs={true} bun={true} serverlessFunctions={false} clientSideRendering={false} serverSideRendering player studio />## See alsodiff --git a/packages/docs/package.json b/packages/docs/package.jsonindex 625321c73d..20abe53b7a 100644--- a/packages/docs/package.json+++ b/packages/docs/package.json@@ -39,6 +39,7 @@"@mediabunny/ac3": "catalog:","@mediabunny/dts": "catalog:","@mediabunny/prores": "catalog:",+ "@mediabunny/server": "catalog:","@radix-ui/react-dropdown-menu": "2.1.16","@react-three/fiber": "catalog:","@remotion-dev/cube-presentation": "https://www.remotion.pro/pkg/cube-transition/1.0.2/docs",diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.tsindex d04b94b83c..7acda71fb0 100644--- a/packages/docs/sidebars.ts+++ b/packages/docs/sidebars.ts@@ -1015,6 +1015,7 @@ const sidebars: SidebarsConfig = {id: 'whisper-webgpu/index',},items: [+ 'whisper-webgpu/node','whisper-webgpu/can-use-whisper-webgpu','whisper-webgpu/get-available-models','whisper-webgpu/clear-stale-models',diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.tsindex 3e4d23052e..b08d38b964 100644--- a/packages/docs/src/data/articles.ts+++ b/packages/docs/src/data/articles.ts@@ -9305,7 +9305,7 @@ export const articles = [title: '@remotion/whisper-webgpu',relativePath: 'docs/whisper-webgpu/index.mdx',compId: 'articles-docs-whisper-webgpu-index',- crumb: 'Transcribe audio in browser',+ crumb: 'Transcribe audio locally',noAi: false,slug: 'whisper-webgpu/index',},@@ -9327,6 +9327,15 @@ export const articles = [noAi: false,slug: 'whisper-webgpu/load-whisper-model',},+ {+ id: 'whisper-webgpu/node',+ title: 'Using @remotion/whisper-webgpu in Node.js',+ relativePath: 'docs/whisper-webgpu/node.mdx',+ compId: 'articles-docs-whisper-webgpu-node',+ crumb: '@remotion/whisper-webgpu',+ noAi: false,+ slug: 'whisper-webgpu/node',+ },{id: 'whisper-webgpu/remove-whisper-model',title: 'removeWhisperModel()',diff --git a/packages/docs/static/generated/articles-docs-whisper-webgpu-node.png b/packages/docs/static/generated/articles-docs-whisper-webgpu-node.pngnew file mode 100644index 0000000000..d4be72f1aaBinary files /dev/null and b/packages/docs/static/generated/articles-docs-whisper-webgpu-node.png differdiff --git a/packages/whisper-webgpu/README.md b/packages/whisper-webgpu/README.mdindex 335bed9b24..afc9dd9ab4 100644--- a/packages/whisper-webgpu/README.md+++ b/packages/whisper-webgpu/README.md@@ -1,6 +1,6 @@# @remotion/whisper-webgpu-Transcribe audio in the browser using Whisper and WebGPU+Transcribe audio in the browser or Node.js using Whisper and WebGPU[](https://npmcharts.com/compare/@remotion/whisper-webgpu?minimal=true)diff --git a/packages/whisper-webgpu/package.json b/packages/whisper-webgpu/package.jsonindex f5d5a95b27..cdb8764a0a 100644--- a/packages/whisper-webgpu/package.json+++ b/packages/whisper-webgpu/package.json@@ -1,7 +1,7 @@{"name": "@remotion/whisper-webgpu","version": "4.0.526",- "description": "Transcribe audio in the browser using Whisper and WebGPU",+ "description": "Transcribe audio using Whisper and WebGPU","main": "dist/index.js","types": "dist/index.d.ts","module": "dist/esm/index.mjs",diff --git a/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts b/packages/whisper-webgpu/src/can-use-whisper-webgpu.tsindex 1c71d634b8..f2c6c09b2d 100644--- a/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts+++ b/packages/whisper-webgpu/src/can-use-whisper-webgpu.ts@@ -17,11 +17,29 @@ export type CanUseWhisperWebGpuResult =export const canUseWhisperWebGpu =async (): Promise<CanUseWhisperWebGpuResult> => {if (typeof window === 'undefined') {+ if (typeof process !== 'undefined' && process.release?.name === 'node') {+ const supportsNativeWebGpu =+ (process.platform === 'darwin' &&+ (process.arch === 'arm64' || process.arch === 'x64')) ||+ (process.platform === 'win32' &&+ (process.arch === 'arm64' || process.arch === 'x64')) ||+ (process.platform === 'linux' && process.arch === 'x64');++ if (supportsNativeWebGpu) {+ return {supported: true};+ }++ return {+ supported: false,+ reason: WhisperWebGpuUnsupportedReason.WebGpuUnavailable,+ detailedReason: `The native WebGPU backend is not available on ${process.platform} ${process.arch}.`,+ };+ }+return {supported: false,reason: WhisperWebGpuUnsupportedReason.WindowUndefined,- detailedReason:- '`window` is not defined. @remotion/whisper-webgpu is intended for browser environments.',+ detailedReason: 'WebGPU is not available in this environment.',};}diff --git a/packages/whisper-webgpu/src/test/transcribe.test.ts b/packages/whisper-webgpu/src/test/transcribe.test.tsindex d81f198826..2289fe2c9d 100644--- a/packages/whisper-webgpu/src/test/transcribe.test.ts+++ b/packages/whisper-webgpu/src/test/transcribe.test.ts@@ -349,6 +349,7 @@ test('transcribes with word timestamps using WebGPU', async () => {remotePathTemplate: 'models/{model}/',});expect(transformersEnvironment).toEqual(originalTransformersEnvironment);+ expect(await canUseWhisperWebGpu()).toEqual({supported: true});const originalWindow = Object.getOwnPropertyDescriptor(globalThis, 'window');const originalNavigator = Object.getOwnPropertyDescriptor(