🎥 Make videos programmatically with React
remotion.dev/convert: Add language selection for transcription
Jonny Burgerf5a46d48884e
diff --git a/packages/convert/app/components/transcribe/App.tsx b/packages/convert/app/components/transcribe/App.tsxindex a0d852cc3a..daa81eeebf 100644--- a/packages/convert/app/components/transcribe/App.tsx+++ b/packages/convert/app/components/transcribe/App.tsx@@ -7,6 +7,7 @@ import {import React, {useEffect, useState} from 'react';import type {Source} from '~/lib/convert-state';import Display from './display';+import type {WhisperLanguage} from './languages';import ModelSelector from './modelSelector';import type {TranscriptionState} from './state';import TranscribeAudio from './transcribeAudio';@@ -20,10 +21,15 @@ const Transcribe: React.FC<{const [selectedModel, setSelectedModel] =useState<WhisperWebGpuModel>('tiny.en');+ const [selectedLanguage, setSelectedLanguage] =+ useState<WhisperLanguage>('en');const [cachedModels, setCachedModels] = useState<WhisperWebGpuModel[] | null>(null,);const [staleModelsCleared, setStaleModelsCleared] = useState(false);+ const selectedModelInfo = getAvailableModels().find(+ (model) => model.name === selectedModel,+ );useEffect(() => {let cancelled = false;@@ -81,6 +87,8 @@ const Transcribe: React.FC<{<ModelSelectorselectedModel={selectedModel}setSelectedModel={setSelectedModel}+ selectedLanguage={selectedLanguage}+ setSelectedLanguage={setSelectedLanguage}disabled={state.type === 'initializing'}cachedModels={cachedModels}/>@@ -90,6 +98,7 @@ const Transcribe: React.FC<{<TranscribeAudiosource={src}selectedModel={selectedModel}+ language={selectedModelInfo?.multilingual ? selectedLanguage : null}name={name}state={state}setState={setState}diff --git a/packages/convert/app/components/transcribe/languages.ts b/packages/convert/app/components/transcribe/languages.tsnew file mode 100644index 0000000000..2cc173c45f--- /dev/null+++ b/packages/convert/app/components/transcribe/languages.ts@@ -0,0 +1,103 @@+export const WHISPER_LANGUAGES = [+ ['af', 'Afrikaans'],+ ['sq', 'Albanian'],+ ['am', 'Amharic'],+ ['ar', 'Arabic'],+ ['hy', 'Armenian'],+ ['as', 'Assamese'],+ ['az', 'Azerbaijani'],+ ['ba', 'Bashkir'],+ ['eu', 'Basque'],+ ['be', 'Belarusian'],+ ['bn', 'Bengali'],+ ['bs', 'Bosnian'],+ ['br', 'Breton'],+ ['bg', 'Bulgarian'],+ ['my', 'Burmese'],+ ['ca', 'Catalan'],+ ['zh', 'Chinese'],+ ['hr', 'Croatian'],+ ['cs', 'Czech'],+ ['da', 'Danish'],+ ['nl', 'Dutch'],+ ['en', 'English'],+ ['et', 'Estonian'],+ ['fo', 'Faroese'],+ ['fi', 'Finnish'],+ ['fr', 'French'],+ ['gl', 'Galician'],+ ['ka', 'Georgian'],+ ['de', 'German'],+ ['el', 'Greek'],+ ['gu', 'Gujarati'],+ ['ht', 'Haitian Creole'],+ ['ha', 'Hausa'],+ ['haw', 'Hawaiian'],+ ['he', 'Hebrew'],+ ['hi', 'Hindi'],+ ['hu', 'Hungarian'],+ ['is', 'Icelandic'],+ ['id', 'Indonesian'],+ ['it', 'Italian'],+ ['ja', 'Japanese'],+ ['jw', 'Javanese'],+ ['kn', 'Kannada'],+ ['kk', 'Kazakh'],+ ['km', 'Khmer'],+ ['ko', 'Korean'],+ ['lo', 'Lao'],+ ['la', 'Latin'],+ ['lv', 'Latvian'],+ ['ln', 'Lingala'],+ ['lt', 'Lithuanian'],+ ['lb', 'Luxembourgish'],+ ['mk', 'Macedonian'],+ ['mg', 'Malagasy'],+ ['ms', 'Malay'],+ ['ml', 'Malayalam'],+ ['mt', 'Maltese'],+ ['mi', 'Maori'],+ ['mr', 'Marathi'],+ ['mn', 'Mongolian'],+ ['ne', 'Nepali'],+ ['no', 'Norwegian'],+ ['nn', 'Nynorsk'],+ ['oc', 'Occitan'],+ ['ps', 'Pashto'],+ ['fa', 'Persian'],+ ['pl', 'Polish'],+ ['pt', 'Portuguese'],+ ['pa', 'Punjabi'],+ ['ro', 'Romanian'],+ ['ru', 'Russian'],+ ['sa', 'Sanskrit'],+ ['sr', 'Serbian'],+ ['sn', 'Shona'],+ ['sd', 'Sindhi'],+ ['si', 'Sinhala'],+ ['sk', 'Slovak'],+ ['sl', 'Slovenian'],+ ['so', 'Somali'],+ ['es', 'Spanish'],+ ['su', 'Sundanese'],+ ['sw', 'Swahili'],+ ['sv', 'Swedish'],+ ['tl', 'Tagalog'],+ ['tg', 'Tajik'],+ ['ta', 'Tamil'],+ ['tt', 'Tatar'],+ ['te', 'Telugu'],+ ['th', 'Thai'],+ ['bo', 'Tibetan'],+ ['tr', 'Turkish'],+ ['tk', 'Turkmen'],+ ['uk', 'Ukrainian'],+ ['ur', 'Urdu'],+ ['uz', 'Uzbek'],+ ['vi', 'Vietnamese'],+ ['cy', 'Welsh'],+ ['yi', 'Yiddish'],+ ['yo', 'Yoruba'],+] as const;++export type WhisperLanguage = (typeof WHISPER_LANGUAGES)[number][0];diff --git a/packages/convert/app/components/transcribe/modelSelector.tsx b/packages/convert/app/components/transcribe/modelSelector.tsxindex db97d62c13..1de6f9e90e 100644--- a/packages/convert/app/components/transcribe/modelSelector.tsx+++ b/packages/convert/app/components/transcribe/modelSelector.tsx@@ -12,18 +12,27 @@ import {} from '@remotion/whisper-webgpu';import {formatBytes} from '../../lib/format-bytes';import {Label} from '../ui/label';+import {WHISPER_LANGUAGES, type WhisperLanguage} from './languages';export default function ModelSelector({selectedModel,setSelectedModel,disabled,cachedModels,+ selectedLanguage,+ setSelectedLanguage,}: {readonly selectedModel: WhisperWebGpuModel;readonly setSelectedModel: (model: WhisperWebGpuModel) => void;readonly disabled: boolean;readonly cachedModels: WhisperWebGpuModel[] | null;+ readonly selectedLanguage: WhisperLanguage;+ readonly setSelectedLanguage: (language: WhisperLanguage) => void;}) {+ const selectedModelInfo = getAvailableModels().find(+ (model) => model.name === selectedModel,+ );+return (<div className="flex flex-col gap-5"><div className="flex items-end gap-2">@@ -59,6 +68,33 @@ export default function ModelSelector({</Select></div></div>+ {selectedModelInfo?.multilingual ? (+ <div className="flex items-end gap-2">+ <div className="grid w-full max-w-sm items-center gap-1.5">+ <Label htmlFor="language">Spoken language</Label>+ <Select+ disabled={disabled}+ value={selectedLanguage}+ onValueChange={(value) =>+ setSelectedLanguage(value as WhisperLanguage)+ }+ >+ <SelectTrigger id="language">+ <SelectValue placeholder="Select a language" />+ </SelectTrigger>+ <SelectContent>+ <SelectGroup>+ {WHISPER_LANGUAGES.map(([code, language]) => (+ <SelectItem key={code} value={code}>+ {language}+ </SelectItem>+ ))}+ </SelectGroup>+ </SelectContent>+ </Select>+ </div>+ </div>+ ) : null}</div>);}diff --git a/packages/convert/app/components/transcribe/transcribeAudio.tsx b/packages/convert/app/components/transcribe/transcribeAudio.tsxindex 83d355fbdf..4cdf96f267 100644--- a/packages/convert/app/components/transcribe/transcribeAudio.tsx+++ b/packages/convert/app/components/transcribe/transcribeAudio.tsx@@ -10,6 +10,7 @@ import type {Source} from '~/lib/convert-state';import {formatBytes} from '~/lib/format-bytes';import {Button} from '../ui/button';import {Card} from '../ui/card';+import type {WhisperLanguage} from './languages';import type {TranscriptionState} from './state';const sourceToBlob = (source: Source) => {@@ -23,12 +24,14 @@ const sourceToBlob = (source: Source) => {export default function TranscribeAudio({source,selectedModel,+ language,name,state,setState,}: {readonly source: Source;readonly selectedModel: WhisperWebGpuModel;+ readonly language: WhisperLanguage | null;readonly name: string;readonly state: TranscriptionState;readonly setState: React.Dispatch<React.SetStateAction<TranscriptionState>>;@@ -59,6 +62,7 @@ export default function TranscribeAudio({const transcription = await transcribe({channelWaveform: waveform,model: selectedModel,+ ...(language === null ? {} : {language}),});setState(() => ({@@ -72,7 +76,7 @@ export default function TranscribeAudio({message: error instanceof Error ? error.message : String(error),});}- }, [selectedModel, source, setState]);+ }, [language, selectedModel, source, setState]);return (<div>diff --git a/packages/convert/test/transcribe-webgpu.test.tsx b/packages/convert/test/transcribe-webgpu.test.tsxindex 46b62f864b..594b21c2e4 100644--- a/packages/convert/test/transcribe-webgpu.test.tsx+++ b/packages/convert/test/transcribe-webgpu.test.tsx@@ -11,6 +11,8 @@ test('offers WebGPU model downloads', () => {cachedModels={['tiny.en']}selectedModel="tiny.en"setSelectedModel={() => undefined}+ selectedLanguage="en"+ setSelectedLanguage={() => undefined}disabled={false}/>,);@@ -18,6 +20,26 @@ test('offers WebGPU model downloads', () => {const model = rendered.getByRole('combobox', {name: 'Whisper model'});expect(model.textContent).toContain('119.7 MB WebGPU');expect(model.textContent).toContain('Downloaded');+ expect(+ rendered.queryByRole('combobox', {name: 'Spoken language'}),+ ).toBeNull();+});++test('offers a language selector for multilingual models', () => {+ const rendered = render(+ <ModelSelector+ cachedModels={[]}+ selectedModel="small"+ setSelectedModel={() => undefined}+ selectedLanguage="de"+ setSelectedLanguage={() => undefined}+ disabled={false}+ />,+ );++ expect(+ rendered.getByRole('combobox', {name: 'Spoken language'}).textContent,+ ).toContain('German');});test('offers transcription actions underneath the transcript', async () => {