Skip to content

🎥 Make videos programmatically with React

remotion.dev/convert: Add language selection for transcription

Jonny Burgerf5a46d48884e
diff --git a/packages/convert/app/components/transcribe/App.tsx b/packages/convert/app/components/transcribe/App.tsx
index a0d852cc3a..daa81eeebf 100644
--- a/packages/convert/app/components/transcribe/App.tsx
+++ b/packages/convert/app/components/transcribe/App.tsx
@@ -7,6 +7,7 @@ import {
import React, {useEffect, useState} from 'react';
import type {Source} from '~/lib/convert-state';
import Display from './display';
+import type {WhisperLanguage} from './languages';
import ModelSelector from './modelSelector';
import type {TranscriptionState} from './state';
import TranscribeAudio from './transcribeAudio';
@@ -20,10 +21,15 @@ const Transcribe: React.FC<{
const [selectedModel, setSelectedModel] =
useState<WhisperWebGpuModel>('tiny.en');
+ const [selectedLanguage, setSelectedLanguage] =
+ useState<WhisperLanguage>('en');
const [cachedModels, setCachedModels] = useState<WhisperWebGpuModel[] | null>(
null,
);
const [staleModelsCleared, setStaleModelsCleared] = useState(false);
+ const selectedModelInfo = getAvailableModels().find(
+ (model) => model.name === selectedModel,
+ );
useEffect(() => {
let cancelled = false;
@@ -81,6 +87,8 @@ const Transcribe: React.FC<{
<ModelSelector
selectedModel={selectedModel}
setSelectedModel={setSelectedModel}
+ selectedLanguage={selectedLanguage}
+ setSelectedLanguage={setSelectedLanguage}
disabled={state.type === 'initializing'}
cachedModels={cachedModels}
/>
@@ -90,6 +98,7 @@ const Transcribe: React.FC<{
<TranscribeAudio
source={src}
selectedModel={selectedModel}
+ language={selectedModelInfo?.multilingual ? selectedLanguage : null}
name={name}
state={state}
setState={setState}
diff --git a/packages/convert/app/components/transcribe/languages.ts b/packages/convert/app/components/transcribe/languages.ts
new file mode 100644
index 0000000000..2cc173c45f
--- /dev/null
+++ b/packages/convert/app/components/transcribe/languages.ts
@@ -0,0 +1,103 @@
+export const WHISPER_LANGUAGES = [
+ ['af', 'Afrikaans'],
+ ['sq', 'Albanian'],
+ ['am', 'Amharic'],
+ ['ar', 'Arabic'],
+ ['hy', 'Armenian'],
+ ['as', 'Assamese'],
+ ['az', 'Azerbaijani'],
+ ['ba', 'Bashkir'],
+ ['eu', 'Basque'],
+ ['be', 'Belarusian'],
+ ['bn', 'Bengali'],
+ ['bs', 'Bosnian'],
+ ['br', 'Breton'],
+ ['bg', 'Bulgarian'],
+ ['my', 'Burmese'],
+ ['ca', 'Catalan'],
+ ['zh', 'Chinese'],
+ ['hr', 'Croatian'],
+ ['cs', 'Czech'],
+ ['da', 'Danish'],
+ ['nl', 'Dutch'],
+ ['en', 'English'],
+ ['et', 'Estonian'],
+ ['fo', 'Faroese'],
+ ['fi', 'Finnish'],
+ ['fr', 'French'],
+ ['gl', 'Galician'],
+ ['ka', 'Georgian'],
+ ['de', 'German'],
+ ['el', 'Greek'],
+ ['gu', 'Gujarati'],
+ ['ht', 'Haitian Creole'],
+ ['ha', 'Hausa'],
+ ['haw', 'Hawaiian'],
+ ['he', 'Hebrew'],
+ ['hi', 'Hindi'],
+ ['hu', 'Hungarian'],
+ ['is', 'Icelandic'],
+ ['id', 'Indonesian'],
+ ['it', 'Italian'],
+ ['ja', 'Japanese'],
+ ['jw', 'Javanese'],
+ ['kn', 'Kannada'],
+ ['kk', 'Kazakh'],
+ ['km', 'Khmer'],
+ ['ko', 'Korean'],
+ ['lo', 'Lao'],
+ ['la', 'Latin'],
+ ['lv', 'Latvian'],
+ ['ln', 'Lingala'],
+ ['lt', 'Lithuanian'],
+ ['lb', 'Luxembourgish'],
+ ['mk', 'Macedonian'],
+ ['mg', 'Malagasy'],
+ ['ms', 'Malay'],
+ ['ml', 'Malayalam'],
+ ['mt', 'Maltese'],
+ ['mi', 'Maori'],
+ ['mr', 'Marathi'],
+ ['mn', 'Mongolian'],
+ ['ne', 'Nepali'],
+ ['no', 'Norwegian'],
+ ['nn', 'Nynorsk'],
+ ['oc', 'Occitan'],
+ ['ps', 'Pashto'],
+ ['fa', 'Persian'],
+ ['pl', 'Polish'],
+ ['pt', 'Portuguese'],
+ ['pa', 'Punjabi'],
+ ['ro', 'Romanian'],
+ ['ru', 'Russian'],
+ ['sa', 'Sanskrit'],
+ ['sr', 'Serbian'],
+ ['sn', 'Shona'],
+ ['sd', 'Sindhi'],
+ ['si', 'Sinhala'],
+ ['sk', 'Slovak'],
+ ['sl', 'Slovenian'],
+ ['so', 'Somali'],
+ ['es', 'Spanish'],
+ ['su', 'Sundanese'],
+ ['sw', 'Swahili'],
+ ['sv', 'Swedish'],
+ ['tl', 'Tagalog'],
+ ['tg', 'Tajik'],
+ ['ta', 'Tamil'],
+ ['tt', 'Tatar'],
+ ['te', 'Telugu'],
+ ['th', 'Thai'],
+ ['bo', 'Tibetan'],
+ ['tr', 'Turkish'],
+ ['tk', 'Turkmen'],
+ ['uk', 'Ukrainian'],
+ ['ur', 'Urdu'],
+ ['uz', 'Uzbek'],
+ ['vi', 'Vietnamese'],
+ ['cy', 'Welsh'],
+ ['yi', 'Yiddish'],
+ ['yo', 'Yoruba'],
+] as const;
+
+export type WhisperLanguage = (typeof WHISPER_LANGUAGES)[number][0];
diff --git a/packages/convert/app/components/transcribe/modelSelector.tsx b/packages/convert/app/components/transcribe/modelSelector.tsx
index db97d62c13..1de6f9e90e 100644
--- a/packages/convert/app/components/transcribe/modelSelector.tsx
+++ b/packages/convert/app/components/transcribe/modelSelector.tsx
@@ -12,18 +12,27 @@ import {
} from '@remotion/whisper-webgpu';
import {formatBytes} from '../../lib/format-bytes';
import {Label} from '../ui/label';
+import {WHISPER_LANGUAGES, type WhisperLanguage} from './languages';
export default function ModelSelector({
selectedModel,
setSelectedModel,
disabled,
cachedModels,
+ selectedLanguage,
+ setSelectedLanguage,
}: {
readonly selectedModel: WhisperWebGpuModel;
readonly setSelectedModel: (model: WhisperWebGpuModel) => void;
readonly disabled: boolean;
readonly cachedModels: WhisperWebGpuModel[] | null;
+ readonly selectedLanguage: WhisperLanguage;
+ readonly setSelectedLanguage: (language: WhisperLanguage) => void;
}) {
+ const selectedModelInfo = getAvailableModels().find(
+ (model) => model.name === selectedModel,
+ );
+
return (
<div className="flex flex-col gap-5">
<div className="flex items-end gap-2">
@@ -59,6 +68,33 @@ export default function ModelSelector({
</Select>
</div>
</div>
+ {selectedModelInfo?.multilingual ? (
+ <div className="flex items-end gap-2">
+ <div className="grid w-full max-w-sm items-center gap-1.5">
+ <Label htmlFor="language">Spoken language</Label>
+ <Select
+ disabled={disabled}
+ value={selectedLanguage}
+ onValueChange={(value) =>
+ setSelectedLanguage(value as WhisperLanguage)
+ }
+ >
+ <SelectTrigger id="language">
+ <SelectValue placeholder="Select a language" />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectGroup>
+ {WHISPER_LANGUAGES.map(([code, language]) => (
+ <SelectItem key={code} value={code}>
+ {language}
+ </SelectItem>
+ ))}
+ </SelectGroup>
+ </SelectContent>
+ </Select>
+ </div>
+ </div>
+ ) : null}
</div>
);
}
diff --git a/packages/convert/app/components/transcribe/transcribeAudio.tsx b/packages/convert/app/components/transcribe/transcribeAudio.tsx
index 83d355fbdf..4cdf96f267 100644
--- a/packages/convert/app/components/transcribe/transcribeAudio.tsx
+++ b/packages/convert/app/components/transcribe/transcribeAudio.tsx
@@ -10,6 +10,7 @@ import type {Source} from '~/lib/convert-state';
import {formatBytes} from '~/lib/format-bytes';
import {Button} from '../ui/button';
import {Card} from '../ui/card';
+import type {WhisperLanguage} from './languages';
import type {TranscriptionState} from './state';
const sourceToBlob = (source: Source) => {
@@ -23,12 +24,14 @@ const sourceToBlob = (source: Source) => {
export default function TranscribeAudio({
source,
selectedModel,
+ language,
name,
state,
setState,
}: {
readonly source: Source;
readonly selectedModel: WhisperWebGpuModel;
+ readonly language: WhisperLanguage | null;
readonly name: string;
readonly state: TranscriptionState;
readonly setState: React.Dispatch<React.SetStateAction<TranscriptionState>>;
@@ -59,6 +62,7 @@ export default function TranscribeAudio({
const transcription = await transcribe({
channelWaveform: waveform,
model: selectedModel,
+ ...(language === null ? {} : {language}),
});
setState(() => ({
@@ -72,7 +76,7 @@ export default function TranscribeAudio({
message: error instanceof Error ? error.message : String(error),
});
}
- }, [selectedModel, source, setState]);
+ }, [language, selectedModel, source, setState]);
return (
<div>
diff --git a/packages/convert/test/transcribe-webgpu.test.tsx b/packages/convert/test/transcribe-webgpu.test.tsx
index 46b62f864b..594b21c2e4 100644
--- a/packages/convert/test/transcribe-webgpu.test.tsx
+++ b/packages/convert/test/transcribe-webgpu.test.tsx
@@ -11,6 +11,8 @@ test('offers WebGPU model downloads', () => {
cachedModels={['tiny.en']}
selectedModel="tiny.en"
setSelectedModel={() => undefined}
+ selectedLanguage="en"
+ setSelectedLanguage={() => undefined}
disabled={false}
/>,
);
@@ -18,6 +20,26 @@ test('offers WebGPU model downloads', () => {
const model = rendered.getByRole('combobox', {name: 'Whisper model'});
expect(model.textContent).toContain('119.7 MB WebGPU');
expect(model.textContent).toContain('Downloaded');
+ expect(
+ rendered.queryByRole('combobox', {name: 'Spoken language'}),
+ ).toBeNull();
+});
+
+test('offers a language selector for multilingual models', () => {
+ const rendered = render(
+ <ModelSelector
+ cachedModels={[]}
+ selectedModel="small"
+ setSelectedModel={() => undefined}
+ selectedLanguage="de"
+ setSelectedLanguage={() => undefined}
+ disabled={false}
+ />,
+ );
+
+ expect(
+ rendered.getByRole('combobox', {name: 'Spoken language'}).textContent,
+ ).toContain('German');
});
test('offers transcription actions underneath the transcript', async () => {