Skip to content

🎥 Make videos programmatically with React

Studio: Show caption model downloads before installation

Igor Samokhovetscc63cfe28cf9
diff --git a/packages/studio/src/components/OptionalPackageModal.tsx b/packages/studio/src/components/OptionalPackageModal.tsx
index cc578d80be..cc73e81e91 100644
--- a/packages/studio/src/components/OptionalPackageModal.tsx
+++ b/packages/studio/src/components/OptionalPackageModal.tsx
@@ -18,6 +18,9 @@ type InstallState =
const panelStyle: React.CSSProperties = {
width: 'min(520px, calc(100vw - 40px))',
+ maxHeight: 'calc(100vh - 40px)',
+ display: 'flex',
+ flexDirection: 'column',
};
const contentStyle: React.CSSProperties = {
@@ -26,6 +29,8 @@ const contentStyle: React.CSSProperties = {
fontSize: 14,
lineHeight: 1.5,
padding: 16,
+ minHeight: 0,
+ overflowY: 'auto',
};
const codeStyle: React.CSSProperties = {
@@ -101,9 +106,18 @@ export const useOptionalPackageInstalled = (packageName: string) => {
export const OptionalPackageModal: React.FC<{
readonly ariaLabel: string;
readonly children: React.ReactNode;
+ readonly installButtonText: string;
+ readonly installMessage: React.ReactNode | null;
readonly packageName: string;
readonly title: string;
-}> = ({ariaLabel, children, packageName, title}) => {
+}> = ({
+ ariaLabel,
+ children,
+ installButtonText,
+ installMessage,
+ packageName,
+ title,
+}) => {
const {setSelectedModal} = useContext(SetSelectedModalContext);
const installed = useOptionalPackageInstalled(packageName);
const [installState, setInstallState] = useState<InstallState>({
@@ -150,14 +164,18 @@ export const OptionalPackageModal: React.FC<{
</div>
) : (
<>
- This requires installing{' '}
- {packageNames.map((name, index) => (
- <React.Fragment key={name}>
- {index > 0 ? ' and ' : null}
- <code style={codeStyle}>{name}</code>
- </React.Fragment>
- ))}
- . Continue?
+ {installMessage ?? (
+ <>
+ This requires installing{' '}
+ {packageNames.map((name, index) => (
+ <React.Fragment key={name}>
+ {index > 0 ? ' and ' : null}
+ <code style={codeStyle}>{name}</code>
+ </React.Fragment>
+ ))}
+ . Continue?
+ </>
+ )}
{installState.type === 'error' ? (
<div style={errorStyle}>{installState.error.message}</div>
) : null}
@@ -172,7 +190,7 @@ export const OptionalPackageModal: React.FC<{
</Button>
<Spacing x={1} />
<ModalButton disabled={busy} onClick={install} autoFocus>
- {installState.type === 'error' ? 'Retry' : 'Continue'}
+ {installState.type === 'error' ? 'Retry' : installButtonText}
</ModalButton>
</Row>
</ModalFooterContainer>
diff --git a/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx b/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx
index 6bb9245b98..8736f152e0 100644
--- a/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx
+++ b/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx
@@ -1,8 +1,103 @@
import React, {Suspense} from 'react';
+import {LIGHT_TEXT, TRANSPARENT, WHITE} from '../../helpers/colors';
+import {
+ FOCUS_VISIBLE_ONLY_CLASS_NAME,
+ HOVERABLE_CLASS_NAME,
+ hoverableStyle,
+} from '../../helpers/hoverable';
import type {TranscriptionModalState} from '../../state/modals';
import {OptionalPackageModal} from '../OptionalPackageModal';
import {WHISPER_WEBGPU_PACKAGE} from './whisper-webgpu-capability';
+const codeStyle: React.CSSProperties = {
+ color: WHITE,
+ fontFamily: 'monospace',
+ fontSize: 14,
+ lineHeight: 1.5,
+};
+
+const detailsStyle: React.CSSProperties = {
+ marginTop: 12,
+};
+
+const summaryStyle: React.CSSProperties = {
+ cursor: 'default',
+ ...hoverableStyle({
+ idleBackground: TRANSPARENT,
+ hoverBackground: TRANSPARENT,
+ idleColor: LIGHT_TEXT,
+ hoverColor: WHITE,
+ }),
+ fontFamily: 'sans-serif',
+ fontSize: 13,
+ fontWeight: 500,
+ lineHeight: 1.5,
+};
+
+const listStyle: React.CSSProperties = {
+ listStyle: 'none',
+ margin: '8px 0 0',
+ padding: 0,
+};
+
+const rowStyle: React.CSSProperties = {
+ display: 'flex',
+ justifyContent: 'space-between',
+ gap: 12,
+ padding: '2px 0',
+ fontFamily: 'sans-serif',
+ fontSize: 13,
+ lineHeight: 1.5,
+};
+
+const modelTextStyle: React.CSSProperties = {
+ color: LIGHT_TEXT,
+ fontFamily: 'monospace',
+ fontSize: 13,
+ lineHeight: 1.5,
+};
+
+const sizeTextStyle: React.CSSProperties = {
+ color: LIGHT_TEXT,
+ fontFamily: 'sans-serif',
+ fontSize: 13,
+ lineHeight: 1.5,
+};
+
+// Keep these approximate sizes in sync with packages/whisper-webgpu/src/models.ts.
+const modelSizes = [
+ {model: 'tiny', size: '~120 MB'},
+ {model: 'base', size: '~206 MB'},
+ {model: 'small', size: '~586 MB'},
+ {model: 'medium', size: '~1.7 GB'},
+ {model: 'large-v3-turbo', size: '~1.6 GB'},
+];
+
+const INSTALL_MESSAGE = (
+ <>
+ Generate captions on-device with{' '}
+ <code style={codeStyle}>@remotion/whisper-webgpu</code> and{' '}
+ <code style={codeStyle}>Transformers.js</code>. Multilingual models support
+ 99 languages. Models require a separate download before first use.
+ <details style={detailsStyle}>
+ <summary
+ className={`${HOVERABLE_CLASS_NAME} ${FOCUS_VISIBLE_ONLY_CLASS_NAME}`}
+ style={summaryStyle}
+ >
+ Model options and download sizes
+ </summary>
+ <ul style={listStyle}>
+ {modelSizes.map(({model, size}) => (
+ <li key={model} style={rowStyle}>
+ <span style={modelTextStyle}>{model}</span>
+ <span style={sizeTextStyle}>{size}</span>
+ </li>
+ ))}
+ </ul>
+ </details>
+ </>
+);
+
const LazyTranscriptionModal = React.lazy(async () => {
const {TranscriptionModal} = await import('./TranscriptionModal');
return {default: TranscriptionModal};
@@ -14,6 +109,8 @@ export const TranscriptionModalWithOptionalWhisper: React.FC<{
return (
<OptionalPackageModal
ariaLabel="Install transcription package"
+ installButtonText="Install packages"
+ installMessage={INSTALL_MESSAGE}
packageName={WHISPER_WEBGPU_PACKAGE}
title={state.target === null ? 'Transcribe' : 'Generate captions'}
>
diff --git a/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx b/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx
index c86e0a36e3..6f538e594d 100644
--- a/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx
+++ b/packages/studio/src/components/VideoMatting/VideoMattingModalWithOptionalPackage.tsx
@@ -13,6 +13,8 @@ export const VideoMattingModalWithOptionalPackage: React.FC<{
}> = ({state}) => (
<OptionalPackageModal
ariaLabel="Install video matting package to remove background"
+ installButtonText="Continue"
+ installMessage={null}
packageName={VIDEO_MATTING_PACKAGE}
title="Remove background"
>
diff --git a/packages/studio/src/test/whisper-webgpu-capability.test.tsx b/packages/studio/src/test/whisper-webgpu-capability.test.tsx
index 2422967d67..305ff0a608 100644
--- a/packages/studio/src/test/whisper-webgpu-capability.test.tsx
+++ b/packages/studio/src/test/whisper-webgpu-capability.test.tsx
@@ -38,29 +38,6 @@ test('adds the pinned Transformers version to AI package installations', () => {
}
});
-test('asks before installing Whisper for transcription', () => {
- window.remotion_installedPackages = [];
- const {container, getByRole} = render(
- <ModalsProvider>
- <TranscriptionModalWithOptionalWhisper
- state={{
- type: 'transcribe',
- src: '/voice.wav',
- displayName: 'voice.wav',
- audioStreamIndex: null,
- requestInit: null,
- target: null,
- }}
- />
- </ModalsProvider>,
- );
-
- expect(container.textContent).toContain(
- 'This requires installing @remotion/whisper-webgpu and @huggingface/transformers. Continue?',
- );
- expect(getByRole('button', {name: 'Continue'})).toBeDefined();
-});
-
test('uses the same install confirmation for video matting', () => {
window.remotion_installedPackages = [];
const {container, getByRole} = render(
@@ -113,7 +90,7 @@ test('opens transcription after installing Whisper without restarting', async ()
</ModalsProvider>,
);
- fireEvent.click(getByRole('button', {name: 'Continue'}));
+ fireEvent.click(getByRole('button', {name: 'Install packages'}));
const installing = getByText(
`Installing ${WHISPER_WEBGPU_PACKAGE} and ${TRANSFORMERS_PACKAGE}…`,
);