Skip to content

🎥 Make videos programmatically with React

Add Studio transcription jobs

JonnyBurgerdaba99ca6b71
diff --git a/bun.lock b/bun.lock
index 2dac03fb47..6b362d6bcb 100644
--- a/bun.lock
+++ b/bun.lock
@@ -159,6 +159,7 @@
"remotion": "workspace:*",
},
"devDependencies": {
+ "@huggingface/transformers": "catalog:",
"@playwright/test": "1.55.1",
"@remotion/eslint-config-internal": "workspace:*",
"@remotion/studio": "workspace:*",
@@ -1826,6 +1827,7 @@
"devDependencies": {
"@happy-dom/global-registrator": "14.5.1",
"@remotion/eslint-config-internal": "workspace:*",
+ "@remotion/whisper-webgpu": "workspace:*",
"@testing-library/react": "16.1.0",
"@types/semver": "7.5.3",
"@typescript/native-preview": "catalog:",
@@ -1834,9 +1836,13 @@
"react-dom": "catalog:",
},
"peerDependencies": {
+ "@remotion/whisper-webgpu": "workspace:*",
"react": ">=16.8.0",
"react-dom": ">=16.8.0",
},
+ "optionalPeers": [
+ "@remotion/whisper-webgpu",
+ ],
},
"packages/studio-codemods": {
"name": "@remotion/studio-codemods",
@@ -5722,7 +5728,7 @@
"address": ["[email protected]", "", {}, "sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA=="],
- "adm-zip": ["[email protected]", "", {}, "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg=="],
+ "adm-zip": ["[email protected]", "", {}, "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng=="],
"aes-js": ["[email protected]", "", {}, "sha512-e5pEa2kBnBOgR4Y/p20pskXI74UEz7de8ZGVo58asOtvSVG5YAbJeELPZxOmt+Bnz3rX753YKhfIn4X4l1PPRQ=="],
@@ -11742,6 +11748,8 @@
"finalhandler/debug": ["[email protected]", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="],
+ "firefox-profile/adm-zip": ["[email protected]", "", {}, "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg=="],
+
"firefox-profile/fs-extra": ["[email protected]", "", { "dependencies": { "graceful-fs": "4.2.11", "jsonfile": "6.1.0", "universalify": "2.0.0" } }, "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw=="],
"firefox-profile/minimist": ["[email protected]", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="],
@@ -12128,8 +12136,6 @@
"ofetch/ufo": ["[email protected]", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="],
- "onnxruntime-node/adm-zip": ["[email protected]", "", {}, "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng=="],
-
"onnxruntime-web/onnxruntime-common": ["[email protected]", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="],
"openai/@types/node": ["@types/[email protected]", "", {}, "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA=="],
diff --git a/packages/browser-studio/bundle.ts b/packages/browser-studio/bundle.ts
index 82b788ba6d..0ad73888d7 100644
--- a/packages/browser-studio/bundle.ts
+++ b/packages/browser-studio/bundle.ts
@@ -1,6 +1,7 @@
import path from 'path';
import {fileURLToPath} from 'url';
import {build} from 'bun';
+import {BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE} from './src/browser-studio-import-map';
import {getBrowserStudioDependencyVersionsForBuild} from './src/dev/get-dependency-versions-for-build';
import {getBrowserStudioReactRefreshFilesForBuild} from './src/dev/get-react-refresh-files-for-build';
import {getBrowserStudioSetupEnvironmentForBuild} from './src/dev/get-setup-environment-for-build';
@@ -24,6 +25,10 @@ const workspacePackageExports =
const vendorOutput = await build({
define: {'process.env.NODE_ENV': JSON.stringify('development')},
entrypoints: ['src/browser-studio-vendor-entry.ts'],
+ external: [
+ '@huggingface/transformers',
+ BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE,
+ ],
format: 'iife',
minify: true,
naming: '[name].mjs',
@@ -41,6 +46,28 @@ const vendorEntryOutput = vendorOutput.outputs.find(
if (!vendorEntryOutput) {
throw new Error('Browser Studio vendor entry was not generated');
}
+if (
+ !(await vendorEntryOutput.text()).includes(
+ BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE,
+ )
+) {
+ throw new Error(
+ 'The Browser Studio vendor entry must preserve the private Transformers import.',
+ );
+}
+
+const transformersOutput = await build({
+ entrypoints: ['src/browser-studio-transformers-entry.ts'],
+ format: 'esm',
+ minify: true,
+ naming: '[name].mjs',
+ target: 'browser',
+});
+
+if (!transformersOutput.success) {
+ console.log(transformersOutput.logs.join('\n'));
+ process.exit(1);
+}
const rspackBrowserEntry = fileURLToPath(
import.meta.resolve('@rspack/browser'),
@@ -84,8 +111,21 @@ if (!output.success) {
const externalVersionSensitiveImport =
/^[^'"\n]*\bfrom\s*["']@remotion\/(?:player|studio-shared|timeline-utils)["'];?\s*$|^\s*import\s*["']@remotion\/(?:player|studio-shared|timeline-utils)["'];?\s*$/m;
-for (const file of [...output.outputs, ...vendorOutput.outputs]) {
+for (const file of [
+ ...output.outputs,
+ ...vendorOutput.outputs,
+ ...transformersOutput.outputs,
+]) {
const str = await file.text();
+ if (
+ !file.path.includes('browser-studio-transformers-entry') &&
+ (str.includes('onnxruntime-web') || str.includes('transformers.web.js'))
+ ) {
+ throw new Error(
+ 'The Browser Studio bootstrap must not bundle Transformers.js. Keep it in the lazy Transformers entry.',
+ );
+ }
+
if (
path.basename(file.path) === 'browser-studio-vendor-entry.mjs' &&
externalVersionSensitiveImport.test(str)
diff --git a/packages/browser-studio/e2e/app.tsx b/packages/browser-studio/e2e/app.tsx
index defaf8cf60..8ea1cf27d1 100644
--- a/packages/browser-studio/e2e/app.tsx
+++ b/packages/browser-studio/e2e/app.tsx
@@ -15,6 +15,7 @@ const initialElementPayload = StudioProtocolInternals.parseBrowserStudioHash(
window as typeof window & {__browserStudioRemotionVersion: string}
).__browserStudioRemotionVersion = VERSION;
const render = async () => {
+ const source = new URLSearchParams(window.location.search).get('source');
const project = new URLSearchParams(window.location.search).has('github')
? await loadGitHubRepository({
repoUrl: 'https://github.com/remotion-dev/opfs-fixture',
@@ -34,6 +35,13 @@ import {RemotionRoot} from './Root';
void fade;
registerRoot(RemotionRoot);
+${
+ source === 'fallback' ||
+ source === 'transformers-override' ||
+ source === 'transformers-vendor'
+ ? `(globalThis as typeof globalThis & {__loadBrowserStudioTransformers: () => Promise<unknown>}).__loadBrowserStudioTransformers = () => import('@huggingface/transformers');`
+ : ''
+}
`;
}
@@ -42,14 +50,22 @@ registerRoot(RemotionRoot);
throw new Error('Could not find root element');
}
- const source = new URLSearchParams(window.location.search).get('source');
-
createRoot(root).render(
<BrowserStudio
dependencyResolver={
source === 'fallback'
? ({name, version}) => (name === 'react' ? version : null)
- : undefined
+ : source === 'transformers-override'
+ ? ({name}) =>
+ name === '@huggingface/transformers'
+ ? {
+ url: new URL(
+ '/fake-transformers.mjs',
+ window.location.href,
+ ).href,
+ }
+ : null
+ : undefined
}
initialElement={
initialElementPayload === null
diff --git a/packages/browser-studio/e2e/browser-studio.test.ts b/packages/browser-studio/e2e/browser-studio.test.ts
index ed9780e1e6..8ecb82d5cb 100644
--- a/packages/browser-studio/e2e/browser-studio.test.ts
+++ b/packages/browser-studio/e2e/browser-studio.test.ts
@@ -975,8 +975,10 @@ test('installs packages without a server API and preserves undo, redo, and HMR',
test('fetches each HTTP module once when the vendor bundle is overridden', async ({
page,
}) => {
+ const requests: string[] = [];
const workspacePackageRequests: string[] = [];
page.on('request', (request) => {
+ requests.push(request.url());
const requestUrl = new URL(request.url());
if (
requestUrl.pathname.startsWith('/__remotion_browser_studio_workspace__/')
@@ -1009,6 +1011,136 @@ test('fetches each HTTP module once when the vendor bundle is overridden', async
expect(workspacePackageRequests).toContain(
'/__remotion_browser_studio_workspace__/commits/e2e/packages/studio/dist/esm/previewEntry.mjs',
);
+ const transformersUrl = await studio
+ .locator('script[type="importmap"]')
+ .evaluate((element) => {
+ const importMap = JSON.parse(element.textContent ?? '') as {
+ imports: Record<string, string>;
+ };
+ return importMap.imports['@huggingface/transformers'];
+ });
+ expect(requests).not.toContain(transformersUrl);
+ const pipelineType = await studio.locator('body').evaluate(async () => {
+ const loadTransformers = (
+ window as typeof window & {
+ __loadBrowserStudioTransformers: () => Promise<{pipeline: unknown}>;
+ }
+ ).__loadBrowserStudioTransformers;
+ const transformers = await loadTransformers();
+ return typeof transformers.pipeline;
+ });
+ expect(pipelineType).toBe('function');
+ expect(
+ requests.filter((request) => request === transformersUrl),
+ ).toHaveLength(1);
+});
+
+test('loads the local Transformers bundle on demand from the vendor Blob bundle', async ({
+ page,
+}) => {
+ const requests: string[] = [];
+ const vendorBundleRequests: string[] = [];
+ page.on('request', (request) => {
+ requests.push(request.url());
+ const requestUrl = new URL(request.url());
+ if (requestUrl.searchParams.has('browserStudioVendor')) {
+ vendorBundleRequests.push(request.url());
+ }
+ });
+
+ await page.goto('/?source=transformers-vendor');
+ const studio = page.frameLocator('iframe');
+ await expect(studio.getByTitle('/project').getByText('MyComp')).toBeVisible();
+ expect(vendorBundleRequests).toHaveLength(1);
+ expect(
+ await studio.locator('script[type="module"]').getAttribute('src'),
+ ).toMatch(/^blob:/);
+ const transformersUrls = await studio
+ .locator('script[type="importmap"]')
+ .evaluate((element) => {
+ const importMap = JSON.parse(element.textContent ?? '') as {
+ imports: Record<string, string>;
+ };
+ return {
+ consumer: importMap.imports['@huggingface/transformers'],
+ whisper:
+ importMap.imports['@remotion/whisper-webgpu/private-transformers'],
+ };
+ });
+ expect(transformersUrls.whisper).not.toBe(transformersUrls.consumer);
+ expect(requests).not.toContain(transformersUrls.consumer);
+ expect(requests).not.toContain(transformersUrls.whisper);
+
+ const loaded = await studio
+ .locator('body')
+ .evaluate(async (_body, privateTransformersPackage) => {
+ const loadTransformers = (
+ window as typeof window & {
+ __loadBrowserStudioTransformers: () => Promise<{
+ env: unknown;
+ pipeline: unknown;
+ }>;
+ }
+ ).__loadBrowserStudioTransformers;
+ const consumerTransformers = await loadTransformers();
+ const whisperTransformers = (await import(
+ privateTransformersPackage
+ )) as {env: unknown; pipeline: unknown};
+ return {
+ consumerPipelineType: typeof consumerTransformers.pipeline,
+ environmentsAreIsolated:
+ consumerTransformers.env !== whisperTransformers.env,
+ whisperPipelineType: typeof whisperTransformers.pipeline,
+ };
+ }, '@remotion/whisper-webgpu/private-transformers');
+
+ expect(loaded.consumerPipelineType).toBe('function');
+ expect(loaded.whisperPipelineType).toBe('function');
+ expect(loaded.environmentsAreIsolated).toBe(true);
+ expect(
+ requests.filter((request) => request === transformersUrls.consumer),
+ ).toHaveLength(1);
+ expect(
+ requests.filter((request) => request === transformersUrls.whisper),
+ ).toHaveLength(1);
+});
+
+test('uses a custom Transformers resolution in the fallback bundle', async ({
+ page,
+}) => {
+ const fakeTransformersRequests: string[] = [];
+ const vendorBundleRequests: string[] = [];
+ page.on('request', (request) => {
+ const requestUrl = new URL(request.url());
+ if (requestUrl.pathname === '/fake-transformers.mjs') {
+ fakeTransformersRequests.push(request.url());
+ }
+
+ if (requestUrl.searchParams.has('browserStudioVendor')) {
+ vendorBundleRequests.push(request.url());
+ }
+ });
+
+ await page.goto('/?source=transformers-override');
+ const studio = page.frameLocator('iframe');
+ await expect(studio.getByTitle('/project').getByText('MyComp')).toBeVisible();
+ expect(vendorBundleRequests).toEqual([]);
+ expect(fakeTransformersRequests).toEqual([]);
+
+ const marker = await studio.locator('body').evaluate(async () => {
+ const loadTransformers = (
+ window as typeof window & {
+ __loadBrowserStudioTransformers: () => Promise<{
+ browserStudioOverrideMarker: number;
+ }>;
+ }
+ ).__loadBrowserStudioTransformers;
+ const transformers = await loadTransformers();
+ return transformers.browserStudioOverrideMarker;
+ });
+
+ expect(marker).toBe(42);
+ expect(fakeTransformersRequests).toHaveLength(1);
});
test('drops and imports an Element payload with the deployment Remotion version', async ({
diff --git a/packages/browser-studio/e2e/server.ts b/packages/browser-studio/e2e/server.ts
index 0d34a36853..b87b4444dd 100644
--- a/packages/browser-studio/e2e/server.ts
+++ b/packages/browser-studio/e2e/server.ts
@@ -93,6 +93,12 @@ Bun.serve({
});
}
+ if (url.pathname === '/fake-transformers.mjs') {
+ return new Response('export const browserStudioOverrideMarker = 42;', {
+ headers: {...headers, 'Content-Type': 'text/javascript'},
+ });
+ }
+
const remotionPackagePath = remotionPackagePaths.find((prefix) =>
url.pathname.startsWith(prefix),
);
diff --git a/packages/browser-studio/package.json b/packages/browser-studio/package.json
index 7b40a8b5fb..ef3f45c602 100644
--- a/packages/browser-studio/package.json
+++ b/packages/browser-studio/package.json
@@ -50,6 +50,7 @@
"./package.json": "./package.json"
},
"devDependencies": {
+ "@huggingface/transformers": "catalog:",
"@playwright/test": "1.55.1",
"@rspack/core": "1.7.11",
"@rspack/plugin-react-refresh": "1.6.1",
diff --git a/packages/browser-studio/src/BrowserStudio.tsx b/packages/browser-studio/src/BrowserStudio.tsx
index 055d1d2bf9..eebf9113eb 100644
--- a/packages/browser-studio/src/BrowserStudio.tsx
+++ b/packages/browser-studio/src/BrowserStudio.tsx
@@ -5,6 +5,12 @@ import {
createBrowserStudioHmrAssetManager,
type BrowserStudioHmrBridge,
} from './browser-studio-hmr-assets';
+import {
+ BROWSER_STUDIO_TRANSFORMERS_PACKAGE,
+ BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE,
+ getBrowserStudioTransformersUrl,
+ getBrowserStudioWhisperTransformersUrl,
+} from './browser-studio-import-map';
import {createBrowserStudioOperations} from './browser-studio-operations';
import {
areBrowserStudioProjectsEqual,
@@ -45,6 +51,13 @@ const localVendorEntry = new URL(
'./browser-studio-vendor-entry.mjs',
import.meta.url,
).href;
+const localTransformersEntry = new URL(
+ './browser-studio-transformers-entry.mjs',
+ import.meta.url,
+).href;
+const localWhisperTransformersEntry = getBrowserStudioWhisperTransformersUrl(
+ localTransformersEntry,
+);
const localVendorEntryWithMarker = `${localVendorEntry}?browserStudioVendor`;
const browserStudioPointerLeaveEvent = 'remotion-browser-studio-pointerleave';
@@ -377,6 +390,11 @@ export const BrowserStudio: React.FC<BrowserStudioProps> = ({
...configuredDependencyResolutions,
...installedDependencyResolutionsRef.current,
};
+ const transformersUrl = getBrowserStudioTransformersUrl({
+ localUrl: localTransformersEntry,
+ resolution:
+ dependencyResolutions[BROWSER_STUDIO_TRANSFORMERS_PACKAGE] ?? null,
+ });
const hasVendorOverride = Object.entries(dependencyResolutions).some(
([name, resolution]) =>
resolution !== null &&
@@ -618,6 +636,11 @@ export const BrowserStudio: React.FC<BrowserStudioProps> = ({
: bundleUrlRef.current;
const html = studioHtml({
+ importMap: {
+ [BROWSER_STUDIO_TRANSFORMERS_PACKAGE]: transformersUrl,
+ [BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE]:
+ localWhisperTransformersEntry,
+ },
audioLatencyHint: 'playback',
experimentalKeepAudioContextAlive: false,
bundleScriptUrl,
diff --git a/packages/browser-studio/src/browser-studio-import-map.ts b/packages/browser-studio/src/browser-studio-import-map.ts
new file mode 100644
index 0000000000..285dd245dc
--- /dev/null
+++ b/packages/browser-studio/src/browser-studio-import-map.ts
@@ -0,0 +1,44 @@
+import type {BrowserStudioDependencyResolution} from './types';
+
+export const BROWSER_STUDIO_TRANSFORMERS_PACKAGE =
+ '@huggingface/transformers' as const;
+export const BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE =
+ '@remotion/whisper-webgpu/private-transformers' as const;
+
+const getEsmShUrl = (version: string) => {
+ const url = new URL(
+ `https://esm.sh/${BROWSER_STUDIO_TRANSFORMERS_PACKAGE}@${version}`,
+ );
+ url.searchParams.set('dev', '');
+ return url.href;
+};
+
+export const getBrowserStudioTransformersUrl = ({
+ localUrl,
+ resolution,
+}: {
+ readonly localUrl: string;
+ readonly resolution: BrowserStudioDependencyResolution;
+}) => {
+ if (typeof resolution === 'string') {
+ return resolution.startsWith('http://') || resolution.startsWith('https://')
+ ? resolution
+ : getEsmShUrl(resolution);
+ }
+
+ if (resolution?.url) {
+ return resolution.url;
+ }
+
+ if (resolution?.version) {
+ return getEsmShUrl(resolution.version);
+ }
+
+ return localUrl;
+};
+
+export const getBrowserStudioWhisperTransformersUrl = (localUrl: string) => {
+ const url = new URL(localUrl);
+ url.searchParams.set('whisper-webgpu-private', '');
+ return url.href;
+};
diff --git a/packages/browser-studio/src/browser-studio-transformers-entry.ts b/packages/browser-studio/src/browser-studio-transformers-entry.ts
new file mode 100644
index 0000000000..bb03ecc2a5
--- /dev/null
+++ b/packages/browser-studio/src/browser-studio-transformers-entry.ts
@@ -0,0 +1 @@
+export * from '@huggingface/transformers';
diff --git a/packages/browser-studio/src/browser-studio-worker.ts b/packages/browser-studio/src/browser-studio-worker.ts
index f2e1c581a5..270c5c804c 100644
--- a/packages/browser-studio/src/browser-studio-worker.ts
+++ b/packages/browser-studio/src/browser-studio-worker.ts
@@ -2,6 +2,10 @@ import type {HotMiddlewareMessage} from '@remotion/studio-shared';
import {getStudioEntryPoints} from '@remotion/studio-shared/studio-entry-points';
import type * as RspackBrowser from '@rspack/browser';
import {makeBrowserStudioHttpClient} from './browser-studio-http-client';
+import {
+ BROWSER_STUDIO_TRANSFORMERS_PACKAGE,
+ BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE,
+} from './browser-studio-import-map';
import {browserStudioDependencyVersions} from './dependency-versions';
import {studioRenderEntryExternal} from './dev/studio-render-entry-external';
import type {
@@ -411,7 +415,20 @@ const createCompiler = async ({
}),
},
},
- externals: useVendorBundle ? browserStudioVendorExternals : undefined,
+ externals: [
+ ...(useVendorBundle ? [browserStudioVendorExternals] : []),
+ ({request}, callback) => {
+ if (
+ request === BROWSER_STUDIO_TRANSFORMERS_PACKAGE ||
+ request === BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE
+ ) {
+ callback(undefined, request, 'import');
+ return;
+ }
+
+ callback();
+ },
+ ],
externalsType: useVendorBundle ? 'var' : undefined,
mode: 'development',
module: {
diff --git a/packages/browser-studio/src/dev/get-dependency-versions-for-build.ts b/packages/browser-studio/src/dev/get-dependency-versions-for-build.ts
index 84f0fae11a..79ac64103f 100644
--- a/packages/browser-studio/src/dev/get-dependency-versions-for-build.ts
+++ b/packages/browser-studio/src/dev/get-dependency-versions-for-build.ts
@@ -101,6 +101,7 @@ export const getBrowserStudioDependencyVersionsForBuild = (): Record<
}
const dependencySpecs: Record<string, string> = {
+ '@huggingface/transformers': 'catalog:',
[studioPackageJson.name]: studioPackageJson.version,
react: 'catalog:',
'react-dom': 'catalog:',
diff --git a/packages/browser-studio/src/dev/server.ts b/packages/browser-studio/src/dev/server.ts
index 73297b413e..67ce8a05d6 100644
--- a/packages/browser-studio/src/dev/server.ts
+++ b/packages/browser-studio/src/dev/server.ts
@@ -1,6 +1,7 @@
import path from 'path';
import {fileURLToPath} from 'url';
import {build} from 'bun';
+import {BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE} from '../browser-studio-import-map';
import {browserStudioPackageJsonArtifactFilename} from '../workspace-package-exports';
import {getBrowserStudioDependencyVersionsForBuild} from './get-dependency-versions-for-build';
import {getBrowserStudioReactRefreshFilesForBuild} from './get-react-refresh-files-for-build';
@@ -82,6 +83,10 @@ const buildDevAssets = async () => {
const vendorOutput = await build({
define: {'process.env.NODE_ENV': JSON.stringify('development')},
entrypoints: ['src/browser-studio-vendor-entry.ts'],
+ external: [
+ '@huggingface/transformers',
+ BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE,
+ ],
format: 'iife',
naming: '[name].mjs',
outdir: outDir,
@@ -101,6 +106,20 @@ const buildDevAssets = async () => {
throw new Error('Browser Studio vendor entry was not generated');
}
+ const transformersOutput = await build({
+ entrypoints: ['src/browser-studio-transformers-entry.ts'],
+ format: 'esm',
+ naming: '[name].mjs',
+ outdir: outDir,
+ sourcemap: 'linked',
+ target: 'browser',
+ });
+
+ if (!transformersOutput.success) {
+ process.stderr.write(`${transformersOutput.logs.join('\n')}\n`);
+ process.exit(1);
+ }
+
const browserStudioAssetSizes = {
rspackWasm: Bun.file(
path.join(rspackBrowserDist, 'rspack.wasm32-wasi.wasm'),
diff --git a/packages/browser-studio/src/dev/studio-render-entry-external.ts b/packages/browser-studio/src/dev/studio-render-entry-external.ts
index f8e038d344..c0efd7e0df 100644
--- a/packages/browser-studio/src/dev/studio-render-entry-external.ts
+++ b/packages/browser-studio/src/dev/studio-render-entry-external.ts
@@ -1,4 +1,6 @@
export const studioRenderEntryExternal = [
+ '@huggingface/transformers',
+ '@remotion/whisper-webgpu/private-transformers',
'@jridgewell/trace-mapping',
'@remotion/media-utils',
'@remotion/renderer',
diff --git a/packages/browser-studio/src/test/browser-studio-import-map.test.ts b/packages/browser-studio/src/test/browser-studio-import-map.test.ts
new file mode 100644
index 0000000000..5b26ddd28e
--- /dev/null
+++ b/packages/browser-studio/src/test/browser-studio-import-map.test.ts
@@ -0,0 +1,49 @@
+import {expect, test} from 'bun:test';
+import {
+ getBrowserStudioTransformersUrl,
+ getBrowserStudioWhisperTransformersUrl,
+} from '../browser-studio-import-map';
+
+const localUrl = 'https://studio.test/browser-studio-transformers-entry.mjs';
+
+test('uses the local lazy Transformers bundle by default', () => {
+ expect(getBrowserStudioTransformersUrl({localUrl, resolution: null})).toBe(
+ localUrl,
+ );
+});
+
+test('honors custom Transformers URLs', () => {
+ expect(
+ getBrowserStudioTransformersUrl({
+ localUrl,
+ resolution: {url: 'https://cdn.test/transformers.mjs'},
+ }),
+ ).toBe('https://cdn.test/transformers.mjs');
+ expect(
+ getBrowserStudioTransformersUrl({
+ localUrl,
+ resolution: 'https://cdn.test/transformers-string.mjs',
+ }),
+ ).toBe('https://cdn.test/transformers-string.mjs');
+});
+
+test('turns custom Transformers versions into pinned esm.sh URLs', () => {
+ expect(
+ getBrowserStudioTransformersUrl({
+ localUrl,
+ resolution: {version: '4.2.0'},
+ }),
+ ).toBe('https://esm.sh/@huggingface/[email protected]?dev=');
+ expect(
+ getBrowserStudioTransformersUrl({
+ localUrl,
+ resolution: '4.1.0',
+ }),
+ ).toBe('https://esm.sh/@huggingface/[email protected]?dev=');
+});
+
+test('uses a separate Transformers module instance for Whisper', () => {
+ expect(getBrowserStudioWhisperTransformersUrl(localUrl)).toBe(
+ 'https://studio.test/browser-studio-transformers-entry.mjs?whisper-webgpu-private=',
+ );
+});
diff --git a/packages/browser-studio/src/test/dependency-versions.test.ts b/packages/browser-studio/src/test/dependency-versions.test.ts
index fb839bea2d..8dcb826d00 100644
--- a/packages/browser-studio/src/test/dependency-versions.test.ts
+++ b/packages/browser-studio/src/test/dependency-versions.test.ts
@@ -28,15 +28,22 @@ test('browser studio dependency versions are derived from package metadata', ()
const transitionsPackageJson = readPackageJson(
join(repoDir, 'packages', 'transitions', 'package.json'),
);
+ const whisperWebGpuPackageJson = readPackageJson(
+ join(repoDir, 'packages', 'whisper-webgpu', 'package.json'),
+ );
const dependencyVersions = getBrowserStudioDependencyVersionsForBuild();
const reactVersion = rootPackageJson.workspaces?.catalog?.react;
const reactDomVersion = rootPackageJson.workspaces?.catalog?.['react-dom'];
+ const transformersVersion =
+ rootPackageJson.workspaces?.catalog?.['@huggingface/transformers'];
if (
!studioPackageJson.version ||
!transitionsPackageJson.version ||
+ !whisperWebGpuPackageJson.version ||
!reactVersion ||
- !reactDomVersion
+ !reactDomVersion ||
+ !transformersVersion
) {
throw new Error(
'Could not find package metadata for dependency version test',
@@ -49,6 +56,12 @@ test('browser studio dependency versions are derived from package metadata', ()
expect(dependencyVersions['@remotion/transitions']).toBe(
transitionsPackageJson.version,
);
+ expect(dependencyVersions['@remotion/whisper-webgpu']).toBe(
+ whisperWebGpuPackageJson.version,
+ );
+ expect(dependencyVersions['@huggingface/transformers']).toBe(
+ transformersVersion,
+ );
expect(dependencyVersions.react).toBe(reactVersion);
expect(dependencyVersions['react-dom']).toBe(reactDomVersion);
expect(dependencyVersions.memfs).toBeUndefined();
diff --git a/packages/bundler/src/bundle.ts b/packages/bundler/src/bundle.ts
index 963af1b5a1..90df81a276 100644
--- a/packages/bundler/src/bundle.ts
+++ b/packages/bundler/src/bundle.ts
@@ -412,6 +412,7 @@ export const internalBundle = async (
}
const html = indexHtml({
+ importMap: null,
staticHash,
publicPath,
editorName: null,
diff --git a/packages/bundler/src/optional-dependencies.ts b/packages/bundler/src/optional-dependencies.ts
index 804e07d9e3..eb38542474 100644
--- a/packages/bundler/src/optional-dependencies.ts
+++ b/packages/bundler/src/optional-dependencies.ts
@@ -9,16 +9,38 @@ const OPTIONAL_DEPENDENCIES = [
'react-native-reanimated/package.json',
];
+const STUDIO_OPTIONAL_DEPENDENCIES = ['@remotion/whisper-webgpu'];
+
const SOURCE_MAP_IGNORE = ['path', 'fs'];
export class AllowOptionalDependenciesPlugin {
- filter(error: Error) {
+ filter(
+ error: Error & {
+ module?: {
+ resource?: string;
+ resourceResolveData?: {descriptionFileData?: {name?: string}};
+ };
+ },
+ ) {
for (const dependency of OPTIONAL_DEPENDENCIES) {
if (error.message.includes(`Can't resolve '${dependency}'`)) {
return false;
}
}
+ const issuer = error.module?.resource?.replaceAll('\\', '/') ?? '';
+ const issuerPackageName =
+ error.module?.resourceResolveData?.descriptionFileData?.name;
+ for (const dependency of STUDIO_OPTIONAL_DEPENDENCIES) {
+ if (
+ error.message.includes(`Can't resolve '${dependency}'`) &&
+ (issuerPackageName === '@remotion/studio' ||
+ issuer.includes('/@remotion/studio/'))
+ ) {
+ return false;
+ }
+ }
+
for (const dependency of SOURCE_MAP_IGNORE) {
if (
error.message.includes(`Can't resolve '${dependency}'`) &&
diff --git a/packages/bundler/src/rspack-config.ts b/packages/bundler/src/rspack-config.ts
index a7ead49ec8..eb775fe58b 100644
--- a/packages/bundler/src/rspack-config.ts
+++ b/packages/bundler/src/rspack-config.ts
@@ -1,6 +1,7 @@
import {getStudioEntryPoints} from '@remotion/studio-shared/studio-entry-points';
import {ProgressPlugin, rspack} from '@rspack/core';
import ReactRefreshPlugin from '@rspack/plugin-react-refresh';
+import {AllowOptionalDependenciesPlugin} from './optional-dependencies';
import type {
BundlerOverrideFn,
RspackConfiguration,
@@ -120,6 +121,7 @@ export const rspackConfig = async ({
? [
new ReactRefreshPlugin({overlay: false}),
new rspack.HotModuleReplacementPlugin(),
+ new AllowOptionalDependenciesPlugin(),
...extraPlugins,
]
: [
@@ -131,6 +133,7 @@ export const rspackConfig = async ({
}
}
}),
+ new AllowOptionalDependenciesPlugin(),
],
output: getOutputConfig(environment),
resolve: getResolveConfig(),
diff --git a/packages/bundler/src/test/optional-dependencies.test.ts b/packages/bundler/src/test/optional-dependencies.test.ts
new file mode 100644
index 0000000000..d6ff9b9eec
--- /dev/null
+++ b/packages/bundler/src/test/optional-dependencies.test.ts
@@ -0,0 +1,27 @@
+import {expect, test} from 'bun:test';
+import {AllowOptionalDependenciesPlugin} from '../optional-dependencies';
+
+const missingWhisperError = (issuer: string) => {
+ return Object.assign(
+ new Error("Can't resolve '@remotion/whisper-webgpu' in '/project'"),
+ {module: {resource: issuer}},
+ );
+};
+
+test('allows Studio to omit its optional Whisper WebGPU capability', () => {
+ const plugin = new AllowOptionalDependenciesPlugin();
+ expect(
+ plugin.filter(
+ missingWhisperError(
+ '/project/node_modules/@remotion/studio/dist/esm/chunk.js',
+ ),
+ ),
+ ).toBe(false);
+});
+
+test('does not hide missing Whisper WebGPU imports in user code', () => {
+ const plugin = new AllowOptionalDependenciesPlugin();
+ expect(plugin.filter(missingWhisperError('/project/src/Video.tsx'))).toBe(
+ true,
+ );
+});
diff --git a/packages/core/src/CompositionManager.tsx b/packages/core/src/CompositionManager.tsx
index 2848d97d70..98969cc10f 100644
--- a/packages/core/src/CompositionManager.tsx
+++ b/packages/core/src/CompositionManager.tsx
@@ -72,6 +72,8 @@ type EnhancedTSequenceData =
| {
type: 'audio';
src: string;
+ audioStreamIndex: number | null;
+ requestInit: Omit<RequestInit, 'signal'> | null;
// Volume is represented as a comma separated list - if it's a string
// React can more efficiently update and will not rerender if anonymous functions
// are passed.
@@ -87,6 +89,8 @@ type EnhancedTSequenceData =
| {
type: 'video';
src: string;
+ audioStreamIndex: number | null;
+ requestInit: Omit<RequestInit, 'signal'> | null;
volume: string | number;
doesVolumeChange: boolean;
muted: boolean;
diff --git a/packages/core/src/Sequence.tsx b/packages/core/src/Sequence.tsx
index d40f0db0d3..1bd5710f87 100644
--- a/packages/core/src/Sequence.tsx
+++ b/packages/core/src/Sequence.tsx
@@ -508,6 +508,8 @@ const RegularSequenceRefForwardingFunction: React.ForwardRefRenderFunction<
showInTimeline,
timelineOrder: null,
src: isMedia.data.src,
+ audioStreamIndex: isMedia.data.audioStreamIndex,
+ requestInit: isMedia.data.requestInit,
getStack: () => stackRef.current,
startMediaFrom: startMediaFrom ?? isMedia.data.startMediaFrom,
mediaFrameAtSequenceZero,
diff --git a/packages/core/src/audio/AudioForPreview.tsx b/packages/core/src/audio/AudioForPreview.tsx
index 1af9b0cc27..cb6b6de640 100644
--- a/packages/core/src/audio/AudioForPreview.tsx
+++ b/packages/core/src/audio/AudioForPreview.tsx
@@ -192,6 +192,8 @@ const AudioForDevelopmentForwardRefFunction: React.ForwardRefRenderFunction<
documentationLink: 'https://www.remotion.dev/docs/html5-audio',
refForOutline: null,
muted: muted ?? false,
+ audioStreamIndex: audioStreamIndex ?? null,
+ requestInit: null,
});
// putting playback before useVolume
diff --git a/packages/core/src/test/sequence-register-once.test.tsx b/packages/core/src/test/sequence-register-once.test.tsx
index 6f5eac2f24..b29b99ec70 100644
--- a/packages/core/src/test/sequence-register-once.test.tsx
+++ b/packages/core/src/test/sequence-register-once.test.tsx
@@ -227,6 +227,8 @@ const makeMediaInTimelineData = ({
startMediaFrom,
src: 'video.mp4',
playbackRate,
+ audioStreamIndex: 3,
+ requestInit: {credentials: 'include'},
}) as unknown as BasicMediaInTimelineReturnType;
test('Sequence calls registerSequence exactly once on mount', () => {
@@ -1171,6 +1173,8 @@ test('Video media registration accounts for its own negative from', () => {
expect(videoSequence?.startMediaFrom).toBe(15);
expect(videoSequence?.mediaFrameAtSequenceZero).toBe(5);
+ expect(videoSequence?.audioStreamIndex).toBe(3);
+ expect(videoSequence?.requestInit).toEqual({credentials: 'include'});
});
test('Video media registration keeps trimBefore at sequence frame zero', () => {
diff --git a/packages/core/src/test/use-media-in-timeline.test.tsx b/packages/core/src/test/use-media-in-timeline.test.tsx
index d5b9d74e52..249a1e920d 100644
--- a/packages/core/src/test/use-media-in-timeline.test.tsx
+++ b/packages/core/src/test/use-media-in-timeline.test.tsx
@@ -86,6 +86,11 @@ test('useMediaInTimeline registers muted changes and unregisters the sequence',
documentationLink: null,
refForOutline: null,
muted,
+ audioStreamIndex: 2,
+ requestInit: {
+ credentials: 'include',
+ headers: {'x-remotion-test': 'timeline'},
+ },
}),
{
wrapper,
@@ -94,6 +99,11 @@ test('useMediaInTimeline registers muted changes and unregisters the sequence',
);
expect(registerSequence).toHaveBeenCalled();
expect(registerSequence.mock.calls[0]?.[0]).toMatchObject({
+ audioStreamIndex: 2,
+ requestInit: {
+ credentials: 'include',
+ headers: {'x-remotion-test': 'timeline'},
+ },
mediaFrameAtSequenceZero: null,
muted: false,
});
@@ -149,6 +159,8 @@ test('useMediaInTimeline keeps documentation links for custom display names', ()
documentationLink: 'https://www.remotion.dev/docs/html5-video',
refForOutline: null,
muted: false,
+ audioStreamIndex: null,
+ requestInit: null,
}),
{
wrapper,
diff --git a/packages/core/src/use-media-in-timeline.ts b/packages/core/src/use-media-in-timeline.ts
index ae0c54a1e2..f014956663 100644
--- a/packages/core/src/use-media-in-timeline.ts
+++ b/packages/core/src/use-media-in-timeline.ts
@@ -35,6 +35,8 @@ export const useBasicMediaInTimeline = ({
mediaStartsAt,
loop,
muted,
+ audioStreamIndex,
+ requestInit,
}: {
volume: VolumeProp | undefined;
mediaVolume: number;
@@ -48,6 +50,8 @@ export const useBasicMediaInTimeline = ({
mediaStartsAt: number;
loop: boolean;
muted: boolean;
+ audioStreamIndex: number | null;
+ requestInit: Omit<RequestInit, 'signal'> | null;
}) => {
if (!src) {
throw new Error('No src passed');
@@ -113,6 +117,8 @@ export const useBasicMediaInTimeline = ({
src,
playbackRate,
muted,
+ audioStreamIndex,
+ requestInit,
};
}, [
volumes,
@@ -123,6 +129,8 @@ export const useBasicMediaInTimeline = ({
startMediaFrom,
playbackRate,
muted,
+ audioStreamIndex,
+ requestInit,
]);
return memoizedResult;
@@ -148,6 +156,8 @@ export const useMediaInTimeline = ({
documentationLink,
refForOutline,
muted,
+ audioStreamIndex,
+ requestInit,
}: {
volume: VolumeProp | undefined;
mediaVolume: number;
@@ -164,6 +174,8 @@ export const useMediaInTimeline = ({
documentationLink: string | null;
refForOutline: React.RefObject<Element | null> | null;
muted: boolean;
+ audioStreamIndex: number | null;
+ requestInit: Omit<RequestInit, 'signal'> | null;
}) => {
const parentSequence = useContext(SequenceContext);
const startsAt = useMediaStartsAt();
@@ -185,6 +197,8 @@ export const useMediaInTimeline = ({
mediaStartsAt,
loop: false,
muted,
+ audioStreamIndex,
+ requestInit,
});
const {isStudio} = useRemotionEnvironment();
@@ -198,6 +212,8 @@ export const useMediaInTimeline = ({
effectRuntimeValues: null,
type: mediaType,
src,
+ audioStreamIndex,
+ requestInit,
id,
duration,
from: 0,
@@ -242,6 +258,8 @@ export const useMediaInTimeline = ({
finalDisplayName,
refForOutline,
muted,
+ audioStreamIndex,
+ requestInit,
]);
const registrationEnabled =
isStudio ||
diff --git a/packages/core/src/video/VideoForPreview.tsx b/packages/core/src/video/VideoForPreview.tsx
index 9694b0b5c0..b4eb1339d8 100644
--- a/packages/core/src/video/VideoForPreview.tsx
+++ b/packages/core/src/video/VideoForPreview.tsx
@@ -181,6 +181,8 @@ const VideoForDevelopmentRefForwardingFunction: React.ForwardRefRenderFunction<
: 'https://www.remotion.dev/docs/html5-video',
refForOutline: videoRef,
muted: muted ?? false,
+ audioStreamIndex: audioStreamIndex ?? 0,
+ requestInit: null,
});
// putting playback before useVolume
diff --git a/packages/example/e2e/transcription-modal.test.mts b/packages/example/e2e/transcription-modal.test.mts
new file mode 100644
index 0000000000..674cfe684e
--- /dev/null
+++ b/packages/example/e2e/transcription-modal.test.mts
@@ -0,0 +1,227 @@
+import path from 'path';
+import {expect, test} from '@playwright/test';
+import {EXPANDED_SIDEBAR_STATE, STUDIO_URL, exampleDir} from './constants.mts';
+import {startStudio, stopStudio} from './studio-server.mts';
+
+test.use({storageState: EXPANDED_SIDEBAR_STATE});
+
+test.describe('transcription modal', () => {
+ test.beforeEach(async () => {
+ await startStudio();
+ });
+
+ test.afterEach(async () => {
+ await stopStudio();
+ });
+
+ test('supports Video and Audio and configures a Caption[] JSON output', async ({
+ page,
+ }) => {
+ await page.addInitScript(() => {
+ Object.defineProperty(navigator, 'gpu', {
+ configurable: true,
+ value: {requestAdapter: () => Promise.resolve({})},
+ });
+ });
+ await page.route('https://remotion.media/video.mp4', async (route) => {
+ await route.fulfill({
+ contentType: 'video/webm',
+ path: path.join(exampleDir, 'public', 'vp8-vorbis.webm'),
+ });
+ });
+ await page.goto(`${STUDIO_URL}/NewVideo`);
+ await expect(page).toHaveURL(/NewVideo/, {timeout: 15_000});
+ await page.waitForFunction(
+ () => !document.body.innerText.includes('Loading...'),
+ {timeout: 30_000},
+ );
+
+ const video = page.getByText('<Video>', {exact: true}).first();
+ const transcribe = page.getByRole('button', {
+ name: 'Transcribe',
+ exact: true,
+ });
+ await expect(async () => {
+ await video.click();
+ await expect(transcribe).toBeVisible({timeout: 1_000});
+ }).toPass({timeout: 30_000});
+
+ await transcribe.click();
+ const dialog = page.getByRole('dialog');
+ const addToQueueButton = dialog.getByRole('button', {
+ name: 'Transcribe',
+ exact: true,
+ });
+ await expect(dialog).toContainText('Transcribe <Video>');
+ await expect(
+ dialog.getByText('Whisper model', {exact: true}),
+ ).toBeVisible();
+ const model = dialog.getByTitle('Whisper model');
+ const task = dialog.getByRole('button', {
+ name: 'Task: Transcribe',
+ exact: true,
+ });
+ await expect(model).toContainText('tiny.en');
+ await expect(task).toContainText('Transcribe');
+ await expect(task).toBeDisabled();
+ await expect(
+ dialog.getByText(/Downloaded with transcription if needed/),
+ ).toBeVisible();
+ await expect(dialog.getByRole('button', {name: /^Download /})).toHaveCount(
+ 0,
+ );
+ await expect(addToQueueButton).toBeEnabled();
+
+ await model.click();
+ await page.getByRole('button', {name: /^tiny ·/}).click();
+ await expect(model).toContainText('tiny ·');
+ await expect(task).toBeEnabled();
+ await task.click();
+ await page
+ .getByRole('button', {name: 'Task: Translate to English', exact: true})
+ .click();
+ await expect(
+ dialog.getByRole('button', {
+ name: 'Task: Translate to English',
+ exact: true,
+ }),
+ ).toContainText('Translate to English');
+ await expect(
+ dialog.getByText(
+ 'Word timings may be less accurate when translating to English.',
+ {exact: true},
+ ),
+ ).toBeVisible();
+ await expect(addToQueueButton).toBeEnabled();
+
+ await dialog.getByRole('button', {name: 'Other', exact: true}).click();
+ const chunkLength = dialog.getByRole('button', {
+ name: /^Chunk length:/,
+ });
+ const strideLength = dialog.getByRole('button', {
+ name: /^Stride length:/,
+ });
+ await expect(chunkLength).toHaveText('30s');
+ await expect(strideLength).toHaveText('5s');
+
+ const forceFullSequences = dialog.getByRole('checkbox', {
+ name: 'Force full sequences',
+ });
+ const useSampling = dialog.getByRole('checkbox', {
+ name: 'Use sampling',
+ });
+ const repetitionPenalty = dialog.getByRole('button', {
+ name: /^Repetition penalty:/,
+ });
+ const noRepeatNgramSize = dialog.getByRole('button', {
+ name: /^No-repeat n-gram size:/,
+ });
+ await expect(forceFullSequences).not.toBeChecked();
+ await expect(useSampling).not.toBeChecked();
+ await expect(
+ dialog.getByRole('button', {name: /^Temperature:/}),
+ ).toHaveCount(0);
+ await expect(dialog.getByRole('button', {name: /^Top K:/})).toHaveCount(0);
+ await expect(repetitionPenalty).toHaveText('1');
+ await expect(noRepeatNgramSize).toHaveText('0');
+
+ await forceFullSequences.check();
+ await useSampling.check();
+ const temperature = dialog.getByRole('button', {
+ name: /^Temperature:/,
+ });
+ const topK = dialog.getByRole('button', {name: /^Top K:/});
+ await expect(temperature).toHaveText('1');
+ await expect(topK).toHaveText('50');
+
+ await topK.click();
+ const topKInput = dialog.getByRole('textbox', {name: /^Top K:/});
+ await topKInput.fill('1.5');
+ await expect(dialog).toContainText('Top K must be a non-negative integer');
+ await expect(addToQueueButton).toBeDisabled();
+ await topKInput.fill('25');
+ await topKInput.press('Enter');
+ await expect(topK).toHaveText('25');
+ await expect(addToQueueButton).toBeEnabled();
+
+ for (const [setting, inputName, value] of [
+ [temperature, /^Temperature:/, '0.7'],
+ [repetitionPenalty, /^Repetition penalty:/, '1.2'],
+ [noRepeatNgramSize, /^No-repeat n-gram size:/, '3'],
+ ] as const) {
+ await setting.click();
+ const input = dialog.getByRole('textbox', {
+ name: inputName,
+ });
+ await input.fill(value);
+ await input.press('Enter');
+ await expect(setting).toHaveText(value);
+ }
+
+ await chunkLength.click();
+ const chunkLengthInput = dialog.getByRole('textbox', {
+ name: /^Chunk length:/,
+ });
+ await chunkLengthInput.fill('10');
+ await chunkLengthInput.press('Enter');
+ await expect(dialog).toContainText(
+ 'Stride length must be less than half of chunk length',
+ );
+ await expect(addToQueueButton).toBeDisabled();
+
+ await strideLength.click();
+ const strideLengthInput = dialog.getByRole('textbox', {
+ name: /^Stride length:/,
+ });
+ await strideLengthInput.fill('4');
+ await strideLengthInput.press('Enter');
+ await expect(addToQueueButton).toBeEnabled();
+
+ await dialog.getByRole('button', {name: 'Output', exact: true}).click();
+ await expect(
+ dialog.getByText('Output in public/', {exact: true}),
+ ).toBeVisible();
+ await expect(dialog).toContainText('Caption[]');
+
+ const output = dialog.getByRole('textbox', {
+ name: 'Caption output file',
+ });
+ await expect(output).toHaveValue('video-captions.json');
+ await output.fill('captions/new-video.json');
+ await expect(output).toHaveValue('captions/new-video.json');
+
+ await page.keyboard.press('Escape');
+ await expect(dialog).toBeHidden();
+ await page.goto(`${STUDIO_URL}/transcription-audio-e2e`);
+ await expect(page).toHaveURL(/transcription-audio-e2e/, {
+ timeout: 15_000,
+ });
+ const audio = page.getByText('<Audio>', {exact: true}).first();
+ await expect(async () => {
+ await audio.click();
+ await expect(transcribe).toBeVisible({timeout: 1_000});
+ }).toPass({timeout: 30_000});
+
+ await page.goto(`${STUDIO_URL}/transcription-legacy-media-e2e`);
+ await expect(page).toHaveURL(/transcription-legacy-media-e2e/, {
+ timeout: 15_000,
+ });
+
+ for (const name of ['vp8-vorbis.webm', '<Legacy Audio>']) {
+ const timelineItem = page.getByText(name, {exact: true}).first();
+ await expect(timelineItem).toBeVisible({timeout: 15_000});
+ await timelineItem.click({button: 'right'});
+ const menu = page.locator('[data-remotion-menu-tree-id]').last();
+ const contextMenuTranscribe = menu.getByRole('button', {
+ name: 'Transcribe',
+ exact: true,
+ });
+ await expect(contextMenuTranscribe).toBeVisible();
+ await contextMenuTranscribe.click();
+ await expect(page.getByRole('dialog')).toContainText(
+ `Transcribe ${name}`,
+ );
+ await page.keyboard.press('Escape');
+ }
+ });
+});
diff --git a/packages/example/e2e/transcription-resampling.test.mts b/packages/example/e2e/transcription-resampling.test.mts
new file mode 100644
index 0000000000..b35e9af4c0
--- /dev/null
+++ b/packages/example/e2e/transcription-resampling.test.mts
@@ -0,0 +1,141 @@
+import {readFileSync} from 'fs';
+import {createServer} from 'http';
+import path from 'path';
+import {expect, test} from '@playwright/test';
+import {BundlerInternals} from '@remotion/bundler';
+import {exampleDir} from './constants.mts';
+
+// Playwright's open-source Chromium build omits the proprietary AAC decoder.
+// Chrome exercises the same Chromium WebCodecs path with AAC enabled.
+test.use({channel: 'chrome'});
+
+test.describe('transcription media resampling', () => {
+ test('decodes authenticated AAC media into mono 16kHz Float32 PCM', async ({
+ page,
+ }) => {
+ const resamplerPath = path.resolve(
+ exampleDir,
+ '..',
+ 'studio',
+ 'src',
+ 'components',
+ 'Transcription',
+ 'resample-media-to-16-khz.ts',
+ );
+ const result = await BundlerInternals.esbuild.build({
+ bundle: true,
+ entryPoints: [resamplerPath],
+ format: 'iife',
+ globalName: 'RemotionTranscriptionResamplingTest',
+ platform: 'browser',
+ write: false,
+ });
+ const browserBundle = result.outputFiles[0]?.text;
+ if (!browserBundle) {
+ throw new Error('Did not generate the browser resampling bundle.');
+ }
+
+ const media = readFileSync(
+ path.join(exampleDir, 'public', 'demo_smpte_h264_aac.mp4'),
+ );
+ const server = createServer((request, response) => {
+ if (request.url === '/') {
+ response.setHeader('Content-Type', 'text/html');
+ response.end('<!doctype html><title>Resampling test</title>');
+ return;
+ }
+
+ if (request.url !== '/fixture.mp4') {
+ response.statusCode = 404;
+ response.end();
+ return;
+ }
+ if (request.headers['x-remotion-test'] !== 'transcription') {
+ response.statusCode = 401;
+ response.end();
+ return;
+ }
+
+ const range = request.headers.range?.match(/^bytes=(\d+)-(\d*)$/);
+ const start = range ? Number(range[1]) : 0;
+ const requestedEnd = range?.[2] ? Number(range[2]) : media.length - 1;
+ const end = Math.min(requestedEnd, media.length - 1);
+ response.statusCode = range ? 206 : 200;
+ response.setHeader('Accept-Ranges', 'bytes');
+ response.setHeader('Content-Type', 'video/mp4');
+ response.setHeader('Content-Length', end - start + 1);
+ if (range) {
+ response.setHeader(
+ 'Content-Range',
+ `bytes ${start}-${end}/${media.length}`,
+ );
+ }
+
+ response.end(
+ request.method === 'HEAD' ? undefined : media.subarray(start, end + 1),
+ );
+ });
+ await new Promise<void>((resolve) =>
+ server.listen(0, '127.0.0.1', resolve),
+ );
+
+ try {
+ const address = server.address();
+ if (address === null || typeof address === 'string') {
+ throw new Error('Could not determine the media server port.');
+ }
+
+ await page.goto(`http://127.0.0.1:${address.port}`);
+ await page.addScriptTag({content: browserBundle});
+ const waveform = await page.evaluate(async () => {
+ const progress: number[] = [];
+ const bundle = window as typeof window & {
+ RemotionTranscriptionResamplingTest: {
+ resampleMediaTo16Khz: (options: {
+ src: string;
+ audioStreamIndex: number | null;
+ requestInit: Omit<RequestInit, 'signal'> | null;
+ onProgress: (value: number) => void;
+ }) => Promise<Float32Array>;
+ };
+ };
+ const pcm =
+ await bundle.RemotionTranscriptionResamplingTest.resampleMediaTo16Khz(
+ {
+ src: '/fixture.mp4',
+ audioStreamIndex: null,
+ requestInit: {
+ headers: {'x-remotion-test': 'transcription'},
+ },
+ onProgress: (value) => progress.push(value),
+ },
+ );
+
+ let peak = 0;
+ for (const sample of pcm) {
+ peak = Math.max(peak, Math.abs(sample));
+ }
+
+ return {
+ isFloat32Array: pcm instanceof Float32Array,
+ length: pcm.length,
+ peak,
+ progress,
+ };
+ });
+
+ expect(waveform.isFloat32Array).toBe(true);
+ // The fixture is 10 seconds long. Around 160,000 samples proves that the
+ // stereo 48kHz AAC track was converted to mono PCM at 16kHz.
+ expect(waveform.length).toBeGreaterThan(159_000);
+ expect(waveform.length).toBeLessThan(161_000);
+ expect(waveform.peak).toBeGreaterThan(0);
+ expect(waveform.progress[0]).toBe(0);
+ expect(waveform.progress.at(-1)).toBe(1);
+ } finally {
+ await new Promise<void>((resolve, reject) => {
+ server.close((error) => (error ? reject(error) : resolve()));
+ });
+ }
+ });
+});
diff --git a/packages/example/src/E2eTestRoot.tsx b/packages/example/src/E2eTestRoot.tsx
index 9a157a71ae..7af9e6564c 100644
--- a/packages/example/src/E2eTestRoot.tsx
+++ b/packages/example/src/E2eTestRoot.tsx
@@ -23,6 +23,10 @@ import {RotationKeyframeE2e} from './RotationKeyframeE2e';
import {SchemaTest, schemaTestSchema} from './SchemaTest';
import {TimelineNegativeFromResize} from './TimelineNegativeFromResize';
import {TimelineVirtualizationTestbed} from './TimelineVirtualizationTestbed';
+import {
+ TranscriptionAudioE2e,
+ TranscriptionLegacyMediaE2e,
+} from './TranscriptionE2e';
import {VisualControls} from './VisualControls';
import {VisualMode3D} from './VisualMode3D';
import {AffineFrameClock} from './VisualModeTests/AffineFrameClock';
@@ -264,6 +268,22 @@ export const E2eTestRoot: React.FC = () => {
fps={30}
durationInFrames={90}
/>
+ <Composition
+ id="transcription-audio-e2e"
+ component={TranscriptionAudioE2e}
+ width={1080}
+ height={1080}
+ fps={30}
+ durationInFrames={90}
+ />
+ <Composition
+ id="transcription-legacy-media-e2e"
+ component={TranscriptionLegacyMediaE2e}
+ width={1080}
+ height={1080}
+ fps={30}
+ durationInFrames={90}
+ />
<NewVideoComp />
</>
);
diff --git a/packages/example/src/TranscriptionE2e.tsx b/packages/example/src/TranscriptionE2e.tsx
new file mode 100644
index 0000000000..fdca2a71f0
--- /dev/null
+++ b/packages/example/src/TranscriptionE2e.tsx
@@ -0,0 +1,16 @@
+import {Audio as MediaAudio} from '@remotion/media';
+import React from 'react';
+import {Audio as LegacyAudio, staticFile, Video as LegacyVideo} from 'remotion';
+
+export const TranscriptionAudioE2e: React.FC = () => {
+ return <MediaAudio name="<Audio>" src={staticFile('chirp.wav')} />;
+};
+
+export const TranscriptionLegacyMediaE2e: React.FC = () => {
+ return (
+ <>
+ <LegacyVideo name="vp8-vorbis.webm" src={staticFile('vp8-vorbis.webm')} />
+ <LegacyAudio name="<Legacy Audio>" src={staticFile('chirp.wav')} />
+ </>
+ );
+};
diff --git a/packages/media/src/audio/audio.tsx b/packages/media/src/audio/audio.tsx
index 9f112992ce..f91ee4856c 100644
--- a/packages/media/src/audio/audio.tsx
+++ b/packages/media/src/audio/audio.tsx
@@ -9,6 +9,7 @@ import {
type SequenceControls,
type InteractivitySchema,
} from 'remotion';
+import {resolveRequestInit} from '../request-init';
import {getLoopDisplay} from '../show-in-timeline';
import {validateToneFrequency} from '../validate-tone-frequency';
import {AudioForPreview} from './audio-for-preview';
@@ -85,6 +86,13 @@ const AudioInner: React.FC<
const [mediaVolume] = Internals.useMediaVolumeState();
const mediaStartsAt = Internals.useMediaStartsAt();
const videoConfig = useVideoConfig();
+ const [resolvedRequestInit] = useState(
+ () =>
+ resolveRequestInit({
+ credentials: props.credentials,
+ requestInit: props.requestInit,
+ }) ?? null,
+ );
const sequenceDurationInFrames = Math.min(
durationInFrames ?? Infinity,
Math.max(0, videoConfig.durationInFrames - (from ?? 0)),
@@ -103,6 +111,8 @@ const AudioInner: React.FC<
mediaStartsAt,
loop: props.loop ?? false,
muted: props.muted ?? false,
+ audioStreamIndex: props.audioStreamIndex ?? null,
+ requestInit: resolvedRequestInit,
});
// TODO: Redundant with what we do in the Studio
diff --git a/packages/media/src/video/video.tsx b/packages/media/src/video/video.tsx
index 9046bf97ff..156e579655 100644
--- a/packages/media/src/video/video.tsx
+++ b/packages/media/src/video/video.tsx
@@ -9,6 +9,7 @@ import {
type SequenceControls,
type InteractivitySchema,
} from 'remotion';
+import {resolveRequestInit} from '../request-init';
import {getLoopDisplay} from '../show-in-timeline';
import {validateToneFrequency} from '../validate-tone-frequency';
import {getVideoSequenceDuration} from './get-video-sequence-duration';
@@ -267,6 +268,9 @@ const VideoInner: React.FC<
const [mediaVolume] = Internals.useMediaVolumeState();
const mediaStartsAt = Internals.useMediaStartsAt();
const videoConfig = useVideoConfig();
+ const [resolvedRequestInit] = useState(
+ () => resolveRequestInit({credentials, requestInit}) ?? null,
+ );
const sequenceDurationInFrames = Math.min(
durationInFrames ?? Infinity,
Math.max(0, videoConfig.durationInFrames - (from ?? 0)),
@@ -292,6 +296,8 @@ const VideoInner: React.FC<
mediaStartsAt,
loop: loop ?? false,
muted: muted ?? false,
+ audioStreamIndex: audioStreamIndex ?? 0,
+ requestInit: resolvedRequestInit,
});
// TODO: Redundant with what we do in the Studio
diff --git a/packages/studio-server/src/preview-server/api-routes.ts b/packages/studio-server/src/preview-server/api-routes.ts
index f47df69391..08b843350d 100644
--- a/packages/studio-server/src/preview-server/api-routes.ts
+++ b/packages/studio-server/src/preview-server/api-routes.ts
@@ -27,7 +27,6 @@ import {duplicateJsxNodeHandler} from './routes/duplicate-jsx-node';
import {findInFileHandler} from './routes/find-in-file';
import {insertElementHandler} from './routes/insert-element';
import {insertJsxElementHandler} from './routes/insert-jsx-element';
-import {handleInstallPackage} from './routes/install-dependency';
import {logStudioErrorHandler} from './routes/log-studio-error';
import {moveKeyframesHandler} from './routes/move-keyframes';
import {openInEditorHandler} from './routes/open-in-editor';
@@ -67,7 +66,10 @@ import {updateElementInstallTargetHandler} from './routes/update-element-install
import {updateSequenceKeyframeSettingsHandler} from './routes/update-sequence-keyframe-settings';
import {handleUpgradeRemotion} from './routes/upgrade-remotion';
-type StandardApiRoute = Exclude<keyof ApiRoutes, '/api/update-config'>;
+type StandardApiRoute = Exclude<
+ keyof ApiRoutes,
+ '/api/install-package' | '/api/update-config'
+>;
export const allApiRoutes: {
[key in StandardApiRoute]: ApiHandler<
@@ -131,7 +133,6 @@ export const allApiRoutes: {
'/api/restart-studio': handleRestartStudio,
'/api/default-coding-agent-info': getDefaultCodingAgentInfoHandler,
'/api/default-editor-info': getDefaultEditorInfoHandler,
- '/api/install-package': handleInstallPackage,
'/api/insert-jsx-element': insertJsxElementHandler,
'/api/insert-element': insertElementHandler,
'/api/prepare-element-install': prepareElementInstallHandler,
diff --git a/packages/studio-server/src/preview-server/routes/install-dependency.ts b/packages/studio-server/src/preview-server/routes/install-dependency.ts
index 35ecbd2a7e..2997d79c7c 100644
--- a/packages/studio-server/src/preview-server/routes/install-dependency.ts
+++ b/packages/studio-server/src/preview-server/routes/install-dependency.ts
@@ -29,10 +29,14 @@ export const getPackageInstallSpec = (
return extraVersion === null ? name : `${name}@${extraVersion}`;
};
-export const handleInstallPackage: ApiHandler<
- InstallPackageRequest,
- InstallPackageResponse
-> = async ({logLevel, remotionRoot, input: {dependencies}}) => {
+export const handleInstallPackage = async ({
+ logLevel,
+ remotionRoot,
+ input: {dependencies},
+ invalidateBundle,
+}: Parameters<ApiHandler<InstallPackageRequest, InstallPackageResponse>>[0] & {
+ readonly invalidateBundle: () => Promise<void>;
+}): Promise<InstallPackageResponse> => {
for (const dependency of dependencies) {
if (!isValidPackageName(dependency.name)) {
return Promise.reject(
@@ -110,6 +114,7 @@ export const handleInstallPackage: ApiHandler<
),
);
});
+ await invalidateBundle();
RenderInternals.Log.info(
{indent: false, logLevel},
RenderInternals.chalk.gray('╰─ '),
diff --git a/packages/studio-server/src/preview-server/start-server.ts b/packages/studio-server/src/preview-server/start-server.ts
index 1fcadbf57b..aabd44ca0a 100644
--- a/packages/studio-server/src/preview-server/start-server.ts
+++ b/packages/studio-server/src/preview-server/start-server.ts
@@ -243,6 +243,25 @@ export const startServer = async (options: {
setWatchIgnoreNextChangePlugin(watchIgnorePlugin);
const wdmMiddleware = wdm(compiler, options.logLevel);
+ const invalidateBundle = () => {
+ return new Promise<void>((resolve, reject) => {
+ const watching = compiler?.watching;
+ if (!watching) {
+ reject(new Error('The Studio bundle is not being watched.'));
+ return;
+ }
+
+ watching.invalidate((error) => {
+ if (error) {
+ reject(error);
+ return;
+ }
+
+ resolve();
+ });
+ });
+ };
+
const liveEventsServer = makeLiveEventsRouter(options.logLevel, () => {
const undoStack = getUndoStack();
const redoStack = getRedoStack();
@@ -304,6 +323,7 @@ export const startServer = async (options: {
getDefaultCodingAgent: options.getDefaultCodingAgent,
getDefaultEditor: options.getDefaultEditor,
configFile: options.configFile,
+ invalidateBundle,
});
})
.catch((err) => {
diff --git a/packages/studio-server/src/routes.ts b/packages/studio-server/src/routes.ts
index 00d79d4c82..d6f53eb9f3 100644
--- a/packages/studio-server/src/routes.ts
+++ b/packages/studio-server/src/routes.ts
@@ -17,6 +17,8 @@ import {RenderInternals} from '@remotion/renderer';
import type {
ApiRoutes,
GitSource,
+ InstallPackageRequest,
+ InstallPackageResponse,
RenderDefaults,
RenderJob,
StudioRuntimeConfig,
@@ -38,6 +40,7 @@ import {handleRequest} from './preview-server/handler';
import type {LiveEventsServer} from './preview-server/live-events';
import {fetchFolder, getFiles} from './preview-server/public-folder';
import {handleAppIcon} from './preview-server/routes/app-icon';
+import {handleInstallPackage} from './preview-server/routes/install-dependency';
import {getEditorName} from './preview-server/routes/open-in-editor';
import {updateConfigHandler} from './preview-server/routes/update-config';
import {serveStatic} from './preview-server/serve-static';
@@ -185,6 +188,7 @@ const handleFallback = async ({
response.end(
BundlerInternals.indexHtml({
+ importMap: null,
staticHash: hash,
publicPath: '/',
editorName: displayName,
@@ -431,6 +435,7 @@ export const handleRoutes = ({
getDefaultCodingAgent,
getDefaultEditor,
configFile,
+ invalidateBundle,
}: {
staticHash: string;
staticHashPrefix: string;
@@ -459,6 +464,7 @@ export const handleRoutes = ({
getDefaultCodingAgent: () => DefaultCodingAgent | null;
getDefaultEditor: () => DefaultEditor | null;
configFile: string | null;
+ invalidateBundle: () => Promise<void>;
}): Promise<void> => {
const url = new URL(request.url as string, 'http://localhost');
@@ -573,6 +579,23 @@ export const handleRoutes = ({
});
}
+ if (url.pathname === '/api/install-package') {
+ return handleRequest<InstallPackageRequest, InstallPackageResponse>({
+ remotionRoot,
+ entryPoint,
+ handler: (params) => handleInstallPackage({...params, invalidateBundle}),
+ request,
+ response,
+ logLevel,
+ methods,
+ binariesDirectory,
+ publicDir,
+ configFile,
+ getDefaultCodingAgent,
+ getDefaultEditor,
+ });
+ }
+
for (const [key, value] of Object.entries(allApiRoutes)) {
if (url.pathname === key) {
return handleRequest({
diff --git a/packages/studio-server/src/test/file-source-route.test.ts b/packages/studio-server/src/test/file-source-route.test.ts
index d69fbb17c0..0e833b0d7f 100644
--- a/packages/studio-server/src/test/file-source-route.test.ts
+++ b/packages/studio-server/src/test/file-source-route.test.ts
@@ -77,6 +77,7 @@ test('serves file source from an origin-less GET request', async () => {
getCurrentInputProps: () => ({}),
getDefaultCodingAgent: () => null,
getDefaultEditor: () => null,
+ invalidateBundle: () => Promise.resolve(),
getEnvVariables: () => ({}),
getRenderDefaults: () => ({}) as RenderDefaults,
getRenderQueue: () => [],
diff --git a/packages/studio-server/src/test/install-dependency.test.ts b/packages/studio-server/src/test/install-dependency.test.ts
index 8d8cf94207..c7e0fba416 100644
--- a/packages/studio-server/src/test/install-dependency.test.ts
+++ b/packages/studio-server/src/test/install-dependency.test.ts
@@ -98,6 +98,7 @@ test('installs without running dependency lifecycle scripts', async () => {
nub: 'nub.lock',
};
const temporaryDirectories: string[] = [];
+ let invalidations = 0;
try {
for (const manager of Object.keys(lockfiles) as PackageManager[]) {
@@ -118,6 +119,10 @@ test('installs without running dependency lifecycle scripts', async () => {
dependencies: [{name: 'lodash', version: '4.17.21'}],
},
logLevel: 'error',
+ invalidateBundle: () => {
+ invalidations++;
+ return Promise.resolve();
+ },
methods: {
addJob: () => undefined,
cancelJob: () => undefined,
@@ -143,6 +148,8 @@ test('installs without running dependency lifecycle scripts', async () => {
} else {
expect(call.args).toContain('--ignore-scripts');
}
+
+ expect(invalidations).toBe(temporaryDirectories.length);
}
} finally {
spawnSpy.mockRestore();
diff --git a/packages/studio-server/src/test/upload-output-security.test.ts b/packages/studio-server/src/test/upload-output-security.test.ts
index cd14a87570..b37ff0d7db 100644
--- a/packages/studio-server/src/test/upload-output-security.test.ts
+++ b/packages/studio-server/src/test/upload-output-security.test.ts
@@ -80,6 +80,7 @@ const requestWriteRoute = async ({
getCurrentInputProps: () => ({}),
getDefaultCodingAgent: () => null,
getDefaultEditor: () => null,
+ invalidateBundle: () => Promise.resolve(),
getEnvVariables: () => ({}),
getExperimentalKeepAudioContextAlive: () => false,
getNumberOfAudioTags: () => 0,
diff --git a/packages/studio-shared/src/studio-html.ts b/packages/studio-shared/src/studio-html.ts
index d02a2fbbe7..1e11634b49 100644
--- a/packages/studio-shared/src/studio-html.ts
+++ b/packages/studio-shared/src/studio-html.ts
@@ -33,6 +33,7 @@ export type StudioHtmlOptions = {
mode: 'dev' | 'bundle';
bundleScriptUrl?: string;
bundleScriptType?: 'classic' | 'module';
+ importMap: Record<string, string> | null;
readOnlyStudio?: boolean;
studioRuntimeConfig?: StudioRuntimeConfig;
};
@@ -65,6 +66,7 @@ export const studioHtml = ({
mode,
bundleScriptUrl,
bundleScriptType,
+ importMap,
readOnlyStudio,
studioRuntimeConfig,
}: StudioHtmlOptions) => {
@@ -87,6 +89,11 @@ export const studioHtml = ({
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
+ ${
+ importMap
+ ? `<script type="importmap">${JSON.stringify({imports: importMap}).replaceAll('<', '\\u003c')}</script>`
+ : ''
+ }
${
includeFavicon
? `<link id="__remotion_favicon" rel="icon" type="image/png" href="${publicPath}favicon.ico" />`
diff --git a/packages/studio-shared/src/test/studio-html.test.ts b/packages/studio-shared/src/test/studio-html.test.ts
index 4d06235b9d..a71f25666a 100644
--- a/packages/studio-shared/src/test/studio-html.test.ts
+++ b/packages/studio-shared/src/test/studio-html.test.ts
@@ -3,16 +3,19 @@ import {studioHtml} from '../studio-html';
const makeHtml = ({
bundleScriptType,
+ importMap,
publicPath,
staticHash,
publicFolderExists,
}: {
bundleScriptType?: 'module';
+ importMap: Record<string, string> | null;
publicPath: string;
staticHash: string;
publicFolderExists: string;
}) => {
return studioHtml({
+ importMap,
publicPath,
staticHash,
editorName: null,
@@ -49,6 +52,7 @@ const makeHtml = ({
test('makes relative bundles resolve public assets from the document URL', () => {
const html = makeHtml({
+ importMap: null,
publicPath: './',
staticHash: './public',
publicFolderExists: './public',
@@ -77,6 +81,7 @@ test('makes relative bundles resolve public assets from the document URL', () =>
test('preserves explicitly absolute bundle paths', () => {
const html = makeHtml({
+ importMap: null,
publicPath: '/sites/alpha/',
staticHash: '/sites/alpha/public',
publicFolderExists: '/sites/alpha/public',
@@ -94,6 +99,7 @@ test('preserves explicitly absolute bundle paths', () => {
test('marks an explicitly requested module bundle', () => {
const html = makeHtml({
bundleScriptType: 'module',
+ importMap: null,
publicPath: './',
staticHash: './public',
publicFolderExists: './public',
@@ -101,3 +107,22 @@ test('marks an explicitly requested module bundle', () => {
expect(html).toContain('<script type="module" src="./bundle.js"></script>');
});
+
+test('places an escaped import map before the bundle script', () => {
+ const html = makeHtml({
+ bundleScriptType: 'module',
+ importMap: {
+ '@huggingface/transformers':
+ 'https://example.com/transformers.mjs?</script>',
+ },
+ publicPath: './',
+ staticHash: './public',
+ publicFolderExists: './public',
+ });
+ const importMap =
+ '<script type="importmap">{"imports":{"@huggingface/transformers":"https://example.com/transformers.mjs?\\u003c/script>"}}</script>';
+ const bundleScript = '<script type="module" src="./bundle.js"></script>';
+
+ expect(html).toContain(importMap);
+ expect(html.indexOf(importMap)).toBeLessThan(html.indexOf(bundleScript));
+});
diff --git a/packages/studio/bundle.ts b/packages/studio/bundle.ts
index 4d4b10fad8..10b9d7c012 100644
--- a/packages/studio/bundle.ts
+++ b/packages/studio/bundle.ts
@@ -15,6 +15,7 @@ const external = [
'@remotion/renderer/client',
'@remotion/renderer/pure',
'@remotion/web-renderer',
+ '@remotion/whisper-webgpu',
'@remotion/renderer/error-handling',
'@jridgewell/trace-mapping',
'zod',
@@ -42,10 +43,12 @@ await buildPackage({
{
path: 'src/internals.ts',
target: 'browser',
+ splitting: true,
},
{
path: 'src/previewEntry.tsx',
target: 'browser',
+ splitting: true,
},
],
});
diff --git a/packages/studio/package.json b/packages/studio/package.json
index a4fbb63aa3..2652d6b57d 100644
--- a/packages/studio/package.json
+++ b/packages/studio/package.json
@@ -20,9 +20,15 @@
"url": "https://github.com/remotion-dev/remotion/issues"
},
"peerDependencies": {
+ "@remotion/whisper-webgpu": "workspace:*",
"react": ">=16.8.0",
"react-dom": ">=16.8.0"
},
+ "peerDependenciesMeta": {
+ "@remotion/whisper-webgpu": {
+ "optional": true
+ }
+ },
"dependencies": {
"@tanstack/react-virtual": "3.14.9",
"@remotion/canvas": "workspace:*",
@@ -47,6 +53,7 @@
},
"devDependencies": {
"@happy-dom/global-registrator": "14.5.1",
+ "@remotion/whisper-webgpu": "workspace:*",
"react": "catalog:",
"react-dom": "catalog:",
"@testing-library/react": "16.1.0",
diff --git a/packages/studio/src/components/Checkbox.tsx b/packages/studio/src/components/Checkbox.tsx
index b315b5635a..b29be5c4a9 100644
--- a/packages/studio/src/components/Checkbox.tsx
+++ b/packages/studio/src/components/Checkbox.tsx
@@ -23,9 +23,10 @@ export const Checkbox: React.FC<{
readonly checked: boolean;
readonly onChange: React.ChangeEventHandler<HTMLInputElement>;
readonly name: string;
+ readonly inputId?: string;
readonly disabled?: boolean;
readonly variant?: CheckboxVariant;
-}> = ({checked, onChange, disabled, name, variant = 'default'}) => {
+}> = ({checked, onChange, disabled, inputId, name, variant = 'default'}) => {
const size = SIZES[variant];
const checkmarkSize = CHECKMARK_SIZES[variant];
@@ -71,6 +72,7 @@ export const Checkbox: React.FC<{
return (
<div style={background}>
<input
+ id={inputId}
style={input}
type={'checkbox'}
checked={checked}
diff --git a/packages/studio/src/components/EditorContexts.tsx b/packages/studio/src/components/EditorContexts.tsx
index 4fe4b1a165..0999b4a695 100644
--- a/packages/studio/src/components/EditorContexts.tsx
+++ b/packages/studio/src/components/EditorContexts.tsx
@@ -12,6 +12,7 @@ import {ExpandedTracksProvider} from './ExpandedTracksProvider';
import {ZodProvider} from './get-zod-if-possible';
import {MediaVolumeProvider} from './MediaVolumeProvider';
import {ModalsProvider} from './ModalsProvider';
+import {CaptionQueueProcessorLoader} from './RenderQueue/CaptionQueueProcessorLoader';
import {ClientRenderQueueProcessor} from './RenderQueue/ClientRenderQueueProcessor';
import {RenderQueueContextProvider} from './RenderQueue/context';
import {SetTimelineInOutProvider} from './SetTimelineInOutProvider';
@@ -35,6 +36,7 @@ export const EditorContexts: React.FC<{
<VisualControlsUndoSync />
<RenderQueueContextProvider>
<ClientRenderQueueProcessor />
+ <CaptionQueueProcessorLoader />
<KeybindingContextProvider>
<CheckerboardProvider>
<ZoomGesturesProvider>
diff --git a/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx b/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx
index 4fec352996..3235c1f29f 100644
--- a/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx
+++ b/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx
@@ -7,7 +7,9 @@ import {AudioIcon} from '../../icons/audio';
import {DuplicateIcon} from '../../icons/duplicate';
import {ScissorsIcon} from '../../icons/scissors';
import {SnowflakeIcon} from '../../icons/snowflake';
+import {TranscriptionIcon} from '../../icons/transcription';
import {TrashIcon} from '../../icons/trash';
+import {SetSelectedModalContext} from '../../state/modals';
import {useConfirmationDialog} from '../ConfirmationDialog';
import {
hasSequenceControls,
@@ -28,11 +30,12 @@ import {
} from '../Timeline/TimelineSelection';
import {useDeleteTimelineItems} from '../Timeline/use-delete-timeline-items';
import {getSequenceFreezeFrameMenuItem} from '../Timeline/use-sequence-freeze-frame-menu-item';
+import {isTranscribableSequence} from '../Transcription/is-transcribable-sequence';
import {AlignmentControls} from './AlignmentControls';
import {
- InspectorQuickActionsSection,
- InspectorQuickAction,
InspectorMessage,
+ InspectorQuickAction,
+ InspectorQuickActionsSection,
InspectorSectionHeader,
} from './common';
import {
@@ -129,8 +132,12 @@ const SequenceSourceQuickActions: React.FC<{
const {previewServerState} = useContext(StudioServerConnectionCtx);
const {propStatuses} = useContext(Internals.VisualModePropStatusesContext);
const {setPropStatuses} = useContext(Internals.VisualModeSettersContext);
+ const {setSelectedModal} = useContext(SetSelectedModalContext);
const confirm = useConfirmationDialog();
const deleteTimelineItems = useDeleteTimelineItems();
+ const transcribableMedia = isTranscribableSequence(track.sequence)
+ ? track.sequence
+ : null;
const propStatusesForOverride = useMemo(
() =>
Internals.getPropStatusesCtx(
@@ -154,6 +161,25 @@ const SequenceSourceQuickActions: React.FC<{
});
const sourceActionsDisabled =
previewServerState.type !== 'connected' || !isStudioInteractivityEnabled();
+ const transcribeDisabledReason =
+ previewServerState.type !== 'connected'
+ ? 'Transcription requires a Studio server connection'
+ : !isStudioInteractivityEnabled()
+ ? 'Transcription is unavailable in read-only Studio'
+ : undefined;
+ const onTranscribe = useCallback(() => {
+ if (sourceActionsDisabled || transcribableMedia === null) {
+ return;
+ }
+
+ setSelectedModal({
+ type: 'transcribe',
+ src: transcribableMedia.src,
+ displayName: transcribableMedia.displayName,
+ audioStreamIndex: transcribableMedia.audioStreamIndex,
+ requestInit: transcribableMedia.requestInit,
+ });
+ }, [setSelectedModal, sourceActionsDisabled, transcribableMedia]);
const onDuplicate = useCallback(() => {
if (sourceActionsDisabled) {
return;
@@ -197,6 +223,18 @@ const SequenceSourceQuickActions: React.FC<{
return (
<>
+ {transcribableMedia ? (
+ <InspectorQuickAction
+ disabled={sourceActionsDisabled}
+ onClick={onTranscribe}
+ title={transcribeDisabledReason}
+ renderIcon={(color) => (
+ <TranscriptionIcon style={actionIconStyle} color={color} />
+ )}
+ >
+ Transcribe
+ </InspectorQuickAction>
+ ) : null}
{freezeFrameMenuItem?.type === 'item' ? (
<InspectorQuickAction
disabled={Boolean(freezeFrameMenuItem.disabled)}
diff --git a/packages/studio/src/components/ModalContainer.tsx b/packages/studio/src/components/ModalContainer.tsx
index 0a8a252bc7..1be627f0a9 100644
--- a/packages/studio/src/components/ModalContainer.tsx
+++ b/packages/studio/src/components/ModalContainer.tsx
@@ -39,6 +39,7 @@ const panel: React.CSSProperties = {
};
export const ModalContainer: React.FC<{
+ readonly ariaLabel?: string;
readonly ariaLabelledBy?: string;
readonly onEscape: () => void;
readonly onOutsideClick: () => void;
@@ -46,6 +47,7 @@ export const ModalContainer: React.FC<{
readonly noZIndex?: boolean;
readonly panelStyle?: React.CSSProperties;
}> = ({
+ ariaLabel,
ariaLabelledBy,
children,
onEscape,
@@ -60,6 +62,7 @@ export const ModalContainer: React.FC<{
return (
<div
+ aria-label={ariaLabel}
aria-labelledby={ariaLabelledBy ?? undefined}
aria-modal="true"
className="css-reset"
diff --git a/packages/studio/src/components/Modals.tsx b/packages/studio/src/components/Modals.tsx
index fc31627bba..968d9f2c5c 100644
--- a/packages/studio/src/components/Modals.tsx
+++ b/packages/studio/src/components/Modals.tsx
@@ -29,6 +29,7 @@ import {RenderModalWithLoader} from './RenderModal/ServerRenderModal';
import {WebRenderModalWithLoader} from './RenderModal/WebRenderModal';
import {SettingsModal} from './SettingsModal';
import {SvgImportDialog} from './SvgImportDialog';
+import {TranscriptionModalWithOptionalWhisper} from './Transcription/TranscriptionModalWithOptionalWhisper';
export const Modals: React.FC<{
readonly readOnlyStudio: boolean;
@@ -259,6 +260,9 @@ export const Modals: React.FC<{
{modalContextType && modalContextType.type === 'render-progress' && (
<RenderStatusModal jobId={modalContextType.jobId} />
)}
+ {modalContextType && modalContextType.type === 'transcribe' ? (
+ <TranscriptionModalWithOptionalWhisper state={modalContextType} />
+ ) : null}
{modalContextType && modalContextType.type === 'fix-computed-value' && (
<FixComputedValueModal state={modalContextType} />
diff --git a/packages/studio/src/components/NewComposition/ComboBox.tsx b/packages/studio/src/components/NewComposition/ComboBox.tsx
index 92d8f098ab..167ba02143 100644
--- a/packages/studio/src/components/NewComposition/ComboBox.tsx
+++ b/packages/studio/src/components/NewComposition/ComboBox.tsx
@@ -116,6 +116,7 @@ export const Combobox: React.FC<{
readonly title: string;
readonly size?: ComboboxSize;
readonly unhoveredIconColor?: string;
+ readonly disabled?: boolean;
}> = ({
values,
selectedId,
@@ -123,6 +124,7 @@ export const Combobox: React.FC<{
title,
size: controlSize = 'default',
unhoveredIconColor = LIGHT_TEXT,
+ disabled = false,
}) => {
const [hovered, setIsHovered] = useState(false);
const [opened, setOpened] = useState(false);
@@ -294,8 +296,9 @@ export const Combobox: React.FC<{
: hovered
? WHITE_ALPHA_05
: BLACK_ALPHA_60,
+ opacity: disabled ? 0.5 : 1,
};
- }, [controlSize, customStyle, hovered, opened]);
+ }, [controlSize, customStyle, disabled, hovered, opened]);
const selectedLabelStyle =
controlSize === 'small'
@@ -309,6 +312,7 @@ export const Combobox: React.FC<{
<button
ref={ref}
title={title}
+ disabled={disabled}
tabIndex={tabIndex}
type="button"
style={style}
diff --git a/packages/studio/src/components/NewComposition/DismissableModal.tsx b/packages/studio/src/components/NewComposition/DismissableModal.tsx
index 2610ff0d59..5cee08b8ac 100644
--- a/packages/studio/src/components/NewComposition/DismissableModal.tsx
+++ b/packages/studio/src/components/NewComposition/DismissableModal.tsx
@@ -5,7 +5,8 @@ import {ModalContainer} from '../ModalContainer';
export const DismissableModal: React.FC<{
readonly children: React.ReactNode;
readonly panelStyle?: React.CSSProperties;
-}> = ({children, panelStyle}) => {
+ readonly ariaLabel?: string;
+}> = ({children, panelStyle, ariaLabel}) => {
const {setSelectedModal} = useContext(SetSelectedModalContext);
const onQuit = useCallback(() => {
@@ -14,6 +15,7 @@ export const DismissableModal: React.FC<{
return (
<ModalContainer
+ ariaLabel={ariaLabel}
onOutsideClick={onQuit}
onEscape={onQuit}
panelStyle={panelStyle}
diff --git a/packages/studio/src/components/RenderModal/NumberSetting.tsx b/packages/studio/src/components/RenderModal/NumberSetting.tsx
index 12b1d1258a..e79cbf4bf2 100644
--- a/packages/studio/src/components/RenderModal/NumberSetting.tsx
+++ b/packages/studio/src/components/RenderModal/NumberSetting.tsx
@@ -3,9 +3,17 @@ import React, {useCallback} from 'react';
import {Spacing} from '../layout';
import {InputDragger} from '../NewComposition/InputDragger';
import {RightAlignInput} from '../NewComposition/RemInput';
+import {InfoBubble} from './InfoBubble';
import {label, optionRow, rightRow} from './layout';
import {OptionExplainerBubble} from './OptionExplainerBubble';
+type NumberSettingHint =
+ | AvailableOptions
+ | {
+ readonly content: React.ReactNode;
+ readonly title: string;
+ };
+
export const NumberSetting: React.FC<{
readonly name: string;
readonly value: number;
@@ -14,7 +22,7 @@ export const NumberSetting: React.FC<{
readonly min: number;
readonly step: number;
readonly formatter?: (value: string | number) => string;
- readonly hint?: AvailableOptions;
+ readonly hint?: NumberSettingHint;
}> = ({name, value, step, hint, onValueChanged, max, min, formatter}) => {
const onTextChanged = useCallback(
(e: string) => {
@@ -44,7 +52,11 @@ export const NumberSetting: React.FC<{
{hint ? (
<>
<Spacing x={0.5} />
- <OptionExplainerBubble id={hint} />
+ {typeof hint === 'string' ? (
+ <OptionExplainerBubble id={hint} />
+ ) : (
+ <InfoBubble title={hint.title}>{hint.content}</InfoBubble>
+ )}
</>
) : null}
</div>
@@ -53,6 +65,7 @@ export const NumberSetting: React.FC<{
<InputDragger
value={value}
name={name.toLowerCase()}
+ aria-label={`${name}: ${formatter ? formatter(value) : value}`}
onTextChange={onTextChanged}
onValueChange={onValueChange}
step={step}
diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueItem.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueItem.tsx
new file mode 100644
index 0000000000..e659490f79
--- /dev/null
+++ b/packages/studio/src/components/RenderQueue/CaptionQueueItem.tsx
@@ -0,0 +1,235 @@
+import React, {
+ useCallback,
+ useContext,
+ useEffect,
+ useMemo,
+ useState,
+} from 'react';
+import {
+ CURRENT_COLOR,
+ FAIL_COLOR,
+ getBackgroundFromHoverState,
+ LIGHT_TEXT,
+} from '../../helpers/colors';
+import {pushUrl} from '../../helpers/url-state';
+import type {RenderInlineAction} from '../InlineAction';
+import {InlineAction} from '../InlineAction';
+import {Row, Spacing} from '../layout';
+import {useSelectAsset} from '../use-select-asset';
+import type {CaptionJob} from './caption-job-types';
+import {
+ CircularProgress,
+ RENDER_STATUS_INDICATOR_SIZE,
+} from './CircularProgress';
+import {RenderQueueContext} from './context';
+import {
+ renderQueueItemSubtitleStyle,
+ RENDER_QUEUE_ITEM_SELECTED_CLASSNAME,
+} from './item-style';
+import {SuccessIcon} from './SuccessIcon';
+
+const container: React.CSSProperties = {
+ padding: 12,
+ display: 'flex',
+ flexDirection: 'row',
+ paddingBottom: 10,
+ paddingRight: 4,
+ marginBottom: 1,
+ marginLeft: 4,
+ borderRadius: 4,
+ width: 'calc(100% - 4px)',
+};
+
+const title: React.CSSProperties = {
+ fontSize: 13,
+ lineHeight: 1,
+};
+
+const right: React.CSSProperties = {
+ flex: 1,
+ display: 'flex',
+ flexDirection: 'column',
+ overflow: 'hidden',
+};
+
+const subtitleContainer: React.CSSProperties = {
+ maxWidth: '100%',
+ flex: 1,
+ display: 'flex',
+ overflow: 'hidden',
+};
+
+const subtitle: React.CSSProperties = {
+ ...renderQueueItemSubtitleStyle,
+ cursor: 'default',
+};
+
+const statusIcon: React.CSSProperties = {
+ height: RENDER_STATUS_INDICATOR_SIZE,
+ width: RENDER_STATUS_INDICATOR_SIZE,
+};
+
+const removeIcon: React.CSSProperties = {
+ height: 16,
+ color: CURRENT_COLOR,
+};
+
+const CaptionJobStatus: React.FC<{readonly job: CaptionJob}> = ({job}) => {
+ if (job.status === 'running') {
+ return (
+ <div
+ style={statusIcon}
+ role="progressbar"
+ aria-label="Caption job progress"
+ aria-valuetext={job.progress.message}
+ aria-valuemin={0}
+ aria-valuemax={100}
+ aria-valuenow={Math.round(job.progress.value * 100)}
+ >
+ <CircularProgress progress={Math.max(0.07, job.progress.value)} />
+ </div>
+ );
+ }
+
+ if (job.status === 'done') {
+ return <SuccessIcon />;
+ }
+
+ if (job.status === 'failed') {
+ return (
+ <svg style={statusIcon} viewBox="0 0 512 512">
+ <path
+ fill={FAIL_COLOR}
+ d="M0 160V352L160 512H352L512 352V160L352 0H160L0 160zm353.9 32l-17 17-47 47 47 47 17 17L320 353.9l-17-17-47-47-47 47-17 17L158.1 320l17-17 47-47-47-47-17-17L192 158.1l17 17 47 47 47-47 17-17L353.9 192z"
+ />
+ </svg>
+ );
+ }
+
+ return (
+ <svg style={statusIcon} viewBox="0 0 512 512">
+ <path
+ fill={LIGHT_TEXT}
+ d="M256 512C114.6 512 0 397.4 0 256S114.6 0 256 0S512 114.6 512 256s-114.6 256-256 256zM232 120V256c0 8 4 15.5 10.7 20l96 64c11 7.4 25.9 4.4 33.3-6.7s4.4-25.9-6.7-33.3L280 243.2V120c0-13.3-10.7-24-24-24s-24 10.7-24 24z"
+ />
+ </svg>
+ );
+};
+
+export const CaptionQueueItem: React.FC<{
+ readonly job: CaptionJob;
+ readonly selected: boolean;
+}> = ({job, selected}) => {
+ const [hovered, setHovered] = useState(false);
+ const {removeCaptionJob} = useContext(RenderQueueContext);
+ const selectAsset = useSelectAsset();
+ const isHoverable = job.status === 'done';
+
+ const containerStyle: React.CSSProperties = useMemo(() => {
+ return {
+ ...container,
+ backgroundColor: getBackgroundFromHoverState({
+ hovered: isHoverable && hovered,
+ selected,
+ }),
+ userSelect: 'none',
+ WebkitUserSelect: 'none',
+ };
+ }, [hovered, isHoverable, selected]);
+
+ const onPointerEnter = useCallback(() => {
+ setHovered(true);
+ }, []);
+
+ const onPointerLeave = useCallback(() => {
+ setHovered(false);
+ }, []);
+
+ const scrollCurrentIntoView = useCallback(() => {
+ document
+ .querySelector(`.${RENDER_QUEUE_ITEM_SELECTED_CLASSNAME}`)
+ ?.scrollIntoView({behavior: 'smooth'});
+ }, []);
+
+ const onClick = useCallback(() => {
+ if (job.status !== 'done') {
+ return;
+ }
+
+ selectAsset(job.outName);
+ pushUrl(`/assets/${job.outName}`);
+ }, [job, selectAsset]);
+
+ useEffect(() => {
+ if (selected) {
+ scrollCurrentIntoView();
+ }
+ }, [scrollCurrentIntoView, selected]);
+
+ const onRemove: React.MouseEventHandler = useCallback(
+ (event) => {
+ event.stopPropagation();
+ removeCaptionJob(job.id);
+ },
+ [job.id, removeCaptionJob],
+ );
+
+ const renderRemoveAction: RenderInlineAction = useCallback((color) => {
+ return (
+ <svg style={removeIcon} viewBox="0 0 320 512">
+ <path
+ fill={color}
+ d="M310.6 150.6c12.5-12.5 12.5-32.8 0-45.3s-32.8-12.5-45.3 0L160 210.7 54.6 105.4c-12.5-12.5-32.8-12.5-45.3 0s-12.5 32.8 0 45.3L114.7 256 9.4 361.4c-12.5 12.5-12.5 32.8 0 45.3s32.8 12.5 45.3 0L160 301.3 265.4 406.6c12.5 12.5 32.8 12.5 45.3 0s12.5-32.8 0-45.3L205.3 256 310.6 150.6z"
+ />
+ </svg>
+ );
+ }, []);
+
+ const message = useMemo(() => {
+ if (job.status === 'idle') {
+ return 'Queued for transcription';
+ }
+
+ if (job.status === 'running') {
+ return job.progress.message;
+ }
+
+ if (job.status === 'failed') {
+ return job.error.message;
+ }
+
+ return job.outName;
+ }, [job]);
+
+ return (
+ <Row
+ data-render-queue-item={job.id}
+ style={containerStyle}
+ align="center"
+ onClick={onClick}
+ onPointerEnter={onPointerEnter}
+ onPointerLeave={onPointerLeave}
+ className={selected ? RENDER_QUEUE_ITEM_SELECTED_CLASSNAME : undefined}
+ >
+ <CaptionJobStatus job={job} />
+ <Spacing x={1} />
+ <div style={right}>
+ <div style={title}>{job.displayName}</div>
+ <div style={subtitleContainer}>
+ <span style={subtitle} title={message}>
+ {message}
+ </span>
+ </div>
+ </div>
+ <Spacing x={1} />
+ {job.status === 'running' ? null : (
+ <InlineAction
+ renderAction={renderRemoveAction}
+ onClick={onRemove}
+ title="Remove"
+ variant={null}
+ />
+ )}
+ </Row>
+ );
+};
diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx
new file mode 100644
index 0000000000..09ed7cda8d
--- /dev/null
+++ b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx
@@ -0,0 +1,143 @@
+import {formatBytes} from '@remotion/studio-shared';
+import {
+ canUseWhisperWebGpu,
+ clearStaleModels,
+ disposeWhisperModel,
+ isWhisperModelCached,
+ loadWhisperModel,
+ toCaptions,
+ transcribe,
+} from '@remotion/whisper-webgpu';
+import {useCallback, useContext, useEffect} from 'react';
+import {writeStaticFile} from '../../api/write-static-file';
+import {resampleMediaTo16Khz} from '../Transcription/resample-media-to-16-khz';
+import type {CaptionJob} from './caption-job-types';
+import {RenderQueueContext} from './context';
+
+export const CaptionQueueProcessor: React.FC = () => {
+ const {
+ updateCaptionJobProgress,
+ markCaptionJobDone,
+ markCaptionJobFailed,
+ setProcessCaptionJobCallback,
+ } = useContext(RenderQueueContext);
+
+ const processJob = useCallback(
+ async (job: CaptionJob) => {
+ let captionCount: number | null = null;
+ let processingError: Error | null = null;
+ try {
+ updateCaptionJobProgress(job.id, {
+ message: 'Checking WebGPU support...',
+ value: 0,
+ });
+ const support = await canUseWhisperWebGpu();
+ if (!support.supported) {
+ throw new Error(support.detailedReason);
+ }
+
+ updateCaptionJobProgress(job.id, {
+ message: `Checking ${job.model}...`,
+ value: 0.02,
+ });
+ await clearStaleModels();
+ const modelIsCached = await isWhisperModelCached({model: job.model});
+ await loadWhisperModel({
+ model: job.model,
+ onProgress: (progress) => {
+ const percentage =
+ progress.progress === null
+ ? null
+ : `${Math.round(progress.progress * 100)}%`;
+ const bytes =
+ progress.loadedBytes === null || progress.totalBytes === null
+ ? null
+ : `${formatBytes(progress.loadedBytes)} / ${formatBytes(progress.totalBytes)}`;
+ updateCaptionJobProgress(job.id, {
+ message: [
+ `${modelIsCached ? 'Loading' : 'Downloading'} ${job.model}…`,
+ percentage,
+ bytes,
+ ]
+ .filter(Boolean)
+ .join(' · '),
+ value: 0.03 + (progress.progress ?? 0) * 0.27,
+ });
+ },
+ });
+
+ updateCaptionJobProgress(job.id, {
+ message: `Preparing ${job.displayName}...`,
+ value: 0.3,
+ });
+ const channelWaveform = await resampleMediaTo16Khz({
+ src: job.src,
+ audioStreamIndex: job.audioStreamIndex,
+ requestInit: job.requestInit,
+ onProgress: (progress) => {
+ updateCaptionJobProgress(job.id, {
+ message: `Preparing ${job.displayName}...`,
+ value: 0.3 + progress * 0.2,
+ });
+ },
+ });
+
+ updateCaptionJobProgress(job.id, {
+ message: `Transcribing ${job.displayName}...`,
+ value: 0.5,
+ });
+ const transcription = await transcribe({
+ channelWaveform,
+ model: job.model,
+ task: job.task,
+ chunkLengthInSeconds: job.chunkLengthInSeconds,
+ strideLengthInSeconds: job.strideLengthInSeconds,
+ forceFullSequences: job.forceFullSequences,
+ doSample: job.doSample,
+ temperature: job.temperature,
+ topK: job.topK,
+ repetitionPenalty: job.repetitionPenalty,
+ noRepeatNgramSize: job.noRepeatNgramSize,
+ ...(job.language === null ? {} : {language: job.language}),
+ });
+ const {captions} = toCaptions({
+ whisperWebGpuOutput: transcription,
+ });
+
+ updateCaptionJobProgress(job.id, {
+ message: `Saving ${job.outName}...`,
+ value: 0.95,
+ });
+ await writeStaticFile({
+ contents: JSON.stringify(captions, null, 2),
+ filePath: job.outName,
+ });
+ captionCount = captions.length;
+ } catch (error) {
+ processingError =
+ error instanceof Error ? error : new Error(String(error));
+ }
+
+ try {
+ await disposeWhisperModel({model: job.model});
+ } catch {
+ // The pipeline is removed from the in-memory registry before disposal.
+ // A cleanup failure should not hide a successfully written caption file.
+ }
+
+ if (processingError) {
+ markCaptionJobFailed(job.id, processingError);
+ } else {
+ markCaptionJobDone(job.id, captionCount ?? 0);
+ }
+ },
+ [markCaptionJobDone, markCaptionJobFailed, updateCaptionJobProgress],
+ );
+
+ useEffect(() => {
+ setProcessCaptionJobCallback(processJob);
+ return () => setProcessCaptionJobCallback(null);
+ }, [processJob, setProcessCaptionJobCallback]);
+
+ return null;
+};
diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx
new file mode 100644
index 0000000000..6f8ba1753e
--- /dev/null
+++ b/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx
@@ -0,0 +1,21 @@
+import React, {Suspense, useContext} from 'react';
+import {isWhisperWebGpuInstalled} from '../Transcription/whisper-webgpu-capability';
+import {RenderQueueContext} from './context';
+
+const LazyCaptionQueueProcessor = React.lazy(async () => {
+ const {CaptionQueueProcessor} = await import('./CaptionQueueProcessor');
+ return {default: CaptionQueueProcessor};
+});
+
+export const CaptionQueueProcessorLoader: React.FC = () => {
+ const {captionJobs} = useContext(RenderQueueContext);
+ if (!isWhisperWebGpuInstalled() || captionJobs.length === 0) {
+ return null;
+ }
+
+ return (
+ <Suspense fallback={null}>
+ <LazyCaptionQueueProcessor />
+ </Suspense>
+ );
+};
diff --git a/packages/studio/src/components/RenderQueue/RenderQueueItem.tsx b/packages/studio/src/components/RenderQueue/RenderQueueItem.tsx
index cd2b20e127..5d3f3c89b7 100644
--- a/packages/studio/src/components/RenderQueue/RenderQueueItem.tsx
+++ b/packages/studio/src/components/RenderQueue/RenderQueueItem.tsx
@@ -16,6 +16,7 @@ import type {ClientRenderJob} from './client-side-render-types';
import {isRestoredClientJob} from './client-side-render-types';
import type {AnyRenderJob} from './context';
import {isClientRenderJob} from './context';
+import {RENDER_QUEUE_ITEM_SELECTED_CLASSNAME} from './item-style';
import {RenderQueueCancelledMessage} from './RenderQueueCancelledMessage';
import {
RenderQueueCopyToClipboard,
@@ -64,8 +65,6 @@ const subtitle: React.CSSProperties = {
overflow: 'hidden',
};
-const SELECTED_CLASSNAME = '__remotion_selected_classname';
-
export const RenderQueueItem: React.FC<{
readonly job: AnyRenderJob;
readonly selected: boolean;
@@ -104,7 +103,7 @@ export const RenderQueueItem: React.FC<{
const scrollCurrentIntoView = useCallback(() => {
document
- .querySelector(`.${SELECTED_CLASSNAME}`)
+ .querySelector(`.${RENDER_QUEUE_ITEM_SELECTED_CLASSNAME}`)
?.scrollIntoView({behavior: 'smooth'});
}, []);
@@ -202,7 +201,7 @@ export const RenderQueueItem: React.FC<{
draggable={canDrag}
onDragStart={onDragStart}
onDragEnd={onDragEnd}
- className={selected ? SELECTED_CLASSNAME : undefined}
+ className={selected ? RENDER_QUEUE_ITEM_SELECTED_CLASSNAME : undefined}
>
<RenderQueueItemStatus job={job} />
<Spacing x={1} />
diff --git a/packages/studio/src/components/RenderQueue/caption-job-types.ts b/packages/studio/src/components/RenderQueue/caption-job-types.ts
new file mode 100644
index 0000000000..37d5fad37a
--- /dev/null
+++ b/packages/studio/src/components/RenderQueue/caption-job-types.ts
@@ -0,0 +1,47 @@
+import type {
+ WhisperWebGpuModel,
+ WhisperWebGpuTask,
+} from '@remotion/whisper-webgpu';
+
+export type CaptionJobProgress = {
+ message: string;
+ value: number;
+};
+
+type CaptionJobStatus =
+ | {status: 'idle'}
+ | {status: 'running'; progress: CaptionJobProgress}
+ | {status: 'done'; captionCount: number}
+ | {
+ status: 'failed';
+ error: {message: string; stack: string | undefined};
+ };
+
+export type AddCaptionJobParams = {
+ src: string;
+ displayName: string;
+ audioStreamIndex: number | null;
+ requestInit: Omit<RequestInit, 'signal'> | null;
+ outName: string;
+ model: WhisperWebGpuModel;
+ language: string | null;
+ task: WhisperWebGpuTask;
+ chunkLengthInSeconds: number;
+ strideLengthInSeconds: number;
+ forceFullSequences: boolean;
+ doSample: boolean;
+ temperature: number;
+ topK: number;
+ repetitionPenalty: number;
+ noRepeatNgramSize: number;
+};
+
+export type CaptionJob = AddCaptionJobParams & {
+ id: string;
+ type: 'caption';
+ startedAt: number;
+} & CaptionJobStatus;
+
+export const isCaptionJob = (job: {type: string}): job is CaptionJob => {
+ return job.type === 'caption';
+};
diff --git a/packages/studio/src/components/RenderQueue/context.tsx b/packages/studio/src/components/RenderQueue/context.tsx
index f360b7ddea..86b7d3d503 100644
--- a/packages/studio/src/components/RenderQueue/context.tsx
+++ b/packages/studio/src/components/RenderQueue/context.tsx
@@ -8,6 +8,11 @@ import React, {
useRef,
useState,
} from 'react';
+import type {
+ AddCaptionJobParams,
+ CaptionJob,
+ CaptionJobProgress,
+} from './caption-job-types';
import {
type AddClientStillJobParams,
type AddClientVideoJobParams,
@@ -55,6 +60,7 @@ type RenderQueueContextType = {
jobs: AnyRenderJob[];
serverJobs: RenderJob[];
clientJobs: ClientRenderJob[];
+ captionJobs: CaptionJob[];
addClientStillJob: (
params: AddClientStillJobParams,
compositionRef: CompositionRef,
@@ -80,6 +86,17 @@ type RenderQueueContextType = {
setProcessJobCallback: (
callback: ((job: ClientRenderJob) => Promise<void>) | null,
) => void;
+ addCaptionJob: (params: AddCaptionJobParams) => string;
+ updateCaptionJobProgress: (
+ jobId: string,
+ progress: CaptionJobProgress,
+ ) => void;
+ markCaptionJobDone: (jobId: string, captionCount: number) => void;
+ markCaptionJobFailed: (jobId: string, error: Error) => void;
+ removeCaptionJob: (jobId: string) => void;
+ setProcessCaptionJobCallback: (
+ callback: ((job: CaptionJob) => Promise<void>) | null,
+ ) => void;
getAbortController: (jobId: string) => AbortController;
getCompositionForJob: (jobId: string) => CompositionRef | undefined;
};
@@ -91,6 +108,7 @@ export const RenderQueueContext = React.createContext<RenderQueueContextType>({
jobs: [],
serverJobs: [],
clientJobs: [],
+ captionJobs: [],
addClientStillJob: noopString,
addClientVideoJob: noopString,
updateClientJobProgress: noop,
@@ -101,6 +119,12 @@ export const RenderQueueContext = React.createContext<RenderQueueContextType>({
removeClientJob: noop,
cancelClientJob: noop,
setProcessJobCallback: noop,
+ addCaptionJob: noopString,
+ updateCaptionJobProgress: noop,
+ markCaptionJobDone: noop,
+ markCaptionJobFailed: noop,
+ removeCaptionJob: noop,
+ setProcessCaptionJobCallback: noop,
getAbortController: () => new AbortController(),
getCompositionForJob: () => undefined,
});
@@ -119,21 +143,66 @@ export const RenderQueueContextProvider: React.FC<{
const [clientJobs, setClientJobs] = useState<ClientRenderJob[]>(
restorePersistedClientRenders,
);
+ const [captionJobs, setCaptionJobs] = useState<CaptionJob[]>([]);
const [currentlyProcessing, setCurrentlyProcessing] = useState<string | null>(
null,
);
+ const lastLocalJobStartedAt = useRef(
+ clientJobs.reduce((latest, job) => Math.max(latest, job.startedAt), 0),
+ );
const processJobCallbackRef = useRef<
((job: ClientRenderJob) => Promise<void>) | null
>(null);
+ const processCaptionJobCallbackRef = useRef<
+ ((job: CaptionJob) => Promise<void>) | null
+ >(null);
+ const [processorCallbacksVersion, setProcessorCallbacksVersion] = useState(0);
+
+ const getLocalJobStartedAt = useCallback(() => {
+ const startedAt = Math.max(Date.now(), lastLocalJobStartedAt.current + 1);
+ lastLocalJobStartedAt.current = startedAt;
+ return startedAt;
+ }, []);
- // Process next job when state changes
+ // Process the next local job when state changes. Server renders have their own
+ // queue and are intentionally independent from browser-local work.
useEffect(() => {
if (currentlyProcessing) {
return;
}
- const nextJob = clientJobs.find((job) => job.status === 'idle');
- if (!nextJob || !processJobCallbackRef.current) {
+ const nextJob = [...clientJobs, ...captionJobs]
+ .filter((job) => job.status === 'idle')
+ .sort((a, b) => a.startedAt - b.startedAt)[0];
+ if (!nextJob) {
+ return;
+ }
+
+ if (nextJob.type === 'caption') {
+ if (!processCaptionJobCallbackRef.current) {
+ return;
+ }
+
+ setCurrentlyProcessing(nextJob.id);
+ setCaptionJobs((prev) =>
+ prev.map((job) =>
+ job.id === nextJob.id
+ ? {
+ ...job,
+ status: 'running',
+ progress: {
+ message: 'Starting transcription...',
+ value: 0,
+ },
+ }
+ : job,
+ ),
+ );
+ processCaptionJobCallbackRef.current(nextJob);
+ return;
+ }
+
+ if (!processJobCallbackRef.current) {
return;
}
@@ -156,7 +225,7 @@ export const RenderQueueContextProvider: React.FC<{
),
);
processJobCallbackRef.current(nextJob);
- }, [clientJobs, currentlyProcessing]);
+ }, [captionJobs, clientJobs, currentlyProcessing, processorCallbacksVersion]);
const addClientStillJob = useCallback(
(
@@ -169,14 +238,14 @@ export const RenderQueueContextProvider: React.FC<{
const newJob: ClientStillRenderJob = {
...params,
id,
- startedAt: Date.now(),
+ startedAt: getLocalJobStartedAt(),
status: 'idle',
};
setClientJobs((prev) => [...prev, newJob]);
return id;
},
- [],
+ [getLocalJobStartedAt],
);
const addClientVideoJob = useCallback(
@@ -190,14 +259,14 @@ export const RenderQueueContextProvider: React.FC<{
const newJob: ClientVideoRenderJob = {
...params,
id,
- startedAt: Date.now(),
+ startedAt: getLocalJobStartedAt(),
status: 'idle',
};
setClientJobs((prev) => [...prev, newJob]);
return id;
},
- [],
+ [getLocalJobStartedAt],
);
const updateClientJobProgress = useCallback(
@@ -237,7 +306,7 @@ export const RenderQueueContextProvider: React.FC<{
job.id === jobId ? {...job, status: 'done', metadata, getBlob} : job,
),
);
- setCurrentlyProcessing(null);
+ setCurrentlyProcessing((current) => (current === jobId ? null : current));
},
[],
);
@@ -258,7 +327,7 @@ export const RenderQueueContextProvider: React.FC<{
: job,
),
);
- setCurrentlyProcessing(null);
+ setCurrentlyProcessing((current) => (current === jobId ? null : current));
},
[],
);
@@ -277,7 +346,7 @@ export const RenderQueueContextProvider: React.FC<{
: job,
),
);
- setCurrentlyProcessing(null);
+ setCurrentlyProcessing((current) => (current === jobId ? null : current));
}, []);
const removeClientJob = useCallback((jobId: string): void => {
@@ -300,6 +369,90 @@ export const RenderQueueContextProvider: React.FC<{
const setProcessJobCallback = useCallback(
(callback: ((job: ClientRenderJob) => Promise<void>) | null): void => {
processJobCallbackRef.current = callback;
+ setProcessorCallbacksVersion((version) => version + 1);
+ },
+ [],
+ );
+
+ const addCaptionJob = useCallback(
+ (params: AddCaptionJobParams): string => {
+ const id = `caption-${Date.now()}-${Math.random()
+ .toString(36)
+ .substring(2, 11)}`;
+ const newJob: CaptionJob = {
+ ...params,
+ id,
+ type: 'caption',
+ startedAt: getLocalJobStartedAt(),
+ status: 'idle',
+ };
+
+ setCaptionJobs((prev) => [...prev, newJob]);
+ return id;
+ },
+ [getLocalJobStartedAt],
+ );
+
+ const updateCaptionJobProgress = useCallback(
+ (jobId: string, progress: CaptionJobProgress): void => {
+ setCaptionJobs((prev) =>
+ prev.map((job) =>
+ job.id === jobId
+ ? ({...job, status: 'running', progress} as CaptionJob)
+ : job,
+ ),
+ );
+ },
+ [],
+ );
+
+ const markCaptionJobDone = useCallback(
+ (jobId: string, captionCount: number): void => {
+ setCaptionJobs((prev) =>
+ prev.map((job) =>
+ job.id === jobId
+ ? ({...job, status: 'done', captionCount} as CaptionJob)
+ : job,
+ ),
+ );
+ setCurrentlyProcessing((current) => (current === jobId ? null : current));
+ },
+ [],
+ );
+
+ const markCaptionJobFailed = useCallback(
+ (jobId: string, error: Error): void => {
+ setCaptionJobs((prev) =>
+ prev.map((job) =>
+ job.id === jobId
+ ? ({
+ ...job,
+ status: 'failed',
+ error: {message: error.message, stack: error.stack},
+ } as CaptionJob)
+ : job,
+ ),
+ );
+ setCurrentlyProcessing((current) => (current === jobId ? null : current));
+ },
+ [],
+ );
+
+ const removeCaptionJob = useCallback((jobId: string): void => {
+ setCaptionJobs((prev) => {
+ const jobToRemove = prev.find((job) => job.id === jobId);
+ if (jobToRemove?.status === 'running') {
+ return prev;
+ }
+
+ return prev.filter((job) => job.id !== jobId);
+ });
+ }, []);
+
+ const setProcessCaptionJobCallback = useCallback(
+ (callback: ((job: CaptionJob) => Promise<void>) | null): void => {
+ processCaptionJobCallbackRef.current = callback;
+ setProcessorCallbacksVersion((version) => version + 1);
},
[],
);
@@ -344,6 +497,7 @@ export const RenderQueueContextProvider: React.FC<{
),
serverJobs,
clientJobs,
+ captionJobs,
addClientStillJob,
addClientVideoJob,
updateClientJobProgress,
@@ -354,12 +508,19 @@ export const RenderQueueContextProvider: React.FC<{
removeClientJob,
cancelClientJob,
setProcessJobCallback,
+ addCaptionJob,
+ updateCaptionJobProgress,
+ markCaptionJobDone,
+ markCaptionJobFailed,
+ removeCaptionJob,
+ setProcessCaptionJobCallback,
getAbortController,
getCompositionForJob,
};
}, [
serverJobs,
clientJobs,
+ captionJobs,
addClientStillJob,
addClientVideoJob,
updateClientJobProgress,
@@ -370,6 +531,12 @@ export const RenderQueueContextProvider: React.FC<{
removeClientJob,
cancelClientJob,
setProcessJobCallback,
+ addCaptionJob,
+ updateCaptionJobProgress,
+ markCaptionJobDone,
+ markCaptionJobFailed,
+ removeCaptionJob,
+ setProcessCaptionJobCallback,
]);
return (
diff --git a/packages/studio/src/components/RenderQueue/index.tsx b/packages/studio/src/components/RenderQueue/index.tsx
index bccec61a09..df032d3025 100644
--- a/packages/studio/src/components/RenderQueue/index.tsx
+++ b/packages/studio/src/components/RenderQueue/index.tsx
@@ -3,6 +3,8 @@ import {Internals} from 'remotion';
import {BACKGROUND, LIGHT_TEXT} from '../../helpers/colors';
import {Spacing} from '../layout';
import {VERTICAL_SCROLLBAR_CLASSNAME} from '../Menu/is-menu-item';
+import {isCaptionJob} from './caption-job-types';
+import {CaptionQueueItem} from './CaptionQueueItem';
import {RenderQueueContext} from './context';
import {RenderQueueItem} from './RenderQueueItem';
@@ -33,10 +35,14 @@ const renderQueue: React.CSSProperties = {
};
export const RenderQueue: React.FC = () => {
- const {jobs} = useContext(RenderQueueContext);
+ const {jobs, captionJobs} = useContext(RenderQueueContext);
const {canvasContent} = useContext(Internals.CompositionManager);
- const previousJobCount = React.useRef(jobs.length);
- const jobCount = jobs.length;
+ const allJobs = useMemo(
+ () => [...jobs, ...captionJobs].sort((a, b) => a.startedAt - b.startedAt),
+ [jobs, captionJobs],
+ );
+ const previousJobCount = React.useRef(allJobs.length);
+ const jobCount = allJobs.length;
const divRef = React.useRef<HTMLDivElement>(null);
@@ -62,16 +68,33 @@ export const RenderQueue: React.FC = () => {
}
if (canvasContent.type === 'output') {
- for (let i = 0; i < jobs.length; i++) {
- const job = jobs[i];
+ for (let i = 0; i < allJobs.length; i++) {
+ const job = allJobs[i];
+ if (isCaptionJob(job)) {
+ continue;
+ }
+
if (job.status === 'done' && canvasContent.path === `/${job.outName}`) {
return i;
}
}
}
+ if (canvasContent.type === 'asset') {
+ for (let i = 0; i < allJobs.length; i++) {
+ const job = allJobs[i];
+ if (
+ isCaptionJob(job) &&
+ job.status === 'done' &&
+ canvasContent.asset === job.outName
+ ) {
+ return i;
+ }
+ }
+ }
+
return -1;
- }, [canvasContent, jobs]);
+ }, [allJobs, canvasContent]);
if (jobCount === 0) {
return (
@@ -89,12 +112,18 @@ export const RenderQueue: React.FC = () => {
style={renderQueue}
className={['css-reset', VERTICAL_SCROLLBAR_CLASSNAME].join(' ')}
>
- {jobs.map((j, index) => {
- return (
+ {allJobs.map((job, index) => {
+ return isCaptionJob(job) ? (
+ <CaptionQueueItem
+ key={job.id}
+ job={job}
+ selected={selectedJob === index}
+ />
+ ) : (
<RenderQueueItem
- key={j.id}
+ key={job.id}
selected={selectedJob === index}
- job={j}
+ job={job}
/>
);
})}
diff --git a/packages/studio/src/components/RenderQueue/item-style.ts b/packages/studio/src/components/RenderQueue/item-style.ts
index a3fe3e7ffa..e7894358a6 100644
--- a/packages/studio/src/components/RenderQueue/item-style.ts
+++ b/packages/studio/src/components/RenderQueue/item-style.ts
@@ -2,6 +2,9 @@ import type React from 'react';
import {LIGHT_TEXT} from '../../helpers/colors';
import {SPACING_UNIT} from '../layout';
+export const RENDER_QUEUE_ITEM_SELECTED_CLASSNAME =
+ '__remotion_selected_classname';
+
export const renderQueueItemSubtitleStyle: React.CSSProperties = {
fontSize: 13,
color: LIGHT_TEXT,
diff --git a/packages/studio/src/components/RendersTab.tsx b/packages/studio/src/components/RendersTab.tsx
index c78c58a423..e5bb85dd63 100644
--- a/packages/studio/src/components/RendersTab.tsx
+++ b/packages/studio/src/components/RendersTab.tsx
@@ -28,9 +28,14 @@ export const RendersTab: React.FC<{
readonly selected: boolean;
readonly onClick: MouseEventHandler<HTMLDivElement>;
}> = ({selected, onClick}) => {
- const {jobs} = useContext(RenderQueueContext);
- const failedJobs = jobs.filter((j) => j.status === 'failed').length;
- const jobCount = jobs.length;
+ const {jobs, captionJobs} = useContext(RenderQueueContext);
+ const failedJobs = [...jobs, ...captionJobs].filter(
+ (job) => job.status === 'failed',
+ ).length;
+ const jobCount = jobs.length + captionJobs.length;
+ const hasRunningCaptionJob = captionJobs.some(
+ (job) => job.status === 'running',
+ );
const badgeStyle: React.CSSProperties = useMemo(() => {
return {
@@ -46,7 +51,7 @@ export const RendersTab: React.FC<{
return (
<Tab selected={selected} onClick={onClick}>
<div style={row}>
- Renders
+ {hasRunningCaptionJob ? 'Jobs' : 'Renders'}
{jobCount > 0 ? (
<>
<Flex />
diff --git a/packages/studio/src/components/SelectedOutlineElement.tsx b/packages/studio/src/components/SelectedOutlineElement.tsx
index a3b35cba47..74360dbc54 100644
--- a/packages/studio/src/components/SelectedOutlineElement.tsx
+++ b/packages/studio/src/components/SelectedOutlineElement.tsx
@@ -37,6 +37,7 @@ import {
} from './Timeline/TimelineSelection';
import {getOriginalLocationFromStack} from './Timeline/TimelineStack/get-stack';
import {useDeleteTimelineItems} from './Timeline/use-delete-timeline-items';
+import {isTranscribableSequence} from './Transcription/is-transcribable-sequence';
import {
useDefaultCodingAgentInfo,
useEditorOpening,
@@ -224,6 +225,11 @@ const SelectedOutlineElementUnmemoized: React.FC<
previewServerState.type === 'connected';
const canCrop = contextMenuTarget.canCrop && !sourceEditDisabled;
const canRotate = !sourceEditDisabled;
+ const transcribableMedia = isTranscribableSequence(
+ contextMenuTarget.sequence,
+ )
+ ? contextMenuTarget.sequence
+ : null;
return getSequenceContextMenuItems({
assetLinkInfo,
@@ -282,6 +288,20 @@ const SelectedOutlineElementUnmemoized: React.FC<
confirm,
).catch(() => undefined);
},
+ onTranscribe:
+ sourceEditingEnabled &&
+ previewServerState.type === 'connected' &&
+ transcribableMedia !== null
+ ? () => {
+ setSelectedModal({
+ type: 'transcribe',
+ src: transcribableMedia.src,
+ displayName: transcribableMedia.displayName,
+ audioStreamIndex: transcribableMedia.audioStreamIndex,
+ requestInit: transcribableMedia.requestInit,
+ });
+ }
+ : null,
openInCodingAgent: (codingAgentId, codingAgentName, contextForAgents) => {
launchCodingAgent(codingAgentId, contextForAgents)
.then((response) => {
diff --git a/packages/studio/src/components/Tabs/vertical.tsx b/packages/studio/src/components/Tabs/vertical.tsx
index 6ec07d215f..1ddfd5e973 100644
--- a/packages/studio/src/components/Tabs/vertical.tsx
+++ b/packages/studio/src/components/Tabs/vertical.tsx
@@ -28,7 +28,8 @@ export const VerticalTab: React.FC<{
readonly renderIcon?: (color: string) => React.ReactNode;
readonly style?: React.CSSProperties;
readonly selected: boolean;
-}> = ({children, onClick, renderIcon, style, selected}) => {
+ readonly autoFocus?: boolean;
+}> = ({children, onClick, renderIcon, style, selected, autoFocus}) => {
const [hovered, setHovered] = useState(false);
const {tabIndex} = useZIndex();
@@ -54,6 +55,7 @@ export const VerticalTab: React.FC<{
return (
<button
+ autoFocus={autoFocus}
style={definiteStyle}
type="button"
onClick={onClick}
diff --git a/packages/studio/src/components/Timeline/TimelineSequence.tsx b/packages/studio/src/components/Timeline/TimelineSequence.tsx
index c5e86cd277..2bd3be86d8 100644
--- a/packages/studio/src/components/Timeline/TimelineSequence.tsx
+++ b/packages/studio/src/components/Timeline/TimelineSequence.tsx
@@ -41,6 +41,7 @@ import {AudioWaveform} from '../AudioWaveform';
import {useConfirmationDialog} from '../ConfirmationDialog';
import {ContextMenu} from '../ContextMenu';
import {useSelectComposition} from '../InitialCompositionLoader';
+import {isTranscribableSequence} from '../Transcription/is-transcribable-sequence';
import {useSelectAsset} from '../use-select-asset';
import {disableSequenceInteractivity} from './disable-sequence-interactivity';
import {duplicateSequencesFromSource} from './duplicate-selected-timeline-item';
@@ -606,6 +607,20 @@ const TimelineSequenceInner: React.FC<{
s.type === 'audio' || s.type === 'video' || s.type === 'image'
? s.src
: null;
+ const transcribableMedia = isTranscribableSequence(s) ? s : null;
+ const onTranscribe = useCallback(() => {
+ if (!previewInteractive || transcribableMedia === null) {
+ return;
+ }
+
+ setSelectedModal({
+ type: 'transcribe',
+ src: transcribableMedia.src,
+ displayName: transcribableMedia.displayName,
+ audioStreamIndex: transcribableMedia.audioStreamIndex,
+ requestInit: transcribableMedia.requestInit,
+ });
+ }, [previewInteractive, setSelectedModal, transcribableMedia]);
const onDuplicateSequenceFromSource = useCallback(() => {
if (!validatedLocation?.source || !nodePathInfo || duplicateDisabled) {
return;
@@ -737,6 +752,8 @@ const TimelineSequenceInner: React.FC<{
onDeleteSequenceFromSource,
onDisableSequenceInteractivity,
onDuplicateSequenceFromSource,
+ onTranscribe:
+ previewInteractive && transcribableMedia !== null ? onTranscribe : null,
openInCodingAgent,
openInEditor,
originalLocation,
@@ -767,6 +784,7 @@ const TimelineSequenceInner: React.FC<{
onDisableSequenceInteractivity,
onDuplicateSequenceFromSource,
onDuplicateSelectedSequences,
+ onTranscribe,
openInCodingAgent,
openInEditor,
originalLocation,
@@ -782,6 +800,7 @@ const TimelineSequenceInner: React.FC<{
sequenceFrameOffset,
setPropStatuses,
setSelectedModal,
+ transcribableMedia,
validatedLocation?.source,
]);
const {frozenFrame} = s;
diff --git a/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx b/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx
index a36cbe1c3c..4b6628f6fa 100644
--- a/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx
+++ b/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx
@@ -69,6 +69,7 @@ import {
cropFieldKeys,
rotateFieldKey,
} from '../selected-outline-types';
+import {isTranscribableSequence} from '../Transcription/is-transcribable-sequence';
import {useSelectAsset} from '../use-select-asset';
import {disableSequenceInteractivity} from './disable-sequence-interactivity';
import {duplicateSequencesFromSource} from './duplicate-selected-timeline-item';
@@ -812,6 +813,22 @@ const TimelineSequenceItemInner: React.FC<{
sequence.type === 'image'
? sequence.src
: null;
+ const transcribableMedia = isTranscribableSequence(sequence)
+ ? sequence
+ : null;
+ const onTranscribe = useCallback(() => {
+ if (!previewInteractive || transcribableMedia === null) {
+ return;
+ }
+
+ setSelectedModal({
+ type: 'transcribe',
+ src: transcribableMedia.src,
+ displayName: transcribableMedia.displayName,
+ audioStreamIndex: transcribableMedia.audioStreamIndex,
+ requestInit: transcribableMedia.requestInit,
+ });
+ }, [previewInteractive, setSelectedModal, transcribableMedia]);
const isExpanded =
previewConnected && nodePathInfo !== null && getIsExpanded(nodePathInfo);
@@ -1186,6 +1203,8 @@ const TimelineSequenceItemInner: React.FC<{
onDeleteSequenceFromSource,
onDisableSequenceInteractivity,
onDuplicateSequenceFromSource,
+ onTranscribe:
+ previewInteractive && transcribableMedia !== null ? onTranscribe : null,
openInCodingAgent,
openInEditor,
originalLocation,
@@ -1296,6 +1315,7 @@ const TimelineSequenceItemInner: React.FC<{
onDuplicateSelectedSequences,
onRenameSequence,
onSelect,
+ onTranscribe,
openInCodingAgent,
openInEditor,
originalLocation,
@@ -1311,6 +1331,7 @@ const TimelineSequenceItemInner: React.FC<{
sequenceFrameOffset,
setSelectedModal,
setPropStatuses,
+ transcribableMedia,
validatedLocation?.source,
]);
const canDropEffect =
diff --git a/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts b/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts
index b2dd49095d..72993b5fd8 100644
--- a/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts
+++ b/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts
@@ -96,6 +96,7 @@ export const getSequenceContextMenuItems = ({
onDeleteSequenceFromSource,
onDisableSequenceInteractivity,
onDuplicateSequenceFromSource,
+ onTranscribe,
openInCodingAgent,
openInEditor,
originalLocation,
@@ -116,6 +117,7 @@ export const getSequenceContextMenuItems = ({
readonly onDeleteSequenceFromSource: () => void;
readonly onDisableSequenceInteractivity: () => void;
readonly onDuplicateSequenceFromSource: () => void;
+ readonly onTranscribe: (() => void) | null;
readonly openInCodingAgent: (
codingAgentId: DefaultCodingAgent,
codingAgentName: string,
@@ -251,7 +253,21 @@ export const getSequenceContextMenuItems = ({
value: 'show-asset',
}
: null,
- assetLinkInfo
+ onTranscribe
+ ? {
+ type: 'item' as const,
+ id: 'transcribe',
+ keyHint: null,
+ label: 'Transcribe',
+ leftItem: null,
+ disabled: false,
+ onClick: onTranscribe,
+ quickSwitcherLabel: null,
+ subMenu: null,
+ value: 'transcribe',
+ }
+ : null,
+ assetLinkInfo || onTranscribe
? {
type: 'divider' as const,
id: 'sequence-link-divider',
diff --git a/packages/studio/src/components/Transcription/TranscriptionModal.tsx b/packages/studio/src/components/Transcription/TranscriptionModal.tsx
new file mode 100644
index 0000000000..1c98ab4f61
--- /dev/null
+++ b/packages/studio/src/components/Transcription/TranscriptionModal.tsx
@@ -0,0 +1,844 @@
+import {formatBytes} from '@remotion/studio-shared';
+import {
+ canUseWhisperWebGpu,
+ getAvailableModels,
+ type WhisperWebGpuModel,
+ type WhisperWebGpuTask,
+} from '@remotion/whisper-webgpu';
+import React, {
+ useCallback,
+ useContext,
+ useEffect,
+ useMemo,
+ useState,
+} from 'react';
+import {BLUE_DISABLED, LIGHT_TEXT, WHITE} from '../../helpers/colors';
+import {Checkmark} from '../../icons/Checkmark';
+import type {TranscriptionModalState} from '../../state/modals';
+import {SetSelectedModalContext} from '../../state/modals';
+import {SidebarContext} from '../../state/sidebar';
+import {Button} from '../Button';
+import {Checkbox} from '../Checkbox';
+import {Spacing} from '../layout';
+import {VERTICAL_SCROLLBAR_CLASSNAME} from '../Menu/is-menu-item';
+import {ModalHeader} from '../ModalHeader';
+import type {ComboboxValue} from '../NewComposition/ComboBox';
+import {Combobox} from '../NewComposition/ComboBox';
+import {DismissableModal} from '../NewComposition/DismissableModal';
+import {RemotionInput} from '../NewComposition/RemInput';
+import {ValidationMessage} from '../NewComposition/ValidationMessage';
+import {optionsSidebarTabs} from '../options-sidebar-tabs';
+import {persistSelectedOptionsSidebarPanel} from '../OptionsPanel';
+import {InfoBubble} from '../RenderModal/InfoBubble';
+import {input, label, optionRow, rightRow} from '../RenderModal/layout';
+import {NumberSetting} from '../RenderModal/NumberSetting';
+import {
+ buttonStyle,
+ container,
+ flexer,
+ optionsPanel,
+ outerModalStyle,
+} from '../RenderModal/render-modals';
+import {RenderModalHr} from '../RenderModal/RenderModalHr';
+import {RenderQueueContext} from '../RenderQueue/context';
+import {useStaticFiles} from '../use-static-files';
+import {
+ getDefaultCaptionOutputName,
+ validateCaptionOutputName,
+} from './caption-output-name';
+import {WHISPER_LANGUAGES, type WhisperLanguage} from './whisper-languages';
+
+const TRANSCRIPTION_OUTPUT_MESSAGE_ID = 'remotion-transcription-output-message';
+const TRANSCRIPTION_CHUNK_MESSAGE_ID = 'remotion-transcription-chunk-message';
+const TRANSCRIPTION_DECODING_MESSAGE_ID =
+ 'remotion-transcription-decoding-message';
+const TRANSCRIPTION_TASK_MESSAGE_ID = 'remotion-transcription-task-message';
+const DEFAULT_CHUNK_LENGTH_IN_SECONDS = 30;
+const DEFAULT_STRIDE_LENGTH_IN_SECONDS = 5;
+const DEFAULT_TEMPERATURE = 1;
+const DEFAULT_TOP_K = 50;
+const DEFAULT_REPETITION_PENALTY = 1;
+const DEFAULT_NO_REPEAT_NGRAM_SIZE = 0;
+const MAX_CHUNK_LENGTH_IN_SECONDS = 30;
+
+type SupportState =
+ | {type: 'checking'}
+ | {type: 'supported'}
+ | {type: 'unsupported'; message: string};
+
+const AVAILABLE_MODELS = getAvailableModels();
+
+const settingsPanel: React.CSSProperties = {
+ ...optionsPanel,
+ flexDirection: 'column',
+ paddingTop: 16,
+};
+
+const controlStyle: React.CSSProperties = {
+ width: 330,
+ maxWidth: '100%',
+};
+
+const nestedLabelStyle: React.CSSProperties = {
+ color: 'inherit',
+ fontFamily: 'inherit',
+ fontSize: 'inherit',
+ lineHeight: 'inherit',
+};
+
+const taskMessageRow: React.CSSProperties = {
+ display: 'flex',
+ justifyContent: 'flex-end',
+ padding: '0 16px 8px',
+};
+
+const outputRow: React.CSSProperties = {
+ ...optionRow,
+ paddingBottom: 12,
+};
+
+const tooltipContent: React.CSSProperties = {
+ color: LIGHT_TEXT,
+ fontSize: 13,
+ lineHeight: 1.5,
+ maxWidth: 360,
+ padding: 12,
+};
+
+const tooltipInlineCode: React.CSSProperties = {
+ color: WHITE,
+ fontFamily: 'monospace',
+ fontSize: 'inherit',
+ lineHeight: 'inherit',
+};
+
+const statusText: React.CSSProperties = {
+ color: LIGHT_TEXT,
+ fontFamily: 'sans-serif',
+ fontSize: 13,
+ fontVariantNumeric: 'tabular-nums',
+ lineHeight: 1.5,
+ textAlign: 'right',
+};
+
+const TranscriptionSettingLabel: React.FC<{
+ readonly children: React.ReactNode;
+ readonly inputId: string | null;
+ readonly name: string;
+}> = ({children, inputId, name}) => {
+ return (
+ <div style={label}>
+ {inputId === null ? (
+ name
+ ) : (
+ <label htmlFor={inputId} style={nestedLabelStyle}>
+ {name}
+ </label>
+ )}
+ <Spacing x={0.5} />
+ <InfoBubble title={`Learn more about ${name}`}>
+ <div style={tooltipContent}>{children}</div>
+ </InfoBubble>
+ </div>
+ );
+};
+
+const ModelSettings: React.FC<{
+ readonly selectedLanguage: WhisperLanguage;
+ readonly selectedModel: WhisperWebGpuModel;
+ readonly selectedTask: WhisperWebGpuTask;
+ readonly setSelectedLanguage: (language: WhisperLanguage) => void;
+ readonly setSelectedModel: (model: WhisperWebGpuModel) => void;
+ readonly setSelectedTask: (task: WhisperWebGpuTask) => void;
+ readonly supportState: SupportState;
+}> = ({
+ selectedLanguage,
+ selectedModel,
+ selectedTask,
+ setSelectedLanguage,
+ setSelectedModel,
+ setSelectedTask,
+ supportState,
+}) => {
+ const selectedModelInfo = AVAILABLE_MODELS.find(
+ ({name}) => name === selectedModel,
+ );
+ if (!selectedModelInfo) {
+ throw new Error(`Unknown Whisper model: ${selectedModel}`);
+ }
+
+ const modelOptions = useMemo((): ComboboxValue[] => {
+ return AVAILABLE_MODELS.map((model): ComboboxValue => {
+ return {
+ type: 'item',
+ id: model.name,
+ value: model.name,
+ label: `${model.name} · ${formatBytes(model.webGpuDownloadSize)}`,
+ leftItem: model.name === selectedModel ? <Checkmark /> : null,
+ keyHint: null,
+ quickSwitcherLabel: null,
+ subMenu: null,
+ disabled: false,
+ onClick: () => setSelectedModel(model.name),
+ };
+ });
+ }, [selectedModel, setSelectedModel]);
+
+ const languageOptions = useMemo((): ComboboxValue[] => {
+ return WHISPER_LANGUAGES.map(([languageCode, languageName]) => ({
+ type: 'item',
+ id: languageCode,
+ value: languageCode,
+ label: languageName,
+ leftItem: languageCode === selectedLanguage ? <Checkmark /> : null,
+ keyHint: null,
+ quickSwitcherLabel: null,
+ subMenu: null,
+ disabled: false,
+ onClick: () => setSelectedLanguage(languageCode),
+ }));
+ }, [selectedLanguage, setSelectedLanguage]);
+ const effectiveTask = selectedModelInfo.supportsTranslation
+ ? selectedTask
+ : 'transcribe';
+ const taskOptions = useMemo((): ComboboxValue[] => {
+ return [
+ {
+ type: 'item',
+ id: 'transcribe',
+ value: 'transcribe',
+ label: (
+ <>
+ <span aria-hidden="true" style={nestedLabelStyle}>
+ Transcribe
+ </span>
+ <span style={{position: 'absolute', clip: 'rect(0 0 0 0)'}}>
+ Task: Transcribe
+ </span>
+ </>
+ ),
+ leftItem: effectiveTask === 'transcribe' ? <Checkmark /> : null,
+ keyHint: null,
+ quickSwitcherLabel: null,
+ subMenu: null,
+ disabled: false,
+ onClick: () => setSelectedTask('transcribe'),
+ },
+ {
+ type: 'item',
+ id: 'translate',
+ value: 'translate',
+ label: (
+ <>
+ <span aria-hidden="true" style={nestedLabelStyle}>
+ Translate to English
+ </span>
+ <span style={{position: 'absolute', clip: 'rect(0 0 0 0)'}}>
+ Task: Translate to English
+ </span>
+ </>
+ ),
+ leftItem: effectiveTask === 'translate' ? <Checkmark /> : null,
+ keyHint: null,
+ quickSwitcherLabel: null,
+ subMenu: null,
+ disabled: false,
+ onClick: () => setSelectedTask('translate'),
+ },
+ ];
+ }, [effectiveTask, setSelectedTask]);
+
+ useEffect(() => {
+ if (!selectedModelInfo.supportsTranslation) {
+ setSelectedTask('transcribe');
+ }
+ }, [selectedModelInfo.supportsTranslation, setSelectedTask]);
+
+ return (
+ <>
+ <div style={optionRow}>
+ <div style={label}>Whisper model</div>
+ <div style={rightRow}>
+ <Combobox
+ values={modelOptions}
+ selectedId={selectedModel}
+ title="Whisper model"
+ disabled={false}
+ style={controlStyle}
+ />
+ </div>
+ </div>
+ {selectedModelInfo.multilingual ? (
+ <div style={optionRow}>
+ <div style={label}>Spoken language</div>
+ <div style={rightRow}>
+ <Combobox
+ values={languageOptions}
+ selectedId={selectedLanguage}
+ title="Spoken language"
+ disabled={false}
+ style={controlStyle}
+ />
+ </div>
+ </div>
+ ) : null}
+ <div style={optionRow}>
+ <div style={label}>Task</div>
+ <div style={rightRow}>
+ <Combobox
+ values={taskOptions}
+ selectedId={effectiveTask}
+ title="Task"
+ disabled={!selectedModelInfo.supportsTranslation}
+ style={controlStyle}
+ />
+ </div>
+ </div>
+ {effectiveTask === 'translate' ? (
+ <div
+ id={TRANSCRIPTION_TASK_MESSAGE_ID}
+ aria-live="polite"
+ style={taskMessageRow}
+ >
+ <ValidationMessage
+ align="flex-end"
+ message="Word timings may be less accurate when translating to English."
+ type="warning"
+ />
+ </div>
+ ) : null}
+ <div style={optionRow}>
+ <TranscriptionSettingLabel inputId={null} name="Model download">
+ If necessary, Studio downloads the model in the background when the
+ job starts. Follow its progress in Jobs. Transcription runs locally
+ using WebGPU; the media does not leave your device.
+ </TranscriptionSettingLabel>
+ <div style={rightRow}>
+ <div style={statusText}>
+ {`${formatBytes(selectedModelInfo.webGpuDownloadSize)} · Downloaded with transcription if needed`}
+ </div>
+ </div>
+ </div>
+ {supportState.type === 'unsupported' ? (
+ <div style={{padding: '0 16px'}}>
+ <ValidationMessage
+ align="flex-end"
+ message={supportState.message}
+ type="error"
+ />
+ </div>
+ ) : null}
+ </>
+ );
+};
+
+const OutputSettings: React.FC<{
+ readonly exists: boolean;
+ readonly onOutNameChange: React.ChangeEventHandler<HTMLInputElement>;
+ readonly outName: string;
+ readonly validationMessage: string | null;
+}> = ({exists, onOutNameChange, outName, validationMessage}) => {
+ return (
+ <div style={outputRow}>
+ <TranscriptionSettingLabel inputId={null} name="Output in public/">
+ Studio writes a JSON array compatible with{' '}
+ <code style={tooltipInlineCode}>Caption[]</code>. Load it from your
+ composition with <code style={tooltipInlineCode}>staticFile()</code>.
+ </TranscriptionSettingLabel>
+ <div style={rightRow}>
+ <div style={controlStyle}>
+ <RemotionInput
+ aria-label="Caption output file"
+ aria-describedby={
+ validationMessage || exists
+ ? TRANSCRIPTION_OUTPUT_MESSAGE_ID
+ : undefined
+ }
+ aria-invalid={validationMessage ? true : undefined}
+ status={validationMessage ? 'error' : exists ? 'warning' : 'ok'}
+ style={input}
+ type="text"
+ value={outName}
+ onChange={onOutNameChange}
+ rightAlign
+ />
+ {validationMessage || exists ? (
+ <div id={TRANSCRIPTION_OUTPUT_MESSAGE_ID} aria-live="polite">
+ <Spacing y={1} block />
+ <ValidationMessage
+ align="flex-end"
+ message={validationMessage ?? 'Exists, will be overwritten'}
+ type={validationMessage ? 'error' : 'warning'}
+ />
+ </div>
+ ) : null}
+ </div>
+ </div>
+ </div>
+ );
+};
+
+const AdvancedSettings: React.FC<{
+ readonly chunkLengthInSeconds: number;
+ readonly doSample: boolean;
+ readonly forceFullSequences: boolean;
+ readonly noRepeatNgramSize: number;
+ readonly repetitionPenalty: number;
+ readonly setChunkLengthInSeconds: React.Dispatch<
+ React.SetStateAction<number>
+ >;
+ readonly setDoSample: React.Dispatch<React.SetStateAction<boolean>>;
+ readonly setForceFullSequences: React.Dispatch<React.SetStateAction<boolean>>;
+ readonly setNoRepeatNgramSize: React.Dispatch<React.SetStateAction<number>>;
+ readonly setRepetitionPenalty: React.Dispatch<React.SetStateAction<number>>;
+ readonly setStrideLengthInSeconds: React.Dispatch<
+ React.SetStateAction<number>
+ >;
+ readonly setTemperature: React.Dispatch<React.SetStateAction<number>>;
+ readonly setTopK: React.Dispatch<React.SetStateAction<number>>;
+ readonly strideLengthInSeconds: number;
+ readonly temperature: number;
+ readonly topK: number;
+ readonly decodingValidationMessage: string | null;
+ readonly validationMessage: string | null;
+}> = ({
+ chunkLengthInSeconds,
+ doSample,
+ forceFullSequences,
+ noRepeatNgramSize,
+ repetitionPenalty,
+ setChunkLengthInSeconds,
+ setDoSample,
+ setForceFullSequences,
+ setNoRepeatNgramSize,
+ setRepetitionPenalty,
+ setStrideLengthInSeconds,
+ setTemperature,
+ setTopK,
+ strideLengthInSeconds,
+ temperature,
+ topK,
+ decodingValidationMessage,
+ validationMessage,
+}) => {
+ const onForceFullSequencesChange: React.ChangeEventHandler<HTMLInputElement> =
+ useCallback(
+ (event) => setForceFullSequences(event.target.checked),
+ [setForceFullSequences],
+ );
+ const onDoSampleChange: React.ChangeEventHandler<HTMLInputElement> =
+ useCallback((event) => setDoSample(event.target.checked), [setDoSample]);
+
+ return (
+ <>
+ <div
+ aria-describedby={
+ validationMessage ? TRANSCRIPTION_CHUNK_MESSAGE_ID : undefined
+ }
+ aria-invalid={validationMessage ? true : undefined}
+ aria-label="Chunk settings"
+ role="group"
+ >
+ <NumberSetting
+ formatter={(value) => `${value}s`}
+ hint={{
+ content: (
+ <div style={tooltipContent}>
+ Studio splits long audio into chunks of up to 30 seconds. Longer
+ chunks need fewer transcription passes.
+ </div>
+ ),
+ title: 'Learn more about Chunk length',
+ }}
+ max={MAX_CHUNK_LENGTH_IN_SECONDS}
+ min={1}
+ name="Chunk length"
+ onValueChanged={setChunkLengthInSeconds}
+ step={1}
+ value={chunkLengthInSeconds}
+ />
+ <NumberSetting
+ formatter={(value) => `${value}s`}
+ hint={{
+ content: (
+ <div style={tooltipContent}>
+ Overlaps both sides of each chunk to help preserve words at the
+ boundaries.
+ </div>
+ ),
+ title: 'Learn more about Stride length',
+ }}
+ min={0}
+ name="Stride length"
+ onValueChanged={setStrideLengthInSeconds}
+ step={1}
+ value={strideLengthInSeconds}
+ />
+ </div>
+ {validationMessage ? (
+ <div
+ id={TRANSCRIPTION_CHUNK_MESSAGE_ID}
+ aria-live="polite"
+ style={{padding: '0 16px'}}
+ >
+ <ValidationMessage
+ align="flex-end"
+ message={validationMessage}
+ type="error"
+ />
+ </div>
+ ) : null}
+ <div
+ aria-describedby={
+ decodingValidationMessage
+ ? TRANSCRIPTION_DECODING_MESSAGE_ID
+ : undefined
+ }
+ aria-invalid={decodingValidationMessage ? true : undefined}
+ aria-label="Decoding settings"
+ role="group"
+ >
+ <RenderModalHr />
+ <div style={optionRow}>
+ <TranscriptionSettingLabel
+ inputId="force-full-sequences"
+ name="Force full sequences"
+ >
+ Makes the job fail if Whisper leaves an incomplete timestamp
+ sequence.
+ </TranscriptionSettingLabel>
+ <div style={rightRow}>
+ <Checkbox
+ checked={forceFullSequences}
+ inputId="force-full-sequences"
+ name="force-full-sequences"
+ onChange={onForceFullSequencesChange}
+ />
+ </div>
+ </div>
+ <RenderModalHr />
+ <div style={optionRow}>
+ <TranscriptionSettingLabel inputId="use-sampling" name="Use sampling">
+ Makes decoding nondeterministic.
+ </TranscriptionSettingLabel>
+ <div style={rightRow}>
+ <Checkbox
+ checked={doSample}
+ inputId="use-sampling"
+ name="use-sampling"
+ onChange={onDoSampleChange}
+ />
+ </div>
+ </div>
+ {doSample ? (
+ <>
+ <NumberSetting
+ hint={{
+ content: (
+ <div style={tooltipContent}>
+ Controls randomness when sampling.
+ </div>
+ ),
+ title: 'Learn more about Temperature',
+ }}
+ min={0.1}
+ name="Temperature"
+ onValueChanged={setTemperature}
+ step={0.1}
+ value={temperature}
+ />
+ <NumberSetting
+ hint={{
+ content: (
+ <div style={tooltipContent}>
+ Limits the candidate tokens when sampling.
+ </div>
+ ),
+ title: 'Learn more about Top K',
+ }}
+ min={0}
+ name="Top K"
+ onValueChanged={setTopK}
+ step={1}
+ value={topK}
+ />
+ </>
+ ) : null}
+ <NumberSetting
+ hint={{
+ content: (
+ <div style={tooltipContent}>
+ Discourages repeated tokens. A value of 1 disables the penalty.
+ </div>
+ ),
+ title: 'Learn more about Repetition penalty',
+ }}
+ min={0.1}
+ name="Repetition penalty"
+ onValueChanged={setRepetitionPenalty}
+ step={0.1}
+ value={repetitionPenalty}
+ />
+ <NumberSetting
+ hint={{
+ content: (
+ <div style={tooltipContent}>
+ Prevents repeated phrases of this many tokens. A value of 0
+ disables the filter.
+ </div>
+ ),
+ title: 'Learn more about No-repeat n-gram size',
+ }}
+ min={0}
+ name="No-repeat n-gram size"
+ onValueChanged={setNoRepeatNgramSize}
+ step={1}
+ value={noRepeatNgramSize}
+ />
+ </div>
+ {decodingValidationMessage ? (
+ <div
+ id={TRANSCRIPTION_DECODING_MESSAGE_ID}
+ aria-live="polite"
+ style={{padding: '0 16px'}}
+ >
+ <ValidationMessage
+ align="flex-end"
+ message={decodingValidationMessage}
+ type="error"
+ />
+ </div>
+ ) : null}
+ </>
+ );
+};
+
+export const TranscriptionModal: React.FC<TranscriptionModalState> = ({
+ audioStreamIndex,
+ displayName,
+ requestInit,
+ src,
+}) => {
+ const [selectedModel, setSelectedModel] =
+ useState<WhisperWebGpuModel>('tiny.en');
+ const [selectedLanguage, setSelectedLanguage] =
+ useState<WhisperLanguage>('en');
+ const [selectedTask, setSelectedTask] =
+ useState<WhisperWebGpuTask>('transcribe');
+ const [chunkLengthInSeconds, setChunkLengthInSeconds] = useState(
+ DEFAULT_CHUNK_LENGTH_IN_SECONDS,
+ );
+ const [strideLengthInSeconds, setStrideLengthInSeconds] = useState(
+ DEFAULT_STRIDE_LENGTH_IN_SECONDS,
+ );
+ const [forceFullSequences, setForceFullSequences] = useState(false);
+ const [doSample, setDoSample] = useState(false);
+ const [temperature, setTemperature] = useState(DEFAULT_TEMPERATURE);
+ const [topK, setTopK] = useState(DEFAULT_TOP_K);
+ const [repetitionPenalty, setRepetitionPenalty] = useState(
+ DEFAULT_REPETITION_PENALTY,
+ );
+ const [noRepeatNgramSize, setNoRepeatNgramSize] = useState(
+ DEFAULT_NO_REPEAT_NGRAM_SIZE,
+ );
+ const [supportState, setSupportState] = useState<SupportState>({
+ type: 'checking',
+ });
+ const [outName, setOutName] = useState(() =>
+ getDefaultCaptionOutputName(src, displayName),
+ );
+ const {addCaptionJob, captionJobs} = useContext(RenderQueueContext);
+ const {setSelectedModal} = useContext(SetSelectedModalContext);
+ const {setSidebarCollapsedState} = useContext(SidebarContext);
+ const staticFiles = useStaticFiles();
+
+ useEffect(() => {
+ let cancelled = false;
+ canUseWhisperWebGpu().then((result) => {
+ if (cancelled) {
+ return;
+ }
+
+ setSupportState(
+ result.supported
+ ? {type: 'supported'}
+ : {type: 'unsupported', message: result.detailedReason},
+ );
+ });
+
+ return () => {
+ cancelled = true;
+ };
+ }, []);
+
+ const normalizedOutName = outName.normalize('NFC').toLowerCase();
+ const queuedOutputExists = captionJobs.some(
+ (job) =>
+ (job.status === 'idle' || job.status === 'running') &&
+ job.outName.normalize('NFC').toLowerCase() === normalizedOutName,
+ );
+ const outputValidationMessage =
+ validateCaptionOutputName(outName) ??
+ (queuedOutputExists
+ ? 'Another caption job is already using this output file'
+ : null);
+ const exists = staticFiles.some(
+ (file) => file.name.normalize('NFC').toLowerCase() === normalizedOutName,
+ );
+ const chunkValidationMessage =
+ strideLengthInSeconds * 2 >= chunkLengthInSeconds
+ ? 'Stride length must be less than half of chunk length'
+ : null;
+ const decodingValidationMessage = !Number.isFinite(temperature)
+ ? 'Temperature must be a finite number greater than 0'
+ : temperature <= 0
+ ? 'Temperature must be greater than 0'
+ : !Number.isInteger(topK) || topK < 0
+ ? 'Top K must be a non-negative integer'
+ : !Number.isFinite(repetitionPenalty)
+ ? 'Repetition penalty must be a finite number greater than 0'
+ : repetitionPenalty <= 0
+ ? 'Repetition penalty must be greater than 0'
+ : !Number.isInteger(noRepeatNgramSize) || noRepeatNgramSize < 0
+ ? 'No-repeat n-gram size must be a non-negative integer'
+ : null;
+ const canTranscribe =
+ supportState.type === 'supported' &&
+ outputValidationMessage === null &&
+ chunkValidationMessage === null &&
+ decodingValidationMessage === null;
+ const transcribeDisabledReason =
+ supportState.type === 'checking'
+ ? 'Checking WebGPU support'
+ : supportState.type === 'unsupported'
+ ? supportState.message
+ : (outputValidationMessage ??
+ chunkValidationMessage ??
+ decodingValidationMessage ??
+ undefined);
+
+ const onOutNameChange = useCallback(
+ (event: React.ChangeEvent<HTMLInputElement>) => {
+ setOutName(event.target.value);
+ },
+ [],
+ );
+
+ const onAddToQueue = useCallback(() => {
+ if (!canTranscribe) {
+ return;
+ }
+
+ const modelInfo = AVAILABLE_MODELS.find(({name}) => name === selectedModel);
+ if (!modelInfo) {
+ throw new Error(`Unknown Whisper model: ${selectedModel}`);
+ }
+
+ addCaptionJob({
+ src,
+ displayName,
+ audioStreamIndex,
+ requestInit,
+ outName,
+ model: selectedModel,
+ language: modelInfo.multilingual ? selectedLanguage : null,
+ task: modelInfo.supportsTranslation ? selectedTask : 'transcribe',
+ chunkLengthInSeconds,
+ strideLengthInSeconds,
+ forceFullSequences,
+ doSample,
+ temperature,
+ topK,
+ repetitionPenalty,
+ noRepeatNgramSize,
+ });
+ setSidebarCollapsedState({left: null, right: 'expanded'});
+ persistSelectedOptionsSidebarPanel('renders');
+ optionsSidebarTabs.current?.selectRendersPanel();
+ setSelectedModal(null);
+ }, [
+ addCaptionJob,
+ audioStreamIndex,
+ canTranscribe,
+ chunkLengthInSeconds,
+ displayName,
+ doSample,
+ forceFullSequences,
+ noRepeatNgramSize,
+ outName,
+ repetitionPenalty,
+ requestInit,
+ src,
+ selectedLanguage,
+ selectedModel,
+ selectedTask,
+ setSelectedModal,
+ setSidebarCollapsedState,
+ strideLengthInSeconds,
+ temperature,
+ topK,
+ ]);
+
+ return (
+ <DismissableModal ariaLabel={`Transcribe ${displayName}`}>
+ <div style={{...outerModalStyle, outline: 'none'}}>
+ <ModalHeader title={`Transcribe ${displayName}`} />
+ <div style={container}>
+ <div style={flexer} />
+ <Button
+ onClick={onAddToQueue}
+ disabled={!canTranscribe}
+ title={transcribeDisabledReason}
+ style={{
+ ...buttonStyle,
+ backgroundColor: canTranscribe
+ ? buttonStyle.backgroundColor
+ : BLUE_DISABLED,
+ }}
+ >
+ Transcribe
+ </Button>
+ </div>
+ <div style={settingsPanel} className={VERTICAL_SCROLLBAR_CLASSNAME}>
+ <OutputSettings
+ exists={exists}
+ onOutNameChange={onOutNameChange}
+ outName={outName}
+ validationMessage={outputValidationMessage}
+ />
+ <RenderModalHr />
+ <ModelSettings
+ selectedLanguage={selectedLanguage}
+ selectedModel={selectedModel}
+ selectedTask={selectedTask}
+ setSelectedLanguage={setSelectedLanguage}
+ setSelectedModel={setSelectedModel}
+ setSelectedTask={setSelectedTask}
+ supportState={supportState}
+ />
+ <RenderModalHr />
+ <AdvancedSettings
+ chunkLengthInSeconds={chunkLengthInSeconds}
+ decodingValidationMessage={decodingValidationMessage}
+ doSample={doSample}
+ forceFullSequences={forceFullSequences}
+ noRepeatNgramSize={noRepeatNgramSize}
+ repetitionPenalty={repetitionPenalty}
+ setChunkLengthInSeconds={setChunkLengthInSeconds}
+ setDoSample={setDoSample}
+ setForceFullSequences={setForceFullSequences}
+ setNoRepeatNgramSize={setNoRepeatNgramSize}
+ setRepetitionPenalty={setRepetitionPenalty}
+ setStrideLengthInSeconds={setStrideLengthInSeconds}
+ setTemperature={setTemperature}
+ setTopK={setTopK}
+ strideLengthInSeconds={strideLengthInSeconds}
+ temperature={temperature}
+ topK={topK}
+ validationMessage={chunkValidationMessage}
+ />
+ </div>
+ </div>
+ </DismissableModal>
+ );
+};
diff --git a/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx b/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx
new file mode 100644
index 0000000000..509996233e
--- /dev/null
+++ b/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx
@@ -0,0 +1,142 @@
+import React, {Suspense, useCallback, useContext, useState} from 'react';
+import {installPackages} from '../../api/install-package';
+import {LIGHT_TEXT, WARNING_COLOR, WHITE} from '../../helpers/colors';
+import type {TranscriptionModalState} from '../../state/modals';
+import {SetSelectedModalContext} from '../../state/modals';
+import {Button} from '../Button';
+import {Flex, Row, Spacing} from '../layout';
+import {ModalButton} from '../ModalButton';
+import {ModalContainer} from '../ModalContainer';
+import {ModalFooterContainer} from '../ModalFooter';
+import {ModalHeader} from '../ModalHeader';
+import {Spinner} from '../Spinner';
+import {
+ isWhisperWebGpuInstalled,
+ WHISPER_WEBGPU_PACKAGE,
+} from './whisper-webgpu-capability';
+
+const LazyTranscriptionModal = React.lazy(async () => {
+ const {TranscriptionModal} = await import('./TranscriptionModal');
+ return {default: TranscriptionModal};
+});
+
+type InstallState =
+ | {type: 'idle'}
+ | {type: 'installing'}
+ | {type: 'error'; error: Error};
+
+const panelStyle: React.CSSProperties = {
+ width: 'min(520px, calc(100vw - 40px))',
+};
+
+const contentStyle: React.CSSProperties = {
+ color: WHITE,
+ fontFamily: 'sans-serif',
+ fontSize: 14,
+ lineHeight: 1.5,
+ padding: 16,
+};
+
+const codeStyle: React.CSSProperties = {
+ color: 'inherit',
+ fontFamily: 'monospace',
+ fontSize: 'inherit',
+};
+
+const statusStyle: React.CSSProperties = {
+ alignItems: 'center',
+ color: LIGHT_TEXT,
+ display: 'flex',
+ gap: 10,
+};
+
+const errorStyle: React.CSSProperties = {
+ color: WARNING_COLOR,
+ marginTop: 12,
+ whiteSpace: 'pre-wrap',
+};
+
+const cancelStyle: React.CSSProperties = {
+ minWidth: 90,
+};
+
+export const TranscriptionModalWithOptionalWhisper: React.FC<{
+ readonly state: TranscriptionModalState;
+}> = ({state}) => {
+ const {setSelectedModal} = useContext(SetSelectedModalContext);
+ const [installed, setInstalled] = useState(isWhisperWebGpuInstalled);
+ const [installState, setInstallState] = useState<InstallState>({
+ type: 'idle',
+ });
+
+ const busy = installState.type === 'installing';
+ const dismiss = useCallback(() => {
+ if (!busy) {
+ setSelectedModal(null);
+ }
+ }, [busy, setSelectedModal]);
+
+ const install = useCallback(async () => {
+ setInstallState({type: 'installing'});
+ try {
+ await installPackages([{name: WHISPER_WEBGPU_PACKAGE, version: null}]);
+ window.remotion_installedPackages = Array.from(
+ new Set([
+ ...(window.remotion_installedPackages ?? []),
+ WHISPER_WEBGPU_PACKAGE,
+ ]),
+ );
+
+ setInstalled(true);
+ } catch (error) {
+ setInstallState({type: 'error', error: error as Error});
+ }
+ }, []);
+
+ if (installed) {
+ return (
+ <Suspense fallback={null}>
+ <LazyTranscriptionModal {...state} />
+ </Suspense>
+ );
+ }
+
+ return (
+ <ModalContainer
+ ariaLabel="Install transcription package"
+ onEscape={dismiss}
+ onOutsideClick={dismiss}
+ panelStyle={panelStyle}
+ >
+ <ModalHeader title="Transcribe" onClose={dismiss} />
+ <div style={contentStyle}>
+ {busy ? (
+ <div style={statusStyle}>
+ <Spinner duration={0.5} size={18} />
+ {`Installing ${WHISPER_WEBGPU_PACKAGE}…`}
+ </div>
+ ) : (
+ <>
+ This requires installing{' '}
+ <code style={codeStyle}>{WHISPER_WEBGPU_PACKAGE}</code>. Continue?
+ {installState.type === 'error' ? (
+ <div style={errorStyle}>{installState.error.message}</div>
+ ) : null}
+ </>
+ )}
+ </div>
+ <ModalFooterContainer>
+ <Row align="center">
+ <Flex />
+ <Button disabled={busy} onClick={dismiss} style={cancelStyle}>
+ Cancel
+ </Button>
+ <Spacing x={1} />
+ <ModalButton disabled={busy} onClick={install} autoFocus>
+ {installState.type === 'error' ? 'Retry' : 'Continue'}
+ </ModalButton>
+ </Row>
+ </ModalFooterContainer>
+ </ModalContainer>
+ );
+};
diff --git a/packages/studio/src/components/Transcription/caption-output-name.ts b/packages/studio/src/components/Transcription/caption-output-name.ts
new file mode 100644
index 0000000000..458f2bb38a
--- /dev/null
+++ b/packages/studio/src/components/Transcription/caption-output-name.ts
@@ -0,0 +1,86 @@
+const invalidCharacters = ['?', '*', '+', ':', '%'];
+
+export const getDefaultCaptionOutputName = (
+ src: string,
+ displayName = 'captions',
+): string => {
+ let pathname = displayName;
+ if (!/^(data|blob):/i.test(src.trimStart())) {
+ pathname = src.split(/[?#]/)[0] ?? src;
+ try {
+ pathname = new URL(
+ src,
+ typeof window === 'undefined'
+ ? 'http://localhost'
+ : window.location.href,
+ ).pathname;
+ } catch {
+ // Fall back to the source as-is if it cannot be parsed as a URL.
+ }
+ }
+
+ const lastPathSegment = pathname.split('/').filter(Boolean).at(-1);
+ let decoded = lastPathSegment ?? 'captions';
+ try {
+ decoded = decodeURIComponent(decoded);
+ } catch {
+ // Keep the encoded name if it contains an invalid escape sequence.
+ }
+
+ const withoutExtension = decoded.replace(/\.[^/.]+$/, '');
+ const safeBaseName = withoutExtension
+ .replace(/[^a-zA-Z0-9-_ ]/g, '-')
+ .trim()
+ .replace(/\s+/g, '-')
+ .replace(/^-+|-+$/g, '');
+
+ return `${safeBaseName || 'captions'}-captions.json`;
+};
+
+export const validateCaptionOutputName = (outName: string): string | null => {
+ if (outName.trim() === '') {
+ return 'Enter an output file';
+ }
+
+ if (outName !== outName.trim()) {
+ return 'The output file must not start or end with whitespace';
+ }
+
+ if (outName.startsWith('/') || /^[a-zA-Z]:/.test(outName)) {
+ return 'The output file must be relative to the public folder';
+ }
+
+ if (outName.includes('\\')) {
+ return 'Use forward slashes in the output file';
+ }
+
+ if (outName.includes('\0')) {
+ return 'The output file must not contain null characters';
+ }
+
+ const segments = outName.split('/');
+ if (segments.some((segment) => segment === '')) {
+ return 'The output file must not contain empty path segments';
+ }
+
+ if (segments.some((segment) => segment === '.')) {
+ return 'The output file must not contain "." path segments';
+ }
+
+ if (segments.some((segment) => segment.startsWith('..'))) {
+ return 'The output file must stay inside the public folder';
+ }
+
+ const invalidCharacter = invalidCharacters.find((character) =>
+ outName.includes(character),
+ );
+ if (invalidCharacter) {
+ return `The output file must not contain "${invalidCharacter}"`;
+ }
+
+ if (!outName.toLowerCase().endsWith('.json')) {
+ return 'The output file must end in .json';
+ }
+
+ return null;
+};
diff --git a/packages/studio/src/components/Transcription/is-transcribable-sequence.ts b/packages/studio/src/components/Transcription/is-transcribable-sequence.ts
new file mode 100644
index 0000000000..e8cae50662
--- /dev/null
+++ b/packages/studio/src/components/Transcription/is-transcribable-sequence.ts
@@ -0,0 +1,29 @@
+import type {TSequence} from 'remotion';
+
+export type TranscribableSequence =
+ | Extract<TSequence, {type: 'audio'}>
+ | Extract<TSequence, {type: 'video'}>;
+
+const modernComponentIdentities = {
+ audio: 'dev.remotion.media.Audio',
+ video: 'dev.remotion.media.Video',
+} as const;
+
+const legacyDocumentationLinks = {
+ audio: 'https://www.remotion.dev/docs/html5-audio',
+ video: 'https://www.remotion.dev/docs/html5-video',
+} as const;
+
+export const isTranscribableSequence = (
+ sequence: TSequence,
+): sequence is TranscribableSequence => {
+ if (sequence.type !== 'audio' && sequence.type !== 'video') {
+ return false;
+ }
+
+ return (
+ sequence.controls?.componentIdentity ===
+ modernComponentIdentities[sequence.type] ||
+ sequence.documentationLink === legacyDocumentationLinks[sequence.type]
+ );
+};
diff --git a/packages/studio/src/components/Transcription/resample-media-to-16-khz.ts b/packages/studio/src/components/Transcription/resample-media-to-16-khz.ts
new file mode 100644
index 0000000000..b150618a83
--- /dev/null
+++ b/packages/studio/src/components/Transcription/resample-media-to-16-khz.ts
@@ -0,0 +1,124 @@
+import {WHISPER_WEBGPU_SAMPLE_RATE} from '@remotion/whisper-webgpu';
+import {
+ ALL_FORMATS,
+ Conversion,
+ Input,
+ NullTarget,
+ Output,
+ UrlSource,
+ WavOutputFormat,
+} from 'mediabunny';
+
+export const resampleMediaTo16Khz = async ({
+ src,
+ audioStreamIndex,
+ requestInit,
+ onProgress,
+}: {
+ readonly src: string;
+ readonly audioStreamIndex: number | null;
+ readonly requestInit: Omit<RequestInit, 'signal'> | null;
+ readonly onProgress: (progress: number) => void;
+}): Promise<Float32Array> => {
+ const input = new Input({
+ formats: ALL_FORMATS,
+ source: new UrlSource(src, requestInit === null ? {} : {requestInit}),
+ });
+ const waveformChunks: Array<{
+ readonly startFrame: number;
+ readonly waveform: Float32Array;
+ }> = [];
+
+ try {
+ onProgress(0);
+ const audioTracks = await input.getAudioTracks();
+ const primaryVideoTrack =
+ audioStreamIndex === null ? await input.getPrimaryVideoTrack() : null;
+ const audioTrack =
+ audioStreamIndex !== null
+ ? (audioTracks[audioStreamIndex] ?? null)
+ : primaryVideoTrack
+ ? await primaryVideoTrack.getPrimaryPairableAudioTrack()
+ : (audioTracks[0] ?? null);
+ if (audioTrack === null) {
+ throw new Error('The selected media does not have an audio track.');
+ }
+
+ if (await audioTrack.isLive()) {
+ throw new Error('Live media cannot be transcribed.');
+ }
+
+ const output = new Output({
+ format: new WavOutputFormat(),
+ target: new NullTarget(),
+ });
+ const conversion = await Conversion.init({
+ input,
+ output,
+ tracks: 'all',
+ video: {discard: true},
+ audio: (track) => {
+ if (track.id !== audioTrack.id) {
+ return {discard: true};
+ }
+
+ return {
+ codec: 'pcm-f32',
+ forceTranscode: true,
+ numberOfChannels: 1,
+ process: (sample) => {
+ const floats = new Float32Array(
+ sample.allocationSize({format: 'f32', planeIndex: 0}) /
+ Float32Array.BYTES_PER_ELEMENT,
+ );
+ sample.copyTo(floats, {format: 'f32', planeIndex: 0});
+ waveformChunks.push({
+ startFrame: Math.round(
+ sample.timestamp * WHISPER_WEBGPU_SAMPLE_RATE,
+ ),
+ waveform: floats,
+ });
+ return sample;
+ },
+ sampleFormat: 'f32',
+ sampleRate: WHISPER_WEBGPU_SAMPLE_RATE,
+ };
+ },
+ });
+
+ if (!conversion.isValid) {
+ throw new Error(
+ 'The audio track cannot be decoded in this browser. Try converting the media to WAV first.',
+ );
+ }
+
+ conversion.onProgress = (progress) => onProgress(progress);
+ await conversion.execute();
+ onProgress(1);
+ } finally {
+ input.dispose();
+ }
+
+ const waveformLength = waveformChunks.reduce(
+ (maximum, chunk) =>
+ Math.max(maximum, chunk.startFrame + chunk.waveform.length),
+ 0,
+ );
+ const waveform = new Float32Array(waveformLength);
+ for (const chunk of waveformChunks) {
+ const destinationStart = Math.max(0, chunk.startFrame);
+ const sourceStart = Math.max(0, -chunk.startFrame);
+ const availableLength = Math.min(
+ chunk.waveform.length - sourceStart,
+ waveform.length - destinationStart,
+ );
+ if (availableLength > 0) {
+ waveform.set(
+ chunk.waveform.subarray(sourceStart, sourceStart + availableLength),
+ destinationStart,
+ );
+ }
+ }
+
+ return waveform;
+};
diff --git a/packages/studio/src/components/Transcription/whisper-languages.ts b/packages/studio/src/components/Transcription/whisper-languages.ts
new file mode 100644
index 0000000000..2cc173c45f
--- /dev/null
+++ b/packages/studio/src/components/Transcription/whisper-languages.ts
@@ -0,0 +1,103 @@
+export const WHISPER_LANGUAGES = [
+ ['af', 'Afrikaans'],
+ ['sq', 'Albanian'],
+ ['am', 'Amharic'],
+ ['ar', 'Arabic'],
+ ['hy', 'Armenian'],
+ ['as', 'Assamese'],
+ ['az', 'Azerbaijani'],
+ ['ba', 'Bashkir'],
+ ['eu', 'Basque'],
+ ['be', 'Belarusian'],
+ ['bn', 'Bengali'],
+ ['bs', 'Bosnian'],
+ ['br', 'Breton'],
+ ['bg', 'Bulgarian'],
+ ['my', 'Burmese'],
+ ['ca', 'Catalan'],
+ ['zh', 'Chinese'],
+ ['hr', 'Croatian'],
+ ['cs', 'Czech'],
+ ['da', 'Danish'],
+ ['nl', 'Dutch'],
+ ['en', 'English'],
+ ['et', 'Estonian'],
+ ['fo', 'Faroese'],
+ ['fi', 'Finnish'],
+ ['fr', 'French'],
+ ['gl', 'Galician'],
+ ['ka', 'Georgian'],
+ ['de', 'German'],
+ ['el', 'Greek'],
+ ['gu', 'Gujarati'],
+ ['ht', 'Haitian Creole'],
+ ['ha', 'Hausa'],
+ ['haw', 'Hawaiian'],
+ ['he', 'Hebrew'],
+ ['hi', 'Hindi'],
+ ['hu', 'Hungarian'],
+ ['is', 'Icelandic'],
+ ['id', 'Indonesian'],
+ ['it', 'Italian'],
+ ['ja', 'Japanese'],
+ ['jw', 'Javanese'],
+ ['kn', 'Kannada'],
+ ['kk', 'Kazakh'],
+ ['km', 'Khmer'],
+ ['ko', 'Korean'],
+ ['lo', 'Lao'],
+ ['la', 'Latin'],
+ ['lv', 'Latvian'],
+ ['ln', 'Lingala'],
+ ['lt', 'Lithuanian'],
+ ['lb', 'Luxembourgish'],
+ ['mk', 'Macedonian'],
+ ['mg', 'Malagasy'],
+ ['ms', 'Malay'],
+ ['ml', 'Malayalam'],
+ ['mt', 'Maltese'],
+ ['mi', 'Maori'],
+ ['mr', 'Marathi'],
+ ['mn', 'Mongolian'],
+ ['ne', 'Nepali'],
+ ['no', 'Norwegian'],
+ ['nn', 'Nynorsk'],
+ ['oc', 'Occitan'],
+ ['ps', 'Pashto'],
+ ['fa', 'Persian'],
+ ['pl', 'Polish'],
+ ['pt', 'Portuguese'],
+ ['pa', 'Punjabi'],
+ ['ro', 'Romanian'],
+ ['ru', 'Russian'],
+ ['sa', 'Sanskrit'],
+ ['sr', 'Serbian'],
+ ['sn', 'Shona'],
+ ['sd', 'Sindhi'],
+ ['si', 'Sinhala'],
+ ['sk', 'Slovak'],
+ ['sl', 'Slovenian'],
+ ['so', 'Somali'],
+ ['es', 'Spanish'],
+ ['su', 'Sundanese'],
+ ['sw', 'Swahili'],
+ ['sv', 'Swedish'],
+ ['tl', 'Tagalog'],
+ ['tg', 'Tajik'],
+ ['ta', 'Tamil'],
+ ['tt', 'Tatar'],
+ ['te', 'Telugu'],
+ ['th', 'Thai'],
+ ['bo', 'Tibetan'],
+ ['tr', 'Turkish'],
+ ['tk', 'Turkmen'],
+ ['uk', 'Ukrainian'],
+ ['ur', 'Urdu'],
+ ['uz', 'Uzbek'],
+ ['vi', 'Vietnamese'],
+ ['cy', 'Welsh'],
+ ['yi', 'Yiddish'],
+ ['yo', 'Yoruba'],
+] as const;
+
+export type WhisperLanguage = (typeof WHISPER_LANGUAGES)[number][0];
diff --git a/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts b/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts
new file mode 100644
index 0000000000..46a9d8ee6d
--- /dev/null
+++ b/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts
@@ -0,0 +1,7 @@
+export const WHISPER_WEBGPU_PACKAGE = '@remotion/whisper-webgpu';
+
+export const isWhisperWebGpuInstalled = () => {
+ return (
+ window.remotion_installedPackages?.includes(WHISPER_WEBGPU_PACKAGE) ?? false
+ );
+};
diff --git a/packages/studio/src/icons/transcription.tsx b/packages/studio/src/icons/transcription.tsx
new file mode 100644
index 0000000000..3b6bc21b43
--- /dev/null
+++ b/packages/studio/src/icons/transcription.tsx
@@ -0,0 +1,14 @@
+import type {SVGProps} from 'react';
+import {CURRENT_COLOR_LOWERCASE} from '../helpers/colors';
+
+export const TranscriptionIcon = ({
+ color,
+ ...props
+}: SVGProps<SVGSVGElement> & {readonly color?: string}) => (
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" {...props}>
+ <path
+ d="M12 15.75a4 4 0 0 0 4-4V7a4 4 0 0 0-8 0v4.75a4 4 0 0 0 4 4Zm-1.9-8.5V7a1.9 1.9 0 0 1 3.8 0v4.75a1.9 1.9 0 0 1-3.8 0v-.25h1.15a1.05 1.05 0 1 0 0-2.1H10.1V7.25Zm7.45 3.45a1.05 1.05 0 0 1 1.05 1.05 6.6 6.6 0 0 1-5.55 6.52v1.63h2.2a1.05 1.05 0 1 1 0 2.1h-6.5a1.05 1.05 0 1 1 0-2.1h2.2v-1.63a6.6 6.6 0 0 1-5.55-6.52 1.05 1.05 0 1 1 2.1 0 4.5 4.5 0 0 0 9 0 1.05 1.05 0 0 1 1.05-1.05Z"
+ fill={color ?? CURRENT_COLOR_LOWERCASE}
+ />
+ </svg>
+);
diff --git a/packages/studio/src/state/modals.ts b/packages/studio/src/state/modals.ts
index c25afedc95..1e2f148c45 100644
--- a/packages/studio/src/state/modals.ts
+++ b/packages/studio/src/state/modals.ts
@@ -140,6 +140,14 @@ export type AddEffectModalState = {
clientId: string;
};
+export type TranscriptionModalState = {
+ type: 'transcribe';
+ src: string;
+ displayName: string;
+ audioStreamIndex: number | null;
+ requestInit: Omit<RequestInit, 'signal'> | null;
+};
+
export type CanvasCaptureImport = {
readonly data: CanvasCaptureData;
readonly durationInSeconds: number;
@@ -255,6 +263,7 @@ export type ModalState =
}
| ElementInstallModalState
| AddEffectModalState
+ | TranscriptionModalState
| ConfirmationDialogState
| SvgImportDialogState;
diff --git a/packages/studio/src/test/caption-output-name.test.ts b/packages/studio/src/test/caption-output-name.test.ts
new file mode 100644
index 0000000000..510ba96200
--- /dev/null
+++ b/packages/studio/src/test/caption-output-name.test.ts
@@ -0,0 +1,50 @@
+import {expect, test} from 'bun:test';
+import {
+ getDefaultCaptionOutputName,
+ validateCaptionOutputName,
+} from '../components/Transcription/caption-output-name';
+
+test('derives a caption JSON name from local and remote media sources', () => {
+ expect(getDefaultCaptionOutputName('/interview.mp4')).toBe(
+ 'interview-captions.json',
+ );
+ expect(
+ getDefaultCaptionOutputName(
+ 'https://example.com/media/My%20Interview.wav?token=secret',
+ ),
+ ).toBe('My-Interview-captions.json');
+ expect(
+ getDefaultCaptionOutputName(
+ `data:audio/wav;base64,${'A'.repeat(1_000_000)}`,
+ '<Audio>',
+ ),
+ ).toBe('Audio-captions.json');
+ expect(
+ getDefaultCaptionOutputName(
+ ` data:audio/wav;base64,${'A'.repeat(1_000_000)}`,
+ '<Audio>',
+ ),
+ ).toBe('Audio-captions.json');
+});
+
+test('accepts safe nested Caption[] output paths', () => {
+ expect(validateCaptionOutputName('captions/interview.json')).toBeNull();
+ expect(validateCaptionOutputName('captions/interview.JSON')).toBeNull();
+});
+
+test.each([
+ '',
+ ' captions.json',
+ '/captions.json',
+ 'C:/captions.json',
+ 'captions\\output.json',
+ 'captions\0output.json',
+ 'captions//output.json',
+ 'captions/./output.json',
+ 'captions/../output.json',
+ 'captions/..output.json',
+ 'captions?.json',
+ 'captions.txt',
+])('rejects unsafe or unsupported output path %s', (outName) => {
+ expect(validateCaptionOutputName(outName)).not.toBeNull();
+});
diff --git a/packages/studio/src/test/caption-queue-processor.test.tsx b/packages/studio/src/test/caption-queue-processor.test.tsx
new file mode 100644
index 0000000000..a0c20e06ed
--- /dev/null
+++ b/packages/studio/src/test/caption-queue-processor.test.tsx
@@ -0,0 +1,368 @@
+import {afterEach, expect, mock, test} from 'bun:test';
+import {act, cleanup, render, waitFor} from '@testing-library/react';
+import {type ContextType, useContext} from 'react';
+import {makeBrowserStudioOperations} from './make-browser-studio-operations';
+
+let modelLoadCall:
+ | {
+ model: string;
+ }
+ | undefined;
+let transcriptionCall:
+ | {
+ channelWaveform: Float32Array;
+ options: Record<string, unknown>;
+ }
+ | undefined;
+let cacheCheck:
+ | {
+ model: string;
+ }
+ | undefined;
+const getCacheCheck = () => cacheCheck;
+const getModelLoadCall = () => modelLoadCall;
+let modelIsCached = false;
+let modelLoadProgressCallback:
+ | ((progress: {
+ status: string;
+ file: string | null;
+ progress: number | null;
+ loadedBytes: number | null;
+ totalBytes: number | null;
+ }) => void)
+ | undefined;
+let resolveModelLoading: (() => void) | null = null;
+let disposalCalls = 0;
+const resamplingCalls: Array<{
+ src: string;
+ audioStreamIndex: number | null;
+ requestInit: Omit<RequestInit, 'signal'> | null;
+}> = [];
+
+const resampledWaveform = new Float32Array(32_000).fill(0.25);
+
+mock.module('../components/Transcription/resample-media-to-16-khz', () => ({
+ resampleMediaTo16Khz: ({
+ src,
+ audioStreamIndex,
+ requestInit,
+ onProgress,
+ }: {
+ src: string;
+ audioStreamIndex: number | null;
+ requestInit: Omit<RequestInit, 'signal'> | null;
+ onProgress: (progress: number) => void;
+ }) => {
+ resamplingCalls.push({src, audioStreamIndex, requestInit});
+ onProgress(0.5);
+ return Promise.resolve(resampledWaveform);
+ },
+}));
+
+mock.module('@remotion/whisper-webgpu', () => ({
+ canUseWhisperWebGpu: () => Promise.resolve({supported: true}),
+ clearStaleModels: () => Promise.resolve(),
+ disposeWhisperModel: () => {
+ disposalCalls++;
+ return Promise.resolve();
+ },
+ getAvailableModels: () => [
+ {
+ name: 'tiny',
+ modelId: 'onnx-community/whisper-tiny_timestamped',
+ parameters: 39_000_000,
+ multilingual: true,
+ supportsTranslation: true,
+ webGpuDownloadSize: 119_699_015,
+ },
+ {
+ name: 'tiny.en',
+ modelId: 'onnx-community/whisper-tiny.en_timestamped',
+ parameters: 39_000_000,
+ multilingual: false,
+ supportsTranslation: false,
+ webGpuDownloadSize: 119_697_479,
+ },
+ ],
+ isWhisperModelCached: ({model}: {model: string}) => {
+ cacheCheck = {model};
+ return Promise.resolve(modelIsCached);
+ },
+ loadWhisperModel: async ({
+ model,
+ onProgress,
+ }: {
+ model: string;
+ onProgress?: typeof modelLoadProgressCallback;
+ }) => {
+ modelLoadCall = {model};
+ modelLoadProgressCallback = onProgress;
+ await new Promise<void>((resolve) => {
+ resolveModelLoading = resolve;
+ });
+ modelLoadProgressCallback?.({
+ status: 'ready',
+ file: null,
+ progress: 1,
+ loadedBytes: 119_699_015,
+ totalBytes: 119_699_015,
+ });
+ return {alreadyLoaded: false};
+ },
+ toCaptions: () => ({
+ captions: [
+ {
+ text: 'Hello',
+ startMs: 125,
+ endMs: 625,
+ timestampMs: 375,
+ confidence: null,
+ },
+ {
+ text: ' world.',
+ startMs: 750,
+ endMs: 1500,
+ timestampMs: 1125,
+ confidence: null,
+ },
+ ],
+ }),
+ transcribe: ({
+ channelWaveform,
+ ...options
+ }: {
+ channelWaveform: Float32Array;
+ [key: string]: unknown;
+ }) => {
+ transcriptionCall = {channelWaveform, options};
+ return Promise.resolve({
+ text: ' Hello world.',
+ chunks: [
+ {text: ' Hello', timestamp: [0.125, 0.625]},
+ {text: ' world.', timestamp: [0.75, 1.5]},
+ ],
+ });
+ },
+ WHISPER_WEBGPU_SAMPLE_RATE: 16_000,
+}));
+
+afterEach(async () => {
+ cleanup();
+ resolveModelLoading?.();
+ const {disposeWhisperModel} = await import('@remotion/whisper-webgpu');
+ await disposeWhisperModel({model: 'tiny'});
+ resolveModelLoading = null;
+});
+
+test('downloads a model in the queued job before transcribing', async () => {
+ disposalCalls = 0;
+ resamplingCalls.length = 0;
+ modelLoadCall = undefined;
+ modelLoadProgressCallback = undefined;
+ transcriptionCall = undefined;
+ cacheCheck = undefined;
+ modelIsCached = false;
+ resolveModelLoading = null;
+ const {CaptionQueueProcessor} =
+ await import('../components/RenderQueue/CaptionQueueProcessor');
+ const {RenderQueueContext, RenderQueueContextProvider} =
+ await import('../components/RenderQueue/context');
+ const originalGpuDescriptor = Object.getOwnPropertyDescriptor(
+ navigator,
+ 'gpu',
+ );
+ const originalSecureContextDescriptor = Object.getOwnPropertyDescriptor(
+ window,
+ 'isSecureContext',
+ );
+ const originalBrowserStudioDescriptor = Object.getOwnPropertyDescriptor(
+ window,
+ 'remotion_browserStudio',
+ );
+ const writtenFiles: Array<{
+ contents: string | ArrayBuffer;
+ filePath: string;
+ }> = [];
+ Object.defineProperty(navigator, 'gpu', {
+ configurable: true,
+ value: {requestAdapter: () => Promise.resolve({})},
+ });
+ Object.defineProperty(window, 'isSecureContext', {
+ configurable: true,
+ value: true,
+ });
+ Object.defineProperty(window, 'remotion_browserStudio', {
+ configurable: true,
+ value: makeBrowserStudioOperations({
+ writeStaticFile: ({contents, filePath}) => {
+ writtenFiles.push({contents, filePath});
+ return Promise.resolve();
+ },
+ }),
+ });
+ let currentContext: ContextType<typeof RenderQueueContext> | null = null;
+ const ReadContext = () => {
+ currentContext = useContext(RenderQueueContext);
+ return null;
+ };
+
+ try {
+ render(
+ <RenderQueueContextProvider>
+ <CaptionQueueProcessor />
+ <ReadContext />
+ </RenderQueueContextProvider>,
+ );
+
+ const getContext = () => {
+ if (currentContext === null) {
+ throw new Error('Render queue context is not mounted');
+ }
+
+ return currentContext;
+ };
+
+ let jobId = '';
+ act(() => {
+ jobId = getContext().addCaptionJob({
+ displayName: 'interview.wav',
+ audioStreamIndex: 2,
+ requestInit: {
+ credentials: 'include',
+ headers: {'x-remotion-test': 'transcription'},
+ },
+ language: 'de',
+ model: 'tiny',
+ chunkLengthInSeconds: 20,
+ strideLengthInSeconds: 3,
+ task: 'translate',
+ forceFullSequences: true,
+ doSample: true,
+ temperature: 0.7,
+ topK: 25,
+ repetitionPenalty: 1.2,
+ noRepeatNgramSize: 3,
+ outName: 'captions/interview.json',
+ src: '/static/interview.wav',
+ });
+ });
+
+ await waitFor(() => expect(modelLoadCall).toBeDefined());
+ expect(getCacheCheck()).toEqual({model: 'tiny'});
+ expect(resamplingCalls).toEqual([]);
+ expect(transcriptionCall).toBeUndefined();
+ expect(writtenFiles).toEqual([]);
+
+ act(() => {
+ modelLoadProgressCallback?.({
+ status: 'loading',
+ file: null,
+ progress: 0.5,
+ loadedBytes: 119_699_015 / 2,
+ totalBytes: 119_699_015,
+ });
+ });
+ await waitFor(() => {
+ const job = getContext().captionJobs.find(({id}) => id === jobId);
+ if (job?.status !== 'running') {
+ throw new Error('Caption job is not running');
+ }
+
+ expect(job.progress.message).toContain('Downloading tiny…');
+ expect(job.progress.message).toContain('50%');
+ expect(job.progress.value).toBeCloseTo(0.165);
+ });
+ expect(resamplingCalls).toEqual([]);
+ expect(transcriptionCall).toBeUndefined();
+ expect(writtenFiles).toEqual([]);
+
+ act(() => resolveModelLoading?.());
+ await waitFor(() => {
+ expect(
+ getContext().captionJobs.find((job) => job.id === jobId),
+ ).toMatchObject({status: 'done', captionCount: 2});
+ });
+
+ expect(resamplingCalls).toEqual([
+ {
+ src: '/static/interview.wav',
+ audioStreamIndex: 2,
+ requestInit: {
+ credentials: 'include',
+ headers: {'x-remotion-test': 'transcription'},
+ },
+ },
+ ]);
+ expect(getModelLoadCall()).toEqual({model: 'tiny'});
+ await waitFor(() => {
+ expect(transcriptionCall).toEqual({
+ channelWaveform: resampledWaveform,
+ options: {
+ model: 'tiny',
+ language: 'de',
+ task: 'translate',
+ chunkLengthInSeconds: 20,
+ strideLengthInSeconds: 3,
+ forceFullSequences: true,
+ doSample: true,
+ temperature: 0.7,
+ topK: 25,
+ repetitionPenalty: 1.2,
+ noRepeatNgramSize: 3,
+ },
+ });
+ });
+ expect(writtenFiles).toEqual([
+ {
+ filePath: 'captions/interview.json',
+ contents: JSON.stringify(
+ [
+ {
+ text: 'Hello',
+ startMs: 125,
+ endMs: 625,
+ timestampMs: 375,
+ confidence: null,
+ },
+ {
+ text: ' world.',
+ startMs: 750,
+ endMs: 1500,
+ timestampMs: 1125,
+ confidence: null,
+ },
+ ],
+ null,
+ 2,
+ ),
+ },
+ ]);
+ expect(disposalCalls).toBe(1);
+ } finally {
+ if (originalGpuDescriptor) {
+ Object.defineProperty(navigator, 'gpu', originalGpuDescriptor);
+ } else {
+ Reflect.deleteProperty(navigator, 'gpu');
+ }
+
+ if (originalSecureContextDescriptor) {
+ Object.defineProperty(
+ window,
+ 'isSecureContext',
+ originalSecureContextDescriptor,
+ );
+ } else {
+ Reflect.deleteProperty(window, 'isSecureContext');
+ }
+
+ if (originalBrowserStudioDescriptor) {
+ Object.defineProperty(
+ window,
+ 'remotion_browserStudio',
+ originalBrowserStudioDescriptor,
+ );
+ } else {
+ Reflect.deleteProperty(window, 'remotion_browserStudio');
+ }
+ }
+});
diff --git a/packages/studio/src/test/render-queue-scheduler.test.tsx b/packages/studio/src/test/render-queue-scheduler.test.tsx
new file mode 100644
index 0000000000..9aa2e84d01
--- /dev/null
+++ b/packages/studio/src/test/render-queue-scheduler.test.tsx
@@ -0,0 +1,152 @@
+import {afterEach, expect, spyOn, test} from 'bun:test';
+import {act, cleanup, render, waitFor} from '@testing-library/react';
+import {type ContextType, useContext} from 'react';
+import type {AddCaptionJobParams} from '../components/RenderQueue/caption-job-types';
+import type {
+ AddClientStillJobParams,
+ CompositionRef,
+} from '../components/RenderQueue/client-render-queue';
+import {
+ RenderQueueContext,
+ RenderQueueContextProvider,
+} from '../components/RenderQueue/context';
+
+afterEach(cleanup);
+
+const compositionRef: CompositionRef = {
+ component: () => null,
+ calculateMetadata: null,
+ width: 100,
+ height: 100,
+ fps: 30,
+ durationInFrames: 30,
+ defaultProps: {},
+};
+
+const makeCaptionParams = (displayName: string): AddCaptionJobParams => ({
+ src: `/${displayName}`,
+ displayName,
+ audioStreamIndex: null,
+ requestInit: null,
+ outName: `${displayName}.json`,
+ model: 'tiny.en',
+ language: null,
+ chunkLengthInSeconds: 30,
+ strideLengthInSeconds: 5,
+ task: 'transcribe',
+ forceFullSequences: false,
+ doSample: false,
+ temperature: 1,
+ topK: 50,
+ repetitionPenalty: 1,
+ noRepeatNgramSize: 0,
+});
+
+const makeStillParams = (compositionId: string): AddClientStillJobParams => ({
+ type: 'client-still',
+ compositionId,
+ outName: `${compositionId}.png`,
+ imageFormat: 'png',
+ frame: 0,
+ inputProps: {},
+ delayRenderTimeout: 30_000,
+ mediaCacheSizeInBytes: null,
+ logLevel: 'info',
+ licenseKey: null,
+ scale: 1,
+ allowHtmlInCanvas: false,
+});
+
+test('processes client renders and captions in one local FIFO', async () => {
+ const dateNow = spyOn(Date, 'now').mockReturnValue(1_000);
+ let currentContext: ContextType<typeof RenderQueueContext> | null = null;
+ const processedJobs: string[] = [];
+ const ReadContext = () => {
+ currentContext = useContext(RenderQueueContext);
+ return null;
+ };
+
+ try {
+ render(
+ <RenderQueueContextProvider>
+ <ReadContext />
+ </RenderQueueContextProvider>,
+ );
+
+ const getContext = () => {
+ if (currentContext === null) {
+ throw new Error('Render queue context is not mounted');
+ }
+
+ return currentContext;
+ };
+
+ let firstCaptionId = '';
+ let renderId = '';
+ let secondCaptionId = '';
+ act(() => {
+ getContext().setProcessCaptionJobCallback((job) => {
+ processedJobs.push(job.id);
+ return Promise.resolve();
+ });
+ getContext().setProcessJobCallback((job) => {
+ processedJobs.push(job.id);
+ return Promise.resolve();
+ });
+ firstCaptionId = getContext().addCaptionJob(
+ makeCaptionParams('first.mp3'),
+ );
+ renderId = getContext().addClientStillJob(
+ makeStillParams('composition'),
+ compositionRef,
+ );
+ secondCaptionId = getContext().addCaptionJob(
+ makeCaptionParams('second.mp3'),
+ );
+ });
+
+ await waitFor(() => expect(processedJobs).toEqual([firstCaptionId]));
+ expect(
+ getContext().clientJobs.find((job) => job.id === renderId)?.status,
+ ).toBe('idle');
+ expect(
+ getContext().captionJobs.find((job) => job.id === secondCaptionId)
+ ?.status,
+ ).toBe('idle');
+
+ act(() => getContext().markCaptionJobDone(firstCaptionId, 1));
+ await waitFor(() =>
+ expect(processedJobs).toEqual([firstCaptionId, renderId]),
+ );
+ expect(
+ getContext().captionJobs.find((job) => job.id === secondCaptionId)
+ ?.status,
+ ).toBe('idle');
+
+ act(() => getContext().markClientJobSaving(renderId));
+ expect(processedJobs).toEqual([firstCaptionId, renderId]);
+ expect(
+ getContext().captionJobs.find((job) => job.id === secondCaptionId)
+ ?.status,
+ ).toBe('idle');
+
+ act(() =>
+ getContext().markClientJobDone(renderId, {
+ width: 100,
+ height: 100,
+ sizeInBytes: 1,
+ }),
+ );
+ await waitFor(() =>
+ expect(processedJobs).toEqual([
+ firstCaptionId,
+ renderId,
+ secondCaptionId,
+ ]),
+ );
+
+ act(() => getContext().markCaptionJobDone(secondCaptionId, 1));
+ } finally {
+ dateNow.mockRestore();
+ }
+});
diff --git a/packages/studio/src/test/renders-tab-caption-jobs.test.tsx b/packages/studio/src/test/renders-tab-caption-jobs.test.tsx
new file mode 100644
index 0000000000..3ec45e1169
--- /dev/null
+++ b/packages/studio/src/test/renders-tab-caption-jobs.test.tsx
@@ -0,0 +1,119 @@
+import {afterEach, expect, spyOn, test} from 'bun:test';
+import {cleanup, fireEvent, render, screen} from '@testing-library/react';
+import type {SetStateAction} from 'react';
+import type {CanvasContent} from 'remotion';
+import {Internals} from 'remotion';
+import type {CaptionJob} from '../components/RenderQueue/caption-job-types';
+import {CaptionQueueItem} from '../components/RenderQueue/CaptionQueueItem';
+import {RenderQueueContext} from '../components/RenderQueue/context';
+import {RendersTab} from '../components/RendersTab';
+import {FolderContextProvider} from '../state/folders';
+
+afterEach(cleanup);
+
+const renderWithCaptionJob = (job: CaptionJob) => {
+ let canvasContent: CanvasContent | null = null;
+ render(
+ <Internals.CompositionSetters.Provider
+ value={
+ {
+ setCanvasContent: (value: SetStateAction<CanvasContent | null>) => {
+ canvasContent =
+ typeof value === 'function' ? value(canvasContent) : value;
+ },
+ } as never
+ }
+ >
+ <FolderContextProvider>
+ <RenderQueueContext.Provider
+ value={
+ {
+ jobs: [],
+ captionJobs: [job],
+ } as never
+ }
+ >
+ <>
+ <RendersTab selected={false} onClick={() => undefined} />
+ <CaptionQueueItem job={job} selected={false} />
+ </>
+ </RenderQueueContext.Provider>
+ </FolderContextProvider>
+ </Internals.CompositionSetters.Provider>,
+ );
+
+ return {getCanvasContent: () => canvasContent};
+};
+
+const baseJob = {
+ id: 'caption-job',
+ type: 'caption',
+ startedAt: 0,
+ src: '/audio.mp3',
+ displayName: 'audio.mp3',
+ audioStreamIndex: null,
+ requestInit: null,
+ outName: 'dialogue-captions.json',
+ model: 'tiny.en',
+ language: null,
+ chunkLengthInSeconds: 30,
+ strideLengthInSeconds: 5,
+ task: 'transcribe',
+ forceFullSequences: false,
+ doSample: false,
+ temperature: 1,
+ topK: 50,
+ repetitionPenalty: 1,
+ noRepeatNgramSize: 0,
+} as const;
+
+test('labels the renders tab as Jobs while a caption job is running', () => {
+ renderWithCaptionJob({
+ ...baseJob,
+ status: 'running',
+ progress: {
+ message: 'Downloading tiny.en… · 50% · 57.1 MB / 114.2 MB',
+ value: 0.165,
+ },
+ });
+
+ expect(screen.getByText('Jobs')).toBeTruthy();
+ expect(screen.getByText(/Downloading tiny\.en… · 50%/)).toBeTruthy();
+ const progress = screen.getByRole('progressbar', {
+ name: 'Caption job progress',
+ });
+ expect(progress.getAttribute('aria-valuenow')).toBe('17');
+ expect(progress.getAttribute('aria-valuetext')).toContain(
+ 'Downloading tiny.en… · 50%',
+ );
+});
+
+test('opens a completed caption output as an asset without showing its caption count', () => {
+ const pushState = spyOn(window.history, 'pushState').mockImplementation(
+ () => undefined,
+ );
+
+ try {
+ const {getCanvasContent} = renderWithCaptionJob({
+ ...baseJob,
+ status: 'done',
+ captionCount: 10,
+ });
+
+ expect(screen.getByText('Renders')).toBeTruthy();
+ expect(screen.queryByText(/10 captions/)).toBeNull();
+ expect(screen.getByText('dialogue-captions.json')).toBeTruthy();
+ fireEvent.click(screen.getByText('dialogue-captions.json'));
+ expect(getCanvasContent()).toEqual({
+ type: 'asset',
+ asset: 'dialogue-captions.json',
+ });
+ expect(pushState).toHaveBeenCalledWith(
+ {},
+ 'Studio',
+ '/assets/dialogue-captions.json',
+ );
+ } finally {
+ pushState.mockRestore();
+ }
+});
diff --git a/packages/studio/src/test/sequence-context-menu-items.test.ts b/packages/studio/src/test/sequence-context-menu-items.test.ts
index a94fdc3a1f..0e10558d7e 100644
--- a/packages/studio/src/test/sequence-context-menu-items.test.ts
+++ b/packages/studio/src/test/sequence-context-menu-items.test.ts
@@ -10,6 +10,7 @@ import {
isSequenceVisibleAtTimelinePosition,
shouldShowFreezeFrameMenuItem,
} from '../components/Timeline/use-sequence-freeze-frame-menu-item';
+import {isTranscribableSequence} from '../components/Transcription/is-transcribable-sequence';
const originalWindowDescriptor = Object.getOwnPropertyDescriptor(
globalThis,
@@ -69,6 +70,44 @@ const renameSequenceItem: ComboboxValue = {
const noop = () => undefined;
+test('recognizes current and legacy Audio and Video timeline sequences', () => {
+ for (const [type, componentIdentity, documentationLink] of [
+ [
+ 'audio',
+ 'dev.remotion.media.Audio',
+ 'https://www.remotion.dev/docs/html5-audio',
+ ],
+ [
+ 'video',
+ 'dev.remotion.media.Video',
+ 'https://www.remotion.dev/docs/html5-video',
+ ],
+ ] as const) {
+ expect(
+ isTranscribableSequence({
+ type,
+ controls: {componentIdentity},
+ documentationLink: null,
+ } as unknown as TSequence),
+ ).toBe(true);
+ expect(
+ isTranscribableSequence({
+ type,
+ controls: null,
+ documentationLink,
+ } as unknown as TSequence),
+ ).toBe(true);
+ }
+
+ expect(
+ isTranscribableSequence({
+ type: 'video',
+ controls: null,
+ documentationLink: 'https://www.remotion.dev/docs/offthreadvideo',
+ } as unknown as TSequence),
+ ).toBe(false);
+});
+
test('copy context menu items write their agent context to the clipboard', () => {
const copiedTexts: string[] = [];
Object.defineProperty(globalThis, 'navigator', {
@@ -141,6 +180,7 @@ const expectNoExtraDividers = (items: ComboboxValue[]) => {
test('sequence context menu normalizes dividers before source actions', () => {
installTestWindow();
+ let transcribed = false;
const items = getSequenceContextMenuItems({
assetLinkInfo: null,
@@ -156,6 +196,9 @@ test('sequence context menu normalizes dividers before source actions', () => {
onDeleteSequenceFromSource: noop,
onDisableSequenceInteractivity: noop,
onDuplicateSequenceFromSource: noop,
+ onTranscribe: () => {
+ transcribed = true;
+ },
openInCodingAgent: noop,
openInEditor: noop,
originalLocation: null,
@@ -175,7 +218,15 @@ test('sequence context menu normalizes dividers before source actions', () => {
expect(
items.slice(copyContextIndex + 1, renameIndex).map((item) => item.type),
- ).toEqual(['divider']);
+ ).toEqual(['item', 'divider']);
+
+ const transcribe = items.find((item) => item.id === 'transcribe');
+ if (transcribe?.type !== 'item') {
+ throw new Error('Expected Transcribe menu item');
+ }
+
+ transcribe.onClick('transcribe', null);
+ expect(transcribed).toBe(true);
});
test('sequence context menu shares alternate apps without repeating defaults', () => {
@@ -230,6 +281,7 @@ test('sequence context menu shares alternate apps without repeating defaults', (
onDeleteSequenceFromSource: noop,
onDisableSequenceInteractivity: noop,
onDuplicateSequenceFromSource: noop,
+ onTranscribe: null,
openInCodingAgent: (codingAgentId, _codingAgentName, prompt) => {
openedCodingAgents.push({id: codingAgentId, prompt});
},
@@ -333,6 +385,7 @@ test('Interactive.Svg context menu can copy the rendered SVG', () => {
onDeleteSequenceFromSource: noop,
onDisableSequenceInteractivity: noop,
onDuplicateSequenceFromSource: noop,
+ onTranscribe: null,
openInCodingAgent: noop,
openInEditor: noop,
originalLocation: null,
@@ -377,6 +430,7 @@ test('programmatically duplicated sequence menus apply actions to all instances'
onDeleteSequenceFromSource: noop,
onDisableSequenceInteractivity: noop,
onDuplicateSequenceFromSource: noop,
+ onTranscribe: null,
openInCodingAgent: noop,
openInEditor: noop,
originalLocation: null,
@@ -411,6 +465,7 @@ test('read-only sequence menus only contain non-mutating actions', () => {
onDeleteSequenceFromSource: noop,
onDisableSequenceInteractivity: noop,
onDuplicateSequenceFromSource: noop,
+ onTranscribe: null,
openInCodingAgent: noop,
openInEditor: noop,
originalLocation: null,
diff --git a/packages/studio/src/test/sequenced-timeline.test.ts b/packages/studio/src/test/sequenced-timeline.test.ts
index 65c690de39..845d182e27 100644
--- a/packages/studio/src/test/sequenced-timeline.test.ts
+++ b/packages/studio/src/test/sequenced-timeline.test.ts
@@ -145,6 +145,8 @@ test('Should calculate sequences correctly', () => {
nodePathInfo: null,
sequence: {
type: 'video',
+ audioStreamIndex: null,
+ requestInit: null,
src: 'http://localhost:5000/codep1-2.mp4',
id: '0.640722264073593',
duration: 486,
@@ -207,6 +209,8 @@ test('Should calculate sequences correctly', () => {
sequence: {
loopDisplay: undefined,
type: 'video',
+ audioStreamIndex: null,
+ requestInit: null,
src: 'http://localhost:5000/screen3.mp4',
id: '0.5529201074994525',
duration: 486,
@@ -287,6 +291,8 @@ const sequences: TSequence[] = [
},
{
type: 'video',
+ audioStreamIndex: null,
+ requestInit: null,
src: 'http://localhost:5000/codep1-2.mp4',
id: '0.640722264073593',
duration: 10736,
@@ -340,6 +346,8 @@ const sequences: TSequence[] = [
},
{
type: 'video',
+ audioStreamIndex: null,
+ requestInit: null,
src: 'http://localhost:5000/screen3.mp4',
id: '0.5529201074994525',
duration: 23066,
diff --git a/packages/studio/src/test/timeline.test.ts b/packages/studio/src/test/timeline.test.ts
index c805592eee..1792aabcca 100644
--- a/packages/studio/src/test/timeline.test.ts
+++ b/packages/studio/src/test/timeline.test.ts
@@ -244,6 +244,8 @@ test('Should inherit loop display from parent for media tracks', () => {
showInTimeline: true,
timelineOrder: null,
type: 'video',
+ audioStreamIndex: null,
+ requestInit: null,
getStack,
refForOutline: null,
isInsideSeries: false,
@@ -377,6 +379,8 @@ test('Should account for a parent Sequence trimBefore in video thumbnails', () =
showInTimeline: true,
timelineOrder: null,
type: 'video',
+ audioStreamIndex: null,
+ requestInit: null,
getStack,
refForOutline: null,
isInsideSeries: false,
diff --git a/packages/studio/src/test/transcription-modal.test.tsx b/packages/studio/src/test/transcription-modal.test.tsx
new file mode 100644
index 0000000000..8a291a37a9
--- /dev/null
+++ b/packages/studio/src/test/transcription-modal.test.tsx
@@ -0,0 +1,305 @@
+import {expect, test} from 'bun:test';
+import {
+ act,
+ cleanup,
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+} from '@testing-library/react';
+import type {AddCaptionJobParams} from '../components/RenderQueue/caption-job-types';
+
+const selectComboboxItem = async (
+ combobox: HTMLElement,
+ itemName: string | RegExp,
+) => {
+ fireEvent.click(combobox);
+ const item = await screen.findByRole('button', {name: itemName});
+ fireEvent.pointerUp(item, {button: 0});
+};
+
+const setNumberSetting = (name: RegExp, value: number) => {
+ fireEvent.click(screen.getByRole('button', {name}));
+ const input = screen.getByRole('textbox', {name});
+ fireEvent.change(input, {target: {value: String(value)}});
+ fireEvent.blur(input);
+};
+
+test('serializes transcription modal settings into caption jobs', async () => {
+ const {portals} = await import('../components/Menu/portals');
+ const previousPortals = [...portals];
+ const createdPortals: HTMLElement[] = [];
+ const testPortals = Array.from({length: 6}, (_, index) => {
+ const id = `menuportal-${index}`;
+ const existing = document.getElementById(id);
+ if (existing) {
+ return existing;
+ }
+
+ const element = document.createElement('div');
+ element.id = id;
+ document.body.appendChild(element);
+ createdPortals.push(element);
+ return element;
+ });
+ portals.splice(0, portals.length, ...testPortals);
+
+ const originalGpuDescriptor = Object.getOwnPropertyDescriptor(
+ navigator,
+ 'gpu',
+ );
+ const originalSecureContextDescriptor = Object.getOwnPropertyDescriptor(
+ window,
+ 'isSecureContext',
+ );
+ const originalStaticBaseDescriptor = Object.getOwnPropertyDescriptor(
+ window,
+ 'remotion_staticBase',
+ );
+ const originalLocalStorageDescriptor = Object.getOwnPropertyDescriptor(
+ globalThis,
+ 'localStorage',
+ );
+ const localStorageValues = new Map<string, string>();
+ const localStorageMock: Storage = {
+ clear: () => localStorageValues.clear(),
+ getItem: (key: string) => localStorageValues.get(key) ?? null,
+ key: (index: number) => [...localStorageValues.keys()][index] ?? null,
+ removeItem: (key: string) => localStorageValues.delete(key),
+ setItem: (key: string, value: string) => localStorageValues.set(key, value),
+ get length() {
+ return localStorageValues.size;
+ },
+ };
+
+ Object.defineProperty(navigator, 'gpu', {
+ configurable: true,
+ value: {requestAdapter: () => Promise.resolve({})},
+ });
+ Object.defineProperty(window, 'isSecureContext', {
+ configurable: true,
+ value: true,
+ });
+ Object.defineProperty(window, 'remotion_staticBase', {
+ configurable: true,
+ value: '/static-test',
+ });
+ Object.defineProperty(globalThis, 'localStorage', {
+ configurable: true,
+ value: localStorageMock,
+ });
+
+ try {
+ const {TranscriptionModal} =
+ await import('../components/Transcription/TranscriptionModal');
+ const {RenderQueueContext} =
+ await import('../components/RenderQueue/context');
+ const {SetSelectedModalContext} = await import('../state/modals');
+ const {SidebarContext} = await import('../state/sidebar');
+ const submittedJobs: AddCaptionJobParams[] = [];
+ const requestInit = {
+ credentials: 'include',
+ headers: {'x-remotion-test': 'modal-serialization'},
+ } as const;
+
+ render(
+ <SetSelectedModalContext.Provider
+ value={{setSelectedModal: () => undefined}}
+ >
+ <SidebarContext.Provider
+ value={{
+ sidebarCollapsedDuringDrag: null,
+ sidebarCollapsedStateLeft: 'collapsed',
+ sidebarCollapsedStateRight: 'collapsed',
+ setSidebarCollapsedDuringDrag: () => undefined,
+ setSidebarCollapsedState: () => undefined,
+ }}
+ >
+ <RenderQueueContext.Provider
+ value={
+ {
+ captionJobs: [],
+ addCaptionJob: (params: AddCaptionJobParams) => {
+ submittedJobs.push(params);
+ return `caption-job-${submittedJobs.length}`;
+ },
+ } as never
+ }
+ >
+ <TranscriptionModal
+ type="transcribe"
+ audioStreamIndex={2}
+ displayName="interview.wav"
+ requestInit={requestInit}
+ src="/media/interview.wav"
+ />
+ </RenderQueueContext.Provider>
+ </SidebarContext.Provider>
+ </SetSelectedModalContext.Provider>,
+ );
+
+ screen.getByRole('dialog', {
+ name: 'Transcribe interview.wav',
+ });
+ const submit = screen.getByRole('button', {
+ name: /^Transcribe$/,
+ }) as HTMLButtonElement;
+ await waitFor(() => expect(submit.disabled).toBe(false));
+ expect(
+ screen.queryByText('Runs locally as a background job using WebGPU'),
+ ).toBeNull();
+ const outputLabel = screen.getByText('Output in public/');
+ const modelLabel = screen.getByText('Whisper model');
+ expect(
+ outputLabel.compareDocumentPosition(modelLabel) &
+ Node.DOCUMENT_POSITION_FOLLOWING,
+ ).not.toBe(0);
+ const outputHelp = screen.getByTitle('Learn more about Output in public/');
+ expect(screen.queryByText('Caption[]')).toBeNull();
+ fireEvent.pointerUp(outputHelp);
+ const captionType = screen.getByText('Caption[]');
+ expect(captionType.style.fontFamily).toBe('monospace');
+ expect(captionType.style.fontSize).toBe('inherit');
+ expect(captionType.style.lineHeight).toBe('inherit');
+ fireEvent.pointerUp(outputHelp);
+ screen.getByTitle('Learn more about Model download');
+ screen.getByTitle('Learn more about Chunk length');
+ screen.getByTitle('Learn more about Stride length');
+ screen.getByTitle('Learn more about Force full sequences');
+ screen.getByTitle('Learn more about Use sampling');
+ screen.getByTitle('Learn more about Repetition penalty');
+ screen.getByTitle('Learn more about No-repeat n-gram size');
+
+ await selectComboboxItem(screen.getByTitle('Whisper model'), /^tiny ·/);
+ await selectComboboxItem(screen.getByTitle('Spoken language'), 'German');
+ await selectComboboxItem(
+ screen.getByRole('button', {name: 'Task: Transcribe'}),
+ 'Task: Translate to English',
+ );
+ const taskLabel = screen.getByText('Translate to English', {
+ selector: 'span[aria-hidden="true"]',
+ });
+ expect(taskLabel.style.color).toBe('inherit');
+ expect(taskLabel.style.fontFamily).toBe('inherit');
+ expect(taskLabel.style.fontSize).toBe('inherit');
+ expect(taskLabel.style.lineHeight).toBe('inherit');
+ const taskWarning = screen.getByText(
+ 'Word timings may be less accurate when translating to English.',
+ );
+ expect(
+ taskWarning.parentElement?.parentElement?.parentElement?.style.display,
+ ).toBe('flex');
+ expect(
+ taskWarning.parentElement?.parentElement?.parentElement?.style
+ .justifyContent,
+ ).toBe('flex-end');
+
+ const outputInput = screen.getByRole('textbox', {
+ name: 'Caption output file',
+ });
+ expect(
+ outputInput.parentElement?.parentElement?.parentElement?.style
+ .paddingBottom,
+ ).toBe('12px');
+ fireEvent.change(outputInput, {
+ target: {value: 'captions/interview.json'},
+ });
+
+ const forceFullSequences = screen.getByRole('checkbox', {
+ name: 'Force full sequences',
+ });
+ const useSampling = screen.getByRole('checkbox', {
+ name: 'Use sampling',
+ });
+ for (const checkbox of [forceFullSequences, useSampling]) {
+ const checkboxLabel = document.querySelector(
+ `label[for="${checkbox.id}"]`,
+ ) as HTMLLabelElement | null;
+ expect(checkboxLabel?.style.color).toBe('inherit');
+ expect(checkboxLabel?.style.fontFamily).toBe('inherit');
+ expect(checkboxLabel?.style.fontSize).toBe('inherit');
+ expect(checkboxLabel?.style.lineHeight).toBe('inherit');
+ }
+
+ act(() => forceFullSequences.click());
+ act(() => useSampling.click());
+ screen.getByTitle('Learn more about Temperature');
+ screen.getByTitle('Learn more about Top K');
+
+ setNumberSetting(/^Chunk length:/, 22);
+ setNumberSetting(/^Stride length:/, 4);
+ setNumberSetting(/^Temperature:/, 0.7);
+ setNumberSetting(/^Top K:/, 25);
+ setNumberSetting(/^Repetition penalty:/, 1.2);
+ setNumberSetting(/^No-repeat n-gram size:/, 3);
+
+ act(() => useSampling.click());
+ fireEvent.click(submit);
+ act(() => forceFullSequences.click());
+ act(() => useSampling.click());
+ fireEvent.click(submit);
+
+ const commonJob = {
+ audioStreamIndex: 2,
+ chunkLengthInSeconds: 22,
+ displayName: 'interview.wav',
+ language: 'de',
+ model: 'tiny',
+ noRepeatNgramSize: 3,
+ outName: 'captions/interview.json',
+ repetitionPenalty: 1.2,
+ requestInit,
+ src: '/media/interview.wav',
+ strideLengthInSeconds: 4,
+ task: 'translate',
+ temperature: 0.7,
+ topK: 25,
+ } satisfies Omit<AddCaptionJobParams, 'doSample' | 'forceFullSequences'>;
+ expect(submittedJobs).toEqual([
+ {...commonJob, doSample: false, forceFullSequences: true},
+ {...commonJob, doSample: true, forceFullSequences: false},
+ ]);
+ } finally {
+ cleanup();
+ portals.splice(0, portals.length, ...previousPortals);
+ for (const portal of createdPortals) {
+ portal.remove();
+ }
+
+ if (originalGpuDescriptor) {
+ Object.defineProperty(navigator, 'gpu', originalGpuDescriptor);
+ } else {
+ Reflect.deleteProperty(navigator, 'gpu');
+ }
+
+ if (originalSecureContextDescriptor) {
+ Object.defineProperty(
+ window,
+ 'isSecureContext',
+ originalSecureContextDescriptor,
+ );
+ } else {
+ Reflect.deleteProperty(window, 'isSecureContext');
+ }
+
+ if (originalStaticBaseDescriptor) {
+ Object.defineProperty(
+ window,
+ 'remotion_staticBase',
+ originalStaticBaseDescriptor,
+ );
+ } else {
+ Reflect.deleteProperty(window, 'remotion_staticBase');
+ }
+
+ if (originalLocalStorageDescriptor) {
+ Object.defineProperty(
+ globalThis,
+ 'localStorage',
+ originalLocalStorageDescriptor,
+ );
+ } else {
+ Reflect.deleteProperty(globalThis, 'localStorage');
+ }
+ }
+});
diff --git a/packages/studio/src/test/whisper-webgpu-capability.test.tsx b/packages/studio/src/test/whisper-webgpu-capability.test.tsx
new file mode 100644
index 0000000000..1d288680bc
--- /dev/null
+++ b/packages/studio/src/test/whisper-webgpu-capability.test.tsx
@@ -0,0 +1,86 @@
+import {afterEach, expect, spyOn, test} from 'bun:test';
+import {cleanup, fireEvent, render, waitFor} from '@testing-library/react';
+import {ModalsProvider} from '../components/ModalsProvider';
+import {TranscriptionModalWithOptionalWhisper} from '../components/Transcription/TranscriptionModalWithOptionalWhisper';
+import {
+ isWhisperWebGpuInstalled,
+ WHISPER_WEBGPU_PACKAGE,
+} from '../components/Transcription/whisper-webgpu-capability';
+
+const originalInstalledPackages = window.remotion_installedPackages;
+
+afterEach(() => {
+ cleanup();
+ window.remotion_installedPackages = originalInstalledPackages;
+});
+
+test('detects whether Whisper WebGPU is installed in the project', () => {
+ window.remotion_installedPackages = [];
+ expect(isWhisperWebGpuInstalled()).toBe(false);
+ window.remotion_installedPackages = [WHISPER_WEBGPU_PACKAGE];
+ expect(isWhisperWebGpuInstalled()).toBe(true);
+});
+
+test('asks before installing Whisper for transcription', () => {
+ window.remotion_installedPackages = [];
+ const {container, getByRole} = render(
+ <ModalsProvider>
+ <TranscriptionModalWithOptionalWhisper
+ state={{
+ type: 'transcribe',
+ src: '/voice.wav',
+ displayName: 'voice.wav',
+ audioStreamIndex: null,
+ requestInit: null,
+ }}
+ />
+ </ModalsProvider>,
+ );
+
+ expect(container.textContent).toContain(
+ 'This requires installing @remotion/whisper-webgpu. Continue?',
+ );
+ expect(getByRole('button', {name: 'Continue'})).toBeDefined();
+});
+
+test('opens transcription after installing Whisper without restarting', async () => {
+ const originalIsStudio = window.remotion_isStudio;
+ const originalIsReadOnlyStudio = window.remotion_isReadOnlyStudio;
+ const originalStaticBase = window.remotion_staticBase;
+ window.remotion_installedPackages = [];
+ window.remotion_isStudio = true;
+ window.remotion_isReadOnlyStudio = false;
+ window.remotion_staticBase = '';
+ const fetchSpy = spyOn(globalThis, 'fetch').mockResolvedValue(
+ new Response(JSON.stringify({success: true, data: {}})),
+ );
+ try {
+ const {container, getByRole} = render(
+ <ModalsProvider>
+ <TranscriptionModalWithOptionalWhisper
+ state={{
+ type: 'transcribe',
+ src: '/voice.wav',
+ displayName: 'voice.wav',
+ audioStreamIndex: null,
+ requestInit: null,
+ }}
+ />
+ </ModalsProvider>,
+ );
+
+ fireEvent.click(getByRole('button', {name: 'Continue'}));
+ await waitFor(() => {
+ expect(window.remotion_installedPackages).toContain(
+ WHISPER_WEBGPU_PACKAGE,
+ );
+ expect(container.textContent).toContain('Output in public/');
+ });
+ expect(fetchSpy).toHaveBeenCalledTimes(1);
+ } finally {
+ fetchSpy.mockRestore();
+ window.remotion_isStudio = originalIsStudio;
+ window.remotion_isReadOnlyStudio = originalIsReadOnlyStudio;
+ window.remotion_staticBase = originalStaticBase;
+ }
+});
diff --git a/packages/studio/tsconfig.json b/packages/studio/tsconfig.json
index 846851819b..56fe86b3c2 100644
--- a/packages/studio/tsconfig.json
+++ b/packages/studio/tsconfig.json
@@ -41,6 +41,9 @@
},
{
"path": "../web-renderer"
+ },
+ {
+ "path": "../whisper-webgpu"
}
]
}