🎥 Make videos programmatically with React
Add Studio transcription jobs
JonnyBurgerdaba99ca6b71
diff --git a/bun.lock b/bun.lockindex 2dac03fb47..6b362d6bcb 100644--- a/bun.lock+++ b/bun.lock@@ -159,6 +159,7 @@"remotion": "workspace:*",},"devDependencies": {+ "@huggingface/transformers": "catalog:","@playwright/test": "1.55.1","@remotion/eslint-config-internal": "workspace:*","@remotion/studio": "workspace:*",@@ -1826,6 +1827,7 @@"devDependencies": {"@happy-dom/global-registrator": "14.5.1","@remotion/eslint-config-internal": "workspace:*",+ "@remotion/whisper-webgpu": "workspace:*","@testing-library/react": "16.1.0","@types/semver": "7.5.3","@typescript/native-preview": "catalog:",@@ -1834,9 +1836,13 @@"react-dom": "catalog:",},"peerDependencies": {+ "@remotion/whisper-webgpu": "workspace:*","react": ">=16.8.0","react-dom": ">=16.8.0",},+ "optionalPeers": [+ "@remotion/whisper-webgpu",+ ],},"packages/studio-codemods": {"name": "@remotion/studio-codemods",@@ -5722,7 +5728,7 @@"address": ["[email protected]", "", {}, "sha512-aT6camzM4xEA54YVJYSqxz1kv4IHnQZRtThJJHhUMRExaU5spC7jX5ugSwTaTgJliIgs4VhZOk7htClvQ/LmRA=="],- "adm-zip": ["[email protected]", "", {}, "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg=="],+ "adm-zip": ["[email protected]", "", {}, "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng=="],"aes-js": ["[email protected]", "", {}, "sha512-e5pEa2kBnBOgR4Y/p20pskXI74UEz7de8ZGVo58asOtvSVG5YAbJeELPZxOmt+Bnz3rX753YKhfIn4X4l1PPRQ=="],@@ -11742,6 +11748,8 @@"finalhandler/debug": ["[email protected]", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="],+ "firefox-profile/adm-zip": ["[email protected]", "", {}, "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg=="],+"firefox-profile/fs-extra": ["[email protected]", "", { "dependencies": { "graceful-fs": "4.2.11", "jsonfile": "6.1.0", "universalify": "2.0.0" } }, "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw=="],"firefox-profile/minimist": ["[email protected]", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="],@@ -12128,8 +12136,6 @@"ofetch/ufo": ["[email protected]", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="],- "onnxruntime-node/adm-zip": ["[email protected]", "", {}, "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng=="],-"onnxruntime-web/onnxruntime-common": ["[email protected]", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="],"openai/@types/node": ["@types/[email protected]", "", {}, "sha512-93+VvleD3mXwlLI/xASjw0FzKcwzl3OdTCzm1LaRfqgS21gfFtK3zDXM5Op9TeeMsJVOaJ2VRDpT9q4Y3d0AvA=="],diff --git a/packages/browser-studio/bundle.ts b/packages/browser-studio/bundle.tsindex 82b788ba6d..0ad73888d7 100644--- a/packages/browser-studio/bundle.ts+++ b/packages/browser-studio/bundle.ts@@ -1,6 +1,7 @@import path from 'path';import {fileURLToPath} from 'url';import {build} from 'bun';+import {BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE} from './src/browser-studio-import-map';import {getBrowserStudioDependencyVersionsForBuild} from './src/dev/get-dependency-versions-for-build';import {getBrowserStudioReactRefreshFilesForBuild} from './src/dev/get-react-refresh-files-for-build';import {getBrowserStudioSetupEnvironmentForBuild} from './src/dev/get-setup-environment-for-build';@@ -24,6 +25,10 @@ const workspacePackageExports =const vendorOutput = await build({define: {'process.env.NODE_ENV': JSON.stringify('development')},entrypoints: ['src/browser-studio-vendor-entry.ts'],+ external: [+ '@huggingface/transformers',+ BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE,+ ],format: 'iife',minify: true,naming: '[name].mjs',@@ -41,6 +46,28 @@ const vendorEntryOutput = vendorOutput.outputs.find(if (!vendorEntryOutput) {throw new Error('Browser Studio vendor entry was not generated');}+if (+ !(await vendorEntryOutput.text()).includes(+ BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE,+ )+) {+ throw new Error(+ 'The Browser Studio vendor entry must preserve the private Transformers import.',+ );+}++const transformersOutput = await build({+ entrypoints: ['src/browser-studio-transformers-entry.ts'],+ format: 'esm',+ minify: true,+ naming: '[name].mjs',+ target: 'browser',+});++if (!transformersOutput.success) {+ console.log(transformersOutput.logs.join('\n'));+ process.exit(1);+}const rspackBrowserEntry = fileURLToPath(import.meta.resolve('@rspack/browser'),@@ -84,8 +111,21 @@ if (!output.success) {const externalVersionSensitiveImport =/^[^'"\n]*\bfrom\s*["']@remotion\/(?:player|studio-shared|timeline-utils)["'];?\s*$|^\s*import\s*["']@remotion\/(?:player|studio-shared|timeline-utils)["'];?\s*$/m;-for (const file of [...output.outputs, ...vendorOutput.outputs]) {+for (const file of [+ ...output.outputs,+ ...vendorOutput.outputs,+ ...transformersOutput.outputs,+]) {const str = await file.text();+ if (+ !file.path.includes('browser-studio-transformers-entry') &&+ (str.includes('onnxruntime-web') || str.includes('transformers.web.js'))+ ) {+ throw new Error(+ 'The Browser Studio bootstrap must not bundle Transformers.js. Keep it in the lazy Transformers entry.',+ );+ }+if (path.basename(file.path) === 'browser-studio-vendor-entry.mjs' &&externalVersionSensitiveImport.test(str)diff --git a/packages/browser-studio/e2e/app.tsx b/packages/browser-studio/e2e/app.tsxindex defaf8cf60..8ea1cf27d1 100644--- a/packages/browser-studio/e2e/app.tsx+++ b/packages/browser-studio/e2e/app.tsx@@ -15,6 +15,7 @@ const initialElementPayload = StudioProtocolInternals.parseBrowserStudioHash(window as typeof window & {__browserStudioRemotionVersion: string}).__browserStudioRemotionVersion = VERSION;const render = async () => {+ const source = new URLSearchParams(window.location.search).get('source');const project = new URLSearchParams(window.location.search).has('github')? await loadGitHubRepository({repoUrl: 'https://github.com/remotion-dev/opfs-fixture',@@ -34,6 +35,13 @@ import {RemotionRoot} from './Root';void fade;registerRoot(RemotionRoot);+${+ source === 'fallback' ||+ source === 'transformers-override' ||+ source === 'transformers-vendor'+ ? `(globalThis as typeof globalThis & {__loadBrowserStudioTransformers: () => Promise<unknown>}).__loadBrowserStudioTransformers = () => import('@huggingface/transformers');`+ : ''+}`;}@@ -42,14 +50,22 @@ registerRoot(RemotionRoot);throw new Error('Could not find root element');}- const source = new URLSearchParams(window.location.search).get('source');-createRoot(root).render(<BrowserStudiodependencyResolver={source === 'fallback'? ({name, version}) => (name === 'react' ? version : null)- : undefined+ : source === 'transformers-override'+ ? ({name}) =>+ name === '@huggingface/transformers'+ ? {+ url: new URL(+ '/fake-transformers.mjs',+ window.location.href,+ ).href,+ }+ : null+ : undefined}initialElement={initialElementPayload === nulldiff --git a/packages/browser-studio/e2e/browser-studio.test.ts b/packages/browser-studio/e2e/browser-studio.test.tsindex ed9780e1e6..8ecb82d5cb 100644--- a/packages/browser-studio/e2e/browser-studio.test.ts+++ b/packages/browser-studio/e2e/browser-studio.test.ts@@ -975,8 +975,10 @@ test('installs packages without a server API and preserves undo, redo, and HMR',test('fetches each HTTP module once when the vendor bundle is overridden', async ({page,}) => {+ const requests: string[] = [];const workspacePackageRequests: string[] = [];page.on('request', (request) => {+ requests.push(request.url());const requestUrl = new URL(request.url());if (requestUrl.pathname.startsWith('/__remotion_browser_studio_workspace__/')@@ -1009,6 +1011,136 @@ test('fetches each HTTP module once when the vendor bundle is overridden', asyncexpect(workspacePackageRequests).toContain('/__remotion_browser_studio_workspace__/commits/e2e/packages/studio/dist/esm/previewEntry.mjs',);+ const transformersUrl = await studio+ .locator('script[type="importmap"]')+ .evaluate((element) => {+ const importMap = JSON.parse(element.textContent ?? '') as {+ imports: Record<string, string>;+ };+ return importMap.imports['@huggingface/transformers'];+ });+ expect(requests).not.toContain(transformersUrl);+ const pipelineType = await studio.locator('body').evaluate(async () => {+ const loadTransformers = (+ window as typeof window & {+ __loadBrowserStudioTransformers: () => Promise<{pipeline: unknown}>;+ }+ ).__loadBrowserStudioTransformers;+ const transformers = await loadTransformers();+ return typeof transformers.pipeline;+ });+ expect(pipelineType).toBe('function');+ expect(+ requests.filter((request) => request === transformersUrl),+ ).toHaveLength(1);+});++test('loads the local Transformers bundle on demand from the vendor Blob bundle', async ({+ page,+}) => {+ const requests: string[] = [];+ const vendorBundleRequests: string[] = [];+ page.on('request', (request) => {+ requests.push(request.url());+ const requestUrl = new URL(request.url());+ if (requestUrl.searchParams.has('browserStudioVendor')) {+ vendorBundleRequests.push(request.url());+ }+ });++ await page.goto('/?source=transformers-vendor');+ const studio = page.frameLocator('iframe');+ await expect(studio.getByTitle('/project').getByText('MyComp')).toBeVisible();+ expect(vendorBundleRequests).toHaveLength(1);+ expect(+ await studio.locator('script[type="module"]').getAttribute('src'),+ ).toMatch(/^blob:/);+ const transformersUrls = await studio+ .locator('script[type="importmap"]')+ .evaluate((element) => {+ const importMap = JSON.parse(element.textContent ?? '') as {+ imports: Record<string, string>;+ };+ return {+ consumer: importMap.imports['@huggingface/transformers'],+ whisper:+ importMap.imports['@remotion/whisper-webgpu/private-transformers'],+ };+ });+ expect(transformersUrls.whisper).not.toBe(transformersUrls.consumer);+ expect(requests).not.toContain(transformersUrls.consumer);+ expect(requests).not.toContain(transformersUrls.whisper);++ const loaded = await studio+ .locator('body')+ .evaluate(async (_body, privateTransformersPackage) => {+ const loadTransformers = (+ window as typeof window & {+ __loadBrowserStudioTransformers: () => Promise<{+ env: unknown;+ pipeline: unknown;+ }>;+ }+ ).__loadBrowserStudioTransformers;+ const consumerTransformers = await loadTransformers();+ const whisperTransformers = (await import(+ privateTransformersPackage+ )) as {env: unknown; pipeline: unknown};+ return {+ consumerPipelineType: typeof consumerTransformers.pipeline,+ environmentsAreIsolated:+ consumerTransformers.env !== whisperTransformers.env,+ whisperPipelineType: typeof whisperTransformers.pipeline,+ };+ }, '@remotion/whisper-webgpu/private-transformers');++ expect(loaded.consumerPipelineType).toBe('function');+ expect(loaded.whisperPipelineType).toBe('function');+ expect(loaded.environmentsAreIsolated).toBe(true);+ expect(+ requests.filter((request) => request === transformersUrls.consumer),+ ).toHaveLength(1);+ expect(+ requests.filter((request) => request === transformersUrls.whisper),+ ).toHaveLength(1);+});++test('uses a custom Transformers resolution in the fallback bundle', async ({+ page,+}) => {+ const fakeTransformersRequests: string[] = [];+ const vendorBundleRequests: string[] = [];+ page.on('request', (request) => {+ const requestUrl = new URL(request.url());+ if (requestUrl.pathname === '/fake-transformers.mjs') {+ fakeTransformersRequests.push(request.url());+ }++ if (requestUrl.searchParams.has('browserStudioVendor')) {+ vendorBundleRequests.push(request.url());+ }+ });++ await page.goto('/?source=transformers-override');+ const studio = page.frameLocator('iframe');+ await expect(studio.getByTitle('/project').getByText('MyComp')).toBeVisible();+ expect(vendorBundleRequests).toEqual([]);+ expect(fakeTransformersRequests).toEqual([]);++ const marker = await studio.locator('body').evaluate(async () => {+ const loadTransformers = (+ window as typeof window & {+ __loadBrowserStudioTransformers: () => Promise<{+ browserStudioOverrideMarker: number;+ }>;+ }+ ).__loadBrowserStudioTransformers;+ const transformers = await loadTransformers();+ return transformers.browserStudioOverrideMarker;+ });++ expect(marker).toBe(42);+ expect(fakeTransformersRequests).toHaveLength(1);});test('drops and imports an Element payload with the deployment Remotion version', async ({diff --git a/packages/browser-studio/e2e/server.ts b/packages/browser-studio/e2e/server.tsindex 0d34a36853..b87b4444dd 100644--- a/packages/browser-studio/e2e/server.ts+++ b/packages/browser-studio/e2e/server.ts@@ -93,6 +93,12 @@ Bun.serve({});}+ if (url.pathname === '/fake-transformers.mjs') {+ return new Response('export const browserStudioOverrideMarker = 42;', {+ headers: {...headers, 'Content-Type': 'text/javascript'},+ });+ }+const remotionPackagePath = remotionPackagePaths.find((prefix) =>url.pathname.startsWith(prefix),);diff --git a/packages/browser-studio/package.json b/packages/browser-studio/package.jsonindex 7b40a8b5fb..ef3f45c602 100644--- a/packages/browser-studio/package.json+++ b/packages/browser-studio/package.json@@ -50,6 +50,7 @@"./package.json": "./package.json"},"devDependencies": {+ "@huggingface/transformers": "catalog:","@playwright/test": "1.55.1","@rspack/core": "1.7.11","@rspack/plugin-react-refresh": "1.6.1",diff --git a/packages/browser-studio/src/BrowserStudio.tsx b/packages/browser-studio/src/BrowserStudio.tsxindex 055d1d2bf9..eebf9113eb 100644--- a/packages/browser-studio/src/BrowserStudio.tsx+++ b/packages/browser-studio/src/BrowserStudio.tsx@@ -5,6 +5,12 @@ import {createBrowserStudioHmrAssetManager,type BrowserStudioHmrBridge,} from './browser-studio-hmr-assets';+import {+ BROWSER_STUDIO_TRANSFORMERS_PACKAGE,+ BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE,+ getBrowserStudioTransformersUrl,+ getBrowserStudioWhisperTransformersUrl,+} from './browser-studio-import-map';import {createBrowserStudioOperations} from './browser-studio-operations';import {areBrowserStudioProjectsEqual,@@ -45,6 +51,13 @@ const localVendorEntry = new URL('./browser-studio-vendor-entry.mjs',import.meta.url,).href;+const localTransformersEntry = new URL(+ './browser-studio-transformers-entry.mjs',+ import.meta.url,+).href;+const localWhisperTransformersEntry = getBrowserStudioWhisperTransformersUrl(+ localTransformersEntry,+);const localVendorEntryWithMarker = `${localVendorEntry}?browserStudioVendor`;const browserStudioPointerLeaveEvent = 'remotion-browser-studio-pointerleave';@@ -377,6 +390,11 @@ export const BrowserStudio: React.FC<BrowserStudioProps> = ({...configuredDependencyResolutions,...installedDependencyResolutionsRef.current,};+ const transformersUrl = getBrowserStudioTransformersUrl({+ localUrl: localTransformersEntry,+ resolution:+ dependencyResolutions[BROWSER_STUDIO_TRANSFORMERS_PACKAGE] ?? null,+ });const hasVendorOverride = Object.entries(dependencyResolutions).some(([name, resolution]) =>resolution !== null &&@@ -618,6 +636,11 @@ export const BrowserStudio: React.FC<BrowserStudioProps> = ({: bundleUrlRef.current;const html = studioHtml({+ importMap: {+ [BROWSER_STUDIO_TRANSFORMERS_PACKAGE]: transformersUrl,+ [BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE]:+ localWhisperTransformersEntry,+ },audioLatencyHint: 'playback',experimentalKeepAudioContextAlive: false,bundleScriptUrl,diff --git a/packages/browser-studio/src/browser-studio-import-map.ts b/packages/browser-studio/src/browser-studio-import-map.tsnew file mode 100644index 0000000000..285dd245dc--- /dev/null+++ b/packages/browser-studio/src/browser-studio-import-map.ts@@ -0,0 +1,44 @@+import type {BrowserStudioDependencyResolution} from './types';++export const BROWSER_STUDIO_TRANSFORMERS_PACKAGE =+ '@huggingface/transformers' as const;+export const BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE =+ '@remotion/whisper-webgpu/private-transformers' as const;++const getEsmShUrl = (version: string) => {+ const url = new URL(+ `https://esm.sh/${BROWSER_STUDIO_TRANSFORMERS_PACKAGE}@${version}`,+ );+ url.searchParams.set('dev', '');+ return url.href;+};++export const getBrowserStudioTransformersUrl = ({+ localUrl,+ resolution,+}: {+ readonly localUrl: string;+ readonly resolution: BrowserStudioDependencyResolution;+}) => {+ if (typeof resolution === 'string') {+ return resolution.startsWith('http://') || resolution.startsWith('https://')+ ? resolution+ : getEsmShUrl(resolution);+ }++ if (resolution?.url) {+ return resolution.url;+ }++ if (resolution?.version) {+ return getEsmShUrl(resolution.version);+ }++ return localUrl;+};++export const getBrowserStudioWhisperTransformersUrl = (localUrl: string) => {+ const url = new URL(localUrl);+ url.searchParams.set('whisper-webgpu-private', '');+ return url.href;+};diff --git a/packages/browser-studio/src/browser-studio-transformers-entry.ts b/packages/browser-studio/src/browser-studio-transformers-entry.tsnew file mode 100644index 0000000000..bb03ecc2a5--- /dev/null+++ b/packages/browser-studio/src/browser-studio-transformers-entry.ts@@ -0,0 +1 @@+export * from '@huggingface/transformers';diff --git a/packages/browser-studio/src/browser-studio-worker.ts b/packages/browser-studio/src/browser-studio-worker.tsindex f2e1c581a5..270c5c804c 100644--- a/packages/browser-studio/src/browser-studio-worker.ts+++ b/packages/browser-studio/src/browser-studio-worker.ts@@ -2,6 +2,10 @@ import type {HotMiddlewareMessage} from '@remotion/studio-shared';import {getStudioEntryPoints} from '@remotion/studio-shared/studio-entry-points';import type * as RspackBrowser from '@rspack/browser';import {makeBrowserStudioHttpClient} from './browser-studio-http-client';+import {+ BROWSER_STUDIO_TRANSFORMERS_PACKAGE,+ BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE,+} from './browser-studio-import-map';import {browserStudioDependencyVersions} from './dependency-versions';import {studioRenderEntryExternal} from './dev/studio-render-entry-external';import type {@@ -411,7 +415,20 @@ const createCompiler = async ({}),},},- externals: useVendorBundle ? browserStudioVendorExternals : undefined,+ externals: [+ ...(useVendorBundle ? [browserStudioVendorExternals] : []),+ ({request}, callback) => {+ if (+ request === BROWSER_STUDIO_TRANSFORMERS_PACKAGE ||+ request === BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE+ ) {+ callback(undefined, request, 'import');+ return;+ }++ callback();+ },+ ],externalsType: useVendorBundle ? 'var' : undefined,mode: 'development',module: {diff --git a/packages/browser-studio/src/dev/get-dependency-versions-for-build.ts b/packages/browser-studio/src/dev/get-dependency-versions-for-build.tsindex 84f0fae11a..79ac64103f 100644--- a/packages/browser-studio/src/dev/get-dependency-versions-for-build.ts+++ b/packages/browser-studio/src/dev/get-dependency-versions-for-build.ts@@ -101,6 +101,7 @@ export const getBrowserStudioDependencyVersionsForBuild = (): Record<}const dependencySpecs: Record<string, string> = {+ '@huggingface/transformers': 'catalog:',[studioPackageJson.name]: studioPackageJson.version,react: 'catalog:','react-dom': 'catalog:',diff --git a/packages/browser-studio/src/dev/server.ts b/packages/browser-studio/src/dev/server.tsindex 73297b413e..67ce8a05d6 100644--- a/packages/browser-studio/src/dev/server.ts+++ b/packages/browser-studio/src/dev/server.ts@@ -1,6 +1,7 @@import path from 'path';import {fileURLToPath} from 'url';import {build} from 'bun';+import {BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE} from '../browser-studio-import-map';import {browserStudioPackageJsonArtifactFilename} from '../workspace-package-exports';import {getBrowserStudioDependencyVersionsForBuild} from './get-dependency-versions-for-build';import {getBrowserStudioReactRefreshFilesForBuild} from './get-react-refresh-files-for-build';@@ -82,6 +83,10 @@ const buildDevAssets = async () => {const vendorOutput = await build({define: {'process.env.NODE_ENV': JSON.stringify('development')},entrypoints: ['src/browser-studio-vendor-entry.ts'],+ external: [+ '@huggingface/transformers',+ BROWSER_STUDIO_WHISPER_TRANSFORMERS_PACKAGE,+ ],format: 'iife',naming: '[name].mjs',outdir: outDir,@@ -101,6 +106,20 @@ const buildDevAssets = async () => {throw new Error('Browser Studio vendor entry was not generated');}+ const transformersOutput = await build({+ entrypoints: ['src/browser-studio-transformers-entry.ts'],+ format: 'esm',+ naming: '[name].mjs',+ outdir: outDir,+ sourcemap: 'linked',+ target: 'browser',+ });++ if (!transformersOutput.success) {+ process.stderr.write(`${transformersOutput.logs.join('\n')}\n`);+ process.exit(1);+ }+const browserStudioAssetSizes = {rspackWasm: Bun.file(path.join(rspackBrowserDist, 'rspack.wasm32-wasi.wasm'),diff --git a/packages/browser-studio/src/dev/studio-render-entry-external.ts b/packages/browser-studio/src/dev/studio-render-entry-external.tsindex f8e038d344..c0efd7e0df 100644--- a/packages/browser-studio/src/dev/studio-render-entry-external.ts+++ b/packages/browser-studio/src/dev/studio-render-entry-external.ts@@ -1,4 +1,6 @@export const studioRenderEntryExternal = [+ '@huggingface/transformers',+ '@remotion/whisper-webgpu/private-transformers','@jridgewell/trace-mapping','@remotion/media-utils','@remotion/renderer',diff --git a/packages/browser-studio/src/test/browser-studio-import-map.test.ts b/packages/browser-studio/src/test/browser-studio-import-map.test.tsnew file mode 100644index 0000000000..5b26ddd28e--- /dev/null+++ b/packages/browser-studio/src/test/browser-studio-import-map.test.ts@@ -0,0 +1,49 @@+import {expect, test} from 'bun:test';+import {+ getBrowserStudioTransformersUrl,+ getBrowserStudioWhisperTransformersUrl,+} from '../browser-studio-import-map';++const localUrl = 'https://studio.test/browser-studio-transformers-entry.mjs';++test('uses the local lazy Transformers bundle by default', () => {+ expect(getBrowserStudioTransformersUrl({localUrl, resolution: null})).toBe(+ localUrl,+ );+});++test('honors custom Transformers URLs', () => {+ expect(+ getBrowserStudioTransformersUrl({+ localUrl,+ resolution: {url: 'https://cdn.test/transformers.mjs'},+ }),+ ).toBe('https://cdn.test/transformers.mjs');+ expect(+ getBrowserStudioTransformersUrl({+ localUrl,+ resolution: 'https://cdn.test/transformers-string.mjs',+ }),+ ).toBe('https://cdn.test/transformers-string.mjs');+});++test('turns custom Transformers versions into pinned esm.sh URLs', () => {+ expect(+ getBrowserStudioTransformersUrl({+ localUrl,+ resolution: {version: '4.2.0'},+ }),+ ).toBe('https://esm.sh/@huggingface/[email protected]?dev=');+ expect(+ getBrowserStudioTransformersUrl({+ localUrl,+ resolution: '4.1.0',+ }),+ ).toBe('https://esm.sh/@huggingface/[email protected]?dev=');+});++test('uses a separate Transformers module instance for Whisper', () => {+ expect(getBrowserStudioWhisperTransformersUrl(localUrl)).toBe(+ 'https://studio.test/browser-studio-transformers-entry.mjs?whisper-webgpu-private=',+ );+});diff --git a/packages/browser-studio/src/test/dependency-versions.test.ts b/packages/browser-studio/src/test/dependency-versions.test.tsindex fb839bea2d..8dcb826d00 100644--- a/packages/browser-studio/src/test/dependency-versions.test.ts+++ b/packages/browser-studio/src/test/dependency-versions.test.ts@@ -28,15 +28,22 @@ test('browser studio dependency versions are derived from package metadata', ()const transitionsPackageJson = readPackageJson(join(repoDir, 'packages', 'transitions', 'package.json'),);+ const whisperWebGpuPackageJson = readPackageJson(+ join(repoDir, 'packages', 'whisper-webgpu', 'package.json'),+ );const dependencyVersions = getBrowserStudioDependencyVersionsForBuild();const reactVersion = rootPackageJson.workspaces?.catalog?.react;const reactDomVersion = rootPackageJson.workspaces?.catalog?.['react-dom'];+ const transformersVersion =+ rootPackageJson.workspaces?.catalog?.['@huggingface/transformers'];if (!studioPackageJson.version ||!transitionsPackageJson.version ||+ !whisperWebGpuPackageJson.version ||!reactVersion ||- !reactDomVersion+ !reactDomVersion ||+ !transformersVersion) {throw new Error('Could not find package metadata for dependency version test',@@ -49,6 +56,12 @@ test('browser studio dependency versions are derived from package metadata', ()expect(dependencyVersions['@remotion/transitions']).toBe(transitionsPackageJson.version,);+ expect(dependencyVersions['@remotion/whisper-webgpu']).toBe(+ whisperWebGpuPackageJson.version,+ );+ expect(dependencyVersions['@huggingface/transformers']).toBe(+ transformersVersion,+ );expect(dependencyVersions.react).toBe(reactVersion);expect(dependencyVersions['react-dom']).toBe(reactDomVersion);expect(dependencyVersions.memfs).toBeUndefined();diff --git a/packages/bundler/src/bundle.ts b/packages/bundler/src/bundle.tsindex 963af1b5a1..90df81a276 100644--- a/packages/bundler/src/bundle.ts+++ b/packages/bundler/src/bundle.ts@@ -412,6 +412,7 @@ export const internalBundle = async (}const html = indexHtml({+ importMap: null,staticHash,publicPath,editorName: null,diff --git a/packages/bundler/src/optional-dependencies.ts b/packages/bundler/src/optional-dependencies.tsindex 804e07d9e3..eb38542474 100644--- a/packages/bundler/src/optional-dependencies.ts+++ b/packages/bundler/src/optional-dependencies.ts@@ -9,16 +9,38 @@ const OPTIONAL_DEPENDENCIES = ['react-native-reanimated/package.json',];+const STUDIO_OPTIONAL_DEPENDENCIES = ['@remotion/whisper-webgpu'];+const SOURCE_MAP_IGNORE = ['path', 'fs'];export class AllowOptionalDependenciesPlugin {- filter(error: Error) {+ filter(+ error: Error & {+ module?: {+ resource?: string;+ resourceResolveData?: {descriptionFileData?: {name?: string}};+ };+ },+ ) {for (const dependency of OPTIONAL_DEPENDENCIES) {if (error.message.includes(`Can't resolve '${dependency}'`)) {return false;}}+ const issuer = error.module?.resource?.replaceAll('\\', '/') ?? '';+ const issuerPackageName =+ error.module?.resourceResolveData?.descriptionFileData?.name;+ for (const dependency of STUDIO_OPTIONAL_DEPENDENCIES) {+ if (+ error.message.includes(`Can't resolve '${dependency}'`) &&+ (issuerPackageName === '@remotion/studio' ||+ issuer.includes('/@remotion/studio/'))+ ) {+ return false;+ }+ }+for (const dependency of SOURCE_MAP_IGNORE) {if (error.message.includes(`Can't resolve '${dependency}'`) &&diff --git a/packages/bundler/src/rspack-config.ts b/packages/bundler/src/rspack-config.tsindex a7ead49ec8..eb775fe58b 100644--- a/packages/bundler/src/rspack-config.ts+++ b/packages/bundler/src/rspack-config.ts@@ -1,6 +1,7 @@import {getStudioEntryPoints} from '@remotion/studio-shared/studio-entry-points';import {ProgressPlugin, rspack} from '@rspack/core';import ReactRefreshPlugin from '@rspack/plugin-react-refresh';+import {AllowOptionalDependenciesPlugin} from './optional-dependencies';import type {BundlerOverrideFn,RspackConfiguration,@@ -120,6 +121,7 @@ export const rspackConfig = async ({? [new ReactRefreshPlugin({overlay: false}),new rspack.HotModuleReplacementPlugin(),+ new AllowOptionalDependenciesPlugin(),...extraPlugins,]: [@@ -131,6 +133,7 @@ export const rspackConfig = async ({}}}),+ new AllowOptionalDependenciesPlugin(),],output: getOutputConfig(environment),resolve: getResolveConfig(),diff --git a/packages/bundler/src/test/optional-dependencies.test.ts b/packages/bundler/src/test/optional-dependencies.test.tsnew file mode 100644index 0000000000..d6ff9b9eec--- /dev/null+++ b/packages/bundler/src/test/optional-dependencies.test.ts@@ -0,0 +1,27 @@+import {expect, test} from 'bun:test';+import {AllowOptionalDependenciesPlugin} from '../optional-dependencies';++const missingWhisperError = (issuer: string) => {+ return Object.assign(+ new Error("Can't resolve '@remotion/whisper-webgpu' in '/project'"),+ {module: {resource: issuer}},+ );+};++test('allows Studio to omit its optional Whisper WebGPU capability', () => {+ const plugin = new AllowOptionalDependenciesPlugin();+ expect(+ plugin.filter(+ missingWhisperError(+ '/project/node_modules/@remotion/studio/dist/esm/chunk.js',+ ),+ ),+ ).toBe(false);+});++test('does not hide missing Whisper WebGPU imports in user code', () => {+ const plugin = new AllowOptionalDependenciesPlugin();+ expect(plugin.filter(missingWhisperError('/project/src/Video.tsx'))).toBe(+ true,+ );+});diff --git a/packages/core/src/CompositionManager.tsx b/packages/core/src/CompositionManager.tsxindex 2848d97d70..98969cc10f 100644--- a/packages/core/src/CompositionManager.tsx+++ b/packages/core/src/CompositionManager.tsx@@ -72,6 +72,8 @@ type EnhancedTSequenceData =| {type: 'audio';src: string;+ audioStreamIndex: number | null;+ requestInit: Omit<RequestInit, 'signal'> | null;// Volume is represented as a comma separated list - if it's a string// React can more efficiently update and will not rerender if anonymous functions// are passed.@@ -87,6 +89,8 @@ type EnhancedTSequenceData =| {type: 'video';src: string;+ audioStreamIndex: number | null;+ requestInit: Omit<RequestInit, 'signal'> | null;volume: string | number;doesVolumeChange: boolean;muted: boolean;diff --git a/packages/core/src/Sequence.tsx b/packages/core/src/Sequence.tsxindex d40f0db0d3..1bd5710f87 100644--- a/packages/core/src/Sequence.tsx+++ b/packages/core/src/Sequence.tsx@@ -508,6 +508,8 @@ const RegularSequenceRefForwardingFunction: React.ForwardRefRenderFunction<showInTimeline,timelineOrder: null,src: isMedia.data.src,+ audioStreamIndex: isMedia.data.audioStreamIndex,+ requestInit: isMedia.data.requestInit,getStack: () => stackRef.current,startMediaFrom: startMediaFrom ?? isMedia.data.startMediaFrom,mediaFrameAtSequenceZero,diff --git a/packages/core/src/audio/AudioForPreview.tsx b/packages/core/src/audio/AudioForPreview.tsxindex 1af9b0cc27..cb6b6de640 100644--- a/packages/core/src/audio/AudioForPreview.tsx+++ b/packages/core/src/audio/AudioForPreview.tsx@@ -192,6 +192,8 @@ const AudioForDevelopmentForwardRefFunction: React.ForwardRefRenderFunction<documentationLink: 'https://www.remotion.dev/docs/html5-audio',refForOutline: null,muted: muted ?? false,+ audioStreamIndex: audioStreamIndex ?? null,+ requestInit: null,});// putting playback before useVolumediff --git a/packages/core/src/test/sequence-register-once.test.tsx b/packages/core/src/test/sequence-register-once.test.tsxindex 6f5eac2f24..b29b99ec70 100644--- a/packages/core/src/test/sequence-register-once.test.tsx+++ b/packages/core/src/test/sequence-register-once.test.tsx@@ -227,6 +227,8 @@ const makeMediaInTimelineData = ({startMediaFrom,src: 'video.mp4',playbackRate,+ audioStreamIndex: 3,+ requestInit: {credentials: 'include'},}) as unknown as BasicMediaInTimelineReturnType;test('Sequence calls registerSequence exactly once on mount', () => {@@ -1171,6 +1173,8 @@ test('Video media registration accounts for its own negative from', () => {expect(videoSequence?.startMediaFrom).toBe(15);expect(videoSequence?.mediaFrameAtSequenceZero).toBe(5);+ expect(videoSequence?.audioStreamIndex).toBe(3);+ expect(videoSequence?.requestInit).toEqual({credentials: 'include'});});test('Video media registration keeps trimBefore at sequence frame zero', () => {diff --git a/packages/core/src/test/use-media-in-timeline.test.tsx b/packages/core/src/test/use-media-in-timeline.test.tsxindex d5b9d74e52..249a1e920d 100644--- a/packages/core/src/test/use-media-in-timeline.test.tsx+++ b/packages/core/src/test/use-media-in-timeline.test.tsx@@ -86,6 +86,11 @@ test('useMediaInTimeline registers muted changes and unregisters the sequence',documentationLink: null,refForOutline: null,muted,+ audioStreamIndex: 2,+ requestInit: {+ credentials: 'include',+ headers: {'x-remotion-test': 'timeline'},+ },}),{wrapper,@@ -94,6 +99,11 @@ test('useMediaInTimeline registers muted changes and unregisters the sequence',);expect(registerSequence).toHaveBeenCalled();expect(registerSequence.mock.calls[0]?.[0]).toMatchObject({+ audioStreamIndex: 2,+ requestInit: {+ credentials: 'include',+ headers: {'x-remotion-test': 'timeline'},+ },mediaFrameAtSequenceZero: null,muted: false,});@@ -149,6 +159,8 @@ test('useMediaInTimeline keeps documentation links for custom display names', ()documentationLink: 'https://www.remotion.dev/docs/html5-video',refForOutline: null,muted: false,+ audioStreamIndex: null,+ requestInit: null,}),{wrapper,diff --git a/packages/core/src/use-media-in-timeline.ts b/packages/core/src/use-media-in-timeline.tsindex ae0c54a1e2..f014956663 100644--- a/packages/core/src/use-media-in-timeline.ts+++ b/packages/core/src/use-media-in-timeline.ts@@ -35,6 +35,8 @@ export const useBasicMediaInTimeline = ({mediaStartsAt,loop,muted,+ audioStreamIndex,+ requestInit,}: {volume: VolumeProp | undefined;mediaVolume: number;@@ -48,6 +50,8 @@ export const useBasicMediaInTimeline = ({mediaStartsAt: number;loop: boolean;muted: boolean;+ audioStreamIndex: number | null;+ requestInit: Omit<RequestInit, 'signal'> | null;}) => {if (!src) {throw new Error('No src passed');@@ -113,6 +117,8 @@ export const useBasicMediaInTimeline = ({src,playbackRate,muted,+ audioStreamIndex,+ requestInit,};}, [volumes,@@ -123,6 +129,8 @@ export const useBasicMediaInTimeline = ({startMediaFrom,playbackRate,muted,+ audioStreamIndex,+ requestInit,]);return memoizedResult;@@ -148,6 +156,8 @@ export const useMediaInTimeline = ({documentationLink,refForOutline,muted,+ audioStreamIndex,+ requestInit,}: {volume: VolumeProp | undefined;mediaVolume: number;@@ -164,6 +174,8 @@ export const useMediaInTimeline = ({documentationLink: string | null;refForOutline: React.RefObject<Element | null> | null;muted: boolean;+ audioStreamIndex: number | null;+ requestInit: Omit<RequestInit, 'signal'> | null;}) => {const parentSequence = useContext(SequenceContext);const startsAt = useMediaStartsAt();@@ -185,6 +197,8 @@ export const useMediaInTimeline = ({mediaStartsAt,loop: false,muted,+ audioStreamIndex,+ requestInit,});const {isStudio} = useRemotionEnvironment();@@ -198,6 +212,8 @@ export const useMediaInTimeline = ({effectRuntimeValues: null,type: mediaType,src,+ audioStreamIndex,+ requestInit,id,duration,from: 0,@@ -242,6 +258,8 @@ export const useMediaInTimeline = ({finalDisplayName,refForOutline,muted,+ audioStreamIndex,+ requestInit,]);const registrationEnabled =isStudio ||diff --git a/packages/core/src/video/VideoForPreview.tsx b/packages/core/src/video/VideoForPreview.tsxindex 9694b0b5c0..b4eb1339d8 100644--- a/packages/core/src/video/VideoForPreview.tsx+++ b/packages/core/src/video/VideoForPreview.tsx@@ -181,6 +181,8 @@ const VideoForDevelopmentRefForwardingFunction: React.ForwardRefRenderFunction<: 'https://www.remotion.dev/docs/html5-video',refForOutline: videoRef,muted: muted ?? false,+ audioStreamIndex: audioStreamIndex ?? 0,+ requestInit: null,});// putting playback before useVolumediff --git a/packages/example/e2e/transcription-modal.test.mts b/packages/example/e2e/transcription-modal.test.mtsnew file mode 100644index 0000000000..674cfe684e--- /dev/null+++ b/packages/example/e2e/transcription-modal.test.mts@@ -0,0 +1,227 @@+import path from 'path';+import {expect, test} from '@playwright/test';+import {EXPANDED_SIDEBAR_STATE, STUDIO_URL, exampleDir} from './constants.mts';+import {startStudio, stopStudio} from './studio-server.mts';++test.use({storageState: EXPANDED_SIDEBAR_STATE});++test.describe('transcription modal', () => {+ test.beforeEach(async () => {+ await startStudio();+ });++ test.afterEach(async () => {+ await stopStudio();+ });++ test('supports Video and Audio and configures a Caption[] JSON output', async ({+ page,+ }) => {+ await page.addInitScript(() => {+ Object.defineProperty(navigator, 'gpu', {+ configurable: true,+ value: {requestAdapter: () => Promise.resolve({})},+ });+ });+ await page.route('https://remotion.media/video.mp4', async (route) => {+ await route.fulfill({+ contentType: 'video/webm',+ path: path.join(exampleDir, 'public', 'vp8-vorbis.webm'),+ });+ });+ await page.goto(`${STUDIO_URL}/NewVideo`);+ await expect(page).toHaveURL(/NewVideo/, {timeout: 15_000});+ await page.waitForFunction(+ () => !document.body.innerText.includes('Loading...'),+ {timeout: 30_000},+ );++ const video = page.getByText('<Video>', {exact: true}).first();+ const transcribe = page.getByRole('button', {+ name: 'Transcribe',+ exact: true,+ });+ await expect(async () => {+ await video.click();+ await expect(transcribe).toBeVisible({timeout: 1_000});+ }).toPass({timeout: 30_000});++ await transcribe.click();+ const dialog = page.getByRole('dialog');+ const addToQueueButton = dialog.getByRole('button', {+ name: 'Transcribe',+ exact: true,+ });+ await expect(dialog).toContainText('Transcribe <Video>');+ await expect(+ dialog.getByText('Whisper model', {exact: true}),+ ).toBeVisible();+ const model = dialog.getByTitle('Whisper model');+ const task = dialog.getByRole('button', {+ name: 'Task: Transcribe',+ exact: true,+ });+ await expect(model).toContainText('tiny.en');+ await expect(task).toContainText('Transcribe');+ await expect(task).toBeDisabled();+ await expect(+ dialog.getByText(/Downloaded with transcription if needed/),+ ).toBeVisible();+ await expect(dialog.getByRole('button', {name: /^Download /})).toHaveCount(+ 0,+ );+ await expect(addToQueueButton).toBeEnabled();++ await model.click();+ await page.getByRole('button', {name: /^tiny ·/}).click();+ await expect(model).toContainText('tiny ·');+ await expect(task).toBeEnabled();+ await task.click();+ await page+ .getByRole('button', {name: 'Task: Translate to English', exact: true})+ .click();+ await expect(+ dialog.getByRole('button', {+ name: 'Task: Translate to English',+ exact: true,+ }),+ ).toContainText('Translate to English');+ await expect(+ dialog.getByText(+ 'Word timings may be less accurate when translating to English.',+ {exact: true},+ ),+ ).toBeVisible();+ await expect(addToQueueButton).toBeEnabled();++ await dialog.getByRole('button', {name: 'Other', exact: true}).click();+ const chunkLength = dialog.getByRole('button', {+ name: /^Chunk length:/,+ });+ const strideLength = dialog.getByRole('button', {+ name: /^Stride length:/,+ });+ await expect(chunkLength).toHaveText('30s');+ await expect(strideLength).toHaveText('5s');++ const forceFullSequences = dialog.getByRole('checkbox', {+ name: 'Force full sequences',+ });+ const useSampling = dialog.getByRole('checkbox', {+ name: 'Use sampling',+ });+ const repetitionPenalty = dialog.getByRole('button', {+ name: /^Repetition penalty:/,+ });+ const noRepeatNgramSize = dialog.getByRole('button', {+ name: /^No-repeat n-gram size:/,+ });+ await expect(forceFullSequences).not.toBeChecked();+ await expect(useSampling).not.toBeChecked();+ await expect(+ dialog.getByRole('button', {name: /^Temperature:/}),+ ).toHaveCount(0);+ await expect(dialog.getByRole('button', {name: /^Top K:/})).toHaveCount(0);+ await expect(repetitionPenalty).toHaveText('1');+ await expect(noRepeatNgramSize).toHaveText('0');++ await forceFullSequences.check();+ await useSampling.check();+ const temperature = dialog.getByRole('button', {+ name: /^Temperature:/,+ });+ const topK = dialog.getByRole('button', {name: /^Top K:/});+ await expect(temperature).toHaveText('1');+ await expect(topK).toHaveText('50');++ await topK.click();+ const topKInput = dialog.getByRole('textbox', {name: /^Top K:/});+ await topKInput.fill('1.5');+ await expect(dialog).toContainText('Top K must be a non-negative integer');+ await expect(addToQueueButton).toBeDisabled();+ await topKInput.fill('25');+ await topKInput.press('Enter');+ await expect(topK).toHaveText('25');+ await expect(addToQueueButton).toBeEnabled();++ for (const [setting, inputName, value] of [+ [temperature, /^Temperature:/, '0.7'],+ [repetitionPenalty, /^Repetition penalty:/, '1.2'],+ [noRepeatNgramSize, /^No-repeat n-gram size:/, '3'],+ ] as const) {+ await setting.click();+ const input = dialog.getByRole('textbox', {+ name: inputName,+ });+ await input.fill(value);+ await input.press('Enter');+ await expect(setting).toHaveText(value);+ }++ await chunkLength.click();+ const chunkLengthInput = dialog.getByRole('textbox', {+ name: /^Chunk length:/,+ });+ await chunkLengthInput.fill('10');+ await chunkLengthInput.press('Enter');+ await expect(dialog).toContainText(+ 'Stride length must be less than half of chunk length',+ );+ await expect(addToQueueButton).toBeDisabled();++ await strideLength.click();+ const strideLengthInput = dialog.getByRole('textbox', {+ name: /^Stride length:/,+ });+ await strideLengthInput.fill('4');+ await strideLengthInput.press('Enter');+ await expect(addToQueueButton).toBeEnabled();++ await dialog.getByRole('button', {name: 'Output', exact: true}).click();+ await expect(+ dialog.getByText('Output in public/', {exact: true}),+ ).toBeVisible();+ await expect(dialog).toContainText('Caption[]');++ const output = dialog.getByRole('textbox', {+ name: 'Caption output file',+ });+ await expect(output).toHaveValue('video-captions.json');+ await output.fill('captions/new-video.json');+ await expect(output).toHaveValue('captions/new-video.json');++ await page.keyboard.press('Escape');+ await expect(dialog).toBeHidden();+ await page.goto(`${STUDIO_URL}/transcription-audio-e2e`);+ await expect(page).toHaveURL(/transcription-audio-e2e/, {+ timeout: 15_000,+ });+ const audio = page.getByText('<Audio>', {exact: true}).first();+ await expect(async () => {+ await audio.click();+ await expect(transcribe).toBeVisible({timeout: 1_000});+ }).toPass({timeout: 30_000});++ await page.goto(`${STUDIO_URL}/transcription-legacy-media-e2e`);+ await expect(page).toHaveURL(/transcription-legacy-media-e2e/, {+ timeout: 15_000,+ });++ for (const name of ['vp8-vorbis.webm', '<Legacy Audio>']) {+ const timelineItem = page.getByText(name, {exact: true}).first();+ await expect(timelineItem).toBeVisible({timeout: 15_000});+ await timelineItem.click({button: 'right'});+ const menu = page.locator('[data-remotion-menu-tree-id]').last();+ const contextMenuTranscribe = menu.getByRole('button', {+ name: 'Transcribe',+ exact: true,+ });+ await expect(contextMenuTranscribe).toBeVisible();+ await contextMenuTranscribe.click();+ await expect(page.getByRole('dialog')).toContainText(+ `Transcribe ${name}`,+ );+ await page.keyboard.press('Escape');+ }+ });+});diff --git a/packages/example/e2e/transcription-resampling.test.mts b/packages/example/e2e/transcription-resampling.test.mtsnew file mode 100644index 0000000000..b35e9af4c0--- /dev/null+++ b/packages/example/e2e/transcription-resampling.test.mts@@ -0,0 +1,141 @@+import {readFileSync} from 'fs';+import {createServer} from 'http';+import path from 'path';+import {expect, test} from '@playwright/test';+import {BundlerInternals} from '@remotion/bundler';+import {exampleDir} from './constants.mts';++// Playwright's open-source Chromium build omits the proprietary AAC decoder.+// Chrome exercises the same Chromium WebCodecs path with AAC enabled.+test.use({channel: 'chrome'});++test.describe('transcription media resampling', () => {+ test('decodes authenticated AAC media into mono 16kHz Float32 PCM', async ({+ page,+ }) => {+ const resamplerPath = path.resolve(+ exampleDir,+ '..',+ 'studio',+ 'src',+ 'components',+ 'Transcription',+ 'resample-media-to-16-khz.ts',+ );+ const result = await BundlerInternals.esbuild.build({+ bundle: true,+ entryPoints: [resamplerPath],+ format: 'iife',+ globalName: 'RemotionTranscriptionResamplingTest',+ platform: 'browser',+ write: false,+ });+ const browserBundle = result.outputFiles[0]?.text;+ if (!browserBundle) {+ throw new Error('Did not generate the browser resampling bundle.');+ }++ const media = readFileSync(+ path.join(exampleDir, 'public', 'demo_smpte_h264_aac.mp4'),+ );+ const server = createServer((request, response) => {+ if (request.url === '/') {+ response.setHeader('Content-Type', 'text/html');+ response.end('<!doctype html><title>Resampling test</title>');+ return;+ }++ if (request.url !== '/fixture.mp4') {+ response.statusCode = 404;+ response.end();+ return;+ }+ if (request.headers['x-remotion-test'] !== 'transcription') {+ response.statusCode = 401;+ response.end();+ return;+ }++ const range = request.headers.range?.match(/^bytes=(\d+)-(\d*)$/);+ const start = range ? Number(range[1]) : 0;+ const requestedEnd = range?.[2] ? Number(range[2]) : media.length - 1;+ const end = Math.min(requestedEnd, media.length - 1);+ response.statusCode = range ? 206 : 200;+ response.setHeader('Accept-Ranges', 'bytes');+ response.setHeader('Content-Type', 'video/mp4');+ response.setHeader('Content-Length', end - start + 1);+ if (range) {+ response.setHeader(+ 'Content-Range',+ `bytes ${start}-${end}/${media.length}`,+ );+ }++ response.end(+ request.method === 'HEAD' ? undefined : media.subarray(start, end + 1),+ );+ });+ await new Promise<void>((resolve) =>+ server.listen(0, '127.0.0.1', resolve),+ );++ try {+ const address = server.address();+ if (address === null || typeof address === 'string') {+ throw new Error('Could not determine the media server port.');+ }++ await page.goto(`http://127.0.0.1:${address.port}`);+ await page.addScriptTag({content: browserBundle});+ const waveform = await page.evaluate(async () => {+ const progress: number[] = [];+ const bundle = window as typeof window & {+ RemotionTranscriptionResamplingTest: {+ resampleMediaTo16Khz: (options: {+ src: string;+ audioStreamIndex: number | null;+ requestInit: Omit<RequestInit, 'signal'> | null;+ onProgress: (value: number) => void;+ }) => Promise<Float32Array>;+ };+ };+ const pcm =+ await bundle.RemotionTranscriptionResamplingTest.resampleMediaTo16Khz(+ {+ src: '/fixture.mp4',+ audioStreamIndex: null,+ requestInit: {+ headers: {'x-remotion-test': 'transcription'},+ },+ onProgress: (value) => progress.push(value),+ },+ );++ let peak = 0;+ for (const sample of pcm) {+ peak = Math.max(peak, Math.abs(sample));+ }++ return {+ isFloat32Array: pcm instanceof Float32Array,+ length: pcm.length,+ peak,+ progress,+ };+ });++ expect(waveform.isFloat32Array).toBe(true);+ // The fixture is 10 seconds long. Around 160,000 samples proves that the+ // stereo 48kHz AAC track was converted to mono PCM at 16kHz.+ expect(waveform.length).toBeGreaterThan(159_000);+ expect(waveform.length).toBeLessThan(161_000);+ expect(waveform.peak).toBeGreaterThan(0);+ expect(waveform.progress[0]).toBe(0);+ expect(waveform.progress.at(-1)).toBe(1);+ } finally {+ await new Promise<void>((resolve, reject) => {+ server.close((error) => (error ? reject(error) : resolve()));+ });+ }+ });+});diff --git a/packages/example/src/E2eTestRoot.tsx b/packages/example/src/E2eTestRoot.tsxindex 9a157a71ae..7af9e6564c 100644--- a/packages/example/src/E2eTestRoot.tsx+++ b/packages/example/src/E2eTestRoot.tsx@@ -23,6 +23,10 @@ import {RotationKeyframeE2e} from './RotationKeyframeE2e';import {SchemaTest, schemaTestSchema} from './SchemaTest';import {TimelineNegativeFromResize} from './TimelineNegativeFromResize';import {TimelineVirtualizationTestbed} from './TimelineVirtualizationTestbed';+import {+ TranscriptionAudioE2e,+ TranscriptionLegacyMediaE2e,+} from './TranscriptionE2e';import {VisualControls} from './VisualControls';import {VisualMode3D} from './VisualMode3D';import {AffineFrameClock} from './VisualModeTests/AffineFrameClock';@@ -264,6 +268,22 @@ export const E2eTestRoot: React.FC = () => {fps={30}durationInFrames={90}/>+ <Composition+ id="transcription-audio-e2e"+ component={TranscriptionAudioE2e}+ width={1080}+ height={1080}+ fps={30}+ durationInFrames={90}+ />+ <Composition+ id="transcription-legacy-media-e2e"+ component={TranscriptionLegacyMediaE2e}+ width={1080}+ height={1080}+ fps={30}+ durationInFrames={90}+ /><NewVideoComp /></>);diff --git a/packages/example/src/TranscriptionE2e.tsx b/packages/example/src/TranscriptionE2e.tsxnew file mode 100644index 0000000000..fdca2a71f0--- /dev/null+++ b/packages/example/src/TranscriptionE2e.tsx@@ -0,0 +1,16 @@+import {Audio as MediaAudio} from '@remotion/media';+import React from 'react';+import {Audio as LegacyAudio, staticFile, Video as LegacyVideo} from 'remotion';++export const TranscriptionAudioE2e: React.FC = () => {+ return <MediaAudio name="<Audio>" src={staticFile('chirp.wav')} />;+};++export const TranscriptionLegacyMediaE2e: React.FC = () => {+ return (+ <>+ <LegacyVideo name="vp8-vorbis.webm" src={staticFile('vp8-vorbis.webm')} />+ <LegacyAudio name="<Legacy Audio>" src={staticFile('chirp.wav')} />+ </>+ );+};diff --git a/packages/media/src/audio/audio.tsx b/packages/media/src/audio/audio.tsxindex 9f112992ce..f91ee4856c 100644--- a/packages/media/src/audio/audio.tsx+++ b/packages/media/src/audio/audio.tsx@@ -9,6 +9,7 @@ import {type SequenceControls,type InteractivitySchema,} from 'remotion';+import {resolveRequestInit} from '../request-init';import {getLoopDisplay} from '../show-in-timeline';import {validateToneFrequency} from '../validate-tone-frequency';import {AudioForPreview} from './audio-for-preview';@@ -85,6 +86,13 @@ const AudioInner: React.FC<const [mediaVolume] = Internals.useMediaVolumeState();const mediaStartsAt = Internals.useMediaStartsAt();const videoConfig = useVideoConfig();+ const [resolvedRequestInit] = useState(+ () =>+ resolveRequestInit({+ credentials: props.credentials,+ requestInit: props.requestInit,+ }) ?? null,+ );const sequenceDurationInFrames = Math.min(durationInFrames ?? Infinity,Math.max(0, videoConfig.durationInFrames - (from ?? 0)),@@ -103,6 +111,8 @@ const AudioInner: React.FC<mediaStartsAt,loop: props.loop ?? false,muted: props.muted ?? false,+ audioStreamIndex: props.audioStreamIndex ?? null,+ requestInit: resolvedRequestInit,});// TODO: Redundant with what we do in the Studiodiff --git a/packages/media/src/video/video.tsx b/packages/media/src/video/video.tsxindex 9046bf97ff..156e579655 100644--- a/packages/media/src/video/video.tsx+++ b/packages/media/src/video/video.tsx@@ -9,6 +9,7 @@ import {type SequenceControls,type InteractivitySchema,} from 'remotion';+import {resolveRequestInit} from '../request-init';import {getLoopDisplay} from '../show-in-timeline';import {validateToneFrequency} from '../validate-tone-frequency';import {getVideoSequenceDuration} from './get-video-sequence-duration';@@ -267,6 +268,9 @@ const VideoInner: React.FC<const [mediaVolume] = Internals.useMediaVolumeState();const mediaStartsAt = Internals.useMediaStartsAt();const videoConfig = useVideoConfig();+ const [resolvedRequestInit] = useState(+ () => resolveRequestInit({credentials, requestInit}) ?? null,+ );const sequenceDurationInFrames = Math.min(durationInFrames ?? Infinity,Math.max(0, videoConfig.durationInFrames - (from ?? 0)),@@ -292,6 +296,8 @@ const VideoInner: React.FC<mediaStartsAt,loop: loop ?? false,muted: muted ?? false,+ audioStreamIndex: audioStreamIndex ?? 0,+ requestInit: resolvedRequestInit,});// TODO: Redundant with what we do in the Studiodiff --git a/packages/studio-server/src/preview-server/api-routes.ts b/packages/studio-server/src/preview-server/api-routes.tsindex f47df69391..08b843350d 100644--- a/packages/studio-server/src/preview-server/api-routes.ts+++ b/packages/studio-server/src/preview-server/api-routes.ts@@ -27,7 +27,6 @@ import {duplicateJsxNodeHandler} from './routes/duplicate-jsx-node';import {findInFileHandler} from './routes/find-in-file';import {insertElementHandler} from './routes/insert-element';import {insertJsxElementHandler} from './routes/insert-jsx-element';-import {handleInstallPackage} from './routes/install-dependency';import {logStudioErrorHandler} from './routes/log-studio-error';import {moveKeyframesHandler} from './routes/move-keyframes';import {openInEditorHandler} from './routes/open-in-editor';@@ -67,7 +66,10 @@ import {updateElementInstallTargetHandler} from './routes/update-element-installimport {updateSequenceKeyframeSettingsHandler} from './routes/update-sequence-keyframe-settings';import {handleUpgradeRemotion} from './routes/upgrade-remotion';-type StandardApiRoute = Exclude<keyof ApiRoutes, '/api/update-config'>;+type StandardApiRoute = Exclude<+ keyof ApiRoutes,+ '/api/install-package' | '/api/update-config'+>;export const allApiRoutes: {[key in StandardApiRoute]: ApiHandler<@@ -131,7 +133,6 @@ export const allApiRoutes: {'/api/restart-studio': handleRestartStudio,'/api/default-coding-agent-info': getDefaultCodingAgentInfoHandler,'/api/default-editor-info': getDefaultEditorInfoHandler,- '/api/install-package': handleInstallPackage,'/api/insert-jsx-element': insertJsxElementHandler,'/api/insert-element': insertElementHandler,'/api/prepare-element-install': prepareElementInstallHandler,diff --git a/packages/studio-server/src/preview-server/routes/install-dependency.ts b/packages/studio-server/src/preview-server/routes/install-dependency.tsindex 35ecbd2a7e..2997d79c7c 100644--- a/packages/studio-server/src/preview-server/routes/install-dependency.ts+++ b/packages/studio-server/src/preview-server/routes/install-dependency.ts@@ -29,10 +29,14 @@ export const getPackageInstallSpec = (return extraVersion === null ? name : `${name}@${extraVersion}`;};-export const handleInstallPackage: ApiHandler<- InstallPackageRequest,- InstallPackageResponse-> = async ({logLevel, remotionRoot, input: {dependencies}}) => {+export const handleInstallPackage = async ({+ logLevel,+ remotionRoot,+ input: {dependencies},+ invalidateBundle,+}: Parameters<ApiHandler<InstallPackageRequest, InstallPackageResponse>>[0] & {+ readonly invalidateBundle: () => Promise<void>;+}): Promise<InstallPackageResponse> => {for (const dependency of dependencies) {if (!isValidPackageName(dependency.name)) {return Promise.reject(@@ -110,6 +114,7 @@ export const handleInstallPackage: ApiHandler<),);});+ await invalidateBundle();RenderInternals.Log.info({indent: false, logLevel},RenderInternals.chalk.gray('╰─ '),diff --git a/packages/studio-server/src/preview-server/start-server.ts b/packages/studio-server/src/preview-server/start-server.tsindex 1fcadbf57b..aabd44ca0a 100644--- a/packages/studio-server/src/preview-server/start-server.ts+++ b/packages/studio-server/src/preview-server/start-server.ts@@ -243,6 +243,25 @@ export const startServer = async (options: {setWatchIgnoreNextChangePlugin(watchIgnorePlugin);const wdmMiddleware = wdm(compiler, options.logLevel);+ const invalidateBundle = () => {+ return new Promise<void>((resolve, reject) => {+ const watching = compiler?.watching;+ if (!watching) {+ reject(new Error('The Studio bundle is not being watched.'));+ return;+ }++ watching.invalidate((error) => {+ if (error) {+ reject(error);+ return;+ }++ resolve();+ });+ });+ };+const liveEventsServer = makeLiveEventsRouter(options.logLevel, () => {const undoStack = getUndoStack();const redoStack = getRedoStack();@@ -304,6 +323,7 @@ export const startServer = async (options: {getDefaultCodingAgent: options.getDefaultCodingAgent,getDefaultEditor: options.getDefaultEditor,configFile: options.configFile,+ invalidateBundle,});}).catch((err) => {diff --git a/packages/studio-server/src/routes.ts b/packages/studio-server/src/routes.tsindex 00d79d4c82..d6f53eb9f3 100644--- a/packages/studio-server/src/routes.ts+++ b/packages/studio-server/src/routes.ts@@ -17,6 +17,8 @@ import {RenderInternals} from '@remotion/renderer';import type {ApiRoutes,GitSource,+ InstallPackageRequest,+ InstallPackageResponse,RenderDefaults,RenderJob,StudioRuntimeConfig,@@ -38,6 +40,7 @@ import {handleRequest} from './preview-server/handler';import type {LiveEventsServer} from './preview-server/live-events';import {fetchFolder, getFiles} from './preview-server/public-folder';import {handleAppIcon} from './preview-server/routes/app-icon';+import {handleInstallPackage} from './preview-server/routes/install-dependency';import {getEditorName} from './preview-server/routes/open-in-editor';import {updateConfigHandler} from './preview-server/routes/update-config';import {serveStatic} from './preview-server/serve-static';@@ -185,6 +188,7 @@ const handleFallback = async ({response.end(BundlerInternals.indexHtml({+ importMap: null,staticHash: hash,publicPath: '/',editorName: displayName,@@ -431,6 +435,7 @@ export const handleRoutes = ({getDefaultCodingAgent,getDefaultEditor,configFile,+ invalidateBundle,}: {staticHash: string;staticHashPrefix: string;@@ -459,6 +464,7 @@ export const handleRoutes = ({getDefaultCodingAgent: () => DefaultCodingAgent | null;getDefaultEditor: () => DefaultEditor | null;configFile: string | null;+ invalidateBundle: () => Promise<void>;}): Promise<void> => {const url = new URL(request.url as string, 'http://localhost');@@ -573,6 +579,23 @@ export const handleRoutes = ({});}+ if (url.pathname === '/api/install-package') {+ return handleRequest<InstallPackageRequest, InstallPackageResponse>({+ remotionRoot,+ entryPoint,+ handler: (params) => handleInstallPackage({...params, invalidateBundle}),+ request,+ response,+ logLevel,+ methods,+ binariesDirectory,+ publicDir,+ configFile,+ getDefaultCodingAgent,+ getDefaultEditor,+ });+ }+for (const [key, value] of Object.entries(allApiRoutes)) {if (url.pathname === key) {return handleRequest({diff --git a/packages/studio-server/src/test/file-source-route.test.ts b/packages/studio-server/src/test/file-source-route.test.tsindex d69fbb17c0..0e833b0d7f 100644--- a/packages/studio-server/src/test/file-source-route.test.ts+++ b/packages/studio-server/src/test/file-source-route.test.ts@@ -77,6 +77,7 @@ test('serves file source from an origin-less GET request', async () => {getCurrentInputProps: () => ({}),getDefaultCodingAgent: () => null,getDefaultEditor: () => null,+ invalidateBundle: () => Promise.resolve(),getEnvVariables: () => ({}),getRenderDefaults: () => ({}) as RenderDefaults,getRenderQueue: () => [],diff --git a/packages/studio-server/src/test/install-dependency.test.ts b/packages/studio-server/src/test/install-dependency.test.tsindex 8d8cf94207..c7e0fba416 100644--- a/packages/studio-server/src/test/install-dependency.test.ts+++ b/packages/studio-server/src/test/install-dependency.test.ts@@ -98,6 +98,7 @@ test('installs without running dependency lifecycle scripts', async () => {nub: 'nub.lock',};const temporaryDirectories: string[] = [];+ let invalidations = 0;try {for (const manager of Object.keys(lockfiles) as PackageManager[]) {@@ -118,6 +119,10 @@ test('installs without running dependency lifecycle scripts', async () => {dependencies: [{name: 'lodash', version: '4.17.21'}],},logLevel: 'error',+ invalidateBundle: () => {+ invalidations++;+ return Promise.resolve();+ },methods: {addJob: () => undefined,cancelJob: () => undefined,@@ -143,6 +148,8 @@ test('installs without running dependency lifecycle scripts', async () => {} else {expect(call.args).toContain('--ignore-scripts');}++ expect(invalidations).toBe(temporaryDirectories.length);}} finally {spawnSpy.mockRestore();diff --git a/packages/studio-server/src/test/upload-output-security.test.ts b/packages/studio-server/src/test/upload-output-security.test.tsindex cd14a87570..b37ff0d7db 100644--- a/packages/studio-server/src/test/upload-output-security.test.ts+++ b/packages/studio-server/src/test/upload-output-security.test.ts@@ -80,6 +80,7 @@ const requestWriteRoute = async ({getCurrentInputProps: () => ({}),getDefaultCodingAgent: () => null,getDefaultEditor: () => null,+ invalidateBundle: () => Promise.resolve(),getEnvVariables: () => ({}),getExperimentalKeepAudioContextAlive: () => false,getNumberOfAudioTags: () => 0,diff --git a/packages/studio-shared/src/studio-html.ts b/packages/studio-shared/src/studio-html.tsindex d02a2fbbe7..1e11634b49 100644--- a/packages/studio-shared/src/studio-html.ts+++ b/packages/studio-shared/src/studio-html.ts@@ -33,6 +33,7 @@ export type StudioHtmlOptions = {mode: 'dev' | 'bundle';bundleScriptUrl?: string;bundleScriptType?: 'classic' | 'module';+ importMap: Record<string, string> | null;readOnlyStudio?: boolean;studioRuntimeConfig?: StudioRuntimeConfig;};@@ -65,6 +66,7 @@ export const studioHtml = ({mode,bundleScriptUrl,bundleScriptType,+ importMap,readOnlyStudio,studioRuntimeConfig,}: StudioHtmlOptions) => {@@ -87,6 +89,11 @@ export const studioHtml = ({<head><meta charset="UTF-8" /><meta name="viewport" content="width=device-width, initial-scale=1.0" />+ ${+ importMap+ ? `<script type="importmap">${JSON.stringify({imports: importMap}).replaceAll('<', '\\u003c')}</script>`+ : ''+ }${includeFavicon? `<link id="__remotion_favicon" rel="icon" type="image/png" href="${publicPath}favicon.ico" />`diff --git a/packages/studio-shared/src/test/studio-html.test.ts b/packages/studio-shared/src/test/studio-html.test.tsindex 4d06235b9d..a71f25666a 100644--- a/packages/studio-shared/src/test/studio-html.test.ts+++ b/packages/studio-shared/src/test/studio-html.test.ts@@ -3,16 +3,19 @@ import {studioHtml} from '../studio-html';const makeHtml = ({bundleScriptType,+ importMap,publicPath,staticHash,publicFolderExists,}: {bundleScriptType?: 'module';+ importMap: Record<string, string> | null;publicPath: string;staticHash: string;publicFolderExists: string;}) => {return studioHtml({+ importMap,publicPath,staticHash,editorName: null,@@ -49,6 +52,7 @@ const makeHtml = ({test('makes relative bundles resolve public assets from the document URL', () => {const html = makeHtml({+ importMap: null,publicPath: './',staticHash: './public',publicFolderExists: './public',@@ -77,6 +81,7 @@ test('makes relative bundles resolve public assets from the document URL', () =>test('preserves explicitly absolute bundle paths', () => {const html = makeHtml({+ importMap: null,publicPath: '/sites/alpha/',staticHash: '/sites/alpha/public',publicFolderExists: '/sites/alpha/public',@@ -94,6 +99,7 @@ test('preserves explicitly absolute bundle paths', () => {test('marks an explicitly requested module bundle', () => {const html = makeHtml({bundleScriptType: 'module',+ importMap: null,publicPath: './',staticHash: './public',publicFolderExists: './public',@@ -101,3 +107,22 @@ test('marks an explicitly requested module bundle', () => {expect(html).toContain('<script type="module" src="./bundle.js"></script>');});++test('places an escaped import map before the bundle script', () => {+ const html = makeHtml({+ bundleScriptType: 'module',+ importMap: {+ '@huggingface/transformers':+ 'https://example.com/transformers.mjs?</script>',+ },+ publicPath: './',+ staticHash: './public',+ publicFolderExists: './public',+ });+ const importMap =+ '<script type="importmap">{"imports":{"@huggingface/transformers":"https://example.com/transformers.mjs?\\u003c/script>"}}</script>';+ const bundleScript = '<script type="module" src="./bundle.js"></script>';++ expect(html).toContain(importMap);+ expect(html.indexOf(importMap)).toBeLessThan(html.indexOf(bundleScript));+});diff --git a/packages/studio/bundle.ts b/packages/studio/bundle.tsindex 4d4b10fad8..10b9d7c012 100644--- a/packages/studio/bundle.ts+++ b/packages/studio/bundle.ts@@ -15,6 +15,7 @@ const external = ['@remotion/renderer/client','@remotion/renderer/pure','@remotion/web-renderer',+ '@remotion/whisper-webgpu','@remotion/renderer/error-handling','@jridgewell/trace-mapping','zod',@@ -42,10 +43,12 @@ await buildPackage({{path: 'src/internals.ts',target: 'browser',+ splitting: true,},{path: 'src/previewEntry.tsx',target: 'browser',+ splitting: true,},],});diff --git a/packages/studio/package.json b/packages/studio/package.jsonindex a4fbb63aa3..2652d6b57d 100644--- a/packages/studio/package.json+++ b/packages/studio/package.json@@ -20,9 +20,15 @@"url": "https://github.com/remotion-dev/remotion/issues"},"peerDependencies": {+ "@remotion/whisper-webgpu": "workspace:*","react": ">=16.8.0","react-dom": ">=16.8.0"},+ "peerDependenciesMeta": {+ "@remotion/whisper-webgpu": {+ "optional": true+ }+ },"dependencies": {"@tanstack/react-virtual": "3.14.9","@remotion/canvas": "workspace:*",@@ -47,6 +53,7 @@},"devDependencies": {"@happy-dom/global-registrator": "14.5.1",+ "@remotion/whisper-webgpu": "workspace:*","react": "catalog:","react-dom": "catalog:","@testing-library/react": "16.1.0",diff --git a/packages/studio/src/components/Checkbox.tsx b/packages/studio/src/components/Checkbox.tsxindex b315b5635a..b29be5c4a9 100644--- a/packages/studio/src/components/Checkbox.tsx+++ b/packages/studio/src/components/Checkbox.tsx@@ -23,9 +23,10 @@ export const Checkbox: React.FC<{readonly checked: boolean;readonly onChange: React.ChangeEventHandler<HTMLInputElement>;readonly name: string;+ readonly inputId?: string;readonly disabled?: boolean;readonly variant?: CheckboxVariant;-}> = ({checked, onChange, disabled, name, variant = 'default'}) => {+}> = ({checked, onChange, disabled, inputId, name, variant = 'default'}) => {const size = SIZES[variant];const checkmarkSize = CHECKMARK_SIZES[variant];@@ -71,6 +72,7 @@ export const Checkbox: React.FC<{return (<div style={background}><input+ id={inputId}style={input}type={'checkbox'}checked={checked}diff --git a/packages/studio/src/components/EditorContexts.tsx b/packages/studio/src/components/EditorContexts.tsxindex 4fe4b1a165..0999b4a695 100644--- a/packages/studio/src/components/EditorContexts.tsx+++ b/packages/studio/src/components/EditorContexts.tsx@@ -12,6 +12,7 @@ import {ExpandedTracksProvider} from './ExpandedTracksProvider';import {ZodProvider} from './get-zod-if-possible';import {MediaVolumeProvider} from './MediaVolumeProvider';import {ModalsProvider} from './ModalsProvider';+import {CaptionQueueProcessorLoader} from './RenderQueue/CaptionQueueProcessorLoader';import {ClientRenderQueueProcessor} from './RenderQueue/ClientRenderQueueProcessor';import {RenderQueueContextProvider} from './RenderQueue/context';import {SetTimelineInOutProvider} from './SetTimelineInOutProvider';@@ -35,6 +36,7 @@ export const EditorContexts: React.FC<{<VisualControlsUndoSync /><RenderQueueContextProvider><ClientRenderQueueProcessor />+ <CaptionQueueProcessorLoader /><KeybindingContextProvider><CheckerboardProvider><ZoomGesturesProvider>diff --git a/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx b/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsxindex 4fec352996..3235c1f29f 100644--- a/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx+++ b/packages/studio/src/components/InspectorPanel/SequenceSelectionInspector.tsx@@ -7,7 +7,9 @@ import {AudioIcon} from '../../icons/audio';import {DuplicateIcon} from '../../icons/duplicate';import {ScissorsIcon} from '../../icons/scissors';import {SnowflakeIcon} from '../../icons/snowflake';+import {TranscriptionIcon} from '../../icons/transcription';import {TrashIcon} from '../../icons/trash';+import {SetSelectedModalContext} from '../../state/modals';import {useConfirmationDialog} from '../ConfirmationDialog';import {hasSequenceControls,@@ -28,11 +30,12 @@ import {} from '../Timeline/TimelineSelection';import {useDeleteTimelineItems} from '../Timeline/use-delete-timeline-items';import {getSequenceFreezeFrameMenuItem} from '../Timeline/use-sequence-freeze-frame-menu-item';+import {isTranscribableSequence} from '../Transcription/is-transcribable-sequence';import {AlignmentControls} from './AlignmentControls';import {- InspectorQuickActionsSection,- InspectorQuickAction,InspectorMessage,+ InspectorQuickAction,+ InspectorQuickActionsSection,InspectorSectionHeader,} from './common';import {@@ -129,8 +132,12 @@ const SequenceSourceQuickActions: React.FC<{const {previewServerState} = useContext(StudioServerConnectionCtx);const {propStatuses} = useContext(Internals.VisualModePropStatusesContext);const {setPropStatuses} = useContext(Internals.VisualModeSettersContext);+ const {setSelectedModal} = useContext(SetSelectedModalContext);const confirm = useConfirmationDialog();const deleteTimelineItems = useDeleteTimelineItems();+ const transcribableMedia = isTranscribableSequence(track.sequence)+ ? track.sequence+ : null;const propStatusesForOverride = useMemo(() =>Internals.getPropStatusesCtx(@@ -154,6 +161,25 @@ const SequenceSourceQuickActions: React.FC<{});const sourceActionsDisabled =previewServerState.type !== 'connected' || !isStudioInteractivityEnabled();+ const transcribeDisabledReason =+ previewServerState.type !== 'connected'+ ? 'Transcription requires a Studio server connection'+ : !isStudioInteractivityEnabled()+ ? 'Transcription is unavailable in read-only Studio'+ : undefined;+ const onTranscribe = useCallback(() => {+ if (sourceActionsDisabled || transcribableMedia === null) {+ return;+ }++ setSelectedModal({+ type: 'transcribe',+ src: transcribableMedia.src,+ displayName: transcribableMedia.displayName,+ audioStreamIndex: transcribableMedia.audioStreamIndex,+ requestInit: transcribableMedia.requestInit,+ });+ }, [setSelectedModal, sourceActionsDisabled, transcribableMedia]);const onDuplicate = useCallback(() => {if (sourceActionsDisabled) {return;@@ -197,6 +223,18 @@ const SequenceSourceQuickActions: React.FC<{return (<>+ {transcribableMedia ? (+ <InspectorQuickAction+ disabled={sourceActionsDisabled}+ onClick={onTranscribe}+ title={transcribeDisabledReason}+ renderIcon={(color) => (+ <TranscriptionIcon style={actionIconStyle} color={color} />+ )}+ >+ Transcribe+ </InspectorQuickAction>+ ) : null}{freezeFrameMenuItem?.type === 'item' ? (<InspectorQuickActiondisabled={Boolean(freezeFrameMenuItem.disabled)}diff --git a/packages/studio/src/components/ModalContainer.tsx b/packages/studio/src/components/ModalContainer.tsxindex 0a8a252bc7..1be627f0a9 100644--- a/packages/studio/src/components/ModalContainer.tsx+++ b/packages/studio/src/components/ModalContainer.tsx@@ -39,6 +39,7 @@ const panel: React.CSSProperties = {};export const ModalContainer: React.FC<{+ readonly ariaLabel?: string;readonly ariaLabelledBy?: string;readonly onEscape: () => void;readonly onOutsideClick: () => void;@@ -46,6 +47,7 @@ export const ModalContainer: React.FC<{readonly noZIndex?: boolean;readonly panelStyle?: React.CSSProperties;}> = ({+ ariaLabel,ariaLabelledBy,children,onEscape,@@ -60,6 +62,7 @@ export const ModalContainer: React.FC<{return (<div+ aria-label={ariaLabel}aria-labelledby={ariaLabelledBy ?? undefined}aria-modal="true"className="css-reset"diff --git a/packages/studio/src/components/Modals.tsx b/packages/studio/src/components/Modals.tsxindex fc31627bba..968d9f2c5c 100644--- a/packages/studio/src/components/Modals.tsx+++ b/packages/studio/src/components/Modals.tsx@@ -29,6 +29,7 @@ import {RenderModalWithLoader} from './RenderModal/ServerRenderModal';import {WebRenderModalWithLoader} from './RenderModal/WebRenderModal';import {SettingsModal} from './SettingsModal';import {SvgImportDialog} from './SvgImportDialog';+import {TranscriptionModalWithOptionalWhisper} from './Transcription/TranscriptionModalWithOptionalWhisper';export const Modals: React.FC<{readonly readOnlyStudio: boolean;@@ -259,6 +260,9 @@ export const Modals: React.FC<{{modalContextType && modalContextType.type === 'render-progress' && (<RenderStatusModal jobId={modalContextType.jobId} />)}+ {modalContextType && modalContextType.type === 'transcribe' ? (+ <TranscriptionModalWithOptionalWhisper state={modalContextType} />+ ) : null}{modalContextType && modalContextType.type === 'fix-computed-value' && (<FixComputedValueModal state={modalContextType} />diff --git a/packages/studio/src/components/NewComposition/ComboBox.tsx b/packages/studio/src/components/NewComposition/ComboBox.tsxindex 92d8f098ab..167ba02143 100644--- a/packages/studio/src/components/NewComposition/ComboBox.tsx+++ b/packages/studio/src/components/NewComposition/ComboBox.tsx@@ -116,6 +116,7 @@ export const Combobox: React.FC<{readonly title: string;readonly size?: ComboboxSize;readonly unhoveredIconColor?: string;+ readonly disabled?: boolean;}> = ({values,selectedId,@@ -123,6 +124,7 @@ export const Combobox: React.FC<{title,size: controlSize = 'default',unhoveredIconColor = LIGHT_TEXT,+ disabled = false,}) => {const [hovered, setIsHovered] = useState(false);const [opened, setOpened] = useState(false);@@ -294,8 +296,9 @@ export const Combobox: React.FC<{: hovered? WHITE_ALPHA_05: BLACK_ALPHA_60,+ opacity: disabled ? 0.5 : 1,};- }, [controlSize, customStyle, hovered, opened]);+ }, [controlSize, customStyle, disabled, hovered, opened]);const selectedLabelStyle =controlSize === 'small'@@ -309,6 +312,7 @@ export const Combobox: React.FC<{<buttonref={ref}title={title}+ disabled={disabled}tabIndex={tabIndex}type="button"style={style}diff --git a/packages/studio/src/components/NewComposition/DismissableModal.tsx b/packages/studio/src/components/NewComposition/DismissableModal.tsxindex 2610ff0d59..5cee08b8ac 100644--- a/packages/studio/src/components/NewComposition/DismissableModal.tsx+++ b/packages/studio/src/components/NewComposition/DismissableModal.tsx@@ -5,7 +5,8 @@ import {ModalContainer} from '../ModalContainer';export const DismissableModal: React.FC<{readonly children: React.ReactNode;readonly panelStyle?: React.CSSProperties;-}> = ({children, panelStyle}) => {+ readonly ariaLabel?: string;+}> = ({children, panelStyle, ariaLabel}) => {const {setSelectedModal} = useContext(SetSelectedModalContext);const onQuit = useCallback(() => {@@ -14,6 +15,7 @@ export const DismissableModal: React.FC<{return (<ModalContainer+ ariaLabel={ariaLabel}onOutsideClick={onQuit}onEscape={onQuit}panelStyle={panelStyle}diff --git a/packages/studio/src/components/RenderModal/NumberSetting.tsx b/packages/studio/src/components/RenderModal/NumberSetting.tsxindex 12b1d1258a..e79cbf4bf2 100644--- a/packages/studio/src/components/RenderModal/NumberSetting.tsx+++ b/packages/studio/src/components/RenderModal/NumberSetting.tsx@@ -3,9 +3,17 @@ import React, {useCallback} from 'react';import {Spacing} from '../layout';import {InputDragger} from '../NewComposition/InputDragger';import {RightAlignInput} from '../NewComposition/RemInput';+import {InfoBubble} from './InfoBubble';import {label, optionRow, rightRow} from './layout';import {OptionExplainerBubble} from './OptionExplainerBubble';+type NumberSettingHint =+ | AvailableOptions+ | {+ readonly content: React.ReactNode;+ readonly title: string;+ };+export const NumberSetting: React.FC<{readonly name: string;readonly value: number;@@ -14,7 +22,7 @@ export const NumberSetting: React.FC<{readonly min: number;readonly step: number;readonly formatter?: (value: string | number) => string;- readonly hint?: AvailableOptions;+ readonly hint?: NumberSettingHint;}> = ({name, value, step, hint, onValueChanged, max, min, formatter}) => {const onTextChanged = useCallback((e: string) => {@@ -44,7 +52,11 @@ export const NumberSetting: React.FC<{{hint ? (<><Spacing x={0.5} />- <OptionExplainerBubble id={hint} />+ {typeof hint === 'string' ? (+ <OptionExplainerBubble id={hint} />+ ) : (+ <InfoBubble title={hint.title}>{hint.content}</InfoBubble>+ )}</>) : null}</div>@@ -53,6 +65,7 @@ export const NumberSetting: React.FC<{<InputDraggervalue={value}name={name.toLowerCase()}+ aria-label={`${name}: ${formatter ? formatter(value) : value}`}onTextChange={onTextChanged}onValueChange={onValueChange}step={step}diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueItem.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueItem.tsxnew file mode 100644index 0000000000..e659490f79--- /dev/null+++ b/packages/studio/src/components/RenderQueue/CaptionQueueItem.tsx@@ -0,0 +1,235 @@+import React, {+ useCallback,+ useContext,+ useEffect,+ useMemo,+ useState,+} from 'react';+import {+ CURRENT_COLOR,+ FAIL_COLOR,+ getBackgroundFromHoverState,+ LIGHT_TEXT,+} from '../../helpers/colors';+import {pushUrl} from '../../helpers/url-state';+import type {RenderInlineAction} from '../InlineAction';+import {InlineAction} from '../InlineAction';+import {Row, Spacing} from '../layout';+import {useSelectAsset} from '../use-select-asset';+import type {CaptionJob} from './caption-job-types';+import {+ CircularProgress,+ RENDER_STATUS_INDICATOR_SIZE,+} from './CircularProgress';+import {RenderQueueContext} from './context';+import {+ renderQueueItemSubtitleStyle,+ RENDER_QUEUE_ITEM_SELECTED_CLASSNAME,+} from './item-style';+import {SuccessIcon} from './SuccessIcon';++const container: React.CSSProperties = {+ padding: 12,+ display: 'flex',+ flexDirection: 'row',+ paddingBottom: 10,+ paddingRight: 4,+ marginBottom: 1,+ marginLeft: 4,+ borderRadius: 4,+ width: 'calc(100% - 4px)',+};++const title: React.CSSProperties = {+ fontSize: 13,+ lineHeight: 1,+};++const right: React.CSSProperties = {+ flex: 1,+ display: 'flex',+ flexDirection: 'column',+ overflow: 'hidden',+};++const subtitleContainer: React.CSSProperties = {+ maxWidth: '100%',+ flex: 1,+ display: 'flex',+ overflow: 'hidden',+};++const subtitle: React.CSSProperties = {+ ...renderQueueItemSubtitleStyle,+ cursor: 'default',+};++const statusIcon: React.CSSProperties = {+ height: RENDER_STATUS_INDICATOR_SIZE,+ width: RENDER_STATUS_INDICATOR_SIZE,+};++const removeIcon: React.CSSProperties = {+ height: 16,+ color: CURRENT_COLOR,+};++const CaptionJobStatus: React.FC<{readonly job: CaptionJob}> = ({job}) => {+ if (job.status === 'running') {+ return (+ <div+ style={statusIcon}+ role="progressbar"+ aria-label="Caption job progress"+ aria-valuetext={job.progress.message}+ aria-valuemin={0}+ aria-valuemax={100}+ aria-valuenow={Math.round(job.progress.value * 100)}+ >+ <CircularProgress progress={Math.max(0.07, job.progress.value)} />+ </div>+ );+ }++ if (job.status === 'done') {+ return <SuccessIcon />;+ }++ if (job.status === 'failed') {+ return (+ <svg style={statusIcon} viewBox="0 0 512 512">+ <path+ fill={FAIL_COLOR}+ d="M0 160V352L160 512H352L512 352V160L352 0H160L0 160zm353.9 32l-17 17-47 47 47 47 17 17L320 353.9l-17-17-47-47-47 47-17 17L158.1 320l17-17 47-47-47-47-17-17L192 158.1l17 17 47 47 47-47 17-17L353.9 192z"+ />+ </svg>+ );+ }++ return (+ <svg style={statusIcon} viewBox="0 0 512 512">+ <path+ fill={LIGHT_TEXT}+ d="M256 512C114.6 512 0 397.4 0 256S114.6 0 256 0S512 114.6 512 256s-114.6 256-256 256zM232 120V256c0 8 4 15.5 10.7 20l96 64c11 7.4 25.9 4.4 33.3-6.7s4.4-25.9-6.7-33.3L280 243.2V120c0-13.3-10.7-24-24-24s-24 10.7-24 24z"+ />+ </svg>+ );+};++export const CaptionQueueItem: React.FC<{+ readonly job: CaptionJob;+ readonly selected: boolean;+}> = ({job, selected}) => {+ const [hovered, setHovered] = useState(false);+ const {removeCaptionJob} = useContext(RenderQueueContext);+ const selectAsset = useSelectAsset();+ const isHoverable = job.status === 'done';++ const containerStyle: React.CSSProperties = useMemo(() => {+ return {+ ...container,+ backgroundColor: getBackgroundFromHoverState({+ hovered: isHoverable && hovered,+ selected,+ }),+ userSelect: 'none',+ WebkitUserSelect: 'none',+ };+ }, [hovered, isHoverable, selected]);++ const onPointerEnter = useCallback(() => {+ setHovered(true);+ }, []);++ const onPointerLeave = useCallback(() => {+ setHovered(false);+ }, []);++ const scrollCurrentIntoView = useCallback(() => {+ document+ .querySelector(`.${RENDER_QUEUE_ITEM_SELECTED_CLASSNAME}`)+ ?.scrollIntoView({behavior: 'smooth'});+ }, []);++ const onClick = useCallback(() => {+ if (job.status !== 'done') {+ return;+ }++ selectAsset(job.outName);+ pushUrl(`/assets/${job.outName}`);+ }, [job, selectAsset]);++ useEffect(() => {+ if (selected) {+ scrollCurrentIntoView();+ }+ }, [scrollCurrentIntoView, selected]);++ const onRemove: React.MouseEventHandler = useCallback(+ (event) => {+ event.stopPropagation();+ removeCaptionJob(job.id);+ },+ [job.id, removeCaptionJob],+ );++ const renderRemoveAction: RenderInlineAction = useCallback((color) => {+ return (+ <svg style={removeIcon} viewBox="0 0 320 512">+ <path+ fill={color}+ d="M310.6 150.6c12.5-12.5 12.5-32.8 0-45.3s-32.8-12.5-45.3 0L160 210.7 54.6 105.4c-12.5-12.5-32.8-12.5-45.3 0s-12.5 32.8 0 45.3L114.7 256 9.4 361.4c-12.5 12.5-12.5 32.8 0 45.3s32.8 12.5 45.3 0L160 301.3 265.4 406.6c12.5 12.5 32.8 12.5 45.3 0s12.5-32.8 0-45.3L205.3 256 310.6 150.6z"+ />+ </svg>+ );+ }, []);++ const message = useMemo(() => {+ if (job.status === 'idle') {+ return 'Queued for transcription';+ }++ if (job.status === 'running') {+ return job.progress.message;+ }++ if (job.status === 'failed') {+ return job.error.message;+ }++ return job.outName;+ }, [job]);++ return (+ <Row+ data-render-queue-item={job.id}+ style={containerStyle}+ align="center"+ onClick={onClick}+ onPointerEnter={onPointerEnter}+ onPointerLeave={onPointerLeave}+ className={selected ? RENDER_QUEUE_ITEM_SELECTED_CLASSNAME : undefined}+ >+ <CaptionJobStatus job={job} />+ <Spacing x={1} />+ <div style={right}>+ <div style={title}>{job.displayName}</div>+ <div style={subtitleContainer}>+ <span style={subtitle} title={message}>+ {message}+ </span>+ </div>+ </div>+ <Spacing x={1} />+ {job.status === 'running' ? null : (+ <InlineAction+ renderAction={renderRemoveAction}+ onClick={onRemove}+ title="Remove"+ variant={null}+ />+ )}+ </Row>+ );+};diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsxnew file mode 100644index 0000000000..09ed7cda8d--- /dev/null+++ b/packages/studio/src/components/RenderQueue/CaptionQueueProcessor.tsx@@ -0,0 +1,143 @@+import {formatBytes} from '@remotion/studio-shared';+import {+ canUseWhisperWebGpu,+ clearStaleModels,+ disposeWhisperModel,+ isWhisperModelCached,+ loadWhisperModel,+ toCaptions,+ transcribe,+} from '@remotion/whisper-webgpu';+import {useCallback, useContext, useEffect} from 'react';+import {writeStaticFile} from '../../api/write-static-file';+import {resampleMediaTo16Khz} from '../Transcription/resample-media-to-16-khz';+import type {CaptionJob} from './caption-job-types';+import {RenderQueueContext} from './context';++export const CaptionQueueProcessor: React.FC = () => {+ const {+ updateCaptionJobProgress,+ markCaptionJobDone,+ markCaptionJobFailed,+ setProcessCaptionJobCallback,+ } = useContext(RenderQueueContext);++ const processJob = useCallback(+ async (job: CaptionJob) => {+ let captionCount: number | null = null;+ let processingError: Error | null = null;+ try {+ updateCaptionJobProgress(job.id, {+ message: 'Checking WebGPU support...',+ value: 0,+ });+ const support = await canUseWhisperWebGpu();+ if (!support.supported) {+ throw new Error(support.detailedReason);+ }++ updateCaptionJobProgress(job.id, {+ message: `Checking ${job.model}...`,+ value: 0.02,+ });+ await clearStaleModels();+ const modelIsCached = await isWhisperModelCached({model: job.model});+ await loadWhisperModel({+ model: job.model,+ onProgress: (progress) => {+ const percentage =+ progress.progress === null+ ? null+ : `${Math.round(progress.progress * 100)}%`;+ const bytes =+ progress.loadedBytes === null || progress.totalBytes === null+ ? null+ : `${formatBytes(progress.loadedBytes)} / ${formatBytes(progress.totalBytes)}`;+ updateCaptionJobProgress(job.id, {+ message: [+ `${modelIsCached ? 'Loading' : 'Downloading'} ${job.model}…`,+ percentage,+ bytes,+ ]+ .filter(Boolean)+ .join(' · '),+ value: 0.03 + (progress.progress ?? 0) * 0.27,+ });+ },+ });++ updateCaptionJobProgress(job.id, {+ message: `Preparing ${job.displayName}...`,+ value: 0.3,+ });+ const channelWaveform = await resampleMediaTo16Khz({+ src: job.src,+ audioStreamIndex: job.audioStreamIndex,+ requestInit: job.requestInit,+ onProgress: (progress) => {+ updateCaptionJobProgress(job.id, {+ message: `Preparing ${job.displayName}...`,+ value: 0.3 + progress * 0.2,+ });+ },+ });++ updateCaptionJobProgress(job.id, {+ message: `Transcribing ${job.displayName}...`,+ value: 0.5,+ });+ const transcription = await transcribe({+ channelWaveform,+ model: job.model,+ task: job.task,+ chunkLengthInSeconds: job.chunkLengthInSeconds,+ strideLengthInSeconds: job.strideLengthInSeconds,+ forceFullSequences: job.forceFullSequences,+ doSample: job.doSample,+ temperature: job.temperature,+ topK: job.topK,+ repetitionPenalty: job.repetitionPenalty,+ noRepeatNgramSize: job.noRepeatNgramSize,+ ...(job.language === null ? {} : {language: job.language}),+ });+ const {captions} = toCaptions({+ whisperWebGpuOutput: transcription,+ });++ updateCaptionJobProgress(job.id, {+ message: `Saving ${job.outName}...`,+ value: 0.95,+ });+ await writeStaticFile({+ contents: JSON.stringify(captions, null, 2),+ filePath: job.outName,+ });+ captionCount = captions.length;+ } catch (error) {+ processingError =+ error instanceof Error ? error : new Error(String(error));+ }++ try {+ await disposeWhisperModel({model: job.model});+ } catch {+ // The pipeline is removed from the in-memory registry before disposal.+ // A cleanup failure should not hide a successfully written caption file.+ }++ if (processingError) {+ markCaptionJobFailed(job.id, processingError);+ } else {+ markCaptionJobDone(job.id, captionCount ?? 0);+ }+ },+ [markCaptionJobDone, markCaptionJobFailed, updateCaptionJobProgress],+ );++ useEffect(() => {+ setProcessCaptionJobCallback(processJob);+ return () => setProcessCaptionJobCallback(null);+ }, [processJob, setProcessCaptionJobCallback]);++ return null;+};diff --git a/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx b/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsxnew file mode 100644index 0000000000..6f8ba1753e--- /dev/null+++ b/packages/studio/src/components/RenderQueue/CaptionQueueProcessorLoader.tsx@@ -0,0 +1,21 @@+import React, {Suspense, useContext} from 'react';+import {isWhisperWebGpuInstalled} from '../Transcription/whisper-webgpu-capability';+import {RenderQueueContext} from './context';++const LazyCaptionQueueProcessor = React.lazy(async () => {+ const {CaptionQueueProcessor} = await import('./CaptionQueueProcessor');+ return {default: CaptionQueueProcessor};+});++export const CaptionQueueProcessorLoader: React.FC = () => {+ const {captionJobs} = useContext(RenderQueueContext);+ if (!isWhisperWebGpuInstalled() || captionJobs.length === 0) {+ return null;+ }++ return (+ <Suspense fallback={null}>+ <LazyCaptionQueueProcessor />+ </Suspense>+ );+};diff --git a/packages/studio/src/components/RenderQueue/RenderQueueItem.tsx b/packages/studio/src/components/RenderQueue/RenderQueueItem.tsxindex cd2b20e127..5d3f3c89b7 100644--- a/packages/studio/src/components/RenderQueue/RenderQueueItem.tsx+++ b/packages/studio/src/components/RenderQueue/RenderQueueItem.tsx@@ -16,6 +16,7 @@ import type {ClientRenderJob} from './client-side-render-types';import {isRestoredClientJob} from './client-side-render-types';import type {AnyRenderJob} from './context';import {isClientRenderJob} from './context';+import {RENDER_QUEUE_ITEM_SELECTED_CLASSNAME} from './item-style';import {RenderQueueCancelledMessage} from './RenderQueueCancelledMessage';import {RenderQueueCopyToClipboard,@@ -64,8 +65,6 @@ const subtitle: React.CSSProperties = {overflow: 'hidden',};-const SELECTED_CLASSNAME = '__remotion_selected_classname';-export const RenderQueueItem: React.FC<{readonly job: AnyRenderJob;readonly selected: boolean;@@ -104,7 +103,7 @@ export const RenderQueueItem: React.FC<{const scrollCurrentIntoView = useCallback(() => {document- .querySelector(`.${SELECTED_CLASSNAME}`)+ .querySelector(`.${RENDER_QUEUE_ITEM_SELECTED_CLASSNAME}`)?.scrollIntoView({behavior: 'smooth'});}, []);@@ -202,7 +201,7 @@ export const RenderQueueItem: React.FC<{draggable={canDrag}onDragStart={onDragStart}onDragEnd={onDragEnd}- className={selected ? SELECTED_CLASSNAME : undefined}+ className={selected ? RENDER_QUEUE_ITEM_SELECTED_CLASSNAME : undefined}><RenderQueueItemStatus job={job} /><Spacing x={1} />diff --git a/packages/studio/src/components/RenderQueue/caption-job-types.ts b/packages/studio/src/components/RenderQueue/caption-job-types.tsnew file mode 100644index 0000000000..37d5fad37a--- /dev/null+++ b/packages/studio/src/components/RenderQueue/caption-job-types.ts@@ -0,0 +1,47 @@+import type {+ WhisperWebGpuModel,+ WhisperWebGpuTask,+} from '@remotion/whisper-webgpu';++export type CaptionJobProgress = {+ message: string;+ value: number;+};++type CaptionJobStatus =+ | {status: 'idle'}+ | {status: 'running'; progress: CaptionJobProgress}+ | {status: 'done'; captionCount: number}+ | {+ status: 'failed';+ error: {message: string; stack: string | undefined};+ };++export type AddCaptionJobParams = {+ src: string;+ displayName: string;+ audioStreamIndex: number | null;+ requestInit: Omit<RequestInit, 'signal'> | null;+ outName: string;+ model: WhisperWebGpuModel;+ language: string | null;+ task: WhisperWebGpuTask;+ chunkLengthInSeconds: number;+ strideLengthInSeconds: number;+ forceFullSequences: boolean;+ doSample: boolean;+ temperature: number;+ topK: number;+ repetitionPenalty: number;+ noRepeatNgramSize: number;+};++export type CaptionJob = AddCaptionJobParams & {+ id: string;+ type: 'caption';+ startedAt: number;+} & CaptionJobStatus;++export const isCaptionJob = (job: {type: string}): job is CaptionJob => {+ return job.type === 'caption';+};diff --git a/packages/studio/src/components/RenderQueue/context.tsx b/packages/studio/src/components/RenderQueue/context.tsxindex f360b7ddea..86b7d3d503 100644--- a/packages/studio/src/components/RenderQueue/context.tsx+++ b/packages/studio/src/components/RenderQueue/context.tsx@@ -8,6 +8,11 @@ import React, {useRef,useState,} from 'react';+import type {+ AddCaptionJobParams,+ CaptionJob,+ CaptionJobProgress,+} from './caption-job-types';import {type AddClientStillJobParams,type AddClientVideoJobParams,@@ -55,6 +60,7 @@ type RenderQueueContextType = {jobs: AnyRenderJob[];serverJobs: RenderJob[];clientJobs: ClientRenderJob[];+ captionJobs: CaptionJob[];addClientStillJob: (params: AddClientStillJobParams,compositionRef: CompositionRef,@@ -80,6 +86,17 @@ type RenderQueueContextType = {setProcessJobCallback: (callback: ((job: ClientRenderJob) => Promise<void>) | null,) => void;+ addCaptionJob: (params: AddCaptionJobParams) => string;+ updateCaptionJobProgress: (+ jobId: string,+ progress: CaptionJobProgress,+ ) => void;+ markCaptionJobDone: (jobId: string, captionCount: number) => void;+ markCaptionJobFailed: (jobId: string, error: Error) => void;+ removeCaptionJob: (jobId: string) => void;+ setProcessCaptionJobCallback: (+ callback: ((job: CaptionJob) => Promise<void>) | null,+ ) => void;getAbortController: (jobId: string) => AbortController;getCompositionForJob: (jobId: string) => CompositionRef | undefined;};@@ -91,6 +108,7 @@ export const RenderQueueContext = React.createContext<RenderQueueContextType>({jobs: [],serverJobs: [],clientJobs: [],+ captionJobs: [],addClientStillJob: noopString,addClientVideoJob: noopString,updateClientJobProgress: noop,@@ -101,6 +119,12 @@ export const RenderQueueContext = React.createContext<RenderQueueContextType>({removeClientJob: noop,cancelClientJob: noop,setProcessJobCallback: noop,+ addCaptionJob: noopString,+ updateCaptionJobProgress: noop,+ markCaptionJobDone: noop,+ markCaptionJobFailed: noop,+ removeCaptionJob: noop,+ setProcessCaptionJobCallback: noop,getAbortController: () => new AbortController(),getCompositionForJob: () => undefined,});@@ -119,21 +143,66 @@ export const RenderQueueContextProvider: React.FC<{const [clientJobs, setClientJobs] = useState<ClientRenderJob[]>(restorePersistedClientRenders,);+ const [captionJobs, setCaptionJobs] = useState<CaptionJob[]>([]);const [currentlyProcessing, setCurrentlyProcessing] = useState<string | null>(null,);+ const lastLocalJobStartedAt = useRef(+ clientJobs.reduce((latest, job) => Math.max(latest, job.startedAt), 0),+ );const processJobCallbackRef = useRef<((job: ClientRenderJob) => Promise<void>) | null>(null);+ const processCaptionJobCallbackRef = useRef<+ ((job: CaptionJob) => Promise<void>) | null+ >(null);+ const [processorCallbacksVersion, setProcessorCallbacksVersion] = useState(0);++ const getLocalJobStartedAt = useCallback(() => {+ const startedAt = Math.max(Date.now(), lastLocalJobStartedAt.current + 1);+ lastLocalJobStartedAt.current = startedAt;+ return startedAt;+ }, []);- // Process next job when state changes+ // Process the next local job when state changes. Server renders have their own+ // queue and are intentionally independent from browser-local work.useEffect(() => {if (currentlyProcessing) {return;}- const nextJob = clientJobs.find((job) => job.status === 'idle');- if (!nextJob || !processJobCallbackRef.current) {+ const nextJob = [...clientJobs, ...captionJobs]+ .filter((job) => job.status === 'idle')+ .sort((a, b) => a.startedAt - b.startedAt)[0];+ if (!nextJob) {+ return;+ }++ if (nextJob.type === 'caption') {+ if (!processCaptionJobCallbackRef.current) {+ return;+ }++ setCurrentlyProcessing(nextJob.id);+ setCaptionJobs((prev) =>+ prev.map((job) =>+ job.id === nextJob.id+ ? {+ ...job,+ status: 'running',+ progress: {+ message: 'Starting transcription...',+ value: 0,+ },+ }+ : job,+ ),+ );+ processCaptionJobCallbackRef.current(nextJob);+ return;+ }++ if (!processJobCallbackRef.current) {return;}@@ -156,7 +225,7 @@ export const RenderQueueContextProvider: React.FC<{),);processJobCallbackRef.current(nextJob);- }, [clientJobs, currentlyProcessing]);+ }, [captionJobs, clientJobs, currentlyProcessing, processorCallbacksVersion]);const addClientStillJob = useCallback((@@ -169,14 +238,14 @@ export const RenderQueueContextProvider: React.FC<{const newJob: ClientStillRenderJob = {...params,id,- startedAt: Date.now(),+ startedAt: getLocalJobStartedAt(),status: 'idle',};setClientJobs((prev) => [...prev, newJob]);return id;},- [],+ [getLocalJobStartedAt],);const addClientVideoJob = useCallback(@@ -190,14 +259,14 @@ export const RenderQueueContextProvider: React.FC<{const newJob: ClientVideoRenderJob = {...params,id,- startedAt: Date.now(),+ startedAt: getLocalJobStartedAt(),status: 'idle',};setClientJobs((prev) => [...prev, newJob]);return id;},- [],+ [getLocalJobStartedAt],);const updateClientJobProgress = useCallback(@@ -237,7 +306,7 @@ export const RenderQueueContextProvider: React.FC<{job.id === jobId ? {...job, status: 'done', metadata, getBlob} : job,),);- setCurrentlyProcessing(null);+ setCurrentlyProcessing((current) => (current === jobId ? null : current));},[],);@@ -258,7 +327,7 @@ export const RenderQueueContextProvider: React.FC<{: job,),);- setCurrentlyProcessing(null);+ setCurrentlyProcessing((current) => (current === jobId ? null : current));},[],);@@ -277,7 +346,7 @@ export const RenderQueueContextProvider: React.FC<{: job,),);- setCurrentlyProcessing(null);+ setCurrentlyProcessing((current) => (current === jobId ? null : current));}, []);const removeClientJob = useCallback((jobId: string): void => {@@ -300,6 +369,90 @@ export const RenderQueueContextProvider: React.FC<{const setProcessJobCallback = useCallback((callback: ((job: ClientRenderJob) => Promise<void>) | null): void => {processJobCallbackRef.current = callback;+ setProcessorCallbacksVersion((version) => version + 1);+ },+ [],+ );++ const addCaptionJob = useCallback(+ (params: AddCaptionJobParams): string => {+ const id = `caption-${Date.now()}-${Math.random()+ .toString(36)+ .substring(2, 11)}`;+ const newJob: CaptionJob = {+ ...params,+ id,+ type: 'caption',+ startedAt: getLocalJobStartedAt(),+ status: 'idle',+ };++ setCaptionJobs((prev) => [...prev, newJob]);+ return id;+ },+ [getLocalJobStartedAt],+ );++ const updateCaptionJobProgress = useCallback(+ (jobId: string, progress: CaptionJobProgress): void => {+ setCaptionJobs((prev) =>+ prev.map((job) =>+ job.id === jobId+ ? ({...job, status: 'running', progress} as CaptionJob)+ : job,+ ),+ );+ },+ [],+ );++ const markCaptionJobDone = useCallback(+ (jobId: string, captionCount: number): void => {+ setCaptionJobs((prev) =>+ prev.map((job) =>+ job.id === jobId+ ? ({...job, status: 'done', captionCount} as CaptionJob)+ : job,+ ),+ );+ setCurrentlyProcessing((current) => (current === jobId ? null : current));+ },+ [],+ );++ const markCaptionJobFailed = useCallback(+ (jobId: string, error: Error): void => {+ setCaptionJobs((prev) =>+ prev.map((job) =>+ job.id === jobId+ ? ({+ ...job,+ status: 'failed',+ error: {message: error.message, stack: error.stack},+ } as CaptionJob)+ : job,+ ),+ );+ setCurrentlyProcessing((current) => (current === jobId ? null : current));+ },+ [],+ );++ const removeCaptionJob = useCallback((jobId: string): void => {+ setCaptionJobs((prev) => {+ const jobToRemove = prev.find((job) => job.id === jobId);+ if (jobToRemove?.status === 'running') {+ return prev;+ }++ return prev.filter((job) => job.id !== jobId);+ });+ }, []);++ const setProcessCaptionJobCallback = useCallback(+ (callback: ((job: CaptionJob) => Promise<void>) | null): void => {+ processCaptionJobCallbackRef.current = callback;+ setProcessorCallbacksVersion((version) => version + 1);},[],);@@ -344,6 +497,7 @@ export const RenderQueueContextProvider: React.FC<{),serverJobs,clientJobs,+ captionJobs,addClientStillJob,addClientVideoJob,updateClientJobProgress,@@ -354,12 +508,19 @@ export const RenderQueueContextProvider: React.FC<{removeClientJob,cancelClientJob,setProcessJobCallback,+ addCaptionJob,+ updateCaptionJobProgress,+ markCaptionJobDone,+ markCaptionJobFailed,+ removeCaptionJob,+ setProcessCaptionJobCallback,getAbortController,getCompositionForJob,};}, [serverJobs,clientJobs,+ captionJobs,addClientStillJob,addClientVideoJob,updateClientJobProgress,@@ -370,6 +531,12 @@ export const RenderQueueContextProvider: React.FC<{removeClientJob,cancelClientJob,setProcessJobCallback,+ addCaptionJob,+ updateCaptionJobProgress,+ markCaptionJobDone,+ markCaptionJobFailed,+ removeCaptionJob,+ setProcessCaptionJobCallback,]);return (diff --git a/packages/studio/src/components/RenderQueue/index.tsx b/packages/studio/src/components/RenderQueue/index.tsxindex bccec61a09..df032d3025 100644--- a/packages/studio/src/components/RenderQueue/index.tsx+++ b/packages/studio/src/components/RenderQueue/index.tsx@@ -3,6 +3,8 @@ import {Internals} from 'remotion';import {BACKGROUND, LIGHT_TEXT} from '../../helpers/colors';import {Spacing} from '../layout';import {VERTICAL_SCROLLBAR_CLASSNAME} from '../Menu/is-menu-item';+import {isCaptionJob} from './caption-job-types';+import {CaptionQueueItem} from './CaptionQueueItem';import {RenderQueueContext} from './context';import {RenderQueueItem} from './RenderQueueItem';@@ -33,10 +35,14 @@ const renderQueue: React.CSSProperties = {};export const RenderQueue: React.FC = () => {- const {jobs} = useContext(RenderQueueContext);+ const {jobs, captionJobs} = useContext(RenderQueueContext);const {canvasContent} = useContext(Internals.CompositionManager);- const previousJobCount = React.useRef(jobs.length);- const jobCount = jobs.length;+ const allJobs = useMemo(+ () => [...jobs, ...captionJobs].sort((a, b) => a.startedAt - b.startedAt),+ [jobs, captionJobs],+ );+ const previousJobCount = React.useRef(allJobs.length);+ const jobCount = allJobs.length;const divRef = React.useRef<HTMLDivElement>(null);@@ -62,16 +68,33 @@ export const RenderQueue: React.FC = () => {}if (canvasContent.type === 'output') {- for (let i = 0; i < jobs.length; i++) {- const job = jobs[i];+ for (let i = 0; i < allJobs.length; i++) {+ const job = allJobs[i];+ if (isCaptionJob(job)) {+ continue;+ }+if (job.status === 'done' && canvasContent.path === `/${job.outName}`) {return i;}}}+ if (canvasContent.type === 'asset') {+ for (let i = 0; i < allJobs.length; i++) {+ const job = allJobs[i];+ if (+ isCaptionJob(job) &&+ job.status === 'done' &&+ canvasContent.asset === job.outName+ ) {+ return i;+ }+ }+ }+return -1;- }, [canvasContent, jobs]);+ }, [allJobs, canvasContent]);if (jobCount === 0) {return (@@ -89,12 +112,18 @@ export const RenderQueue: React.FC = () => {style={renderQueue}className={['css-reset', VERTICAL_SCROLLBAR_CLASSNAME].join(' ')}>- {jobs.map((j, index) => {- return (+ {allJobs.map((job, index) => {+ return isCaptionJob(job) ? (+ <CaptionQueueItem+ key={job.id}+ job={job}+ selected={selectedJob === index}+ />+ ) : (<RenderQueueItem- key={j.id}+ key={job.id}selected={selectedJob === index}- job={j}+ job={job}/>);})}diff --git a/packages/studio/src/components/RenderQueue/item-style.ts b/packages/studio/src/components/RenderQueue/item-style.tsindex a3fe3e7ffa..e7894358a6 100644--- a/packages/studio/src/components/RenderQueue/item-style.ts+++ b/packages/studio/src/components/RenderQueue/item-style.ts@@ -2,6 +2,9 @@ import type React from 'react';import {LIGHT_TEXT} from '../../helpers/colors';import {SPACING_UNIT} from '../layout';+export const RENDER_QUEUE_ITEM_SELECTED_CLASSNAME =+ '__remotion_selected_classname';+export const renderQueueItemSubtitleStyle: React.CSSProperties = {fontSize: 13,color: LIGHT_TEXT,diff --git a/packages/studio/src/components/RendersTab.tsx b/packages/studio/src/components/RendersTab.tsxindex c78c58a423..e5bb85dd63 100644--- a/packages/studio/src/components/RendersTab.tsx+++ b/packages/studio/src/components/RendersTab.tsx@@ -28,9 +28,14 @@ export const RendersTab: React.FC<{readonly selected: boolean;readonly onClick: MouseEventHandler<HTMLDivElement>;}> = ({selected, onClick}) => {- const {jobs} = useContext(RenderQueueContext);- const failedJobs = jobs.filter((j) => j.status === 'failed').length;- const jobCount = jobs.length;+ const {jobs, captionJobs} = useContext(RenderQueueContext);+ const failedJobs = [...jobs, ...captionJobs].filter(+ (job) => job.status === 'failed',+ ).length;+ const jobCount = jobs.length + captionJobs.length;+ const hasRunningCaptionJob = captionJobs.some(+ (job) => job.status === 'running',+ );const badgeStyle: React.CSSProperties = useMemo(() => {return {@@ -46,7 +51,7 @@ export const RendersTab: React.FC<{return (<Tab selected={selected} onClick={onClick}><div style={row}>- Renders+ {hasRunningCaptionJob ? 'Jobs' : 'Renders'}{jobCount > 0 ? (<><Flex />diff --git a/packages/studio/src/components/SelectedOutlineElement.tsx b/packages/studio/src/components/SelectedOutlineElement.tsxindex a3b35cba47..74360dbc54 100644--- a/packages/studio/src/components/SelectedOutlineElement.tsx+++ b/packages/studio/src/components/SelectedOutlineElement.tsx@@ -37,6 +37,7 @@ import {} from './Timeline/TimelineSelection';import {getOriginalLocationFromStack} from './Timeline/TimelineStack/get-stack';import {useDeleteTimelineItems} from './Timeline/use-delete-timeline-items';+import {isTranscribableSequence} from './Transcription/is-transcribable-sequence';import {useDefaultCodingAgentInfo,useEditorOpening,@@ -224,6 +225,11 @@ const SelectedOutlineElementUnmemoized: React.FC<previewServerState.type === 'connected';const canCrop = contextMenuTarget.canCrop && !sourceEditDisabled;const canRotate = !sourceEditDisabled;+ const transcribableMedia = isTranscribableSequence(+ contextMenuTarget.sequence,+ )+ ? contextMenuTarget.sequence+ : null;return getSequenceContextMenuItems({assetLinkInfo,@@ -282,6 +288,20 @@ const SelectedOutlineElementUnmemoized: React.FC<confirm,).catch(() => undefined);},+ onTranscribe:+ sourceEditingEnabled &&+ previewServerState.type === 'connected' &&+ transcribableMedia !== null+ ? () => {+ setSelectedModal({+ type: 'transcribe',+ src: transcribableMedia.src,+ displayName: transcribableMedia.displayName,+ audioStreamIndex: transcribableMedia.audioStreamIndex,+ requestInit: transcribableMedia.requestInit,+ });+ }+ : null,openInCodingAgent: (codingAgentId, codingAgentName, contextForAgents) => {launchCodingAgent(codingAgentId, contextForAgents).then((response) => {diff --git a/packages/studio/src/components/Tabs/vertical.tsx b/packages/studio/src/components/Tabs/vertical.tsxindex 6ec07d215f..1ddfd5e973 100644--- a/packages/studio/src/components/Tabs/vertical.tsx+++ b/packages/studio/src/components/Tabs/vertical.tsx@@ -28,7 +28,8 @@ export const VerticalTab: React.FC<{readonly renderIcon?: (color: string) => React.ReactNode;readonly style?: React.CSSProperties;readonly selected: boolean;-}> = ({children, onClick, renderIcon, style, selected}) => {+ readonly autoFocus?: boolean;+}> = ({children, onClick, renderIcon, style, selected, autoFocus}) => {const [hovered, setHovered] = useState(false);const {tabIndex} = useZIndex();@@ -54,6 +55,7 @@ export const VerticalTab: React.FC<{return (<button+ autoFocus={autoFocus}style={definiteStyle}type="button"onClick={onClick}diff --git a/packages/studio/src/components/Timeline/TimelineSequence.tsx b/packages/studio/src/components/Timeline/TimelineSequence.tsxindex c5e86cd277..2bd3be86d8 100644--- a/packages/studio/src/components/Timeline/TimelineSequence.tsx+++ b/packages/studio/src/components/Timeline/TimelineSequence.tsx@@ -41,6 +41,7 @@ import {AudioWaveform} from '../AudioWaveform';import {useConfirmationDialog} from '../ConfirmationDialog';import {ContextMenu} from '../ContextMenu';import {useSelectComposition} from '../InitialCompositionLoader';+import {isTranscribableSequence} from '../Transcription/is-transcribable-sequence';import {useSelectAsset} from '../use-select-asset';import {disableSequenceInteractivity} from './disable-sequence-interactivity';import {duplicateSequencesFromSource} from './duplicate-selected-timeline-item';@@ -606,6 +607,20 @@ const TimelineSequenceInner: React.FC<{s.type === 'audio' || s.type === 'video' || s.type === 'image'? s.src: null;+ const transcribableMedia = isTranscribableSequence(s) ? s : null;+ const onTranscribe = useCallback(() => {+ if (!previewInteractive || transcribableMedia === null) {+ return;+ }++ setSelectedModal({+ type: 'transcribe',+ src: transcribableMedia.src,+ displayName: transcribableMedia.displayName,+ audioStreamIndex: transcribableMedia.audioStreamIndex,+ requestInit: transcribableMedia.requestInit,+ });+ }, [previewInteractive, setSelectedModal, transcribableMedia]);const onDuplicateSequenceFromSource = useCallback(() => {if (!validatedLocation?.source || !nodePathInfo || duplicateDisabled) {return;@@ -737,6 +752,8 @@ const TimelineSequenceInner: React.FC<{onDeleteSequenceFromSource,onDisableSequenceInteractivity,onDuplicateSequenceFromSource,+ onTranscribe:+ previewInteractive && transcribableMedia !== null ? onTranscribe : null,openInCodingAgent,openInEditor,originalLocation,@@ -767,6 +784,7 @@ const TimelineSequenceInner: React.FC<{onDisableSequenceInteractivity,onDuplicateSequenceFromSource,onDuplicateSelectedSequences,+ onTranscribe,openInCodingAgent,openInEditor,originalLocation,@@ -782,6 +800,7 @@ const TimelineSequenceInner: React.FC<{sequenceFrameOffset,setPropStatuses,setSelectedModal,+ transcribableMedia,validatedLocation?.source,]);const {frozenFrame} = s;diff --git a/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx b/packages/studio/src/components/Timeline/TimelineSequenceItem.tsxindex a36cbe1c3c..4b6628f6fa 100644--- a/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx+++ b/packages/studio/src/components/Timeline/TimelineSequenceItem.tsx@@ -69,6 +69,7 @@ import {cropFieldKeys,rotateFieldKey,} from '../selected-outline-types';+import {isTranscribableSequence} from '../Transcription/is-transcribable-sequence';import {useSelectAsset} from '../use-select-asset';import {disableSequenceInteractivity} from './disable-sequence-interactivity';import {duplicateSequencesFromSource} from './duplicate-selected-timeline-item';@@ -812,6 +813,22 @@ const TimelineSequenceItemInner: React.FC<{sequence.type === 'image'? sequence.src: null;+ const transcribableMedia = isTranscribableSequence(sequence)+ ? sequence+ : null;+ const onTranscribe = useCallback(() => {+ if (!previewInteractive || transcribableMedia === null) {+ return;+ }++ setSelectedModal({+ type: 'transcribe',+ src: transcribableMedia.src,+ displayName: transcribableMedia.displayName,+ audioStreamIndex: transcribableMedia.audioStreamIndex,+ requestInit: transcribableMedia.requestInit,+ });+ }, [previewInteractive, setSelectedModal, transcribableMedia]);const isExpanded =previewConnected && nodePathInfo !== null && getIsExpanded(nodePathInfo);@@ -1186,6 +1203,8 @@ const TimelineSequenceItemInner: React.FC<{onDeleteSequenceFromSource,onDisableSequenceInteractivity,onDuplicateSequenceFromSource,+ onTranscribe:+ previewInteractive && transcribableMedia !== null ? onTranscribe : null,openInCodingAgent,openInEditor,originalLocation,@@ -1296,6 +1315,7 @@ const TimelineSequenceItemInner: React.FC<{onDuplicateSelectedSequences,onRenameSequence,onSelect,+ onTranscribe,openInCodingAgent,openInEditor,originalLocation,@@ -1311,6 +1331,7 @@ const TimelineSequenceItemInner: React.FC<{sequenceFrameOffset,setSelectedModal,setPropStatuses,+ transcribableMedia,validatedLocation?.source,]);const canDropEffect =diff --git a/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts b/packages/studio/src/components/Timeline/get-sequence-context-menu-items.tsindex b2dd49095d..72993b5fd8 100644--- a/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts+++ b/packages/studio/src/components/Timeline/get-sequence-context-menu-items.ts@@ -96,6 +96,7 @@ export const getSequenceContextMenuItems = ({onDeleteSequenceFromSource,onDisableSequenceInteractivity,onDuplicateSequenceFromSource,+ onTranscribe,openInCodingAgent,openInEditor,originalLocation,@@ -116,6 +117,7 @@ export const getSequenceContextMenuItems = ({readonly onDeleteSequenceFromSource: () => void;readonly onDisableSequenceInteractivity: () => void;readonly onDuplicateSequenceFromSource: () => void;+ readonly onTranscribe: (() => void) | null;readonly openInCodingAgent: (codingAgentId: DefaultCodingAgent,codingAgentName: string,@@ -251,7 +253,21 @@ export const getSequenceContextMenuItems = ({value: 'show-asset',}: null,- assetLinkInfo+ onTranscribe+ ? {+ type: 'item' as const,+ id: 'transcribe',+ keyHint: null,+ label: 'Transcribe',+ leftItem: null,+ disabled: false,+ onClick: onTranscribe,+ quickSwitcherLabel: null,+ subMenu: null,+ value: 'transcribe',+ }+ : null,+ assetLinkInfo || onTranscribe? {type: 'divider' as const,id: 'sequence-link-divider',diff --git a/packages/studio/src/components/Transcription/TranscriptionModal.tsx b/packages/studio/src/components/Transcription/TranscriptionModal.tsxnew file mode 100644index 0000000000..1c98ab4f61--- /dev/null+++ b/packages/studio/src/components/Transcription/TranscriptionModal.tsx@@ -0,0 +1,844 @@+import {formatBytes} from '@remotion/studio-shared';+import {+ canUseWhisperWebGpu,+ getAvailableModels,+ type WhisperWebGpuModel,+ type WhisperWebGpuTask,+} from '@remotion/whisper-webgpu';+import React, {+ useCallback,+ useContext,+ useEffect,+ useMemo,+ useState,+} from 'react';+import {BLUE_DISABLED, LIGHT_TEXT, WHITE} from '../../helpers/colors';+import {Checkmark} from '../../icons/Checkmark';+import type {TranscriptionModalState} from '../../state/modals';+import {SetSelectedModalContext} from '../../state/modals';+import {SidebarContext} from '../../state/sidebar';+import {Button} from '../Button';+import {Checkbox} from '../Checkbox';+import {Spacing} from '../layout';+import {VERTICAL_SCROLLBAR_CLASSNAME} from '../Menu/is-menu-item';+import {ModalHeader} from '../ModalHeader';+import type {ComboboxValue} from '../NewComposition/ComboBox';+import {Combobox} from '../NewComposition/ComboBox';+import {DismissableModal} from '../NewComposition/DismissableModal';+import {RemotionInput} from '../NewComposition/RemInput';+import {ValidationMessage} from '../NewComposition/ValidationMessage';+import {optionsSidebarTabs} from '../options-sidebar-tabs';+import {persistSelectedOptionsSidebarPanel} from '../OptionsPanel';+import {InfoBubble} from '../RenderModal/InfoBubble';+import {input, label, optionRow, rightRow} from '../RenderModal/layout';+import {NumberSetting} from '../RenderModal/NumberSetting';+import {+ buttonStyle,+ container,+ flexer,+ optionsPanel,+ outerModalStyle,+} from '../RenderModal/render-modals';+import {RenderModalHr} from '../RenderModal/RenderModalHr';+import {RenderQueueContext} from '../RenderQueue/context';+import {useStaticFiles} from '../use-static-files';+import {+ getDefaultCaptionOutputName,+ validateCaptionOutputName,+} from './caption-output-name';+import {WHISPER_LANGUAGES, type WhisperLanguage} from './whisper-languages';++const TRANSCRIPTION_OUTPUT_MESSAGE_ID = 'remotion-transcription-output-message';+const TRANSCRIPTION_CHUNK_MESSAGE_ID = 'remotion-transcription-chunk-message';+const TRANSCRIPTION_DECODING_MESSAGE_ID =+ 'remotion-transcription-decoding-message';+const TRANSCRIPTION_TASK_MESSAGE_ID = 'remotion-transcription-task-message';+const DEFAULT_CHUNK_LENGTH_IN_SECONDS = 30;+const DEFAULT_STRIDE_LENGTH_IN_SECONDS = 5;+const DEFAULT_TEMPERATURE = 1;+const DEFAULT_TOP_K = 50;+const DEFAULT_REPETITION_PENALTY = 1;+const DEFAULT_NO_REPEAT_NGRAM_SIZE = 0;+const MAX_CHUNK_LENGTH_IN_SECONDS = 30;++type SupportState =+ | {type: 'checking'}+ | {type: 'supported'}+ | {type: 'unsupported'; message: string};++const AVAILABLE_MODELS = getAvailableModels();++const settingsPanel: React.CSSProperties = {+ ...optionsPanel,+ flexDirection: 'column',+ paddingTop: 16,+};++const controlStyle: React.CSSProperties = {+ width: 330,+ maxWidth: '100%',+};++const nestedLabelStyle: React.CSSProperties = {+ color: 'inherit',+ fontFamily: 'inherit',+ fontSize: 'inherit',+ lineHeight: 'inherit',+};++const taskMessageRow: React.CSSProperties = {+ display: 'flex',+ justifyContent: 'flex-end',+ padding: '0 16px 8px',+};++const outputRow: React.CSSProperties = {+ ...optionRow,+ paddingBottom: 12,+};++const tooltipContent: React.CSSProperties = {+ color: LIGHT_TEXT,+ fontSize: 13,+ lineHeight: 1.5,+ maxWidth: 360,+ padding: 12,+};++const tooltipInlineCode: React.CSSProperties = {+ color: WHITE,+ fontFamily: 'monospace',+ fontSize: 'inherit',+ lineHeight: 'inherit',+};++const statusText: React.CSSProperties = {+ color: LIGHT_TEXT,+ fontFamily: 'sans-serif',+ fontSize: 13,+ fontVariantNumeric: 'tabular-nums',+ lineHeight: 1.5,+ textAlign: 'right',+};++const TranscriptionSettingLabel: React.FC<{+ readonly children: React.ReactNode;+ readonly inputId: string | null;+ readonly name: string;+}> = ({children, inputId, name}) => {+ return (+ <div style={label}>+ {inputId === null ? (+ name+ ) : (+ <label htmlFor={inputId} style={nestedLabelStyle}>+ {name}+ </label>+ )}+ <Spacing x={0.5} />+ <InfoBubble title={`Learn more about ${name}`}>+ <div style={tooltipContent}>{children}</div>+ </InfoBubble>+ </div>+ );+};++const ModelSettings: React.FC<{+ readonly selectedLanguage: WhisperLanguage;+ readonly selectedModel: WhisperWebGpuModel;+ readonly selectedTask: WhisperWebGpuTask;+ readonly setSelectedLanguage: (language: WhisperLanguage) => void;+ readonly setSelectedModel: (model: WhisperWebGpuModel) => void;+ readonly setSelectedTask: (task: WhisperWebGpuTask) => void;+ readonly supportState: SupportState;+}> = ({+ selectedLanguage,+ selectedModel,+ selectedTask,+ setSelectedLanguage,+ setSelectedModel,+ setSelectedTask,+ supportState,+}) => {+ const selectedModelInfo = AVAILABLE_MODELS.find(+ ({name}) => name === selectedModel,+ );+ if (!selectedModelInfo) {+ throw new Error(`Unknown Whisper model: ${selectedModel}`);+ }++ const modelOptions = useMemo((): ComboboxValue[] => {+ return AVAILABLE_MODELS.map((model): ComboboxValue => {+ return {+ type: 'item',+ id: model.name,+ value: model.name,+ label: `${model.name} · ${formatBytes(model.webGpuDownloadSize)}`,+ leftItem: model.name === selectedModel ? <Checkmark /> : null,+ keyHint: null,+ quickSwitcherLabel: null,+ subMenu: null,+ disabled: false,+ onClick: () => setSelectedModel(model.name),+ };+ });+ }, [selectedModel, setSelectedModel]);++ const languageOptions = useMemo((): ComboboxValue[] => {+ return WHISPER_LANGUAGES.map(([languageCode, languageName]) => ({+ type: 'item',+ id: languageCode,+ value: languageCode,+ label: languageName,+ leftItem: languageCode === selectedLanguage ? <Checkmark /> : null,+ keyHint: null,+ quickSwitcherLabel: null,+ subMenu: null,+ disabled: false,+ onClick: () => setSelectedLanguage(languageCode),+ }));+ }, [selectedLanguage, setSelectedLanguage]);+ const effectiveTask = selectedModelInfo.supportsTranslation+ ? selectedTask+ : 'transcribe';+ const taskOptions = useMemo((): ComboboxValue[] => {+ return [+ {+ type: 'item',+ id: 'transcribe',+ value: 'transcribe',+ label: (+ <>+ <span aria-hidden="true" style={nestedLabelStyle}>+ Transcribe+ </span>+ <span style={{position: 'absolute', clip: 'rect(0 0 0 0)'}}>+ Task: Transcribe+ </span>+ </>+ ),+ leftItem: effectiveTask === 'transcribe' ? <Checkmark /> : null,+ keyHint: null,+ quickSwitcherLabel: null,+ subMenu: null,+ disabled: false,+ onClick: () => setSelectedTask('transcribe'),+ },+ {+ type: 'item',+ id: 'translate',+ value: 'translate',+ label: (+ <>+ <span aria-hidden="true" style={nestedLabelStyle}>+ Translate to English+ </span>+ <span style={{position: 'absolute', clip: 'rect(0 0 0 0)'}}>+ Task: Translate to English+ </span>+ </>+ ),+ leftItem: effectiveTask === 'translate' ? <Checkmark /> : null,+ keyHint: null,+ quickSwitcherLabel: null,+ subMenu: null,+ disabled: false,+ onClick: () => setSelectedTask('translate'),+ },+ ];+ }, [effectiveTask, setSelectedTask]);++ useEffect(() => {+ if (!selectedModelInfo.supportsTranslation) {+ setSelectedTask('transcribe');+ }+ }, [selectedModelInfo.supportsTranslation, setSelectedTask]);++ return (+ <>+ <div style={optionRow}>+ <div style={label}>Whisper model</div>+ <div style={rightRow}>+ <Combobox+ values={modelOptions}+ selectedId={selectedModel}+ title="Whisper model"+ disabled={false}+ style={controlStyle}+ />+ </div>+ </div>+ {selectedModelInfo.multilingual ? (+ <div style={optionRow}>+ <div style={label}>Spoken language</div>+ <div style={rightRow}>+ <Combobox+ values={languageOptions}+ selectedId={selectedLanguage}+ title="Spoken language"+ disabled={false}+ style={controlStyle}+ />+ </div>+ </div>+ ) : null}+ <div style={optionRow}>+ <div style={label}>Task</div>+ <div style={rightRow}>+ <Combobox+ values={taskOptions}+ selectedId={effectiveTask}+ title="Task"+ disabled={!selectedModelInfo.supportsTranslation}+ style={controlStyle}+ />+ </div>+ </div>+ {effectiveTask === 'translate' ? (+ <div+ id={TRANSCRIPTION_TASK_MESSAGE_ID}+ aria-live="polite"+ style={taskMessageRow}+ >+ <ValidationMessage+ align="flex-end"+ message="Word timings may be less accurate when translating to English."+ type="warning"+ />+ </div>+ ) : null}+ <div style={optionRow}>+ <TranscriptionSettingLabel inputId={null} name="Model download">+ If necessary, Studio downloads the model in the background when the+ job starts. Follow its progress in Jobs. Transcription runs locally+ using WebGPU; the media does not leave your device.+ </TranscriptionSettingLabel>+ <div style={rightRow}>+ <div style={statusText}>+ {`${formatBytes(selectedModelInfo.webGpuDownloadSize)} · Downloaded with transcription if needed`}+ </div>+ </div>+ </div>+ {supportState.type === 'unsupported' ? (+ <div style={{padding: '0 16px'}}>+ <ValidationMessage+ align="flex-end"+ message={supportState.message}+ type="error"+ />+ </div>+ ) : null}+ </>+ );+};++const OutputSettings: React.FC<{+ readonly exists: boolean;+ readonly onOutNameChange: React.ChangeEventHandler<HTMLInputElement>;+ readonly outName: string;+ readonly validationMessage: string | null;+}> = ({exists, onOutNameChange, outName, validationMessage}) => {+ return (+ <div style={outputRow}>+ <TranscriptionSettingLabel inputId={null} name="Output in public/">+ Studio writes a JSON array compatible with{' '}+ <code style={tooltipInlineCode}>Caption[]</code>. Load it from your+ composition with <code style={tooltipInlineCode}>staticFile()</code>.+ </TranscriptionSettingLabel>+ <div style={rightRow}>+ <div style={controlStyle}>+ <RemotionInput+ aria-label="Caption output file"+ aria-describedby={+ validationMessage || exists+ ? TRANSCRIPTION_OUTPUT_MESSAGE_ID+ : undefined+ }+ aria-invalid={validationMessage ? true : undefined}+ status={validationMessage ? 'error' : exists ? 'warning' : 'ok'}+ style={input}+ type="text"+ value={outName}+ onChange={onOutNameChange}+ rightAlign+ />+ {validationMessage || exists ? (+ <div id={TRANSCRIPTION_OUTPUT_MESSAGE_ID} aria-live="polite">+ <Spacing y={1} block />+ <ValidationMessage+ align="flex-end"+ message={validationMessage ?? 'Exists, will be overwritten'}+ type={validationMessage ? 'error' : 'warning'}+ />+ </div>+ ) : null}+ </div>+ </div>+ </div>+ );+};++const AdvancedSettings: React.FC<{+ readonly chunkLengthInSeconds: number;+ readonly doSample: boolean;+ readonly forceFullSequences: boolean;+ readonly noRepeatNgramSize: number;+ readonly repetitionPenalty: number;+ readonly setChunkLengthInSeconds: React.Dispatch<+ React.SetStateAction<number>+ >;+ readonly setDoSample: React.Dispatch<React.SetStateAction<boolean>>;+ readonly setForceFullSequences: React.Dispatch<React.SetStateAction<boolean>>;+ readonly setNoRepeatNgramSize: React.Dispatch<React.SetStateAction<number>>;+ readonly setRepetitionPenalty: React.Dispatch<React.SetStateAction<number>>;+ readonly setStrideLengthInSeconds: React.Dispatch<+ React.SetStateAction<number>+ >;+ readonly setTemperature: React.Dispatch<React.SetStateAction<number>>;+ readonly setTopK: React.Dispatch<React.SetStateAction<number>>;+ readonly strideLengthInSeconds: number;+ readonly temperature: number;+ readonly topK: number;+ readonly decodingValidationMessage: string | null;+ readonly validationMessage: string | null;+}> = ({+ chunkLengthInSeconds,+ doSample,+ forceFullSequences,+ noRepeatNgramSize,+ repetitionPenalty,+ setChunkLengthInSeconds,+ setDoSample,+ setForceFullSequences,+ setNoRepeatNgramSize,+ setRepetitionPenalty,+ setStrideLengthInSeconds,+ setTemperature,+ setTopK,+ strideLengthInSeconds,+ temperature,+ topK,+ decodingValidationMessage,+ validationMessage,+}) => {+ const onForceFullSequencesChange: React.ChangeEventHandler<HTMLInputElement> =+ useCallback(+ (event) => setForceFullSequences(event.target.checked),+ [setForceFullSequences],+ );+ const onDoSampleChange: React.ChangeEventHandler<HTMLInputElement> =+ useCallback((event) => setDoSample(event.target.checked), [setDoSample]);++ return (+ <>+ <div+ aria-describedby={+ validationMessage ? TRANSCRIPTION_CHUNK_MESSAGE_ID : undefined+ }+ aria-invalid={validationMessage ? true : undefined}+ aria-label="Chunk settings"+ role="group"+ >+ <NumberSetting+ formatter={(value) => `${value}s`}+ hint={{+ content: (+ <div style={tooltipContent}>+ Studio splits long audio into chunks of up to 30 seconds. Longer+ chunks need fewer transcription passes.+ </div>+ ),+ title: 'Learn more about Chunk length',+ }}+ max={MAX_CHUNK_LENGTH_IN_SECONDS}+ min={1}+ name="Chunk length"+ onValueChanged={setChunkLengthInSeconds}+ step={1}+ value={chunkLengthInSeconds}+ />+ <NumberSetting+ formatter={(value) => `${value}s`}+ hint={{+ content: (+ <div style={tooltipContent}>+ Overlaps both sides of each chunk to help preserve words at the+ boundaries.+ </div>+ ),+ title: 'Learn more about Stride length',+ }}+ min={0}+ name="Stride length"+ onValueChanged={setStrideLengthInSeconds}+ step={1}+ value={strideLengthInSeconds}+ />+ </div>+ {validationMessage ? (+ <div+ id={TRANSCRIPTION_CHUNK_MESSAGE_ID}+ aria-live="polite"+ style={{padding: '0 16px'}}+ >+ <ValidationMessage+ align="flex-end"+ message={validationMessage}+ type="error"+ />+ </div>+ ) : null}+ <div+ aria-describedby={+ decodingValidationMessage+ ? TRANSCRIPTION_DECODING_MESSAGE_ID+ : undefined+ }+ aria-invalid={decodingValidationMessage ? true : undefined}+ aria-label="Decoding settings"+ role="group"+ >+ <RenderModalHr />+ <div style={optionRow}>+ <TranscriptionSettingLabel+ inputId="force-full-sequences"+ name="Force full sequences"+ >+ Makes the job fail if Whisper leaves an incomplete timestamp+ sequence.+ </TranscriptionSettingLabel>+ <div style={rightRow}>+ <Checkbox+ checked={forceFullSequences}+ inputId="force-full-sequences"+ name="force-full-sequences"+ onChange={onForceFullSequencesChange}+ />+ </div>+ </div>+ <RenderModalHr />+ <div style={optionRow}>+ <TranscriptionSettingLabel inputId="use-sampling" name="Use sampling">+ Makes decoding nondeterministic.+ </TranscriptionSettingLabel>+ <div style={rightRow}>+ <Checkbox+ checked={doSample}+ inputId="use-sampling"+ name="use-sampling"+ onChange={onDoSampleChange}+ />+ </div>+ </div>+ {doSample ? (+ <>+ <NumberSetting+ hint={{+ content: (+ <div style={tooltipContent}>+ Controls randomness when sampling.+ </div>+ ),+ title: 'Learn more about Temperature',+ }}+ min={0.1}+ name="Temperature"+ onValueChanged={setTemperature}+ step={0.1}+ value={temperature}+ />+ <NumberSetting+ hint={{+ content: (+ <div style={tooltipContent}>+ Limits the candidate tokens when sampling.+ </div>+ ),+ title: 'Learn more about Top K',+ }}+ min={0}+ name="Top K"+ onValueChanged={setTopK}+ step={1}+ value={topK}+ />+ </>+ ) : null}+ <NumberSetting+ hint={{+ content: (+ <div style={tooltipContent}>+ Discourages repeated tokens. A value of 1 disables the penalty.+ </div>+ ),+ title: 'Learn more about Repetition penalty',+ }}+ min={0.1}+ name="Repetition penalty"+ onValueChanged={setRepetitionPenalty}+ step={0.1}+ value={repetitionPenalty}+ />+ <NumberSetting+ hint={{+ content: (+ <div style={tooltipContent}>+ Prevents repeated phrases of this many tokens. A value of 0+ disables the filter.+ </div>+ ),+ title: 'Learn more about No-repeat n-gram size',+ }}+ min={0}+ name="No-repeat n-gram size"+ onValueChanged={setNoRepeatNgramSize}+ step={1}+ value={noRepeatNgramSize}+ />+ </div>+ {decodingValidationMessage ? (+ <div+ id={TRANSCRIPTION_DECODING_MESSAGE_ID}+ aria-live="polite"+ style={{padding: '0 16px'}}+ >+ <ValidationMessage+ align="flex-end"+ message={decodingValidationMessage}+ type="error"+ />+ </div>+ ) : null}+ </>+ );+};++export const TranscriptionModal: React.FC<TranscriptionModalState> = ({+ audioStreamIndex,+ displayName,+ requestInit,+ src,+}) => {+ const [selectedModel, setSelectedModel] =+ useState<WhisperWebGpuModel>('tiny.en');+ const [selectedLanguage, setSelectedLanguage] =+ useState<WhisperLanguage>('en');+ const [selectedTask, setSelectedTask] =+ useState<WhisperWebGpuTask>('transcribe');+ const [chunkLengthInSeconds, setChunkLengthInSeconds] = useState(+ DEFAULT_CHUNK_LENGTH_IN_SECONDS,+ );+ const [strideLengthInSeconds, setStrideLengthInSeconds] = useState(+ DEFAULT_STRIDE_LENGTH_IN_SECONDS,+ );+ const [forceFullSequences, setForceFullSequences] = useState(false);+ const [doSample, setDoSample] = useState(false);+ const [temperature, setTemperature] = useState(DEFAULT_TEMPERATURE);+ const [topK, setTopK] = useState(DEFAULT_TOP_K);+ const [repetitionPenalty, setRepetitionPenalty] = useState(+ DEFAULT_REPETITION_PENALTY,+ );+ const [noRepeatNgramSize, setNoRepeatNgramSize] = useState(+ DEFAULT_NO_REPEAT_NGRAM_SIZE,+ );+ const [supportState, setSupportState] = useState<SupportState>({+ type: 'checking',+ });+ const [outName, setOutName] = useState(() =>+ getDefaultCaptionOutputName(src, displayName),+ );+ const {addCaptionJob, captionJobs} = useContext(RenderQueueContext);+ const {setSelectedModal} = useContext(SetSelectedModalContext);+ const {setSidebarCollapsedState} = useContext(SidebarContext);+ const staticFiles = useStaticFiles();++ useEffect(() => {+ let cancelled = false;+ canUseWhisperWebGpu().then((result) => {+ if (cancelled) {+ return;+ }++ setSupportState(+ result.supported+ ? {type: 'supported'}+ : {type: 'unsupported', message: result.detailedReason},+ );+ });++ return () => {+ cancelled = true;+ };+ }, []);++ const normalizedOutName = outName.normalize('NFC').toLowerCase();+ const queuedOutputExists = captionJobs.some(+ (job) =>+ (job.status === 'idle' || job.status === 'running') &&+ job.outName.normalize('NFC').toLowerCase() === normalizedOutName,+ );+ const outputValidationMessage =+ validateCaptionOutputName(outName) ??+ (queuedOutputExists+ ? 'Another caption job is already using this output file'+ : null);+ const exists = staticFiles.some(+ (file) => file.name.normalize('NFC').toLowerCase() === normalizedOutName,+ );+ const chunkValidationMessage =+ strideLengthInSeconds * 2 >= chunkLengthInSeconds+ ? 'Stride length must be less than half of chunk length'+ : null;+ const decodingValidationMessage = !Number.isFinite(temperature)+ ? 'Temperature must be a finite number greater than 0'+ : temperature <= 0+ ? 'Temperature must be greater than 0'+ : !Number.isInteger(topK) || topK < 0+ ? 'Top K must be a non-negative integer'+ : !Number.isFinite(repetitionPenalty)+ ? 'Repetition penalty must be a finite number greater than 0'+ : repetitionPenalty <= 0+ ? 'Repetition penalty must be greater than 0'+ : !Number.isInteger(noRepeatNgramSize) || noRepeatNgramSize < 0+ ? 'No-repeat n-gram size must be a non-negative integer'+ : null;+ const canTranscribe =+ supportState.type === 'supported' &&+ outputValidationMessage === null &&+ chunkValidationMessage === null &&+ decodingValidationMessage === null;+ const transcribeDisabledReason =+ supportState.type === 'checking'+ ? 'Checking WebGPU support'+ : supportState.type === 'unsupported'+ ? supportState.message+ : (outputValidationMessage ??+ chunkValidationMessage ??+ decodingValidationMessage ??+ undefined);++ const onOutNameChange = useCallback(+ (event: React.ChangeEvent<HTMLInputElement>) => {+ setOutName(event.target.value);+ },+ [],+ );++ const onAddToQueue = useCallback(() => {+ if (!canTranscribe) {+ return;+ }++ const modelInfo = AVAILABLE_MODELS.find(({name}) => name === selectedModel);+ if (!modelInfo) {+ throw new Error(`Unknown Whisper model: ${selectedModel}`);+ }++ addCaptionJob({+ src,+ displayName,+ audioStreamIndex,+ requestInit,+ outName,+ model: selectedModel,+ language: modelInfo.multilingual ? selectedLanguage : null,+ task: modelInfo.supportsTranslation ? selectedTask : 'transcribe',+ chunkLengthInSeconds,+ strideLengthInSeconds,+ forceFullSequences,+ doSample,+ temperature,+ topK,+ repetitionPenalty,+ noRepeatNgramSize,+ });+ setSidebarCollapsedState({left: null, right: 'expanded'});+ persistSelectedOptionsSidebarPanel('renders');+ optionsSidebarTabs.current?.selectRendersPanel();+ setSelectedModal(null);+ }, [+ addCaptionJob,+ audioStreamIndex,+ canTranscribe,+ chunkLengthInSeconds,+ displayName,+ doSample,+ forceFullSequences,+ noRepeatNgramSize,+ outName,+ repetitionPenalty,+ requestInit,+ src,+ selectedLanguage,+ selectedModel,+ selectedTask,+ setSelectedModal,+ setSidebarCollapsedState,+ strideLengthInSeconds,+ temperature,+ topK,+ ]);++ return (+ <DismissableModal ariaLabel={`Transcribe ${displayName}`}>+ <div style={{...outerModalStyle, outline: 'none'}}>+ <ModalHeader title={`Transcribe ${displayName}`} />+ <div style={container}>+ <div style={flexer} />+ <Button+ onClick={onAddToQueue}+ disabled={!canTranscribe}+ title={transcribeDisabledReason}+ style={{+ ...buttonStyle,+ backgroundColor: canTranscribe+ ? buttonStyle.backgroundColor+ : BLUE_DISABLED,+ }}+ >+ Transcribe+ </Button>+ </div>+ <div style={settingsPanel} className={VERTICAL_SCROLLBAR_CLASSNAME}>+ <OutputSettings+ exists={exists}+ onOutNameChange={onOutNameChange}+ outName={outName}+ validationMessage={outputValidationMessage}+ />+ <RenderModalHr />+ <ModelSettings+ selectedLanguage={selectedLanguage}+ selectedModel={selectedModel}+ selectedTask={selectedTask}+ setSelectedLanguage={setSelectedLanguage}+ setSelectedModel={setSelectedModel}+ setSelectedTask={setSelectedTask}+ supportState={supportState}+ />+ <RenderModalHr />+ <AdvancedSettings+ chunkLengthInSeconds={chunkLengthInSeconds}+ decodingValidationMessage={decodingValidationMessage}+ doSample={doSample}+ forceFullSequences={forceFullSequences}+ noRepeatNgramSize={noRepeatNgramSize}+ repetitionPenalty={repetitionPenalty}+ setChunkLengthInSeconds={setChunkLengthInSeconds}+ setDoSample={setDoSample}+ setForceFullSequences={setForceFullSequences}+ setNoRepeatNgramSize={setNoRepeatNgramSize}+ setRepetitionPenalty={setRepetitionPenalty}+ setStrideLengthInSeconds={setStrideLengthInSeconds}+ setTemperature={setTemperature}+ setTopK={setTopK}+ strideLengthInSeconds={strideLengthInSeconds}+ temperature={temperature}+ topK={topK}+ validationMessage={chunkValidationMessage}+ />+ </div>+ </div>+ </DismissableModal>+ );+};diff --git a/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx b/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsxnew file mode 100644index 0000000000..509996233e--- /dev/null+++ b/packages/studio/src/components/Transcription/TranscriptionModalWithOptionalWhisper.tsx@@ -0,0 +1,142 @@+import React, {Suspense, useCallback, useContext, useState} from 'react';+import {installPackages} from '../../api/install-package';+import {LIGHT_TEXT, WARNING_COLOR, WHITE} from '../../helpers/colors';+import type {TranscriptionModalState} from '../../state/modals';+import {SetSelectedModalContext} from '../../state/modals';+import {Button} from '../Button';+import {Flex, Row, Spacing} from '../layout';+import {ModalButton} from '../ModalButton';+import {ModalContainer} from '../ModalContainer';+import {ModalFooterContainer} from '../ModalFooter';+import {ModalHeader} from '../ModalHeader';+import {Spinner} from '../Spinner';+import {+ isWhisperWebGpuInstalled,+ WHISPER_WEBGPU_PACKAGE,+} from './whisper-webgpu-capability';++const LazyTranscriptionModal = React.lazy(async () => {+ const {TranscriptionModal} = await import('./TranscriptionModal');+ return {default: TranscriptionModal};+});++type InstallState =+ | {type: 'idle'}+ | {type: 'installing'}+ | {type: 'error'; error: Error};++const panelStyle: React.CSSProperties = {+ width: 'min(520px, calc(100vw - 40px))',+};++const contentStyle: React.CSSProperties = {+ color: WHITE,+ fontFamily: 'sans-serif',+ fontSize: 14,+ lineHeight: 1.5,+ padding: 16,+};++const codeStyle: React.CSSProperties = {+ color: 'inherit',+ fontFamily: 'monospace',+ fontSize: 'inherit',+};++const statusStyle: React.CSSProperties = {+ alignItems: 'center',+ color: LIGHT_TEXT,+ display: 'flex',+ gap: 10,+};++const errorStyle: React.CSSProperties = {+ color: WARNING_COLOR,+ marginTop: 12,+ whiteSpace: 'pre-wrap',+};++const cancelStyle: React.CSSProperties = {+ minWidth: 90,+};++export const TranscriptionModalWithOptionalWhisper: React.FC<{+ readonly state: TranscriptionModalState;+}> = ({state}) => {+ const {setSelectedModal} = useContext(SetSelectedModalContext);+ const [installed, setInstalled] = useState(isWhisperWebGpuInstalled);+ const [installState, setInstallState] = useState<InstallState>({+ type: 'idle',+ });++ const busy = installState.type === 'installing';+ const dismiss = useCallback(() => {+ if (!busy) {+ setSelectedModal(null);+ }+ }, [busy, setSelectedModal]);++ const install = useCallback(async () => {+ setInstallState({type: 'installing'});+ try {+ await installPackages([{name: WHISPER_WEBGPU_PACKAGE, version: null}]);+ window.remotion_installedPackages = Array.from(+ new Set([+ ...(window.remotion_installedPackages ?? []),+ WHISPER_WEBGPU_PACKAGE,+ ]),+ );++ setInstalled(true);+ } catch (error) {+ setInstallState({type: 'error', error: error as Error});+ }+ }, []);++ if (installed) {+ return (+ <Suspense fallback={null}>+ <LazyTranscriptionModal {...state} />+ </Suspense>+ );+ }++ return (+ <ModalContainer+ ariaLabel="Install transcription package"+ onEscape={dismiss}+ onOutsideClick={dismiss}+ panelStyle={panelStyle}+ >+ <ModalHeader title="Transcribe" onClose={dismiss} />+ <div style={contentStyle}>+ {busy ? (+ <div style={statusStyle}>+ <Spinner duration={0.5} size={18} />+ {`Installing ${WHISPER_WEBGPU_PACKAGE}…`}+ </div>+ ) : (+ <>+ This requires installing{' '}+ <code style={codeStyle}>{WHISPER_WEBGPU_PACKAGE}</code>. Continue?+ {installState.type === 'error' ? (+ <div style={errorStyle}>{installState.error.message}</div>+ ) : null}+ </>+ )}+ </div>+ <ModalFooterContainer>+ <Row align="center">+ <Flex />+ <Button disabled={busy} onClick={dismiss} style={cancelStyle}>+ Cancel+ </Button>+ <Spacing x={1} />+ <ModalButton disabled={busy} onClick={install} autoFocus>+ {installState.type === 'error' ? 'Retry' : 'Continue'}+ </ModalButton>+ </Row>+ </ModalFooterContainer>+ </ModalContainer>+ );+};diff --git a/packages/studio/src/components/Transcription/caption-output-name.ts b/packages/studio/src/components/Transcription/caption-output-name.tsnew file mode 100644index 0000000000..458f2bb38a--- /dev/null+++ b/packages/studio/src/components/Transcription/caption-output-name.ts@@ -0,0 +1,86 @@+const invalidCharacters = ['?', '*', '+', ':', '%'];++export const getDefaultCaptionOutputName = (+ src: string,+ displayName = 'captions',+): string => {+ let pathname = displayName;+ if (!/^(data|blob):/i.test(src.trimStart())) {+ pathname = src.split(/[?#]/)[0] ?? src;+ try {+ pathname = new URL(+ src,+ typeof window === 'undefined'+ ? 'http://localhost'+ : window.location.href,+ ).pathname;+ } catch {+ // Fall back to the source as-is if it cannot be parsed as a URL.+ }+ }++ const lastPathSegment = pathname.split('/').filter(Boolean).at(-1);+ let decoded = lastPathSegment ?? 'captions';+ try {+ decoded = decodeURIComponent(decoded);+ } catch {+ // Keep the encoded name if it contains an invalid escape sequence.+ }++ const withoutExtension = decoded.replace(/\.[^/.]+$/, '');+ const safeBaseName = withoutExtension+ .replace(/[^a-zA-Z0-9-_ ]/g, '-')+ .trim()+ .replace(/\s+/g, '-')+ .replace(/^-+|-+$/g, '');++ return `${safeBaseName || 'captions'}-captions.json`;+};++export const validateCaptionOutputName = (outName: string): string | null => {+ if (outName.trim() === '') {+ return 'Enter an output file';+ }++ if (outName !== outName.trim()) {+ return 'The output file must not start or end with whitespace';+ }++ if (outName.startsWith('/') || /^[a-zA-Z]:/.test(outName)) {+ return 'The output file must be relative to the public folder';+ }++ if (outName.includes('\\')) {+ return 'Use forward slashes in the output file';+ }++ if (outName.includes('\0')) {+ return 'The output file must not contain null characters';+ }++ const segments = outName.split('/');+ if (segments.some((segment) => segment === '')) {+ return 'The output file must not contain empty path segments';+ }++ if (segments.some((segment) => segment === '.')) {+ return 'The output file must not contain "." path segments';+ }++ if (segments.some((segment) => segment.startsWith('..'))) {+ return 'The output file must stay inside the public folder';+ }++ const invalidCharacter = invalidCharacters.find((character) =>+ outName.includes(character),+ );+ if (invalidCharacter) {+ return `The output file must not contain "${invalidCharacter}"`;+ }++ if (!outName.toLowerCase().endsWith('.json')) {+ return 'The output file must end in .json';+ }++ return null;+};diff --git a/packages/studio/src/components/Transcription/is-transcribable-sequence.ts b/packages/studio/src/components/Transcription/is-transcribable-sequence.tsnew file mode 100644index 0000000000..e8cae50662--- /dev/null+++ b/packages/studio/src/components/Transcription/is-transcribable-sequence.ts@@ -0,0 +1,29 @@+import type {TSequence} from 'remotion';++export type TranscribableSequence =+ | Extract<TSequence, {type: 'audio'}>+ | Extract<TSequence, {type: 'video'}>;++const modernComponentIdentities = {+ audio: 'dev.remotion.media.Audio',+ video: 'dev.remotion.media.Video',+} as const;++const legacyDocumentationLinks = {+ audio: 'https://www.remotion.dev/docs/html5-audio',+ video: 'https://www.remotion.dev/docs/html5-video',+} as const;++export const isTranscribableSequence = (+ sequence: TSequence,+): sequence is TranscribableSequence => {+ if (sequence.type !== 'audio' && sequence.type !== 'video') {+ return false;+ }++ return (+ sequence.controls?.componentIdentity ===+ modernComponentIdentities[sequence.type] ||+ sequence.documentationLink === legacyDocumentationLinks[sequence.type]+ );+};diff --git a/packages/studio/src/components/Transcription/resample-media-to-16-khz.ts b/packages/studio/src/components/Transcription/resample-media-to-16-khz.tsnew file mode 100644index 0000000000..b150618a83--- /dev/null+++ b/packages/studio/src/components/Transcription/resample-media-to-16-khz.ts@@ -0,0 +1,124 @@+import {WHISPER_WEBGPU_SAMPLE_RATE} from '@remotion/whisper-webgpu';+import {+ ALL_FORMATS,+ Conversion,+ Input,+ NullTarget,+ Output,+ UrlSource,+ WavOutputFormat,+} from 'mediabunny';++export const resampleMediaTo16Khz = async ({+ src,+ audioStreamIndex,+ requestInit,+ onProgress,+}: {+ readonly src: string;+ readonly audioStreamIndex: number | null;+ readonly requestInit: Omit<RequestInit, 'signal'> | null;+ readonly onProgress: (progress: number) => void;+}): Promise<Float32Array> => {+ const input = new Input({+ formats: ALL_FORMATS,+ source: new UrlSource(src, requestInit === null ? {} : {requestInit}),+ });+ const waveformChunks: Array<{+ readonly startFrame: number;+ readonly waveform: Float32Array;+ }> = [];++ try {+ onProgress(0);+ const audioTracks = await input.getAudioTracks();+ const primaryVideoTrack =+ audioStreamIndex === null ? await input.getPrimaryVideoTrack() : null;+ const audioTrack =+ audioStreamIndex !== null+ ? (audioTracks[audioStreamIndex] ?? null)+ : primaryVideoTrack+ ? await primaryVideoTrack.getPrimaryPairableAudioTrack()+ : (audioTracks[0] ?? null);+ if (audioTrack === null) {+ throw new Error('The selected media does not have an audio track.');+ }++ if (await audioTrack.isLive()) {+ throw new Error('Live media cannot be transcribed.');+ }++ const output = new Output({+ format: new WavOutputFormat(),+ target: new NullTarget(),+ });+ const conversion = await Conversion.init({+ input,+ output,+ tracks: 'all',+ video: {discard: true},+ audio: (track) => {+ if (track.id !== audioTrack.id) {+ return {discard: true};+ }++ return {+ codec: 'pcm-f32',+ forceTranscode: true,+ numberOfChannels: 1,+ process: (sample) => {+ const floats = new Float32Array(+ sample.allocationSize({format: 'f32', planeIndex: 0}) /+ Float32Array.BYTES_PER_ELEMENT,+ );+ sample.copyTo(floats, {format: 'f32', planeIndex: 0});+ waveformChunks.push({+ startFrame: Math.round(+ sample.timestamp * WHISPER_WEBGPU_SAMPLE_RATE,+ ),+ waveform: floats,+ });+ return sample;+ },+ sampleFormat: 'f32',+ sampleRate: WHISPER_WEBGPU_SAMPLE_RATE,+ };+ },+ });++ if (!conversion.isValid) {+ throw new Error(+ 'The audio track cannot be decoded in this browser. Try converting the media to WAV first.',+ );+ }++ conversion.onProgress = (progress) => onProgress(progress);+ await conversion.execute();+ onProgress(1);+ } finally {+ input.dispose();+ }++ const waveformLength = waveformChunks.reduce(+ (maximum, chunk) =>+ Math.max(maximum, chunk.startFrame + chunk.waveform.length),+ 0,+ );+ const waveform = new Float32Array(waveformLength);+ for (const chunk of waveformChunks) {+ const destinationStart = Math.max(0, chunk.startFrame);+ const sourceStart = Math.max(0, -chunk.startFrame);+ const availableLength = Math.min(+ chunk.waveform.length - sourceStart,+ waveform.length - destinationStart,+ );+ if (availableLength > 0) {+ waveform.set(+ chunk.waveform.subarray(sourceStart, sourceStart + availableLength),+ destinationStart,+ );+ }+ }++ return waveform;+};diff --git a/packages/studio/src/components/Transcription/whisper-languages.ts b/packages/studio/src/components/Transcription/whisper-languages.tsnew file mode 100644index 0000000000..2cc173c45f--- /dev/null+++ b/packages/studio/src/components/Transcription/whisper-languages.ts@@ -0,0 +1,103 @@+export const WHISPER_LANGUAGES = [+ ['af', 'Afrikaans'],+ ['sq', 'Albanian'],+ ['am', 'Amharic'],+ ['ar', 'Arabic'],+ ['hy', 'Armenian'],+ ['as', 'Assamese'],+ ['az', 'Azerbaijani'],+ ['ba', 'Bashkir'],+ ['eu', 'Basque'],+ ['be', 'Belarusian'],+ ['bn', 'Bengali'],+ ['bs', 'Bosnian'],+ ['br', 'Breton'],+ ['bg', 'Bulgarian'],+ ['my', 'Burmese'],+ ['ca', 'Catalan'],+ ['zh', 'Chinese'],+ ['hr', 'Croatian'],+ ['cs', 'Czech'],+ ['da', 'Danish'],+ ['nl', 'Dutch'],+ ['en', 'English'],+ ['et', 'Estonian'],+ ['fo', 'Faroese'],+ ['fi', 'Finnish'],+ ['fr', 'French'],+ ['gl', 'Galician'],+ ['ka', 'Georgian'],+ ['de', 'German'],+ ['el', 'Greek'],+ ['gu', 'Gujarati'],+ ['ht', 'Haitian Creole'],+ ['ha', 'Hausa'],+ ['haw', 'Hawaiian'],+ ['he', 'Hebrew'],+ ['hi', 'Hindi'],+ ['hu', 'Hungarian'],+ ['is', 'Icelandic'],+ ['id', 'Indonesian'],+ ['it', 'Italian'],+ ['ja', 'Japanese'],+ ['jw', 'Javanese'],+ ['kn', 'Kannada'],+ ['kk', 'Kazakh'],+ ['km', 'Khmer'],+ ['ko', 'Korean'],+ ['lo', 'Lao'],+ ['la', 'Latin'],+ ['lv', 'Latvian'],+ ['ln', 'Lingala'],+ ['lt', 'Lithuanian'],+ ['lb', 'Luxembourgish'],+ ['mk', 'Macedonian'],+ ['mg', 'Malagasy'],+ ['ms', 'Malay'],+ ['ml', 'Malayalam'],+ ['mt', 'Maltese'],+ ['mi', 'Maori'],+ ['mr', 'Marathi'],+ ['mn', 'Mongolian'],+ ['ne', 'Nepali'],+ ['no', 'Norwegian'],+ ['nn', 'Nynorsk'],+ ['oc', 'Occitan'],+ ['ps', 'Pashto'],+ ['fa', 'Persian'],+ ['pl', 'Polish'],+ ['pt', 'Portuguese'],+ ['pa', 'Punjabi'],+ ['ro', 'Romanian'],+ ['ru', 'Russian'],+ ['sa', 'Sanskrit'],+ ['sr', 'Serbian'],+ ['sn', 'Shona'],+ ['sd', 'Sindhi'],+ ['si', 'Sinhala'],+ ['sk', 'Slovak'],+ ['sl', 'Slovenian'],+ ['so', 'Somali'],+ ['es', 'Spanish'],+ ['su', 'Sundanese'],+ ['sw', 'Swahili'],+ ['sv', 'Swedish'],+ ['tl', 'Tagalog'],+ ['tg', 'Tajik'],+ ['ta', 'Tamil'],+ ['tt', 'Tatar'],+ ['te', 'Telugu'],+ ['th', 'Thai'],+ ['bo', 'Tibetan'],+ ['tr', 'Turkish'],+ ['tk', 'Turkmen'],+ ['uk', 'Ukrainian'],+ ['ur', 'Urdu'],+ ['uz', 'Uzbek'],+ ['vi', 'Vietnamese'],+ ['cy', 'Welsh'],+ ['yi', 'Yiddish'],+ ['yo', 'Yoruba'],+] as const;++export type WhisperLanguage = (typeof WHISPER_LANGUAGES)[number][0];diff --git a/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts b/packages/studio/src/components/Transcription/whisper-webgpu-capability.tsnew file mode 100644index 0000000000..46a9d8ee6d--- /dev/null+++ b/packages/studio/src/components/Transcription/whisper-webgpu-capability.ts@@ -0,0 +1,7 @@+export const WHISPER_WEBGPU_PACKAGE = '@remotion/whisper-webgpu';++export const isWhisperWebGpuInstalled = () => {+ return (+ window.remotion_installedPackages?.includes(WHISPER_WEBGPU_PACKAGE) ?? false+ );+};diff --git a/packages/studio/src/icons/transcription.tsx b/packages/studio/src/icons/transcription.tsxnew file mode 100644index 0000000000..3b6bc21b43--- /dev/null+++ b/packages/studio/src/icons/transcription.tsx@@ -0,0 +1,14 @@+import type {SVGProps} from 'react';+import {CURRENT_COLOR_LOWERCASE} from '../helpers/colors';++export const TranscriptionIcon = ({+ color,+ ...props+}: SVGProps<SVGSVGElement> & {readonly color?: string}) => (+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" {...props}>+ <path+ d="M12 15.75a4 4 0 0 0 4-4V7a4 4 0 0 0-8 0v4.75a4 4 0 0 0 4 4Zm-1.9-8.5V7a1.9 1.9 0 0 1 3.8 0v4.75a1.9 1.9 0 0 1-3.8 0v-.25h1.15a1.05 1.05 0 1 0 0-2.1H10.1V7.25Zm7.45 3.45a1.05 1.05 0 0 1 1.05 1.05 6.6 6.6 0 0 1-5.55 6.52v1.63h2.2a1.05 1.05 0 1 1 0 2.1h-6.5a1.05 1.05 0 1 1 0-2.1h2.2v-1.63a6.6 6.6 0 0 1-5.55-6.52 1.05 1.05 0 1 1 2.1 0 4.5 4.5 0 0 0 9 0 1.05 1.05 0 0 1 1.05-1.05Z"+ fill={color ?? CURRENT_COLOR_LOWERCASE}+ />+ </svg>+);diff --git a/packages/studio/src/state/modals.ts b/packages/studio/src/state/modals.tsindex c25afedc95..1e2f148c45 100644--- a/packages/studio/src/state/modals.ts+++ b/packages/studio/src/state/modals.ts@@ -140,6 +140,14 @@ export type AddEffectModalState = {clientId: string;};+export type TranscriptionModalState = {+ type: 'transcribe';+ src: string;+ displayName: string;+ audioStreamIndex: number | null;+ requestInit: Omit<RequestInit, 'signal'> | null;+};+export type CanvasCaptureImport = {readonly data: CanvasCaptureData;readonly durationInSeconds: number;@@ -255,6 +263,7 @@ export type ModalState =}| ElementInstallModalState| AddEffectModalState+ | TranscriptionModalState| ConfirmationDialogState| SvgImportDialogState;diff --git a/packages/studio/src/test/caption-output-name.test.ts b/packages/studio/src/test/caption-output-name.test.tsnew file mode 100644index 0000000000..510ba96200--- /dev/null+++ b/packages/studio/src/test/caption-output-name.test.ts@@ -0,0 +1,50 @@+import {expect, test} from 'bun:test';+import {+ getDefaultCaptionOutputName,+ validateCaptionOutputName,+} from '../components/Transcription/caption-output-name';++test('derives a caption JSON name from local and remote media sources', () => {+ expect(getDefaultCaptionOutputName('/interview.mp4')).toBe(+ 'interview-captions.json',+ );+ expect(+ getDefaultCaptionOutputName(+ 'https://example.com/media/My%20Interview.wav?token=secret',+ ),+ ).toBe('My-Interview-captions.json');+ expect(+ getDefaultCaptionOutputName(+ `data:audio/wav;base64,${'A'.repeat(1_000_000)}`,+ '<Audio>',+ ),+ ).toBe('Audio-captions.json');+ expect(+ getDefaultCaptionOutputName(+ ` data:audio/wav;base64,${'A'.repeat(1_000_000)}`,+ '<Audio>',+ ),+ ).toBe('Audio-captions.json');+});++test('accepts safe nested Caption[] output paths', () => {+ expect(validateCaptionOutputName('captions/interview.json')).toBeNull();+ expect(validateCaptionOutputName('captions/interview.JSON')).toBeNull();+});++test.each([+ '',+ ' captions.json',+ '/captions.json',+ 'C:/captions.json',+ 'captions\\output.json',+ 'captions\0output.json',+ 'captions//output.json',+ 'captions/./output.json',+ 'captions/../output.json',+ 'captions/..output.json',+ 'captions?.json',+ 'captions.txt',+])('rejects unsafe or unsupported output path %s', (outName) => {+ expect(validateCaptionOutputName(outName)).not.toBeNull();+});diff --git a/packages/studio/src/test/caption-queue-processor.test.tsx b/packages/studio/src/test/caption-queue-processor.test.tsxnew file mode 100644index 0000000000..a0c20e06ed--- /dev/null+++ b/packages/studio/src/test/caption-queue-processor.test.tsx@@ -0,0 +1,368 @@+import {afterEach, expect, mock, test} from 'bun:test';+import {act, cleanup, render, waitFor} from '@testing-library/react';+import {type ContextType, useContext} from 'react';+import {makeBrowserStudioOperations} from './make-browser-studio-operations';++let modelLoadCall:+ | {+ model: string;+ }+ | undefined;+let transcriptionCall:+ | {+ channelWaveform: Float32Array;+ options: Record<string, unknown>;+ }+ | undefined;+let cacheCheck:+ | {+ model: string;+ }+ | undefined;+const getCacheCheck = () => cacheCheck;+const getModelLoadCall = () => modelLoadCall;+let modelIsCached = false;+let modelLoadProgressCallback:+ | ((progress: {+ status: string;+ file: string | null;+ progress: number | null;+ loadedBytes: number | null;+ totalBytes: number | null;+ }) => void)+ | undefined;+let resolveModelLoading: (() => void) | null = null;+let disposalCalls = 0;+const resamplingCalls: Array<{+ src: string;+ audioStreamIndex: number | null;+ requestInit: Omit<RequestInit, 'signal'> | null;+}> = [];++const resampledWaveform = new Float32Array(32_000).fill(0.25);++mock.module('../components/Transcription/resample-media-to-16-khz', () => ({+ resampleMediaTo16Khz: ({+ src,+ audioStreamIndex,+ requestInit,+ onProgress,+ }: {+ src: string;+ audioStreamIndex: number | null;+ requestInit: Omit<RequestInit, 'signal'> | null;+ onProgress: (progress: number) => void;+ }) => {+ resamplingCalls.push({src, audioStreamIndex, requestInit});+ onProgress(0.5);+ return Promise.resolve(resampledWaveform);+ },+}));++mock.module('@remotion/whisper-webgpu', () => ({+ canUseWhisperWebGpu: () => Promise.resolve({supported: true}),+ clearStaleModels: () => Promise.resolve(),+ disposeWhisperModel: () => {+ disposalCalls++;+ return Promise.resolve();+ },+ getAvailableModels: () => [+ {+ name: 'tiny',+ modelId: 'onnx-community/whisper-tiny_timestamped',+ parameters: 39_000_000,+ multilingual: true,+ supportsTranslation: true,+ webGpuDownloadSize: 119_699_015,+ },+ {+ name: 'tiny.en',+ modelId: 'onnx-community/whisper-tiny.en_timestamped',+ parameters: 39_000_000,+ multilingual: false,+ supportsTranslation: false,+ webGpuDownloadSize: 119_697_479,+ },+ ],+ isWhisperModelCached: ({model}: {model: string}) => {+ cacheCheck = {model};+ return Promise.resolve(modelIsCached);+ },+ loadWhisperModel: async ({+ model,+ onProgress,+ }: {+ model: string;+ onProgress?: typeof modelLoadProgressCallback;+ }) => {+ modelLoadCall = {model};+ modelLoadProgressCallback = onProgress;+ await new Promise<void>((resolve) => {+ resolveModelLoading = resolve;+ });+ modelLoadProgressCallback?.({+ status: 'ready',+ file: null,+ progress: 1,+ loadedBytes: 119_699_015,+ totalBytes: 119_699_015,+ });+ return {alreadyLoaded: false};+ },+ toCaptions: () => ({+ captions: [+ {+ text: 'Hello',+ startMs: 125,+ endMs: 625,+ timestampMs: 375,+ confidence: null,+ },+ {+ text: ' world.',+ startMs: 750,+ endMs: 1500,+ timestampMs: 1125,+ confidence: null,+ },+ ],+ }),+ transcribe: ({+ channelWaveform,+ ...options+ }: {+ channelWaveform: Float32Array;+ [key: string]: unknown;+ }) => {+ transcriptionCall = {channelWaveform, options};+ return Promise.resolve({+ text: ' Hello world.',+ chunks: [+ {text: ' Hello', timestamp: [0.125, 0.625]},+ {text: ' world.', timestamp: [0.75, 1.5]},+ ],+ });+ },+ WHISPER_WEBGPU_SAMPLE_RATE: 16_000,+}));++afterEach(async () => {+ cleanup();+ resolveModelLoading?.();+ const {disposeWhisperModel} = await import('@remotion/whisper-webgpu');+ await disposeWhisperModel({model: 'tiny'});+ resolveModelLoading = null;+});++test('downloads a model in the queued job before transcribing', async () => {+ disposalCalls = 0;+ resamplingCalls.length = 0;+ modelLoadCall = undefined;+ modelLoadProgressCallback = undefined;+ transcriptionCall = undefined;+ cacheCheck = undefined;+ modelIsCached = false;+ resolveModelLoading = null;+ const {CaptionQueueProcessor} =+ await import('../components/RenderQueue/CaptionQueueProcessor');+ const {RenderQueueContext, RenderQueueContextProvider} =+ await import('../components/RenderQueue/context');+ const originalGpuDescriptor = Object.getOwnPropertyDescriptor(+ navigator,+ 'gpu',+ );+ const originalSecureContextDescriptor = Object.getOwnPropertyDescriptor(+ window,+ 'isSecureContext',+ );+ const originalBrowserStudioDescriptor = Object.getOwnPropertyDescriptor(+ window,+ 'remotion_browserStudio',+ );+ const writtenFiles: Array<{+ contents: string | ArrayBuffer;+ filePath: string;+ }> = [];+ Object.defineProperty(navigator, 'gpu', {+ configurable: true,+ value: {requestAdapter: () => Promise.resolve({})},+ });+ Object.defineProperty(window, 'isSecureContext', {+ configurable: true,+ value: true,+ });+ Object.defineProperty(window, 'remotion_browserStudio', {+ configurable: true,+ value: makeBrowserStudioOperations({+ writeStaticFile: ({contents, filePath}) => {+ writtenFiles.push({contents, filePath});+ return Promise.resolve();+ },+ }),+ });+ let currentContext: ContextType<typeof RenderQueueContext> | null = null;+ const ReadContext = () => {+ currentContext = useContext(RenderQueueContext);+ return null;+ };++ try {+ render(+ <RenderQueueContextProvider>+ <CaptionQueueProcessor />+ <ReadContext />+ </RenderQueueContextProvider>,+ );++ const getContext = () => {+ if (currentContext === null) {+ throw new Error('Render queue context is not mounted');+ }++ return currentContext;+ };++ let jobId = '';+ act(() => {+ jobId = getContext().addCaptionJob({+ displayName: 'interview.wav',+ audioStreamIndex: 2,+ requestInit: {+ credentials: 'include',+ headers: {'x-remotion-test': 'transcription'},+ },+ language: 'de',+ model: 'tiny',+ chunkLengthInSeconds: 20,+ strideLengthInSeconds: 3,+ task: 'translate',+ forceFullSequences: true,+ doSample: true,+ temperature: 0.7,+ topK: 25,+ repetitionPenalty: 1.2,+ noRepeatNgramSize: 3,+ outName: 'captions/interview.json',+ src: '/static/interview.wav',+ });+ });++ await waitFor(() => expect(modelLoadCall).toBeDefined());+ expect(getCacheCheck()).toEqual({model: 'tiny'});+ expect(resamplingCalls).toEqual([]);+ expect(transcriptionCall).toBeUndefined();+ expect(writtenFiles).toEqual([]);++ act(() => {+ modelLoadProgressCallback?.({+ status: 'loading',+ file: null,+ progress: 0.5,+ loadedBytes: 119_699_015 / 2,+ totalBytes: 119_699_015,+ });+ });+ await waitFor(() => {+ const job = getContext().captionJobs.find(({id}) => id === jobId);+ if (job?.status !== 'running') {+ throw new Error('Caption job is not running');+ }++ expect(job.progress.message).toContain('Downloading tiny…');+ expect(job.progress.message).toContain('50%');+ expect(job.progress.value).toBeCloseTo(0.165);+ });+ expect(resamplingCalls).toEqual([]);+ expect(transcriptionCall).toBeUndefined();+ expect(writtenFiles).toEqual([]);++ act(() => resolveModelLoading?.());+ await waitFor(() => {+ expect(+ getContext().captionJobs.find((job) => job.id === jobId),+ ).toMatchObject({status: 'done', captionCount: 2});+ });++ expect(resamplingCalls).toEqual([+ {+ src: '/static/interview.wav',+ audioStreamIndex: 2,+ requestInit: {+ credentials: 'include',+ headers: {'x-remotion-test': 'transcription'},+ },+ },+ ]);+ expect(getModelLoadCall()).toEqual({model: 'tiny'});+ await waitFor(() => {+ expect(transcriptionCall).toEqual({+ channelWaveform: resampledWaveform,+ options: {+ model: 'tiny',+ language: 'de',+ task: 'translate',+ chunkLengthInSeconds: 20,+ strideLengthInSeconds: 3,+ forceFullSequences: true,+ doSample: true,+ temperature: 0.7,+ topK: 25,+ repetitionPenalty: 1.2,+ noRepeatNgramSize: 3,+ },+ });+ });+ expect(writtenFiles).toEqual([+ {+ filePath: 'captions/interview.json',+ contents: JSON.stringify(+ [+ {+ text: 'Hello',+ startMs: 125,+ endMs: 625,+ timestampMs: 375,+ confidence: null,+ },+ {+ text: ' world.',+ startMs: 750,+ endMs: 1500,+ timestampMs: 1125,+ confidence: null,+ },+ ],+ null,+ 2,+ ),+ },+ ]);+ expect(disposalCalls).toBe(1);+ } finally {+ if (originalGpuDescriptor) {+ Object.defineProperty(navigator, 'gpu', originalGpuDescriptor);+ } else {+ Reflect.deleteProperty(navigator, 'gpu');+ }++ if (originalSecureContextDescriptor) {+ Object.defineProperty(+ window,+ 'isSecureContext',+ originalSecureContextDescriptor,+ );+ } else {+ Reflect.deleteProperty(window, 'isSecureContext');+ }++ if (originalBrowserStudioDescriptor) {+ Object.defineProperty(+ window,+ 'remotion_browserStudio',+ originalBrowserStudioDescriptor,+ );+ } else {+ Reflect.deleteProperty(window, 'remotion_browserStudio');+ }+ }+});diff --git a/packages/studio/src/test/render-queue-scheduler.test.tsx b/packages/studio/src/test/render-queue-scheduler.test.tsxnew file mode 100644index 0000000000..9aa2e84d01--- /dev/null+++ b/packages/studio/src/test/render-queue-scheduler.test.tsx@@ -0,0 +1,152 @@+import {afterEach, expect, spyOn, test} from 'bun:test';+import {act, cleanup, render, waitFor} from '@testing-library/react';+import {type ContextType, useContext} from 'react';+import type {AddCaptionJobParams} from '../components/RenderQueue/caption-job-types';+import type {+ AddClientStillJobParams,+ CompositionRef,+} from '../components/RenderQueue/client-render-queue';+import {+ RenderQueueContext,+ RenderQueueContextProvider,+} from '../components/RenderQueue/context';++afterEach(cleanup);++const compositionRef: CompositionRef = {+ component: () => null,+ calculateMetadata: null,+ width: 100,+ height: 100,+ fps: 30,+ durationInFrames: 30,+ defaultProps: {},+};++const makeCaptionParams = (displayName: string): AddCaptionJobParams => ({+ src: `/${displayName}`,+ displayName,+ audioStreamIndex: null,+ requestInit: null,+ outName: `${displayName}.json`,+ model: 'tiny.en',+ language: null,+ chunkLengthInSeconds: 30,+ strideLengthInSeconds: 5,+ task: 'transcribe',+ forceFullSequences: false,+ doSample: false,+ temperature: 1,+ topK: 50,+ repetitionPenalty: 1,+ noRepeatNgramSize: 0,+});++const makeStillParams = (compositionId: string): AddClientStillJobParams => ({+ type: 'client-still',+ compositionId,+ outName: `${compositionId}.png`,+ imageFormat: 'png',+ frame: 0,+ inputProps: {},+ delayRenderTimeout: 30_000,+ mediaCacheSizeInBytes: null,+ logLevel: 'info',+ licenseKey: null,+ scale: 1,+ allowHtmlInCanvas: false,+});++test('processes client renders and captions in one local FIFO', async () => {+ const dateNow = spyOn(Date, 'now').mockReturnValue(1_000);+ let currentContext: ContextType<typeof RenderQueueContext> | null = null;+ const processedJobs: string[] = [];+ const ReadContext = () => {+ currentContext = useContext(RenderQueueContext);+ return null;+ };++ try {+ render(+ <RenderQueueContextProvider>+ <ReadContext />+ </RenderQueueContextProvider>,+ );++ const getContext = () => {+ if (currentContext === null) {+ throw new Error('Render queue context is not mounted');+ }++ return currentContext;+ };++ let firstCaptionId = '';+ let renderId = '';+ let secondCaptionId = '';+ act(() => {+ getContext().setProcessCaptionJobCallback((job) => {+ processedJobs.push(job.id);+ return Promise.resolve();+ });+ getContext().setProcessJobCallback((job) => {+ processedJobs.push(job.id);+ return Promise.resolve();+ });+ firstCaptionId = getContext().addCaptionJob(+ makeCaptionParams('first.mp3'),+ );+ renderId = getContext().addClientStillJob(+ makeStillParams('composition'),+ compositionRef,+ );+ secondCaptionId = getContext().addCaptionJob(+ makeCaptionParams('second.mp3'),+ );+ });++ await waitFor(() => expect(processedJobs).toEqual([firstCaptionId]));+ expect(+ getContext().clientJobs.find((job) => job.id === renderId)?.status,+ ).toBe('idle');+ expect(+ getContext().captionJobs.find((job) => job.id === secondCaptionId)+ ?.status,+ ).toBe('idle');++ act(() => getContext().markCaptionJobDone(firstCaptionId, 1));+ await waitFor(() =>+ expect(processedJobs).toEqual([firstCaptionId, renderId]),+ );+ expect(+ getContext().captionJobs.find((job) => job.id === secondCaptionId)+ ?.status,+ ).toBe('idle');++ act(() => getContext().markClientJobSaving(renderId));+ expect(processedJobs).toEqual([firstCaptionId, renderId]);+ expect(+ getContext().captionJobs.find((job) => job.id === secondCaptionId)+ ?.status,+ ).toBe('idle');++ act(() =>+ getContext().markClientJobDone(renderId, {+ width: 100,+ height: 100,+ sizeInBytes: 1,+ }),+ );+ await waitFor(() =>+ expect(processedJobs).toEqual([+ firstCaptionId,+ renderId,+ secondCaptionId,+ ]),+ );++ act(() => getContext().markCaptionJobDone(secondCaptionId, 1));+ } finally {+ dateNow.mockRestore();+ }+});diff --git a/packages/studio/src/test/renders-tab-caption-jobs.test.tsx b/packages/studio/src/test/renders-tab-caption-jobs.test.tsxnew file mode 100644index 0000000000..3ec45e1169--- /dev/null+++ b/packages/studio/src/test/renders-tab-caption-jobs.test.tsx@@ -0,0 +1,119 @@+import {afterEach, expect, spyOn, test} from 'bun:test';+import {cleanup, fireEvent, render, screen} from '@testing-library/react';+import type {SetStateAction} from 'react';+import type {CanvasContent} from 'remotion';+import {Internals} from 'remotion';+import type {CaptionJob} from '../components/RenderQueue/caption-job-types';+import {CaptionQueueItem} from '../components/RenderQueue/CaptionQueueItem';+import {RenderQueueContext} from '../components/RenderQueue/context';+import {RendersTab} from '../components/RendersTab';+import {FolderContextProvider} from '../state/folders';++afterEach(cleanup);++const renderWithCaptionJob = (job: CaptionJob) => {+ let canvasContent: CanvasContent | null = null;+ render(+ <Internals.CompositionSetters.Provider+ value={+ {+ setCanvasContent: (value: SetStateAction<CanvasContent | null>) => {+ canvasContent =+ typeof value === 'function' ? value(canvasContent) : value;+ },+ } as never+ }+ >+ <FolderContextProvider>+ <RenderQueueContext.Provider+ value={+ {+ jobs: [],+ captionJobs: [job],+ } as never+ }+ >+ <>+ <RendersTab selected={false} onClick={() => undefined} />+ <CaptionQueueItem job={job} selected={false} />+ </>+ </RenderQueueContext.Provider>+ </FolderContextProvider>+ </Internals.CompositionSetters.Provider>,+ );++ return {getCanvasContent: () => canvasContent};+};++const baseJob = {+ id: 'caption-job',+ type: 'caption',+ startedAt: 0,+ src: '/audio.mp3',+ displayName: 'audio.mp3',+ audioStreamIndex: null,+ requestInit: null,+ outName: 'dialogue-captions.json',+ model: 'tiny.en',+ language: null,+ chunkLengthInSeconds: 30,+ strideLengthInSeconds: 5,+ task: 'transcribe',+ forceFullSequences: false,+ doSample: false,+ temperature: 1,+ topK: 50,+ repetitionPenalty: 1,+ noRepeatNgramSize: 0,+} as const;++test('labels the renders tab as Jobs while a caption job is running', () => {+ renderWithCaptionJob({+ ...baseJob,+ status: 'running',+ progress: {+ message: 'Downloading tiny.en… · 50% · 57.1 MB / 114.2 MB',+ value: 0.165,+ },+ });++ expect(screen.getByText('Jobs')).toBeTruthy();+ expect(screen.getByText(/Downloading tiny\.en… · 50%/)).toBeTruthy();+ const progress = screen.getByRole('progressbar', {+ name: 'Caption job progress',+ });+ expect(progress.getAttribute('aria-valuenow')).toBe('17');+ expect(progress.getAttribute('aria-valuetext')).toContain(+ 'Downloading tiny.en… · 50%',+ );+});++test('opens a completed caption output as an asset without showing its caption count', () => {+ const pushState = spyOn(window.history, 'pushState').mockImplementation(+ () => undefined,+ );++ try {+ const {getCanvasContent} = renderWithCaptionJob({+ ...baseJob,+ status: 'done',+ captionCount: 10,+ });++ expect(screen.getByText('Renders')).toBeTruthy();+ expect(screen.queryByText(/10 captions/)).toBeNull();+ expect(screen.getByText('dialogue-captions.json')).toBeTruthy();+ fireEvent.click(screen.getByText('dialogue-captions.json'));+ expect(getCanvasContent()).toEqual({+ type: 'asset',+ asset: 'dialogue-captions.json',+ });+ expect(pushState).toHaveBeenCalledWith(+ {},+ 'Studio',+ '/assets/dialogue-captions.json',+ );+ } finally {+ pushState.mockRestore();+ }+});diff --git a/packages/studio/src/test/sequence-context-menu-items.test.ts b/packages/studio/src/test/sequence-context-menu-items.test.tsindex a94fdc3a1f..0e10558d7e 100644--- a/packages/studio/src/test/sequence-context-menu-items.test.ts+++ b/packages/studio/src/test/sequence-context-menu-items.test.ts@@ -10,6 +10,7 @@ import {isSequenceVisibleAtTimelinePosition,shouldShowFreezeFrameMenuItem,} from '../components/Timeline/use-sequence-freeze-frame-menu-item';+import {isTranscribableSequence} from '../components/Transcription/is-transcribable-sequence';const originalWindowDescriptor = Object.getOwnPropertyDescriptor(globalThis,@@ -69,6 +70,44 @@ const renameSequenceItem: ComboboxValue = {const noop = () => undefined;+test('recognizes current and legacy Audio and Video timeline sequences', () => {+ for (const [type, componentIdentity, documentationLink] of [+ [+ 'audio',+ 'dev.remotion.media.Audio',+ 'https://www.remotion.dev/docs/html5-audio',+ ],+ [+ 'video',+ 'dev.remotion.media.Video',+ 'https://www.remotion.dev/docs/html5-video',+ ],+ ] as const) {+ expect(+ isTranscribableSequence({+ type,+ controls: {componentIdentity},+ documentationLink: null,+ } as unknown as TSequence),+ ).toBe(true);+ expect(+ isTranscribableSequence({+ type,+ controls: null,+ documentationLink,+ } as unknown as TSequence),+ ).toBe(true);+ }++ expect(+ isTranscribableSequence({+ type: 'video',+ controls: null,+ documentationLink: 'https://www.remotion.dev/docs/offthreadvideo',+ } as unknown as TSequence),+ ).toBe(false);+});+test('copy context menu items write their agent context to the clipboard', () => {const copiedTexts: string[] = [];Object.defineProperty(globalThis, 'navigator', {@@ -141,6 +180,7 @@ const expectNoExtraDividers = (items: ComboboxValue[]) => {test('sequence context menu normalizes dividers before source actions', () => {installTestWindow();+ let transcribed = false;const items = getSequenceContextMenuItems({assetLinkInfo: null,@@ -156,6 +196,9 @@ test('sequence context menu normalizes dividers before source actions', () => {onDeleteSequenceFromSource: noop,onDisableSequenceInteractivity: noop,onDuplicateSequenceFromSource: noop,+ onTranscribe: () => {+ transcribed = true;+ },openInCodingAgent: noop,openInEditor: noop,originalLocation: null,@@ -175,7 +218,15 @@ test('sequence context menu normalizes dividers before source actions', () => {expect(items.slice(copyContextIndex + 1, renameIndex).map((item) => item.type),- ).toEqual(['divider']);+ ).toEqual(['item', 'divider']);++ const transcribe = items.find((item) => item.id === 'transcribe');+ if (transcribe?.type !== 'item') {+ throw new Error('Expected Transcribe menu item');+ }++ transcribe.onClick('transcribe', null);+ expect(transcribed).toBe(true);});test('sequence context menu shares alternate apps without repeating defaults', () => {@@ -230,6 +281,7 @@ test('sequence context menu shares alternate apps without repeating defaults', (onDeleteSequenceFromSource: noop,onDisableSequenceInteractivity: noop,onDuplicateSequenceFromSource: noop,+ onTranscribe: null,openInCodingAgent: (codingAgentId, _codingAgentName, prompt) => {openedCodingAgents.push({id: codingAgentId, prompt});},@@ -333,6 +385,7 @@ test('Interactive.Svg context menu can copy the rendered SVG', () => {onDeleteSequenceFromSource: noop,onDisableSequenceInteractivity: noop,onDuplicateSequenceFromSource: noop,+ onTranscribe: null,openInCodingAgent: noop,openInEditor: noop,originalLocation: null,@@ -377,6 +430,7 @@ test('programmatically duplicated sequence menus apply actions to all instances'onDeleteSequenceFromSource: noop,onDisableSequenceInteractivity: noop,onDuplicateSequenceFromSource: noop,+ onTranscribe: null,openInCodingAgent: noop,openInEditor: noop,originalLocation: null,@@ -411,6 +465,7 @@ test('read-only sequence menus only contain non-mutating actions', () => {onDeleteSequenceFromSource: noop,onDisableSequenceInteractivity: noop,onDuplicateSequenceFromSource: noop,+ onTranscribe: null,openInCodingAgent: noop,openInEditor: noop,originalLocation: null,diff --git a/packages/studio/src/test/sequenced-timeline.test.ts b/packages/studio/src/test/sequenced-timeline.test.tsindex 65c690de39..845d182e27 100644--- a/packages/studio/src/test/sequenced-timeline.test.ts+++ b/packages/studio/src/test/sequenced-timeline.test.ts@@ -145,6 +145,8 @@ test('Should calculate sequences correctly', () => {nodePathInfo: null,sequence: {type: 'video',+ audioStreamIndex: null,+ requestInit: null,src: 'http://localhost:5000/codep1-2.mp4',id: '0.640722264073593',duration: 486,@@ -207,6 +209,8 @@ test('Should calculate sequences correctly', () => {sequence: {loopDisplay: undefined,type: 'video',+ audioStreamIndex: null,+ requestInit: null,src: 'http://localhost:5000/screen3.mp4',id: '0.5529201074994525',duration: 486,@@ -287,6 +291,8 @@ const sequences: TSequence[] = [},{type: 'video',+ audioStreamIndex: null,+ requestInit: null,src: 'http://localhost:5000/codep1-2.mp4',id: '0.640722264073593',duration: 10736,@@ -340,6 +346,8 @@ const sequences: TSequence[] = [},{type: 'video',+ audioStreamIndex: null,+ requestInit: null,src: 'http://localhost:5000/screen3.mp4',id: '0.5529201074994525',duration: 23066,diff --git a/packages/studio/src/test/timeline.test.ts b/packages/studio/src/test/timeline.test.tsindex c805592eee..1792aabcca 100644--- a/packages/studio/src/test/timeline.test.ts+++ b/packages/studio/src/test/timeline.test.ts@@ -244,6 +244,8 @@ test('Should inherit loop display from parent for media tracks', () => {showInTimeline: true,timelineOrder: null,type: 'video',+ audioStreamIndex: null,+ requestInit: null,getStack,refForOutline: null,isInsideSeries: false,@@ -377,6 +379,8 @@ test('Should account for a parent Sequence trimBefore in video thumbnails', () =showInTimeline: true,timelineOrder: null,type: 'video',+ audioStreamIndex: null,+ requestInit: null,getStack,refForOutline: null,isInsideSeries: false,diff --git a/packages/studio/src/test/transcription-modal.test.tsx b/packages/studio/src/test/transcription-modal.test.tsxnew file mode 100644index 0000000000..8a291a37a9--- /dev/null+++ b/packages/studio/src/test/transcription-modal.test.tsx@@ -0,0 +1,305 @@+import {expect, test} from 'bun:test';+import {+ act,+ cleanup,+ fireEvent,+ render,+ screen,+ waitFor,+} from '@testing-library/react';+import type {AddCaptionJobParams} from '../components/RenderQueue/caption-job-types';++const selectComboboxItem = async (+ combobox: HTMLElement,+ itemName: string | RegExp,+) => {+ fireEvent.click(combobox);+ const item = await screen.findByRole('button', {name: itemName});+ fireEvent.pointerUp(item, {button: 0});+};++const setNumberSetting = (name: RegExp, value: number) => {+ fireEvent.click(screen.getByRole('button', {name}));+ const input = screen.getByRole('textbox', {name});+ fireEvent.change(input, {target: {value: String(value)}});+ fireEvent.blur(input);+};++test('serializes transcription modal settings into caption jobs', async () => {+ const {portals} = await import('../components/Menu/portals');+ const previousPortals = [...portals];+ const createdPortals: HTMLElement[] = [];+ const testPortals = Array.from({length: 6}, (_, index) => {+ const id = `menuportal-${index}`;+ const existing = document.getElementById(id);+ if (existing) {+ return existing;+ }++ const element = document.createElement('div');+ element.id = id;+ document.body.appendChild(element);+ createdPortals.push(element);+ return element;+ });+ portals.splice(0, portals.length, ...testPortals);++ const originalGpuDescriptor = Object.getOwnPropertyDescriptor(+ navigator,+ 'gpu',+ );+ const originalSecureContextDescriptor = Object.getOwnPropertyDescriptor(+ window,+ 'isSecureContext',+ );+ const originalStaticBaseDescriptor = Object.getOwnPropertyDescriptor(+ window,+ 'remotion_staticBase',+ );+ const originalLocalStorageDescriptor = Object.getOwnPropertyDescriptor(+ globalThis,+ 'localStorage',+ );+ const localStorageValues = new Map<string, string>();+ const localStorageMock: Storage = {+ clear: () => localStorageValues.clear(),+ getItem: (key: string) => localStorageValues.get(key) ?? null,+ key: (index: number) => [...localStorageValues.keys()][index] ?? null,+ removeItem: (key: string) => localStorageValues.delete(key),+ setItem: (key: string, value: string) => localStorageValues.set(key, value),+ get length() {+ return localStorageValues.size;+ },+ };++ Object.defineProperty(navigator, 'gpu', {+ configurable: true,+ value: {requestAdapter: () => Promise.resolve({})},+ });+ Object.defineProperty(window, 'isSecureContext', {+ configurable: true,+ value: true,+ });+ Object.defineProperty(window, 'remotion_staticBase', {+ configurable: true,+ value: '/static-test',+ });+ Object.defineProperty(globalThis, 'localStorage', {+ configurable: true,+ value: localStorageMock,+ });++ try {+ const {TranscriptionModal} =+ await import('../components/Transcription/TranscriptionModal');+ const {RenderQueueContext} =+ await import('../components/RenderQueue/context');+ const {SetSelectedModalContext} = await import('../state/modals');+ const {SidebarContext} = await import('../state/sidebar');+ const submittedJobs: AddCaptionJobParams[] = [];+ const requestInit = {+ credentials: 'include',+ headers: {'x-remotion-test': 'modal-serialization'},+ } as const;++ render(+ <SetSelectedModalContext.Provider+ value={{setSelectedModal: () => undefined}}+ >+ <SidebarContext.Provider+ value={{+ sidebarCollapsedDuringDrag: null,+ sidebarCollapsedStateLeft: 'collapsed',+ sidebarCollapsedStateRight: 'collapsed',+ setSidebarCollapsedDuringDrag: () => undefined,+ setSidebarCollapsedState: () => undefined,+ }}+ >+ <RenderQueueContext.Provider+ value={+ {+ captionJobs: [],+ addCaptionJob: (params: AddCaptionJobParams) => {+ submittedJobs.push(params);+ return `caption-job-${submittedJobs.length}`;+ },+ } as never+ }+ >+ <TranscriptionModal+ type="transcribe"+ audioStreamIndex={2}+ displayName="interview.wav"+ requestInit={requestInit}+ src="/media/interview.wav"+ />+ </RenderQueueContext.Provider>+ </SidebarContext.Provider>+ </SetSelectedModalContext.Provider>,+ );++ screen.getByRole('dialog', {+ name: 'Transcribe interview.wav',+ });+ const submit = screen.getByRole('button', {+ name: /^Transcribe$/,+ }) as HTMLButtonElement;+ await waitFor(() => expect(submit.disabled).toBe(false));+ expect(+ screen.queryByText('Runs locally as a background job using WebGPU'),+ ).toBeNull();+ const outputLabel = screen.getByText('Output in public/');+ const modelLabel = screen.getByText('Whisper model');+ expect(+ outputLabel.compareDocumentPosition(modelLabel) &+ Node.DOCUMENT_POSITION_FOLLOWING,+ ).not.toBe(0);+ const outputHelp = screen.getByTitle('Learn more about Output in public/');+ expect(screen.queryByText('Caption[]')).toBeNull();+ fireEvent.pointerUp(outputHelp);+ const captionType = screen.getByText('Caption[]');+ expect(captionType.style.fontFamily).toBe('monospace');+ expect(captionType.style.fontSize).toBe('inherit');+ expect(captionType.style.lineHeight).toBe('inherit');+ fireEvent.pointerUp(outputHelp);+ screen.getByTitle('Learn more about Model download');+ screen.getByTitle('Learn more about Chunk length');+ screen.getByTitle('Learn more about Stride length');+ screen.getByTitle('Learn more about Force full sequences');+ screen.getByTitle('Learn more about Use sampling');+ screen.getByTitle('Learn more about Repetition penalty');+ screen.getByTitle('Learn more about No-repeat n-gram size');++ await selectComboboxItem(screen.getByTitle('Whisper model'), /^tiny ·/);+ await selectComboboxItem(screen.getByTitle('Spoken language'), 'German');+ await selectComboboxItem(+ screen.getByRole('button', {name: 'Task: Transcribe'}),+ 'Task: Translate to English',+ );+ const taskLabel = screen.getByText('Translate to English', {+ selector: 'span[aria-hidden="true"]',+ });+ expect(taskLabel.style.color).toBe('inherit');+ expect(taskLabel.style.fontFamily).toBe('inherit');+ expect(taskLabel.style.fontSize).toBe('inherit');+ expect(taskLabel.style.lineHeight).toBe('inherit');+ const taskWarning = screen.getByText(+ 'Word timings may be less accurate when translating to English.',+ );+ expect(+ taskWarning.parentElement?.parentElement?.parentElement?.style.display,+ ).toBe('flex');+ expect(+ taskWarning.parentElement?.parentElement?.parentElement?.style+ .justifyContent,+ ).toBe('flex-end');++ const outputInput = screen.getByRole('textbox', {+ name: 'Caption output file',+ });+ expect(+ outputInput.parentElement?.parentElement?.parentElement?.style+ .paddingBottom,+ ).toBe('12px');+ fireEvent.change(outputInput, {+ target: {value: 'captions/interview.json'},+ });++ const forceFullSequences = screen.getByRole('checkbox', {+ name: 'Force full sequences',+ });+ const useSampling = screen.getByRole('checkbox', {+ name: 'Use sampling',+ });+ for (const checkbox of [forceFullSequences, useSampling]) {+ const checkboxLabel = document.querySelector(+ `label[for="${checkbox.id}"]`,+ ) as HTMLLabelElement | null;+ expect(checkboxLabel?.style.color).toBe('inherit');+ expect(checkboxLabel?.style.fontFamily).toBe('inherit');+ expect(checkboxLabel?.style.fontSize).toBe('inherit');+ expect(checkboxLabel?.style.lineHeight).toBe('inherit');+ }++ act(() => forceFullSequences.click());+ act(() => useSampling.click());+ screen.getByTitle('Learn more about Temperature');+ screen.getByTitle('Learn more about Top K');++ setNumberSetting(/^Chunk length:/, 22);+ setNumberSetting(/^Stride length:/, 4);+ setNumberSetting(/^Temperature:/, 0.7);+ setNumberSetting(/^Top K:/, 25);+ setNumberSetting(/^Repetition penalty:/, 1.2);+ setNumberSetting(/^No-repeat n-gram size:/, 3);++ act(() => useSampling.click());+ fireEvent.click(submit);+ act(() => forceFullSequences.click());+ act(() => useSampling.click());+ fireEvent.click(submit);++ const commonJob = {+ audioStreamIndex: 2,+ chunkLengthInSeconds: 22,+ displayName: 'interview.wav',+ language: 'de',+ model: 'tiny',+ noRepeatNgramSize: 3,+ outName: 'captions/interview.json',+ repetitionPenalty: 1.2,+ requestInit,+ src: '/media/interview.wav',+ strideLengthInSeconds: 4,+ task: 'translate',+ temperature: 0.7,+ topK: 25,+ } satisfies Omit<AddCaptionJobParams, 'doSample' | 'forceFullSequences'>;+ expect(submittedJobs).toEqual([+ {...commonJob, doSample: false, forceFullSequences: true},+ {...commonJob, doSample: true, forceFullSequences: false},+ ]);+ } finally {+ cleanup();+ portals.splice(0, portals.length, ...previousPortals);+ for (const portal of createdPortals) {+ portal.remove();+ }++ if (originalGpuDescriptor) {+ Object.defineProperty(navigator, 'gpu', originalGpuDescriptor);+ } else {+ Reflect.deleteProperty(navigator, 'gpu');+ }++ if (originalSecureContextDescriptor) {+ Object.defineProperty(+ window,+ 'isSecureContext',+ originalSecureContextDescriptor,+ );+ } else {+ Reflect.deleteProperty(window, 'isSecureContext');+ }++ if (originalStaticBaseDescriptor) {+ Object.defineProperty(+ window,+ 'remotion_staticBase',+ originalStaticBaseDescriptor,+ );+ } else {+ Reflect.deleteProperty(window, 'remotion_staticBase');+ }++ if (originalLocalStorageDescriptor) {+ Object.defineProperty(+ globalThis,+ 'localStorage',+ originalLocalStorageDescriptor,+ );+ } else {+ Reflect.deleteProperty(globalThis, 'localStorage');+ }+ }+});diff --git a/packages/studio/src/test/whisper-webgpu-capability.test.tsx b/packages/studio/src/test/whisper-webgpu-capability.test.tsxnew file mode 100644index 0000000000..1d288680bc--- /dev/null+++ b/packages/studio/src/test/whisper-webgpu-capability.test.tsx@@ -0,0 +1,86 @@+import {afterEach, expect, spyOn, test} from 'bun:test';+import {cleanup, fireEvent, render, waitFor} from '@testing-library/react';+import {ModalsProvider} from '../components/ModalsProvider';+import {TranscriptionModalWithOptionalWhisper} from '../components/Transcription/TranscriptionModalWithOptionalWhisper';+import {+ isWhisperWebGpuInstalled,+ WHISPER_WEBGPU_PACKAGE,+} from '../components/Transcription/whisper-webgpu-capability';++const originalInstalledPackages = window.remotion_installedPackages;++afterEach(() => {+ cleanup();+ window.remotion_installedPackages = originalInstalledPackages;+});++test('detects whether Whisper WebGPU is installed in the project', () => {+ window.remotion_installedPackages = [];+ expect(isWhisperWebGpuInstalled()).toBe(false);+ window.remotion_installedPackages = [WHISPER_WEBGPU_PACKAGE];+ expect(isWhisperWebGpuInstalled()).toBe(true);+});++test('asks before installing Whisper for transcription', () => {+ window.remotion_installedPackages = [];+ const {container, getByRole} = render(+ <ModalsProvider>+ <TranscriptionModalWithOptionalWhisper+ state={{+ type: 'transcribe',+ src: '/voice.wav',+ displayName: 'voice.wav',+ audioStreamIndex: null,+ requestInit: null,+ }}+ />+ </ModalsProvider>,+ );++ expect(container.textContent).toContain(+ 'This requires installing @remotion/whisper-webgpu. Continue?',+ );+ expect(getByRole('button', {name: 'Continue'})).toBeDefined();+});++test('opens transcription after installing Whisper without restarting', async () => {+ const originalIsStudio = window.remotion_isStudio;+ const originalIsReadOnlyStudio = window.remotion_isReadOnlyStudio;+ const originalStaticBase = window.remotion_staticBase;+ window.remotion_installedPackages = [];+ window.remotion_isStudio = true;+ window.remotion_isReadOnlyStudio = false;+ window.remotion_staticBase = '';+ const fetchSpy = spyOn(globalThis, 'fetch').mockResolvedValue(+ new Response(JSON.stringify({success: true, data: {}})),+ );+ try {+ const {container, getByRole} = render(+ <ModalsProvider>+ <TranscriptionModalWithOptionalWhisper+ state={{+ type: 'transcribe',+ src: '/voice.wav',+ displayName: 'voice.wav',+ audioStreamIndex: null,+ requestInit: null,+ }}+ />+ </ModalsProvider>,+ );++ fireEvent.click(getByRole('button', {name: 'Continue'}));+ await waitFor(() => {+ expect(window.remotion_installedPackages).toContain(+ WHISPER_WEBGPU_PACKAGE,+ );+ expect(container.textContent).toContain('Output in public/');+ });+ expect(fetchSpy).toHaveBeenCalledTimes(1);+ } finally {+ fetchSpy.mockRestore();+ window.remotion_isStudio = originalIsStudio;+ window.remotion_isReadOnlyStudio = originalIsReadOnlyStudio;+ window.remotion_staticBase = originalStaticBase;+ }+});diff --git a/packages/studio/tsconfig.json b/packages/studio/tsconfig.jsonindex 846851819b..56fe86b3c2 100644--- a/packages/studio/tsconfig.json+++ b/packages/studio/tsconfig.json@@ -41,6 +41,9 @@},{"path": "../web-renderer"+ },+ {+ "path": "../whisper-webgpu"}]}