π₯ Make videos programmatically with React
@remotion/studio: Add JSON and SRT caption importing
Igor Samokhovets91287f7f1566
diff --git a/bun.lock b/bun.lockindex 6582230b33..f5ece55fd3 100644--- a/bun.lock+++ b/bun.lock@@ -1805,8 +1805,10 @@"@jridgewell/trace-mapping": "0.3.31","@remotion/canvas": "workspace:*","@remotion/captions": "workspace:*",+ "@remotion/elevenlabs": "workspace:*","@remotion/media": "workspace:*","@remotion/media-utils": "workspace:*",+ "@remotion/openai-whisper": "workspace:*","@remotion/player": "workspace:*","@remotion/renderer": "workspace:*","@remotion/studio-codemods": "workspace:*",diff --git a/packages/docs/docs/elevenlabs/TableOfContents.tsx b/packages/docs/docs/elevenlabs/TableOfContents.tsxindex 069651aee2..251b106f10 100644--- a/packages/docs/docs/elevenlabs/TableOfContents.tsx+++ b/packages/docs/docs/elevenlabs/TableOfContents.tsx@@ -13,6 +13,10 @@ export const TableOfContents: React.FC = () => {<code>Caption</code></div></TOCItem>+ <TOCItem link="/docs/elevenlabs/is-elevenlabs-transcript-json">+ <strong>{'isElevenLabsTranscriptJson()'}</strong>+ <div>Detect ElevenLabs Speech to Text JSON</div>+ </TOCItem></Grid></div>);diff --git a/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx b/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdxindex 2a3e06608c..f968d501fe 100644--- a/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx+++ b/packages/docs/docs/elevenlabs/elevenlabs-transcript-to-captions.mdx@@ -63,6 +63,7 @@ An array of [`Caption`](/docs/captions/caption) objects.## See also+- [`isElevenLabsTranscriptJson()`](/docs/elevenlabs/is-elevenlabs-transcript-json)- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts)- [`@remotion/elevenlabs`](/docs/elevenlabs)- [`@remotion/openai-whisper`](/docs/openai-whisper)diff --git a/packages/docs/docs/elevenlabs/index.mdx b/packages/docs/docs/elevenlabs/index.mdxindex d665ed253a..94649d6148 100644--- a/packages/docs/docs/elevenlabs/index.mdx+++ b/packages/docs/docs/elevenlabs/index.mdx@@ -6,7 +6,7 @@ crumb: 'Subtitle videos'# @remotion/elevenlabs<AvailableFrom v="4.0.443" />-The `@remotion/elevenlabs` package converts the output of the [ElevenLabs Speech to Text API](https://elevenlabs.io/docs/api-reference/speech-to-text/convert) into an array of [`Caption`](/docs/captions/caption) objects, making them compatible with other caption-related Remotion APIs.+The `@remotion/elevenlabs` package detects and converts the output of the [ElevenLabs Speech to Text API](https://elevenlabs.io/docs/api-reference/speech-to-text/convert) into an array of [`Caption`](/docs/captions/caption) objects, making them compatible with other caption-related Remotion APIs.import {TableOfContents} from './TableOfContents';diff --git a/packages/docs/docs/elevenlabs/is-elevenlabs-transcript-json.mdx b/packages/docs/docs/elevenlabs/is-elevenlabs-transcript-json.mdxnew file mode 100644index 0000000000..c5a8653e50--- /dev/null+++ b/packages/docs/docs/elevenlabs/is-elevenlabs-transcript-json.mdx@@ -0,0 +1,52 @@+---+image: /generated/articles-docs-elevenlabs-is-elevenlabs-transcript-json.png+title: isElevenLabsTranscriptJson()+crumb: '@remotion/elevenlabs'+---++# isElevenLabsTranscriptJson()<AvailableFrom v="4.0.521" />++Returns whether a value has the top-level fields of an ElevenLabs Speech to Text JSON response.++Use it before [`elevenLabsTranscriptToCaptions()`](/docs/elevenlabs/elevenlabs-transcript-to-captions) when accepting JSON whose source is unknown.++This function detects the format but does not validate every word. The converter validates the fields it uses and throws an actionable error if the recognized input is malformed.++## Example++```ts twoslash title="convert-upload.ts"+import {+ elevenLabsTranscriptToCaptions,+ isElevenLabsTranscriptJson,+} from '@remotion/elevenlabs';++const contents = '{"language_code":"eng","text":"Hello","words":[]}';+const json = JSON.parse(contents);++if (isElevenLabsTranscriptJson(json)) {+ const {captions} = elevenLabsTranscriptToCaptions({transcript: json});+ console.log(captions);+}+```++## API++### `value`++The value to inspect. It may have any JavaScript type.++### Return value++A boolean. It is `true` when the value is an object with `language_code` and either `words` or `text` as top-level fields.++Additional fields do not affect detection.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering serverSideRendering player studio />++## See also++- [`elevenLabsTranscriptToCaptions()`](/docs/elevenlabs/elevenlabs-transcript-to-captions)+- [`isOpenAiWhisperJson()`](/docs/openai-whisper/is-open-ai-whisper-json)+- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts)diff --git a/packages/docs/docs/openai-whisper/TableOfContents.tsx b/packages/docs/docs/openai-whisper/TableOfContents.tsxindex 0488a2a185..a82c9a7932 100644--- a/packages/docs/docs/openai-whisper/TableOfContents.tsx+++ b/packages/docs/docs/openai-whisper/TableOfContents.tsx@@ -6,6 +6,10 @@ export const TableOfContents: React.FC = () => {return (<div><Grid>+ <TOCItem link="/docs/openai-whisper/is-open-ai-whisper-json">+ <strong>{'isOpenAiWhisperJson()'}</strong>+ <div>Detect OpenAI Whisper verbose JSON</div>+ </TOCItem><TOCItem link="/docs/openai-whisper/openai-whisper-api-to-captions"><strong>{'openAiWhisperApiToCaptions()'}</strong><div>diff --git a/packages/docs/docs/openai-whisper/is-open-ai-whisper-json.mdx b/packages/docs/docs/openai-whisper/is-open-ai-whisper-json.mdxnew file mode 100644index 0000000000..8829011cb2--- /dev/null+++ b/packages/docs/docs/openai-whisper/is-open-ai-whisper-json.mdx@@ -0,0 +1,52 @@+---+image: /generated/articles-docs-openai-whisper-is-open-ai-whisper-json.png+title: isOpenAiWhisperJson()+crumb: '@remotion/openai-whisper'+---++# isOpenAiWhisperJson()<AvailableFrom v="4.0.521" />++Returns whether a value has the top-level fields of an OpenAI Whisper verbose JSON response.++Use it before [`openAiWhisperApiToCaptions()`](/docs/openai-whisper/openai-whisper-api-to-captions) when accepting JSON whose source is unknown.++This function detects the format but does not validate every word. The converter validates the fields it uses and throws an actionable error if the recognized input is malformed.++## Example++```ts twoslash title="convert-upload.ts"+import {+ isOpenAiWhisperJson,+ openAiWhisperApiToCaptions,+} from '@remotion/openai-whisper';++const contents = '{"language":"english","text":"Hello","words":[]}';+const json = JSON.parse(contents);++if (isOpenAiWhisperJson(json)) {+ const {captions} = openAiWhisperApiToCaptions({transcription: json});+ console.log(captions);+}+```++## API++### `value`++The value to inspect. It may have any JavaScript type.++### Return value++A boolean. It is `true` when the value is an object without `language_code`, with `text`, and with at least one of `task`, `language`, `duration`, `segments`, or `words` as top-level fields.++Other additional fields do not affect detection.++## Compatibility++<CompatibilityTable chrome firefox safari nodejs bun serverlessFunctions clientSideRendering serverSideRendering player studio />++## See also++- [`openAiWhisperApiToCaptions()`](/docs/openai-whisper/openai-whisper-api-to-captions)+- [`isElevenLabsTranscriptJson()`](/docs/elevenlabs/is-elevenlabs-transcript-json)+- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/openai-whisper/src/openai-whisper-api-to-captions.ts)diff --git a/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx b/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdxindex a690eb6c30..762cde20ee 100644--- a/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx+++ b/packages/docs/docs/openai-whisper/openai-whisper-api-to-captions.mdx@@ -32,6 +32,7 @@ const {captions} = openAiWhisperApiToCaptions({transcription});## See also+- [`isOpenAiWhisperJson()`](/docs/openai-whisper/is-open-ai-whisper-json)- [Source code for this function](https://github.com/remotion-dev/remotion/blob/main/packages/openai-whisper/src/openai-whisper-api-to-captions.ts)- [`@remotion/openai-whisper`](/docs/openai-whisper)- [`@remotion/install-whisper-cpp`](/docs/install-whisper-cpp)diff --git a/packages/docs/elements/captions/index.mdx b/packages/docs/elements/captions/index.mdxindex 1280cdd7c3..25ef2cd6c5 100644--- a/packages/docs/elements/captions/index.mdx+++ b/packages/docs/elements/captions/index.mdx@@ -7,4 +7,14 @@ import {ElementLibrary} from '@site/src/components/Elements/ElementLibrary';# Captions+## Importing captions into Studio<AvailableFrom v="4.0.521" />++<Step>1</Step> Install an Element from the [Captions category](/elements/captions), or add [`Interactive.captionsSchema`](/docs/interactivity-schema#interactivecaptionsschema) to an interactive component that accepts [`Caption[]`](/docs/captions/caption).<br />+<Step>2</Step> Select its caption layer in Studio.<br />+<Step>3</Step> In the Captions inspector, click **Import captions** and choose the JSON or SRT file exported by ElevenLabs or your Whisper transcription tool.++Studio detects and converts Remotion [`Caption[]`](/docs/captions/caption) JSON, ElevenLabs Speech-to-Text JSON, ElevenLabs segmented export JSON, OpenAI Whisper verbose JSON, and [SRT subtitles](/docs/captions/parse-srt). JSON transcripts must include word-level timestamps. SRT cue timestamps work, but cannot provide true word-by-word animation.++Files are processed locally.+<ElementLibrary category="captions" />diff --git a/packages/docs/sidebars.ts b/packages/docs/sidebars.tsindex 03e4a8385f..a0d4cd4b3d 100644--- a/packages/docs/sidebars.ts+++ b/packages/docs/sidebars.ts@@ -354,7 +354,10 @@ const sidebars: SidebarsConfig = {type: 'doc',id: 'elevenlabs/index',},- items: ['elevenlabs/elevenlabs-transcript-to-captions'],+ items: [+ 'elevenlabs/elevenlabs-transcript-to-captions',+ 'elevenlabs/is-elevenlabs-transcript-json',+ ],},{type: 'category',@@ -578,7 +581,10 @@ const sidebars: SidebarsConfig = {type: 'doc',id: 'openai-whisper/index',},- items: ['openai-whisper/openai-whisper-api-to-captions'],+ items: [+ 'openai-whisper/is-open-ai-whisper-json',+ 'openai-whisper/openai-whisper-api-to-captions',+ ],},{type: 'category',diff --git a/packages/docs/src/components/Elements/ElementPage.tsx b/packages/docs/src/components/Elements/ElementPage.tsxindex a8bd4ae5e4..378d11a9fd 100644--- a/packages/docs/src/components/Elements/ElementPage.tsx+++ b/packages/docs/src/components/Elements/ElementPage.tsx@@ -343,6 +343,14 @@ export const ElementPage: React.FC<ElementPageProps> = ({<div className={styles.details}><p className={styles.description}>{description}</p>+ {definition.category === 'captions' ? (+ <p className={styles.description} style={{marginTop: 8}}>+ Have a transcript?{' '}+ <a href="/elements/captions/#importing-captions-into-studio">+ Import JSON or SRT from the Captions inspector in Studio.+ </a>+ </p>+ ) : null}<dl className={styles.metadata}><div><dt>Dimensions</dt>diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.tsindex a492a185f7..ddb63e5b6e 100644--- a/packages/docs/src/data/articles.ts+++ b/packages/docs/src/data/articles.ts@@ -2528,6 +2528,15 @@ export const articles = [noAi: false,slug: 'elevenlabs/index',},+ {+ id: 'elevenlabs/is-elevenlabs-transcript-json',+ title: 'isElevenLabsTranscriptJson()',+ relativePath: 'docs/elevenlabs/is-elevenlabs-transcript-json.mdx',+ compId: 'articles-docs-elevenlabs-is-elevenlabs-transcript-json',+ crumb: '@remotion/elevenlabs',+ noAi: false,+ slug: 'elevenlabs/is-elevenlabs-transcript-json',+ },{id: 'enable-scss/enable-scss',title: 'enableScss()',@@ -5066,6 +5075,15 @@ export const articles = [noAi: false,slug: 'openai-whisper/index',},+ {+ id: 'openai-whisper/is-open-ai-whisper-json',+ title: 'isOpenAiWhisperJson()',+ relativePath: 'docs/openai-whisper/is-open-ai-whisper-json.mdx',+ compId: 'articles-docs-openai-whisper-is-open-ai-whisper-json',+ crumb: '@remotion/openai-whisper',+ noAi: false,+ slug: 'openai-whisper/is-open-ai-whisper-json',+ },{id: 'openai-whisper/openai-whisper-api-to-captions',title: 'openAiWhisperApiToCaptions()',diff --git a/packages/docs/static/generated/articles-docs-elevenlabs-is-elevenlabs-transcript-json.png b/packages/docs/static/generated/articles-docs-elevenlabs-is-elevenlabs-transcript-json.pngnew file mode 100644index 0000000000..e4abd628f3Binary files /dev/null and b/packages/docs/static/generated/articles-docs-elevenlabs-is-elevenlabs-transcript-json.png differdiff --git a/packages/docs/static/generated/articles-docs-openai-whisper-is-open-ai-whisper-json.png b/packages/docs/static/generated/articles-docs-openai-whisper-is-open-ai-whisper-json.pngnew file mode 100644index 0000000000..d0c1aab130Binary files /dev/null and b/packages/docs/static/generated/articles-docs-openai-whisper-is-open-ai-whisper-json.png differdiff --git a/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts b/packages/elevenlabs/src/elevenlabs-transcript-to-captions.tsindex 671644ba95..9df34fdea1 100644--- a/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts+++ b/packages/elevenlabs/src/elevenlabs-transcript-to-captions.ts@@ -9,34 +9,126 @@ export type ElevenLabsTranscriptToCaptionsOutput = {captions: Caption[];};+export const isElevenLabsTranscriptJson = (value: unknown): boolean => {+ return (+ typeof value === 'object' &&+ value !== null &&+ !Array.isArray(value) &&+ 'language_code' in value &&+ ('words' in value || 'text' in value)+ );+};+export const elevenLabsTranscriptToCaptions = ({transcript,}: ElevenLabsTranscriptToCaptionsInput): ElevenLabsTranscriptToCaptionsOutput => {const captions: Caption[] = [];- if (!transcript || !transcript.words || !Array.isArray(transcript.words)) {+ if (+ typeof transcript !== 'object' ||+ transcript === null ||+ Array.isArray(transcript)+ ) {throw new Error(- 'Invalid ElevenLabs transcript. The transcript must be generated with `timestamps_granularity` set to `"word"`. See https://www.remotion.dev/docs/elevenlabs/elevenlabs-transcript-to-captions',+ 'Invalid ElevenLabs transcript: transcript must be an object.',);}- const {words} = transcript;+ const {words}: {words: unknown} = transcript;+ if (!Array.isArray(words)) {+ throw new Error(+ 'Invalid ElevenLabs transcript: words must be an array generated with `timestamps_granularity` set to `"word"`. See https://www.remotion.dev/docs/elevenlabs/elevenlabs-transcript-to-captions',+ );+ }let isFirst = true;for (let i = 0; i < words.length; i++) {- const entry = words[i];+ const value: unknown = words[i];+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {+ throw new Error(+ `Invalid ElevenLabs transcript: words[${i}] must be an object.`,+ );+ }+ const entry = value as Record<string, unknown>;if (entry.type !== 'word') {continue;}- const prevEntry = i > 0 ? words[i - 1] : null;- const hasSpacing = prevEntry !== null && prevEntry.type === 'spacing';+ if (typeof entry.text !== 'string') {+ throw new Error(+ `Invalid ElevenLabs transcript: words[${i}].text must be a string.`,+ );+ }++ if (+ typeof entry.start !== 'number' ||+ !Number.isFinite(entry.start) ||+ entry.start < 0+ ) {+ throw new Error(+ `Invalid ElevenLabs transcript: words[${i}].start must be a finite, non-negative number.`,+ );+ }++ if (+ typeof entry.end !== 'number' ||+ !Number.isFinite(entry.end) ||+ entry.end < 0+ ) {+ throw new Error(+ `Invalid ElevenLabs transcript: words[${i}].end must be a finite, non-negative number.`,+ );+ }++ if (entry.end < entry.start) {+ throw new Error(+ `Invalid ElevenLabs transcript: words[${i}].end must not be earlier than words[${i}].start.`,+ );+ }++ let spacingStart: number | null = null;+ const previousValue: unknown = i > 0 ? words[i - 1] : null;+ if (+ !isFirst &&+ typeof previousValue === 'object' &&+ previousValue !== null &&+ !Array.isArray(previousValue)+ ) {+ const previousEntry = previousValue as Record<string, unknown>;+ if (previousEntry.type === 'spacing') {+ if (+ typeof previousEntry.start !== 'number' ||+ !Number.isFinite(previousEntry.start) ||+ previousEntry.start < 0+ ) {+ throw new Error(+ `Invalid ElevenLabs transcript: words[${i - 1}].start must be a finite, non-negative number.`,+ );+ }++ spacingStart = previousEntry.start;+ }+ }- const startMs =- !isFirst && hasSpacing ? prevEntry.start * 1000 : entry.start * 1000;+ const startMs = (spacingStart ?? entry.start) * 1000;const endMs = entry.end * 1000;+ if (endMs < startMs) {+ throw new Error(+ `Invalid ElevenLabs transcript: words[${i}].end must not be earlier than the caption start.`,+ );+ }++ if (+ captions.length > 0 &&+ startMs < captions[captions.length - 1].startMs+ ) {+ throw new Error(+ `Invalid ElevenLabs transcript: words[${i}].start is out of timestamp order.`,+ );+ }+const text = isFirst ? entry.text : ` ${entry.text}`;captions.push({diff --git a/packages/elevenlabs/src/index.ts b/packages/elevenlabs/src/index.tsindex 50006f266f..75f90603b2 100644--- a/packages/elevenlabs/src/index.ts+++ b/packages/elevenlabs/src/index.ts@@ -2,6 +2,7 @@ export {ElevenLabsTranscriptToCaptionsInput,ElevenLabsTranscriptToCaptionsOutput,elevenLabsTranscriptToCaptions,+ isElevenLabsTranscriptJson,} from './elevenlabs-transcript-to-captions';export {diff --git a/packages/elevenlabs/src/test/elevenlabs-transcript-to-captions.test.ts b/packages/elevenlabs/src/test/elevenlabs-transcript-to-captions.test.tsindex 4152504afb..0af8d82445 100644--- a/packages/elevenlabs/src/test/elevenlabs-transcript-to-captions.test.ts+++ b/packages/elevenlabs/src/test/elevenlabs-transcript-to-captions.test.ts@@ -1,9 +1,21 @@import {expect, test} from 'bun:test';import type {ElevenLabsTranscript} from '../elevenlabs-transcript';-import {elevenLabsTranscriptToCaptions} from '../elevenlabs-transcript-to-captions';+import {+ elevenLabsTranscriptToCaptions,+ isElevenLabsTranscriptJson,+} from '../elevenlabs-transcript-to-captions';import expectedCaptions from './gimme-gimme-captions-snapshot.json';import transcript from './gimme-gimme-transcript.json';+test('recognizes ElevenLabs transcript JSON', () => {+ expect(+ isElevenLabsTranscriptJson({language_code: 'eng', text: 'Hello'}),+ ).toBe(true);+ expect(isElevenLabsTranscriptJson({language: 'english', text: 'Hello'})).toBe(+ false,+ );+});+test('converts elevenlabs transcript to captions', () => {const {captions} = elevenLabsTranscriptToCaptions({transcript: transcript as ElevenLabsTranscript,@@ -33,3 +45,76 @@ test('handles empty words array', () => {expect(captions).toEqual([]);});++test.each([+ {+ name: 'non-object entry',+ words: [null],+ error: 'words[0] must be an object',+ },+ {+ name: 'invalid timestamp',+ words: [{type: 'word', text: 'Hello', start: Number.NaN, end: 1}],+ error: 'words[0].start must be a finite, non-negative number',+ },+ {+ name: 'invalid consumed spacing timestamp',+ words: [+ {type: 'word', text: 'Hello', start: 0, end: 1},+ {type: 'spacing', text: ' ', start: 'later', end: 2},+ {type: 'word', text: 'world', start: 2, end: 3},+ ],+ error: 'words[1].start must be a finite, non-negative number',+ },+ {+ name: 'reversed timestamps',+ words: [{type: 'word', text: 'Hello', start: 2, end: 1}],+ error: 'words[0].end must not be earlier than words[0].start',+ },+ {+ name: 'out-of-order words',+ words: [+ {type: 'word', text: 'Later', start: 2, end: 3},+ {type: 'word', text: 'Earlier', start: 1, end: 2},+ ],+ error: 'words[1].start is out of timestamp order',+ },+])('rejects $name', ({words, error}) => {+ expect(() =>+ elevenLabsTranscriptToCaptions({+ transcript: {words} as unknown as ElevenLabsTranscript,+ }),+ ).toThrow(`Invalid ElevenLabs transcript: ${error}`);+});++test('ignores unknown fields and entry types', () => {+ const {captions} = elevenLabsTranscriptToCaptions({+ transcript: {+ language_code: 'eng',+ language_probability: 1,+ text: 'Hello',+ transcription_id: 'test',+ provider_added_field: true,+ words: [+ {type: 'provider_event', provider_added_field: true},+ {+ type: 'word',+ text: 'Hello',+ start: 0,+ end: 1,+ provider_added_field: true,+ },+ ],+ } as unknown as ElevenLabsTranscript,+ });++ expect(captions).toEqual([+ {+ confidence: null,+ startMs: 0,+ endMs: 1000,+ text: 'Hello',+ timestampMs: 500,+ },+ ]);+});diff --git a/packages/example/e2e/captions-inspector.test.mts b/packages/example/e2e/captions-inspector.test.mtsindex 1aae3ad836..a5d9ca3523 100644--- a/packages/example/e2e/captions-inspector.test.mts+++ b/packages/example/e2e/captions-inspector.test.mts@@ -100,17 +100,92 @@ test.describe('captions inspector', () => {}).toPass({timeout: 30_000});await expect(defaultCaption).toBeEnabled();- await defaultCaption.fill('Editable captions');+ const importCaptionsButton = page.getByRole('button', {+ name: 'Import captions',+ exact: true,+ });+ const importCaptionsInput = page.getByLabel('Import captions file');+ await expect(importCaptionsButton).toBeVisible();++ const sourceBeforeFailedImport = fs.readFileSync(+ elementCaptionsFile,+ 'utf-8',+ );+ await importCaptionsInput.setInputFiles({+ name: 'broken.json',+ mimeType: 'application/json',+ buffer: Buffer.from(+ JSON.stringify({+ language_code: 'eng',+ segments: [+ {+ text: 'Broken',+ start_time: 0,+ end_time: 1,+ words: [{text: 'Broken', end_time: 1}],+ },+ ],+ }),+ ),+ });+ await expect(+ page.getByText(+ /broken\.json:.*segments\[0\]\.words\[0\]\.start_time must be a finite number/,+ ),+ ).toBeVisible();+ expect(fs.readFileSync(elementCaptionsFile, 'utf-8')).toBe(+ sourceBeforeFailedImport,+ );++ await importCaptionsInput.setInputFiles({+ name: 'captions.json',+ mimeType: 'application/json',+ buffer: Buffer.from(+ JSON.stringify({+ language_code: 'eng',+ segments: [+ {+ text: 'Imported captions',+ start_time: 0.1,+ end_time: 0.9,+ words: [+ {text: 'Imported', start_time: 0.1, end_time: 0.4},+ {text: ' ', start_time: 0.4, end_time: 0.5},+ {text: 'captions', start_time: 0.5, end_time: 0.9},+ ],+ },+ ],+ }),+ ),+ });+ await expect(+ page.getByText('Replace captions?', {exact: true}),+ ).toBeVisible();+ expect(fs.readFileSync(elementCaptionsFile, 'utf-8')).toBe(+ sourceBeforeFailedImport,+ );+ await page.getByRole('button', {name: 'Replace captions'}).click();+ await expect(+ page.getByText(+ 'Imported 2 captions from ElevenLabs segmented JSON. Undo is available.',+ ),+ ).toBeVisible();+ await expect(defaultCaption).toHaveValue('Imported');+ await expect+ .poll(() => fs.readFileSync(elementCaptionsFile, 'utf-8'))+ .toMatch(/startMs:\s*100[\s\S]*text:\s*['"]Imported['"]/);+ expect(fs.readFileSync(elementCallSiteFile, 'utf-8')).toBe(+ elementCallSiteSourceBefore,+ );++ await defaultCaption.fill('Edited imported caption');await defaultCaption.blur();await expect.poll(() => {- return /text:\s*['"]Editable captions['"]/.test(+ return /text:\s*['"]Edited imported caption['"]/.test(fs.readFileSync(elementCaptionsFile, 'utf-8'),);}).toBe(true);- expect(fs.readFileSync(elementCallSiteFile, 'utf-8')).toBe(- elementCallSiteSourceBefore,- );});});diff --git a/packages/openai-whisper/src/index.ts b/packages/openai-whisper/src/index.tsindex 11916012c8..1a88934b0e 100644--- a/packages/openai-whisper/src/index.ts+++ b/packages/openai-whisper/src/index.ts@@ -1,6 +1,7 @@export {OpenAiToCaptionsInput,OpenAiToCaptionsOutput,+ isOpenAiWhisperJson,openAiWhisperApiToCaptions,} from './openai-whisper-api-to-captions';diff --git a/packages/openai-whisper/src/openai-whisper-api-to-captions.ts b/packages/openai-whisper/src/openai-whisper-api-to-captions.tsindex 60809ce5aa..920a20cba3 100644--- a/packages/openai-whisper/src/openai-whisper-api-to-captions.ts+++ b/packages/openai-whisper/src/openai-whisper-api-to-captions.ts@@ -9,6 +9,21 @@ export type OpenAiToCaptionsOutput = {captions: Caption[];};+export const isOpenAiWhisperJson = (value: unknown): boolean => {+ return (+ typeof value === 'object' &&+ value !== null &&+ !Array.isArray(value) &&+ !('language_code' in value) &&+ 'text' in value &&+ ('task' in value ||+ 'language' in value ||+ 'duration' in value ||+ 'segments' in value ||+ 'words' in value)+ );+};+const escapeRegex = (text: string) => {return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');};@@ -31,36 +46,107 @@ export const openAiWhisperApiToCaptions = ({}: OpenAiToCaptionsInput): OpenAiToCaptionsOutput => {const captions: Caption[] = [];+ if (+ typeof transcription !== 'object' ||+ transcription === null ||+ Array.isArray(transcription)+ ) {+ throw new Error(+ 'Invalid OpenAI Whisper transcription: transcription must be an object.',+ );+ }++ if (transcription.task && transcription.task !== 'transcribe') {+ throw new Error(+ `Invalid OpenAI Whisper transcription: The transcription does need to be a "transcribe" task. The input you gave is "task": "${transcription.task}"`,+ );+ }+if (!transcription.words) {- if (transcription.task && transcription.task !== 'transcribe') {- throw new Error(- `The transcription does need to be a "transcribe" task. The input you gave is "task": "${transcription.task}"`,- );- }+ throw new Error(+ 'Invalid OpenAI Whisper transcription: The transcription does need to be been generated with `timestamp_granularities: ["word"]`',+ );+ }+ const {words} = transcription as unknown as {words: unknown};+ if (!Array.isArray(words)) {throw new Error(- 'The transcription does need to be been generated with `timestamp_granularities: ["word"]`',+ 'Invalid OpenAI Whisper transcription: words must be an array.',+ );+ }++ if (typeof transcription.text !== 'string') {+ throw new Error(+ 'Invalid OpenAI Whisper transcription: text must be a string.',);}let remainingText = transcription.text;+ let previousStart: number | null = null;++ for (let i = 0; i < words.length; i++) {+ const value: unknown = words[i];+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {+ throw new Error(+ `Invalid OpenAI Whisper transcription: words[${i}] must be an object.`,+ );+ }++ const word = value as Record<string, unknown>;+ if (typeof word.word !== 'string') {+ throw new Error(+ `Invalid OpenAI Whisper transcription: words[${i}].word must be a string.`,+ );+ }++ if (+ typeof word.start !== 'number' ||+ !Number.isFinite(word.start) ||+ word.start < 0+ ) {+ throw new Error(+ `Invalid OpenAI Whisper transcription: words[${i}].start must be a finite, non-negative number.`,+ );+ }++ if (+ typeof word.end !== 'number' ||+ !Number.isFinite(word.end) ||+ word.end < 0+ ) {+ throw new Error(+ `Invalid OpenAI Whisper transcription: words[${i}].end must be a finite, non-negative number.`,+ );+ }++ if (word.end < word.start) {+ throw new Error(+ `Invalid OpenAI Whisper transcription: words[${i}].end must not be earlier than words[${i}].start.`,+ );+ }++ if (previousStart !== null && word.start < previousStart) {+ throw new Error(+ `Invalid OpenAI Whisper transcription: words[${i}].start is out of timestamp order.`,+ );+ }- for (let i = 0; i < transcription.words.length; i++) {- const word = transcription.words[i];+ previousStart = word.start;const firstWord = i === 0;// https://github.com/remotion-dev/remotion/issues/5031+ const wordText = firstWord ? word.word.trimStart() : word.word;if (firstWord) {- word.word = word.word.trimStart();+ word.word = wordText;}const punctuation = `\\?,\\.\\%\\β\\!\\;\\:\\'\\"\\-\\_\\(\\)\\[\\]\\{\\}\\@\\#\\$\\^\\&\\*\\+\\=\\/\\|\\<\\>\\~\`\\u2018\\u2019\\u02bc\\uff07`;- const wordToMatch = word.word.replace(new RegExp(`^[${punctuation}]+`), '');+ const wordToMatch = wordText.replace(new RegExp(`^[${punctuation}]+`), '');const match = new RegExp(`^([\\s?${punctuation}]{0,4})${escapeWordForRegex(wordToMatch)}([${punctuation}]{0,3})?`,).exec(remainingText);if (!match) {throw new Error(- `Unable to parse punctuation from OpenAI Whisper output. Could not find word "${word.word}" in text "${remainingText.slice(0, 100)}". File an issue under https://remotion.dev/issue and post the input for openAiWhisperApiToCaptions() to ask for a fix.`,+ `Unable to parse punctuation from OpenAI Whisper output. Could not find word "${wordText}" in text "${remainingText.slice(0, 100)}". File an issue under https://remotion.dev/issue and post the input for openAiWhisperApiToCaptions() to ask for a fix.`,);}diff --git a/packages/openai-whisper/src/test/regressions.test.ts b/packages/openai-whisper/src/test/regressions.test.tsindex 2863b69365..07d3584040 100644--- a/packages/openai-whisper/src/test/regressions.test.ts+++ b/packages/openai-whisper/src/test/regressions.test.ts@@ -1,6 +1,9 @@import {expect, test} from 'bun:test';import type {OpenAiVerboseTranscription} from '../openai-format';-import {openAiWhisperApiToCaptions} from '../openai-whisper-api-to-captions';+import {+ isOpenAiWhisperJson,+ openAiWhisperApiToCaptions,+} from '../openai-whisper-api-to-captions';const transcript1: OpenAiVerboseTranscription = {task: 'transcribe',@@ -31,6 +34,15 @@ const transcript1: OpenAiVerboseTranscription = {language: 'english',};+test('recognizes OpenAI Whisper JSON', () => {+ expect(+ isOpenAiWhisperJson({language: 'english', text: 'Hello', words: []}),+ ).toBe(true);+ expect(+ isOpenAiWhisperJson({language_code: 'eng', text: 'Hello', words: []}),+ ).toBe(false);+});+const transcript2: OpenAiVerboseTranscription = {task: 'transcribe',text: 'in real-time functions.',@@ -168,3 +180,69 @@ test('Issue 7298 - apostrophe variants', () => {},]);});++test.each([+ {+ name: 'malformed word text',+ words: [{word: 42, start: 0, end: 1}],+ error: 'words[0].word must be a string',+ },+ {+ name: 'malformed word timestamp',+ words: [{word: 'Hello', start: Number.POSITIVE_INFINITY, end: 1}],+ error: 'words[0].start must be a finite, non-negative number',+ },+ {+ name: 'reversed timestamps',+ words: [{word: 'Hello', start: 2, end: 1}],+ error: 'words[0].end must not be earlier than words[0].start',+ },+ {+ name: 'out-of-order words',+ words: [+ {word: 'Hello', start: 2, end: 3},+ {word: 'world', start: 1, end: 2},+ ],+ error: 'words[1].start is out of timestamp order',+ },+])('rejects $name', ({words, error}) => {+ expect(() =>+ openAiWhisperApiToCaptions({+ transcription: {+ text: 'Hello world',+ words,+ } as unknown as OpenAiVerboseTranscription,+ }),+ ).toThrow(`Invalid OpenAI Whisper transcription: ${error}`);+});++test('accepts extra provider fields', () => {+ expect(+ openAiWhisperApiToCaptions({+ transcription: {+ duration: 1,+ language: 'english',+ text: 'Hello',+ words: [+ {+ word: 'Hello',+ start: 0,+ end: 1,+ provider_added_field: true,+ },+ ],+ provider_added_field: true,+ } as unknown as OpenAiVerboseTranscription,+ }),+ ).toEqual({+ captions: [+ {+ confidence: null,+ endMs: 1000,+ startMs: 0,+ text: 'Hello',+ timestampMs: 500,+ },+ ],+ });+});diff --git a/packages/studio/package.json b/packages/studio/package.jsonindex 341f8ec78d..275ef86508 100644--- a/packages/studio/package.json+++ b/packages/studio/package.json@@ -31,9 +31,11 @@"semver": "7.5.3","remotion": "workspace:*","@remotion/captions": "workspace:*",+ "@remotion/elevenlabs": "workspace:*","@remotion/media": "workspace:*","@remotion/player": "workspace:*","@remotion/media-utils": "workspace:*",+ "@remotion/openai-whisper": "workspace:*","@remotion/renderer": "workspace:*","@remotion/web-renderer": "workspace:*","@remotion/studio-shared": "workspace:*",diff --git a/packages/studio/src/components/CaptionInspector.tsx b/packages/studio/src/components/CaptionInspector.tsxindex f312e7111b..a3a1edb600 100644--- a/packages/studio/src/components/CaptionInspector.tsx+++ b/packages/studio/src/components/CaptionInspector.tsx@@ -1,10 +1,28 @@import type {Caption} from '@remotion/captions';-import React from 'react';+import React, {useCallback, useRef, useState} from 'react';import {LIGHT_TEXT} from '../helpers/colors';+import {FOCUS_VISIBLE_ONLY_CLASS_NAME} from '../helpers/hoverable';+import {UploadIcon} from '../icons/upload';import {CaptionTextEditor} from './CaptionTextEditor';+import {useConfirmationDialog} from './ConfirmationDialog';+import {InlineAction} from './InlineAction';import {CollapsibleInspectorSectionHeader} from './InspectorPanel/CollapsibleInspectorSectionHeader';import {InspectorSectionHeader} from './InspectorPanel/common';import {sectionHeaderEnd} from './InspectorPanel/styles';+import {showNotification} from './Notifications/NotificationCenter';+import {parseCaptionFile, type ParsedCaptionFile} from './parse-caption-file';++const importTooltip = `Import captions++Supports Remotion Caption[] JSON, ElevenLabs Speech-to-Text JSON, ElevenLabs segmented export JSON, OpenAI Whisper verbose JSON, and SRT subtitles. JSON transcripts need word-level timestamps. Files are processed locally.`;++const formatLabels: Record<ParsedCaptionFile['format'], string> = {+ remotion: 'Remotion JSON',+ elevenlabs: 'ElevenLabs JSON',+ 'elevenlabs-segments': 'ElevenLabs segmented JSON',+ 'openai-whisper': 'OpenAI Whisper JSON',+ srt: 'SRT',+};const readOnlyStatus: React.CSSProperties = {color: LIGHT_TEXT,@@ -21,6 +39,7 @@ export const CaptionInspector: React.FC<{readonly onTextChange: (captions: Caption[]) => void;readonly onTextSave: ((captions: Caption[]) => void) | null;readonly onTextCancel: (() => void) | null;+ readonly onReplaceCaptions: ((captions: Caption[]) => Promise<void>) | null;readonly onToggle: () => void;readonly readOnly: boolean;readonly readOnlyTitle: string | null;@@ -30,16 +49,94 @@ export const CaptionInspector: React.FC<{onTextChange,onTextSave,onTextCancel,+ onReplaceCaptions,onToggle,readOnly,readOnlyTitle,}) => {+ const fileInput = useRef<HTMLInputElement>(null);+ const [isImporting, setIsImporting] = useState(false);+ const confirm = useConfirmationDialog();++ const importCaptions = useCallback(+ async (event: React.ChangeEvent<HTMLInputElement>) => {+ const file = event.currentTarget.files?.[0];+ event.currentTarget.value = '';+ if (!file || onReplaceCaptions === null) {+ return;+ }++ setIsImporting(true);+ try {+ const parsed = parseCaptionFile({+ fileName: file.name,+ contents: await file.text(),+ });+ if (+ captions.length > 0 &&+ !(await confirm({+ title: 'Replace captions?',+ message: `This will replace ${captions.length} existing ${captions.length === 1 ? 'caption' : 'captions'} with ${parsed.captions.length} imported ${parsed.captions.length === 1 ? 'caption' : 'captions'}. You can undo this action.`,+ confirmLabel: 'Replace captions',+ cancelLabel: 'Cancel',+ }))+ ) {+ return;+ }++ await onReplaceCaptions(parsed.captions);+ const captionLabel =+ parsed.captions.length === 1 ? 'caption' : 'captions';+ showNotification(+ `Imported ${parsed.captions.length} ${captionLabel} from ${formatLabels[parsed.format]}. Undo is available.`,+ 4000,+ );+ } catch (error) {+ showNotification(+ `Could not import ${file.name}: ${error instanceof Error ? error.message : String(error)}`,+ 5000,+ );+ } finally {+ setIsImporting(false);+ }+ },+ [captions.length, confirm, onReplaceCaptions],+ );+return (<><InspectorSectionHeader><CollapsibleInspectorSectionHeaderaction={<div style={sectionHeaderEnd}>+ {onReplaceCaptions === null ? null : (+ <>+ <input+ ref={fileInput}+ accept=".json,.srt"+ aria-label="Import captions file"+ hidden+ onChange={importCaptions}+ type="file"+ />+ <InlineAction+ aria-label="Import captions"+ className={FOCUS_VISIBLE_ONLY_CLASS_NAME}+ disabled={isImporting}+ onClick={() => fileInput.current?.click()}+ renderAction={(color) => (+ <UploadIcon+ aria-hidden="true"+ color={color}+ focusable="false"+ style={{height: 16, width: 16}}+ />+ )}+ title={importTooltip}+ variant={null}+ />+ </>+ )}{readOnly ? (<div style={readOnlyStatus} title={readOnlyTitle ?? undefined}>Read onlydiff --git a/packages/studio/src/components/InlineCaptionInspector.tsx b/packages/studio/src/components/InlineCaptionInspector.tsxindex 2591d1fc1e..d43935b008 100644--- a/packages/studio/src/components/InlineCaptionInspector.tsx+++ b/packages/studio/src/components/InlineCaptionInspector.tsx@@ -15,7 +15,10 @@ import {Internals} from 'remotion';import type {CodePosition} from '../error-overlay/react-overlay/utils/get-source-map';import {StudioServerConnectionCtx} from '../helpers/client-id';import {CaptionInspector} from './CaptionInspector';-import {saveInlineCaptionPatches} from './Timeline/save-sequence-prop';+import {+ saveInlineCaptionPatches,+ saveSequencePropsOrThrow,+} from './Timeline/save-sequence-prop';const serializeCaptions = (captions: Caption[]): string => {return JSON.stringify(captions);@@ -171,6 +174,58 @@ export const InlineCaptionInspector: React.FC<{],);+ const replaceCaptions = useCallback(+ async (nextCaptions: Caption[]) => {+ if (!canSave || clientId === null) {+ throw new Error('Caption importing is unavailable');+ }++ setDraftCaptions(nextCaptions);+ setDragOverrides(+ nodePath,+ 'captions',+ Internals.makeStaticDragOverride(nextCaptions),+ );++ try {+ await saveSequencePropsOrThrow({+ changes: [+ {+ fileName: validatedLocation.source,+ nodePath,+ fieldKey: 'captions',+ value: nextCaptions,+ defaultValue: null,+ schema: controls.schema,+ },+ ],+ addedKeyframes: null,+ movedKeyframes: null,+ setPropStatuses,+ clientId,+ undoLabel: 'Import captions',+ redoLabel: 'Import captions again',+ });+ savedCaptions.current = nextCaptions;+ } catch (error) {+ setDraftCaptions(savedCaptions.current);+ throw error;+ } finally {+ clearDragOverrides(nodePath);+ }+ },+ [+ canSave,+ clearDragOverrides,+ clientId,+ controls.schema,+ nodePath,+ setDragOverrides,+ setPropStatuses,+ validatedLocation.source,+ ],+ );+const readOnlyTitle = readOnlyStudio? 'Caption editing is unavailable in read-only Studio': clientId === null@@ -189,6 +244,7 @@ export const InlineCaptionInspector: React.FC<{onToggle={onToggle}readOnly={!canSave}readOnlyTitle={canSave ? null : readOnlyTitle}+ onReplaceCaptions={canSave ? replaceCaptions : null}/>);};diff --git a/packages/studio/src/components/Timeline/save-prop-queue.ts b/packages/studio/src/components/Timeline/save-prop-queue.tsindex ccc8ba7c6c..4f093e36f9 100644--- a/packages/studio/src/components/Timeline/save-prop-queue.ts+++ b/packages/studio/src/components/Timeline/save-prop-queue.ts@@ -54,14 +54,17 @@ export type EnqueueSaveOptions<TResponse> = {errorLabel: string;};-export const enqueueSavePropChange = <TResponse>({- nodePath,- setPropStatuses,- applyOptimistic,- applyServerResponse,- apiCall,- errorLabel,-}: EnqueueSaveOptions<TResponse>): Promise<void> => {+const enqueueSavePropChangeInternal = <TResponse>(+ {+ nodePath,+ setPropStatuses,+ applyOptimistic,+ applyServerResponse,+ apiCall,+ errorLabel,+ }: EnqueueSaveOptions<TResponse>,+ rejectOnError: boolean,+): Promise<void> => {const q = getQueue(nodePath);if (q.cancelled) {@@ -98,6 +101,10 @@ export const enqueueSavePropChange = <TResponse>({myQueue.cancelled = true;dropQueue(nodePath, myQueue);+ if (rejectOnError) {+ throw err;+ }+showNotification(`${errorLabel}: ${err instanceof Error ? err.message : String(err)}`,4000,@@ -108,3 +115,15 @@ export const enqueueSavePropChange = <TResponse>({myQueue.chain = next;return next;};++export const enqueueSavePropChange = <TResponse>(+ options: EnqueueSaveOptions<TResponse>,+): Promise<void> => {+ return enqueueSavePropChangeInternal(options, false);+};++export const enqueueSavePropChangeOrThrow = <TResponse>(+ options: EnqueueSaveOptions<TResponse>,+): Promise<void> => {+ return enqueueSavePropChangeInternal(options, true);+};diff --git a/packages/studio/src/components/Timeline/save-sequence-prop.ts b/packages/studio/src/components/Timeline/save-sequence-prop.tsindex 8cd844e3ce..e7bef9caf5 100644--- a/packages/studio/src/components/Timeline/save-sequence-prop.ts+++ b/packages/studio/src/components/Timeline/save-sequence-prop.ts@@ -16,7 +16,10 @@ import {type MoveEffectKeyframeChange,type MoveSequenceKeyframeChange,} from './call-move-keyframe';-import {enqueueSavePropChange} from './save-prop-queue';+import {+ enqueueSavePropChange,+ enqueueSavePropChangeOrThrow,+} from './save-prop-queue';export type SetPropStatuses = (nodePath: SequencePropsSubscriptionKey,@@ -102,15 +105,18 @@ export const saveInlineCaptionPatches = ({});};-export const saveSequenceProps = ({- changes,- addedKeyframes,- movedKeyframes,- setPropStatuses,- clientId,- undoLabel,- redoLabel,-}: SaveSequencePropsOptions): Promise<void> => {+const saveSequencePropsInternal = (+ {+ changes,+ addedKeyframes,+ movedKeyframes,+ setPropStatuses,+ clientId,+ undoLabel,+ redoLabel,+ }: SaveSequencePropsOptions,+ rejectOnError: boolean,+): Promise<void> => {const keyframesToAdd = addedKeyframes === null ? [] : addedKeyframes;const sequenceKeyframes =movedKeyframes === null ? [] : movedKeyframes.sequenceKeyframes;@@ -136,7 +142,10 @@ export const saveSequenceProps = ({throw new Error('Expected a sequence prop change');}- return enqueueSavePropChange({+ const enqueue = rejectOnError+ ? enqueueSavePropChangeOrThrow+ : enqueueSavePropChange;+ return enqueue({nodePath: change.nodePath,setPropStatuses,applyOptimistic: (prev) =>@@ -256,3 +265,15 @@ export const saveSequenceProps = ({redoLabel,}).then(() => undefined);};++export const saveSequenceProps = (+ options: SaveSequencePropsOptions,+): Promise<void> => {+ return saveSequencePropsInternal(options, false);+};++export const saveSequencePropsOrThrow = (+ options: SaveSequencePropsOptions,+): Promise<void> => {+ return saveSequencePropsInternal(options, true);+};diff --git a/packages/studio/src/components/parse-caption-file.ts b/packages/studio/src/components/parse-caption-file.tsnew file mode 100644index 0000000000..d5cdcde565--- /dev/null+++ b/packages/studio/src/components/parse-caption-file.ts@@ -0,0 +1,284 @@+import type {Caption} from '@remotion/captions';+import {parseSrt} from '@remotion/captions';+import {+ elevenLabsTranscriptToCaptions,+ isElevenLabsTranscriptJson,+ type ElevenLabsTranscript,+} from '@remotion/elevenlabs';+import {+ isOpenAiWhisperJson,+ openAiWhisperApiToCaptions,+ type OpenAiVerboseTranscription,+} from '@remotion/openai-whisper';++export type ParsedCaptionFile = {+ readonly captions: Caption[];+ readonly format:+ | 'remotion'+ | 'elevenlabs'+ | 'elevenlabs-segments'+ | 'openai-whisper'+ | 'srt';+};++const isObject = (value: unknown): value is Record<string, unknown> => {+ return typeof value === 'object' && value !== null && !Array.isArray(value);+};++const isElevenLabsSegmentedJson = (value: unknown): boolean => {+ return isObject(value) && 'language_code' in value && 'segments' in value;+};++const isFiniteNumber = (value: unknown): value is number => {+ return typeof value === 'number' && Number.isFinite(value);+};++const errorMessage = (error: unknown) => {+ return error instanceof Error ? error.message : String(error);+};++const validateCaptions = (value: unknown): Caption[] => {+ if (!Array.isArray(value)) {+ throw new Error('captions must be an array of Remotion captions.');+ }++ let previousStart: number | null = null;+ for (const [index, caption] of value.entries()) {+ const path = `captions[${index}]`;+ if (!isObject(caption)) {+ throw new Error(`${path} must be an object.`);+ }++ if (typeof caption.text !== 'string') {+ throw new Error(`${path}.text must be a string.`);+ }++ if (!isFiniteNumber(caption.startMs)) {+ throw new Error(`${path}.startMs must be a finite number.`);+ }++ if (caption.startMs < 0) {+ throw new Error(`${path}.startMs must be a non-negative number.`);+ }++ if (!isFiniteNumber(caption.endMs)) {+ throw new Error(`${path}.endMs must be a finite number.`);+ }++ if (caption.endMs < 0) {+ throw new Error(`${path}.endMs must be a non-negative number.`);+ }++ if (caption.endMs < caption.startMs) {+ throw new Error(`${path}.endMs must not be earlier than startMs.`);+ }++ if (caption.timestampMs !== null && !isFiniteNumber(caption.timestampMs)) {+ throw new Error(`${path}.timestampMs must be a finite number or null.`);+ }++ if (caption.confidence !== null && !isFiniteNumber(caption.confidence)) {+ throw new Error(`${path}.confidence must be a finite number or null.`);+ }++ if (+ typeof caption.confidence === 'number' &&+ (caption.confidence < 0 || caption.confidence > 1)+ ) {+ throw new Error(`${path}.confidence must be between 0 and 1.`);+ }++ if (+ caption.pageBreakAfter !== undefined &&+ typeof caption.pageBreakAfter !== 'boolean'+ ) {+ throw new Error(+ `${path}.pageBreakAfter must be a boolean when provided.`,+ );+ }++ if (previousStart !== null && caption.startMs < previousStart) {+ throw new Error(`${path}.startMs is out of timestamp order.`);+ }++ previousStart = caption.startMs;+ }++ return value as Caption[];+};++const convertSegmentedElevenLabsTranscript = (+ transcript: unknown,+): Caption[] => {+ if (!isObject(transcript) || !Array.isArray(transcript.segments)) {+ throw new Error('transcript.segments must be an array.');+ }++ const captions: Caption[] = [];+ let pendingSpacing = '';+ let pendingSpacingStart: number | null = null;+ let previousStart: number | null = null;++ for (const [segmentIndex, segment] of transcript.segments.entries()) {+ const segmentPath = `transcript.segments[${segmentIndex}]`;+ if (!isObject(segment)) {+ throw new Error(`${segmentPath} must be an object.`);+ }++ if (!Array.isArray(segment.words)) {+ throw new Error(`${segmentPath}.words must be an array.`);+ }++ for (const [wordIndex, word] of segment.words.entries()) {+ const path = `${segmentPath}.words[${wordIndex}]`;+ if (!isObject(word)) {+ throw new Error(`${path} must be an object.`);+ }++ if (typeof word.text !== 'string') {+ throw new Error(`${path}.text must be a string.`);+ }++ if (!isFiniteNumber(word.start_time)) {+ throw new Error(`${path}.start_time must be a finite number.`);+ }++ if (word.start_time < 0) {+ throw new Error(`${path}.start_time must be a non-negative number.`);+ }++ if (!isFiniteNumber(word.end_time)) {+ throw new Error(`${path}.end_time must be a finite number.`);+ }++ if (word.end_time < 0) {+ throw new Error(`${path}.end_time must be a non-negative number.`);+ }++ if (word.end_time < word.start_time) {+ throw new Error(+ `${path}.end_time must not be earlier than start_time.`,+ );+ }++ if (previousStart !== null && word.start_time < previousStart) {+ throw new Error(`${path}.start_time is out of timestamp order.`);+ }++ previousStart = word.start_time;+ if (word.text.trim() === '') {+ pendingSpacingStart ??= word.start_time;+ pendingSpacing += word.text;+ continue;+ }++ const startMs =+ captions.length > 0 && pendingSpacingStart !== null+ ? pendingSpacingStart * 1000+ : word.start_time * 1000;+ const endMs = word.end_time * 1000;+ captions.push({+ confidence: null,+ startMs,+ endMs,+ text: `${pendingSpacing}${word.text}`,+ timestampMs: (startMs + endMs) / 2,+ });+ pendingSpacing = '';+ pendingSpacingStart = null;+ }+ }++ return captions;+};++export const parseCaptionFile = ({+ fileName,+ contents,+}: {+ fileName: string;+ contents: string;+}): ParsedCaptionFile => {+ const lowerCaseFileName = fileName.toLowerCase();+ if (lowerCaseFileName.endsWith('.srt')) {+ try {+ return {+ captions: validateCaptions(parseSrt({input: contents}).captions),+ format: 'srt',+ };+ } catch (error) {+ throw new Error(`Invalid SRT file: ${errorMessage(error)}`);+ }+ }++ if (!lowerCaseFileName.endsWith('.json')) {+ throw new Error('Unsupported caption file. Choose a .json or .srt file.');+ }++ let parsed: unknown;+ try {+ parsed = JSON.parse(contents);+ } catch (error) {+ throw new Error(`Invalid JSON: ${errorMessage(error)}`);+ }++ if (Array.isArray(parsed)) {+ return {captions: validateCaptions(parsed), format: 'remotion'};+ }++ if (!isObject(parsed)) {+ throw new Error(+ 'Unsupported transcription JSON. Expected an object or Caption[] array.',+ );+ }++ if (isElevenLabsSegmentedJson(parsed)) {+ try {+ return {+ captions: validateCaptions(+ convertSegmentedElevenLabsTranscript(parsed),+ ),+ format: 'elevenlabs-segments',+ };+ } catch (error) {+ throw new Error(+ `Detected ElevenLabs segmented JSON, but ${errorMessage(error)}`,+ );+ }+ }++ if (isElevenLabsTranscriptJson(parsed)) {+ try {+ const {captions} = elevenLabsTranscriptToCaptions({+ transcript: parsed as ElevenLabsTranscript,+ });+ return {+ captions: validateCaptions(captions),+ format: 'elevenlabs',+ };+ } catch (error) {+ throw new Error(`Detected ElevenLabs JSON, but ${errorMessage(error)}`);+ }+ }++ if (isOpenAiWhisperJson(parsed)) {+ try {+ const {captions} = openAiWhisperApiToCaptions({+ transcription: parsed as unknown as OpenAiVerboseTranscription,+ });+ return {+ captions: validateCaptions(captions),+ format: 'openai-whisper',+ };+ } catch (error) {+ throw new Error(+ `Detected OpenAI Whisper JSON, but ${errorMessage(error)}`,+ );+ }+ }++ const keys = Object.keys(parsed);+ throw new Error(+ `Unsupported transcription JSON. Found top-level keys: ${keys.length === 0 ? '(none)' : keys.join(', ')}.`,+ );+};diff --git a/packages/studio/src/test/parse-caption-file.test.ts b/packages/studio/src/test/parse-caption-file.test.tsnew file mode 100644index 0000000000..d3c8acb9a1--- /dev/null+++ b/packages/studio/src/test/parse-caption-file.test.ts@@ -0,0 +1,225 @@+import {expect, test} from 'bun:test';+import {parseCaptionFile} from '../components/parse-caption-file';++const parseJson = (value: unknown) => {+ return parseCaptionFile({+ fileName: 'captions.json',+ contents: JSON.stringify(value),+ });+};++test('imports supported caption and transcription formats', () => {+ const remotion = parseJson([+ {+ text: 'Hello',+ startMs: 0,+ endMs: 500,+ timestampMs: 250,+ confidence: null,+ },+ ]);+ expect(remotion).toEqual({+ format: 'remotion',+ captions: [+ {+ text: 'Hello',+ startMs: 0,+ endMs: 500,+ timestampMs: 250,+ confidence: null,+ },+ ],+ });+ expect(parseJson([])).toEqual({format: 'remotion', captions: []});++ const elevenLabs = parseJson({+ language_code: 'eng',+ text: 'Hello world',+ words: [+ {text: 'Hello', type: 'word', start: 0, end: 0.4},+ {text: ' ', type: 'spacing', start: 0.4, end: 0.5},+ {text: 'world', type: 'word', start: 0.5, end: 0.9},+ ],+ });+ expect(elevenLabs.format).toBe('elevenlabs');+ expect(elevenLabs.captions.map(({text}) => text)).toEqual([+ 'Hello',+ ' world',+ ]);++ const segmented = parseJson({+ language_code: 'eng',+ segments: [+ {+ text: 'Hello world',+ start_time: 0,+ end_time: 0.9,+ words: [+ {text: 'Hello', start_time: 0, end_time: 0.4},+ {text: ' ', start_time: 0.4, end_time: 0.5},+ {text: 'world', start_time: 0.5, end_time: 0.9},+ ],+ },+ ],+ });+ expect(segmented).toEqual({+ format: 'elevenlabs-segments',+ captions: [+ {+ confidence: null,+ endMs: 400,+ startMs: 0,+ text: 'Hello',+ timestampMs: 200,+ },+ {+ confidence: null,+ endMs: 900,+ startMs: 400,+ text: ' world',+ timestampMs: 650,+ },+ ],+ });++ const openAi = parseJson({+ duration: 0.9,+ language: 'english',+ task: 'transcribe',+ text: 'Hello, world!',+ words: [+ {word: 'Hello', start: 0, end: 0.4},+ {word: 'world', start: 0.5, end: 0.9},+ ],+ });+ expect(openAi.format).toBe('openai-whisper');+ expect(openAi.captions.map(({text}) => text)).toEqual(['Hello,', ' world!']);++ const srt = parseCaptionFile({+ fileName: 'CAPTIONS.SRT',+ contents: '1\n00:00:00,000 --> 00:00:01,000\nHello world\n',+ });+ expect(srt.format).toBe('srt');+ expect(srt.captions[0]).toEqual({+ confidence: 1,+ endMs: 1000,+ startMs: 0,+ text: 'Hello world',+ timestampMs: 500,+ });+});++test.each([+ {+ name: 'invalid JSON',+ fileName: 'captions.json',+ contents: '{',+ error: 'Invalid JSON:',+ },+ {+ name: 'unknown JSON',+ fileName: 'captions.json',+ contents: JSON.stringify({language: 'english', utterances: []}),+ error: 'Found top-level keys: language, utterances',+ },+ {+ name: 'OpenAI response without words',+ fileName: 'captions.json',+ contents: JSON.stringify({+ duration: 1,+ language: 'english',+ text: 'Hello',+ }),+ error: 'timestamp_granularities: ["word"]',+ },+ {+ name: 'malformed ElevenLabs segmented word',+ fileName: 'captions.json',+ contents: JSON.stringify({+ language_code: 'eng',+ segments: [+ {+ text: 'Hello',+ start_time: 0,+ end_time: 1,+ words: [{text: 'Hello', end_time: 1}],+ },+ ],+ }),+ error: 'segments[0].words[0].start_time must be a finite number',+ },+ {+ name: 'malformed ElevenLabs API word',+ fileName: 'captions.json',+ contents: JSON.stringify({+ language_code: 'eng',+ text: 'Hello',+ words: [{type: 'word', text: 'Hello', start: 'now', end: 1}],+ }),+ error:+ 'Detected ElevenLabs JSON, but Invalid ElevenLabs transcript: words[0].start',+ },+ {+ name: 'malformed OpenAI Whisper word',+ fileName: 'captions.json',+ contents: JSON.stringify({+ duration: 1,+ language: 'english',+ text: 'Hello',+ words: [{word: 'Hello', start: 0, end: 'later'}],+ }),+ error:+ 'Detected OpenAI Whisper JSON, but Invalid OpenAI Whisper transcription: words[0].end',+ },+ {+ name: 'invalid canonical confidence',+ fileName: 'captions.json',+ contents: JSON.stringify([+ {+ text: 'Hello',+ startMs: 0,+ endMs: 1,+ timestampMs: 0.5,+ confidence: 2,+ },+ ]),+ error: 'captions[0].confidence must be between 0 and 1',+ },+ {+ name: 'reversed canonical timing',+ fileName: 'captions.json',+ contents: JSON.stringify([+ {+ text: 'Hello',+ startMs: 2,+ endMs: 1,+ timestampMs: 1.5,+ confidence: null,+ },+ ]),+ error: 'captions[0].endMs must not be earlier than startMs',+ },+ {+ name: 'out-of-order canonical captions',+ fileName: 'captions.json',+ contents: JSON.stringify([+ {+ text: 'Later',+ startMs: 2,+ endMs: 3,+ timestampMs: 2.5,+ confidence: null,+ },+ {+ text: 'Earlier',+ startMs: 1,+ endMs: 2,+ timestampMs: 1.5,+ confidence: null,+ },+ ]),+ error: 'captions[1].startMs is out of timestamp order',+ },+])('rejects $name', ({fileName, contents, error}) => {+ expect(() => parseCaptionFile({fileName, contents})).toThrow(error);+});